Implement a TTL on cached fetchurl/fetchTarball results

This is because we don't want to do HTTP requests on every evaluation,
even though we can prevent a full redownload via the cached ETag. The
default is one hour.

3 files changed, 34 insertions, 13 deletions

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 77ca42042d5f..03a3c82863b1 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1631,14 +1631,22 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     string expectedETag;
 
+    int ttl = settings.get("tarball-ttl", 60 * 60);
+    bool skip = false;
+
     if (pathExists(fileLink) && pathExists(dataFile)) {
         storePath = readLink(fileLink);
         store->addTempRoot(storePath);
         if (store->isValidPath(storePath)) {
             auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
-            if (ss.size() >= 2 && ss[0] == url) {
-                printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]);
-                expectedETag = ss[1];
+            if (ss.size() >= 3 && ss[0] == url) {
+                time_t lastChecked;
+                if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0))
+                    skip = true;
+                else if (!ss[1].empty()) {
+                    printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]);
+                    expectedETag = ss[1];
+                }
             }
         } else
             storePath = "";
@@ -1648,19 +1656,22 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     auto p = url.rfind('/');
     if (p != string::npos) name = string(url, p + 1);
 
-    if (expectedETag.empty())
-        printMsg(lvlInfo, format("downloading ‘%1%’...") % url);
-    else
-        printMsg(lvlInfo, format("checking ‘%1%’...") % url);
-    Curl curl;
+    if (!skip) {
 
-    if (curl.fetch(url, expectedETag))
-        storePath = store->addTextToStore(name, curl.data, PathSet(), state.repair);
+        if (expectedETag.empty())
+            printMsg(lvlInfo, format("downloading ‘%1%’...") % url);
+        else
+            printMsg(lvlInfo, format("checking ‘%1%’...") % url);
+        Curl curl;
+
+        if (curl.fetch(url, expectedETag))
+            storePath = store->addTextToStore(name, curl.data, PathSet(), state.repair);
 
-    assert(!storePath.empty());
-    replaceSymlink(storePath, fileLink);
+        assert(!storePath.empty());
+        replaceSymlink(storePath, fileLink);
 
-    writeFile(dataFile, url + "\n" + curl.etag + "\n");
+        writeFile(dataFile, url + "\n" + curl.etag + "\n" + int2String(time(0)) + "\n");
+    }
 
     if (unpack) {
         Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index e382b3aac03a..f900fb290bdb 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -143,6 +143,14 @@ bool Settings::get(const string & name, bool def)
 }
 
 
+int Settings::get(const string & name, int def)
+{
+    int res = def;
+    _get(res, name);
+    return res;
+}
+
+
 void Settings::update()
 {
     _get(tryFallback, "build-fallback");
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 0230a540e655..0a1072e36999 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -27,6 +27,8 @@ struct Settings {
 
     bool get(const string & name, bool def);
 
+    int get(const string & name, int def);
+
     void update();
 
     string pack();