6 files changed, 284 insertions, 80 deletions

diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc
index c03d720bde82..13760490d9c4 100644
--- a/src/libexpr/common-opts.cc
+++ b/src/libexpr/common-opts.cc
@@ -1,5 +1,6 @@
 #include "common-opts.hh"
-#include "../libmain/shared.hh"
+#include "shared.hh"
+#include "download.hh"
 #include "util.hh"
 
 
@@ -53,7 +54,9 @@ bool parseSearchPathArg(Strings::iterator & i,
 
 Path lookupFileArg(EvalState & state, string s)
 {
-    if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
+    if (isUri(s))
+        return downloadFileCached(s, true);
+    else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);
     } else
diff --git a/src/libexpr/download.cc b/src/libexpr/download.cc
new file mode 100644
index 000000000000..18ab6fbcda73
--- /dev/null
+++ b/src/libexpr/download.cc
@@ -0,0 +1,235 @@
+#include "download.hh"
+#include "util.hh"
+#include "globals.hh"
+#include "hash.hh"
+#include "store-api.hh"
+
+#include <curl/curl.h>
+
+namespace nix {
+
+struct Curl
+{
+    CURL * curl;
+    string data;
+    string etag, status, expectedETag;
+
+    struct curl_slist * requestHeaders;
+
+    static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp)
+    {
+        Curl & c(* (Curl *) userp);
+        size_t realSize = size * nmemb;
+        c.data.append((char *) contents, realSize);
+        return realSize;
+    }
+
+    static size_t headerCallback(void * contents, size_t size, size_t nmemb, void * userp)
+    {
+        Curl & c(* (Curl *) userp);
+        size_t realSize = size * nmemb;
+        string line = string((char *) contents, realSize);
+        printMsg(lvlVomit, format("got header: %1%") % trim(line));
+        if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
+            c.etag = "";
+            auto ss = tokenizeString<vector<string>>(line, " ");
+            c.status = ss.size() >= 2 ? ss[1] : "";
+        } else {
+            auto i = line.find(':');
+            if (i != string::npos) {
+                string name = trim(string(line, 0, i));
+                if (name == "ETag") { // FIXME: case
+                    c.etag = trim(string(line, i + 1));
+                    /* Hack to work around a GitHub bug: it sends
+                       ETags, but ignores If-None-Match. So if we get
+                       the expected ETag on a 200 response, then shut
+                       down the connection because we already have the
+                       data. */
+                    printMsg(lvlDebug, format("got ETag: %1%") % c.etag);
+                    if (c.etag == c.expectedETag && c.status == "200") {
+                        printMsg(lvlDebug, format("shutting down on 200 HTTP response with expected ETag"));
+                        return 0;
+                    }
+                }
+            }
+        }
+        return realSize;
+    }
+
+    static int progressCallback(void * clientp, double dltotal, double dlnow, double ultotal, double ulnow)
+    {
+        return _isInterrupted;
+    }
+
+    Curl()
+    {
+        requestHeaders = 0;
+
+        curl = curl_easy_init();
+        if (!curl) throw Error("unable to initialize curl");
+
+        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+        curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str());
+        curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str());
+        curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+
+        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
+        curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl);
+
+        curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback);
+        curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &curl);
+
+        curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback);
+        curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+    }
+
+    ~Curl()
+    {
+        if (curl) curl_easy_cleanup(curl);
+        if (requestHeaders) curl_slist_free_all(requestHeaders);
+    }
+
+    bool fetch(const string & url, const string & expectedETag = "")
+    {
+        curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+
+        data.clear();
+
+        if (requestHeaders) {
+            curl_slist_free_all(requestHeaders);
+            requestHeaders = 0;
+        }
+
+        if (!expectedETag.empty()) {
+            this->expectedETag = expectedETag;
+            requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + expectedETag).c_str());
+        }
+
+        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders);
+
+        CURLcode res = curl_easy_perform(curl);
+        checkInterrupt();
+        if (res == CURLE_WRITE_ERROR && etag == expectedETag) return false;
+        if (res != CURLE_OK)
+            throw DownloadError(format("unable to download ‘%1%’: %2% (%3%)")
+                % url % curl_easy_strerror(res) % res);
+
+        long httpStatus = 0;
+        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus);
+        if (httpStatus == 304) return false;
+
+        return true;
+    }
+};
+
+
+DownloadResult downloadFile(string url, string expectedETag)
+{
+    DownloadResult res;
+    Curl curl;
+    if (curl.fetch(url, expectedETag)) {
+        res.cached = false;
+        res.data = curl.data;
+    } else
+        res.cached = true;
+    res.etag = curl.etag;
+    return res;
+}
+
+
+Path downloadFileCached(const string & url, bool unpack)
+{
+    Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs";
+    createDirs(cacheDir);
+
+    string urlHash = printHash32(hashString(htSHA256, url));
+
+    Path dataFile = cacheDir + "/" + urlHash + ".info";
+    Path fileLink = cacheDir + "/" + urlHash + "-file";
+
+    Path storePath;
+
+    string expectedETag;
+
+    int ttl = settings.get("tarball-ttl", 60 * 60);
+    bool skip = false;
+
+    if (pathExists(fileLink) && pathExists(dataFile)) {
+        storePath = readLink(fileLink);
+        store->addTempRoot(storePath);
+        if (store->isValidPath(storePath)) {
+            auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
+            if (ss.size() >= 3 && ss[0] == url) {
+                time_t lastChecked;
+                if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0))
+                    skip = true;
+                else if (!ss[1].empty()) {
+                    printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]);
+                    expectedETag = ss[1];
+                }
+            }
+        } else
+            storePath = "";
+    }
+
+    string name;
+    auto p = url.rfind('/');
+    if (p != string::npos) name = string(url, p + 1);
+
+    if (!skip) {
+
+        if (storePath.empty())
+            printMsg(lvlInfo, format("downloading ‘%1%’...") % url);
+        else
+            printMsg(lvlInfo, format("checking ‘%1%’...") % url);
+
+        try {
+            auto res = downloadFile(url, expectedETag);
+
+            if (!res.cached)
+                storePath = store->addTextToStore(name, res.data, PathSet(), false);
+
+            assert(!storePath.empty());
+            replaceSymlink(storePath, fileLink);
+
+            writeFile(dataFile, url + "\n" + res.etag + "\n" + int2String(time(0)) + "\n");
+        } catch (DownloadError & e) {
+            if (storePath.empty()) throw;
+            printMsg(lvlError, format("warning: %1%; using cached result") % e.msg());
+        }
+    }
+
+    if (unpack) {
+        Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
+        Path unpackedStorePath;
+        if (pathExists(unpackedLink)) {
+            unpackedStorePath = readLink(unpackedLink);
+            store->addTempRoot(unpackedStorePath);
+            if (!store->isValidPath(unpackedStorePath))
+                unpackedStorePath = "";
+        }
+        if (unpackedStorePath.empty()) {
+            printMsg(lvlInfo, format("unpacking ‘%1%’...") % url);
+            Path tmpDir = createTempDir();
+            AutoDelete autoDelete(tmpDir, true);
+            runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, "");
+            unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, false);
+        }
+        replaceSymlink(unpackedStorePath, unpackedLink);
+        return unpackedStorePath;
+    }
+
+    return storePath;
+}
+
+
+bool isUri(const string & s)
+{
+    size_t pos = s.find("://");
+    if (pos == string::npos) return false;
+    string scheme(s, 0, pos);
+    return scheme == "http" || scheme == "https";
+}
+
+
+}
diff --git a/src/libexpr/download.hh b/src/libexpr/download.hh
new file mode 100644
index 000000000000..28c9117e4227
--- /dev/null
+++ b/src/libexpr/download.hh
@@ -0,0 +1,22 @@
+#pragma once
+
+#include "types.hh"
+#include <string>
+
+namespace nix {
+
+struct DownloadResult
+{
+    bool cached;
+    string data, etag;
+};
+
+DownloadResult downloadFile(string url, string expectedETag = "");
+
+Path downloadFileCached(const string & url, bool unpack);
+
+MakeError(DownloadError, Error)
+
+bool isUri(const string & s);
+
+}
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d8a4193a8e64..301f991b7ab9 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -292,6 +292,11 @@ Path EvalState::checkSourcePath(const Path & path_)
         if (path == i.second || isInDir(path, i.second))
             return path;
 
+    /* Hack to support the chroot dependencies of corepkgs (see
+       corepkgs/config.nix.in). */
+    if (path == settings.nixPrefix && isStorePath(settings.nixPrefix))
+        return path;
+
     throw RestrictedPathError(format("access to path ‘%1%’ is forbidden in restricted mode") % path_);
 }
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 664d6692f51e..26168b2ed420 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -527,6 +527,8 @@ formal
 #include <unistd.h>
 
 #include <eval.hh>
+#include <download.hh>
+#include <store-api.hh>
 
 
 namespace nix {
@@ -611,6 +613,9 @@ void EvalState::addToSearchPath(const string & s, bool warn)
         path = string(s, pos + 1);
     }
 
+    if (isUri(path))
+        path = downloadFileCached(path, true);
+
     path = absPath(path);
     if (pathExists(path)) {
         debug(format("adding path ‘%1%’ to the search path") % path);
@@ -629,16 +634,17 @@ Path EvalState::findFile(const string & path)
 
 Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos)
 {
-    foreach (SearchPath::iterator, i, searchPath) {
+    for (auto & i : searchPath) {
+        assert(!isUri(i.second));
         Path res;
-        if (i->first.empty())
-            res = i->second + "/" + path;
+        if (i.first.empty())
+            res = i.second + "/" + path;
         else {
-            if (path.compare(0, i->first.size(), i->first) != 0 ||
-                (path.size() > i->first.size() && path[i->first.size()] != '/'))
+            if (path.compare(0, i.first.size(), i.first) != 0 ||
+                (path.size() > i.first.size() && path[i.first.size()] != '/'))
                 continue;
-            res = i->second +
-                (path.size() == i->first.size() ? "" : "/" + string(path, i->first.size()));
+            res = i.second +
+                (path.size() == i.first.size() ? "" : "/" + string(path, i.first.size()));
         }
         if (pathExists(res)) return canonPath(res);
     }
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e818496460ea..fe2f1b1e0ae1 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -9,6 +9,7 @@
 #include "json-to-value.hh"
 #include "names.hh"
 #include "eval-inline.hh"
+#include "download.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -18,8 +19,6 @@
 #include <cstring>
 #include <dlfcn.h>
 
-#include <curl/curl.h>
-
 
 namespace nix {
 
@@ -104,7 +103,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
         }
         w.attrs->sort();
         Value fun;
-        state.evalFile(state.findFile("nix/imported-drv-to-derivation.nix"), fun);
+        state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun);
         state.forceFunction(fun, pos);
         mkApp(v, fun, w);
         state.forceAttrs(v, pos);
@@ -1486,53 +1485,6 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a
  *************************************************************/
 
 
-struct Curl
-{
-    CURL * curl;
-    string data;
-
-    static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp)
-    {
-        Curl & c(* (Curl *) userp);
-        size_t realSize = size * nmemb;
-        c.data.append((char *) contents, realSize);
-        return realSize;
-    }
-
-    Curl()
-    {
-        curl = curl_easy_init();
-        if (!curl) throw Error("unable to initialize curl");
-
-        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
-        curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str());
-        curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str());
-
-        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
-        curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl);
-    }
-
-    ~Curl()
-    {
-        if (curl) curl_easy_cleanup(curl);
-    }
-
-    string fetch(const string & url)
-    {
-        curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
-
-        data.clear();
-
-        CURLcode res = curl_easy_perform(curl);
-        if (res != CURLE_OK)
-            throw Error(format("unable to download ‘%1%’: %2%")
-                % url % curl_easy_strerror(res));
-
-        return data;
-    }
-};
-
-
 void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     const string & who, bool unpack)
 {
@@ -1560,25 +1512,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
-    // TODO: cache downloads.
-
-    Curl curl;
-    string data = curl.fetch(url);
-
-    string name;
-    string::size_type p = url.rfind('/');
-    if (p != string::npos) name = string(url, p + 1);
-
-    Path storePath = store->addTextToStore(name, data, PathSet(), state.repair);
-
-    if (unpack) {
-        Path tmpDir = createTempDir();
-        AutoDelete autoDelete(tmpDir, true);
-        runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, "");
-        storePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, state.repair);
-    }
-
-    mkString(v, storePath, singleton<PathSet>(storePath));
+    mkString(v, downloadFileCached(url, unpack), PathSet({url}));
 }
 
 
@@ -1738,8 +1672,7 @@ void EvalState::createBaseEnv()
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
-    string path = findFile("nix/derivation.nix");
-    assert(!path.empty());
+    string path = settings.nixDataDir + "/nix/corepkgs/derivation.nix";
     sDerivationNix = symbols.create(path);
     evalFile(path, v);
     addConstant("derivation", v);