From 0a2bee307b20411f5b0dda0c662b1f9bb9e0e131 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Jul 2015 04:30:16 +0200 Subject: Make a builtin builder This ensures that 1) the derivation doesn't change when Nix changes; 2) the derivation closure doesn't contain Nix and its dependencies; 3) we don't have to rely on ugly chroot hacks. --- corepkgs/config.nix.in | 1 - corepkgs/fetchurl.nix | 15 +-- src/libexpr/download.cc | 236 ----------------------------------------------- src/libexpr/download.hh | 22 ----- src/libexpr/local.mk | 2 +- src/libstore/build.cc | 28 +++++- src/libstore/builtins.cc | 24 +++++ src/libstore/builtins.hh | 9 ++ src/libstore/download.cc | 236 +++++++++++++++++++++++++++++++++++++++++++++++ src/libstore/download.hh | 22 +++++ src/libstore/local.mk | 2 +- 11 files changed, 319 insertions(+), 278 deletions(-) delete mode 100644 src/libexpr/download.cc delete mode 100644 src/libexpr/download.hh create mode 100644 src/libstore/builtins.cc create mode 100644 src/libstore/builtins.hh create mode 100644 src/libstore/download.cc create mode 100644 src/libstore/download.hh diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in index 8918f4ddde8d..90e8edbea833 100644 --- a/corepkgs/config.nix.in +++ b/corepkgs/config.nix.in @@ -12,7 +12,6 @@ in rec { tar = "@tar@"; tarFlags = "@tarFlags@"; tr = "@tr@"; - curl = "@curl@"; nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@"; nixPrefix = "@prefix@"; diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix index 1ce88593cff2..64d1f121f81c 100644 --- a/corepkgs/fetchurl.nix +++ b/corepkgs/fetchurl.nix @@ -5,20 +5,9 @@ with import ; assert (outputHash != "" && outputHashAlgo != "") || md5 != "" || sha1 != "" || sha256 != ""; -let - - builder = builtins.toFile "fetchurl.sh" - ('' - echo "downloading $url into $out" - ${curl} --fail --location --max-redirs 20 --insecure "$url" > "$out" - '' + (if executable then "${coreutils}/chmod +x $out" else "")); - -in - derivation { name = baseNameOf (toString url); - builder = shell; - args = [ "-e" builder ]; + builder = "builtin:fetchurl"; # New-style output content requirements. outputHashAlgo = if outputHashAlgo != "" then outputHashAlgo else @@ -39,6 +28,4 @@ derivation { # by definition pure. "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" ]; - - inherit chrootDeps; } diff --git a/src/libexpr/download.cc b/src/libexpr/download.cc deleted file mode 100644 index 9bf3e13aa9da..000000000000 --- a/src/libexpr/download.cc +++ /dev/null @@ -1,236 +0,0 @@ -#include "download.hh" -#include "util.hh" -#include "globals.hh" -#include "hash.hh" -#include "store-api.hh" - -#include - -namespace nix { - -struct Curl -{ - CURL * curl; - string data; - string etag, status, expectedETag; - - struct curl_slist * requestHeaders; - - static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp) - { - Curl & c(* (Curl *) userp); - size_t realSize = size * nmemb; - c.data.append((char *) contents, realSize); - return realSize; - } - - static size_t headerCallback(void * contents, size_t size, size_t nmemb, void * userp) - { - Curl & c(* (Curl *) userp); - size_t realSize = size * nmemb; - string line = string((char *) contents, realSize); - printMsg(lvlVomit, format("got header: %1%") % trim(line)); - if (line.compare(0, 5, "HTTP/") == 0) { // new response starts - c.etag = ""; - auto ss = tokenizeString>(line, " "); - c.status = ss.size() >= 2 ? ss[1] : ""; - } else { - auto i = line.find(':'); - if (i != string::npos) { - string name = trim(string(line, 0, i)); - if (name == "ETag") { // FIXME: case - c.etag = trim(string(line, i + 1)); - /* Hack to work around a GitHub bug: it sends - ETags, but ignores If-None-Match. So if we get - the expected ETag on a 200 response, then shut - down the connection because we already have the - data. */ - printMsg(lvlDebug, format("got ETag: %1%") % c.etag); - if (c.etag == c.expectedETag && c.status == "200") { - printMsg(lvlDebug, format("shutting down on 200 HTTP response with expected ETag")); - return 0; - } - } - } - } - return realSize; - } - - static int progressCallback(void * clientp, double dltotal, double dlnow, double ultotal, double ulnow) - { - return _isInterrupted; - } - - Curl() - { - requestHeaders = 0; - - curl = curl_easy_init(); - if (!curl) throw Error("unable to initialize curl"); - - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str()); - curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); - curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); - - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl); - - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &curl); - - curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); - } - - ~Curl() - { - if (curl) curl_easy_cleanup(curl); - if (requestHeaders) curl_slist_free_all(requestHeaders); - } - - bool fetch(const string & url, const string & expectedETag = "") - { - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - - data.clear(); - - if (requestHeaders) { - curl_slist_free_all(requestHeaders); - requestHeaders = 0; - } - - if (!expectedETag.empty()) { - this->expectedETag = expectedETag; - requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + expectedETag).c_str()); - } - - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders); - - CURLcode res = curl_easy_perform(curl); - checkInterrupt(); - if (res == CURLE_WRITE_ERROR && etag == expectedETag) return false; - if (res != CURLE_OK) - throw DownloadError(format("unable to download ‘%1%’: %2% (%3%)") - % url % curl_easy_strerror(res) % res); - - long httpStatus = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); - if (httpStatus == 304) return false; - - return true; - } -}; - - -DownloadResult downloadFile(string url, string expectedETag) -{ - DownloadResult res; - Curl curl; - if (curl.fetch(url, expectedETag)) { - res.cached = false; - res.data = curl.data; - } else - res.cached = true; - res.etag = curl.etag; - return res; -} - - -Path downloadFileCached(const string & url, bool unpack) -{ - Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs"; - createDirs(cacheDir); - - string urlHash = printHash32(hashString(htSHA256, url)); - - Path dataFile = cacheDir + "/" + urlHash + ".info"; - Path fileLink = cacheDir + "/" + urlHash + "-file"; - - Path storePath; - - string expectedETag; - - int ttl = settings.get("tarball-ttl", 60 * 60); - bool skip = false; - - if (pathExists(fileLink) && pathExists(dataFile)) { - storePath = readLink(fileLink); - store->addTempRoot(storePath); - if (store->isValidPath(storePath)) { - auto ss = tokenizeString>(readFile(dataFile), "\n"); - if (ss.size() >= 3 && ss[0] == url) { - time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) - skip = true; - else if (!ss[1].empty()) { - printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]); - expectedETag = ss[1]; - } - } - } else - storePath = ""; - } - - string name; - auto p = url.rfind('/'); - if (p != string::npos) name = string(url, p + 1); - - if (!skip) { - - if (storePath.empty()) - printMsg(lvlInfo, format("downloading ‘%1%’...") % url); - else - printMsg(lvlInfo, format("checking ‘%1%’...") % url); - - try { - auto res = downloadFile(url, expectedETag); - - if (!res.cached) - storePath = store->addTextToStore(name, res.data, PathSet(), false); - - assert(!storePath.empty()); - replaceSymlink(storePath, fileLink); - - writeFile(dataFile, url + "\n" + res.etag + "\n" + int2String(time(0)) + "\n"); - } catch (DownloadError & e) { - if (storePath.empty()) throw; - printMsg(lvlError, format("warning: %1%; using cached result") % e.msg()); - } - } - - if (unpack) { - Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; - Path unpackedStorePath; - if (pathExists(unpackedLink)) { - unpackedStorePath = readLink(unpackedLink); - store->addTempRoot(unpackedStorePath); - if (!store->isValidPath(unpackedStorePath)) - unpackedStorePath = ""; - } - if (unpackedStorePath.empty()) { - printMsg(lvlInfo, format("unpacking ‘%1%’...") % url); - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, ""); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, false); - } - replaceSymlink(unpackedStorePath, unpackedLink); - return unpackedStorePath; - } - - return storePath; -} - - -bool isUri(const string & s) -{ - size_t pos = s.find("://"); - if (pos == string::npos) return false; - string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file"; -} - - -} diff --git a/src/libexpr/download.hh b/src/libexpr/download.hh deleted file mode 100644 index 28c9117e4227..000000000000 --- a/src/libexpr/download.hh +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include "types.hh" -#include - -namespace nix { - -struct DownloadResult -{ - bool cached; - string data, etag; -}; - -DownloadResult downloadFile(string url, string expectedETag = ""); - -Path downloadFileCached(const string & url, bool unpack); - -MakeError(DownloadError, Error) - -bool isUri(const string & s); - -} diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 35e84980a6dd..4c1f4de19187 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,7 +8,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc libexpr_LIBS = libutil libstore libformat -libexpr_LDFLAGS = -ldl -lcurl +libexpr_LDFLAGS = -ldl # The dependency on libgc must be propagated (i.e. meaning that # programs/libraries that use libexpr must explicitly pass -lgc), diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 61677d1eb5a2..46d2841dc9e4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -8,6 +8,7 @@ #include "util.hh" #include "archive.hh" #include "affinity.hh" +#include "builtins.hh" #include #include @@ -1269,6 +1270,12 @@ bool substitutesAllowed(const BasicDerivation & drv) } +static bool isBuiltin(const BasicDerivation & drv) +{ + return string(drv.builder, 0, 8) == "builtin:"; +} + + void DerivationGoal::tryToBuild() { trace("trying to build"); @@ -2139,7 +2146,7 @@ void DerivationGoal::startBuilder() #endif { ProcessOptions options; - options.allowVfork = !buildUser.enabled(); + options.allowVfork = !buildUser.enabled() && !isBuiltin(*drv); pid = startProcess([&]() { runChild(); }, options); @@ -2386,7 +2393,9 @@ void DerivationGoal::runChild() const char *builder = "invalid"; string sandboxProfile; - if (useChroot && SANDBOX_ENABLED) { + if (isBuiltin(*drv)) + ; + else if (useChroot && SANDBOX_ENABLED) { /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ PathSet ancestry; @@ -2413,7 +2422,6 @@ void DerivationGoal::runChild() for (auto & i : inputPaths) dirsInChroot[i] = i; - /* TODO: we should factor out the policy cleanly, so we don't have to repeat the constants every time... */ sandboxProfile += "(version 1)\n"; @@ -2517,6 +2525,20 @@ void DerivationGoal::runChild() } /* Execute the program. This should not return. */ + if (isBuiltin(*drv)) { + try { + logType = ltFlat; + if (drv->builder == "builtin:fetchurl") + builtinFetchurl(*drv); + else + throw Error(format("unsupported builtin function ‘%1%’") % string(drv->builder, 8)); + _exit(0); + } catch (std::exception & e) { + writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); + _exit(1); + } + } + execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); throw SysError(format("executing ‘%1%’") % drv->builder); diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc new file mode 100644 index 000000000000..97d6cb943402 --- /dev/null +++ b/src/libstore/builtins.cc @@ -0,0 +1,24 @@ +#include "builtins.hh" +#include "download.hh" + +namespace nix { + +void builtinFetchurl(const BasicDerivation & drv) +{ + auto url = drv.env.find("url"); + if (url == drv.env.end()) throw Error("attribute ‘url’ missing"); + printMsg(lvlInfo, format("downloading ‘%1%’...") % url->second); + auto data = downloadFile(url->second); // FIXME: show progress + + auto out = drv.env.find("out"); + if (out == drv.env.end()) throw Error("attribute ‘url’ missing"); + writeFile(out->second, data.data); + + auto executable = drv.env.find("out"); + if (executable != drv.env.end() && executable->second == "1") { + if (chmod(out->second.c_str(), 0755) == -1) + throw SysError(format("making ‘%1%’ executable") % out->second); + } +} + +} diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh new file mode 100644 index 000000000000..4b2431aa08cf --- /dev/null +++ b/src/libstore/builtins.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "derivations.hh" + +namespace nix { + +void builtinFetchurl(const BasicDerivation & drv); + +} diff --git a/src/libstore/download.cc b/src/libstore/download.cc new file mode 100644 index 000000000000..9bf3e13aa9da --- /dev/null +++ b/src/libstore/download.cc @@ -0,0 +1,236 @@ +#include "download.hh" +#include "util.hh" +#include "globals.hh" +#include "hash.hh" +#include "store-api.hh" + +#include + +namespace nix { + +struct Curl +{ + CURL * curl; + string data; + string etag, status, expectedETag; + + struct curl_slist * requestHeaders; + + static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp) + { + Curl & c(* (Curl *) userp); + size_t realSize = size * nmemb; + c.data.append((char *) contents, realSize); + return realSize; + } + + static size_t headerCallback(void * contents, size_t size, size_t nmemb, void * userp) + { + Curl & c(* (Curl *) userp); + size_t realSize = size * nmemb; + string line = string((char *) contents, realSize); + printMsg(lvlVomit, format("got header: %1%") % trim(line)); + if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + c.etag = ""; + auto ss = tokenizeString>(line, " "); + c.status = ss.size() >= 2 ? ss[1] : ""; + } else { + auto i = line.find(':'); + if (i != string::npos) { + string name = trim(string(line, 0, i)); + if (name == "ETag") { // FIXME: case + c.etag = trim(string(line, i + 1)); + /* Hack to work around a GitHub bug: it sends + ETags, but ignores If-None-Match. So if we get + the expected ETag on a 200 response, then shut + down the connection because we already have the + data. */ + printMsg(lvlDebug, format("got ETag: %1%") % c.etag); + if (c.etag == c.expectedETag && c.status == "200") { + printMsg(lvlDebug, format("shutting down on 200 HTTP response with expected ETag")); + return 0; + } + } + } + } + return realSize; + } + + static int progressCallback(void * clientp, double dltotal, double dlnow, double ultotal, double ulnow) + { + return _isInterrupted; + } + + Curl() + { + requestHeaders = 0; + + curl = curl_easy_init(); + if (!curl) throw Error("unable to initialize curl"); + + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str()); + curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl); + + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &curl); + + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); + } + + ~Curl() + { + if (curl) curl_easy_cleanup(curl); + if (requestHeaders) curl_slist_free_all(requestHeaders); + } + + bool fetch(const string & url, const string & expectedETag = "") + { + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + + data.clear(); + + if (requestHeaders) { + curl_slist_free_all(requestHeaders); + requestHeaders = 0; + } + + if (!expectedETag.empty()) { + this->expectedETag = expectedETag; + requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + expectedETag).c_str()); + } + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders); + + CURLcode res = curl_easy_perform(curl); + checkInterrupt(); + if (res == CURLE_WRITE_ERROR && etag == expectedETag) return false; + if (res != CURLE_OK) + throw DownloadError(format("unable to download ‘%1%’: %2% (%3%)") + % url % curl_easy_strerror(res) % res); + + long httpStatus = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); + if (httpStatus == 304) return false; + + return true; + } +}; + + +DownloadResult downloadFile(string url, string expectedETag) +{ + DownloadResult res; + Curl curl; + if (curl.fetch(url, expectedETag)) { + res.cached = false; + res.data = curl.data; + } else + res.cached = true; + res.etag = curl.etag; + return res; +} + + +Path downloadFileCached(const string & url, bool unpack) +{ + Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs"; + createDirs(cacheDir); + + string urlHash = printHash32(hashString(htSHA256, url)); + + Path dataFile = cacheDir + "/" + urlHash + ".info"; + Path fileLink = cacheDir + "/" + urlHash + "-file"; + + Path storePath; + + string expectedETag; + + int ttl = settings.get("tarball-ttl", 60 * 60); + bool skip = false; + + if (pathExists(fileLink) && pathExists(dataFile)) { + storePath = readLink(fileLink); + store->addTempRoot(storePath); + if (store->isValidPath(storePath)) { + auto ss = tokenizeString>(readFile(dataFile), "\n"); + if (ss.size() >= 3 && ss[0] == url) { + time_t lastChecked; + if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) + skip = true; + else if (!ss[1].empty()) { + printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]); + expectedETag = ss[1]; + } + } + } else + storePath = ""; + } + + string name; + auto p = url.rfind('/'); + if (p != string::npos) name = string(url, p + 1); + + if (!skip) { + + if (storePath.empty()) + printMsg(lvlInfo, format("downloading ‘%1%’...") % url); + else + printMsg(lvlInfo, format("checking ‘%1%’...") % url); + + try { + auto res = downloadFile(url, expectedETag); + + if (!res.cached) + storePath = store->addTextToStore(name, res.data, PathSet(), false); + + assert(!storePath.empty()); + replaceSymlink(storePath, fileLink); + + writeFile(dataFile, url + "\n" + res.etag + "\n" + int2String(time(0)) + "\n"); + } catch (DownloadError & e) { + if (storePath.empty()) throw; + printMsg(lvlError, format("warning: %1%; using cached result") % e.msg()); + } + } + + if (unpack) { + Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; + Path unpackedStorePath; + if (pathExists(unpackedLink)) { + unpackedStorePath = readLink(unpackedLink); + store->addTempRoot(unpackedStorePath); + if (!store->isValidPath(unpackedStorePath)) + unpackedStorePath = ""; + } + if (unpackedStorePath.empty()) { + printMsg(lvlInfo, format("unpacking ‘%1%’...") % url); + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + // FIXME: this requires GNU tar for decompression. + runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, ""); + unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, false); + } + replaceSymlink(unpackedStorePath, unpackedLink); + return unpackedStorePath; + } + + return storePath; +} + + +bool isUri(const string & s) +{ + size_t pos = s.find("://"); + if (pos == string::npos) return false; + string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file"; +} + + +} diff --git a/src/libstore/download.hh b/src/libstore/download.hh new file mode 100644 index 000000000000..28c9117e4227 --- /dev/null +++ b/src/libstore/download.hh @@ -0,0 +1,22 @@ +#pragma once + +#include "types.hh" +#include + +namespace nix { + +struct DownloadResult +{ + bool cached; + string data, etag; +}; + +DownloadResult downloadFile(string url, string expectedETag = ""); + +Path downloadFileCached(const string & url, bool unpack); + +MakeError(DownloadError, Error) + +bool isUri(const string & s); + +} diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 771c06753a65..bf5c256c949e 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc) libstore_LIBS = libutil libformat -libstore_LDFLAGS = -lsqlite3 -lbz2 +libstore_LDFLAGS = -lsqlite3 -lbz2 -lcurl ifeq ($(OS), SunOS) libstore_LDFLAGS += -lsocket -- cgit 1.4.1