diff options
Diffstat (limited to 'src')
34 files changed, 399 insertions, 161 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 3908dfac487d..d7aee288670a 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -233,7 +233,7 @@ int main (int argc, char * * argv) lock = -1; try { - sshStore = openStore("ssh://" + bestMachine->hostName, + sshStore = openStore("ssh-ng://" + bestMachine->hostName, { {"ssh-key", bestMachine->sshKey }, {"max-connections", "1" } }); hostName = bestMachine->hostName; @@ -252,10 +252,10 @@ connected: string line; if (!getline(cin, line)) throw Error("hook caller didn't send inputs"); - auto inputs = tokenizeString<std::list<string>>(line); + auto inputs = tokenizeString<PathSet>(line); if (!getline(cin, line)) throw Error("hook caller didn't send outputs"); - auto outputs = tokenizeString<Strings>(line); + auto outputs = tokenizeString<PathSet>(line); AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + hostName + ".upload-lock", true); auto old = signal(SIGALRM, handleAlarm); alarm(15 * 60); @@ -269,11 +269,15 @@ connected: printError("building ‘%s’ on ‘%s’", drvPath, hostName); sshStore->buildDerivation(drvPath, readDerivation(drvPath)); - std::remove_if(outputs.begin(), outputs.end(), [=](const Path & path) { return store->isValidPath(path); }); - if (!outputs.empty()) { - setenv("NIX_HELD_LOCKS", concatStringsSep(" ", outputs).c_str(), 1); /* FIXME: ugly */ - copyPaths(ref<Store>(sshStore), store, outputs); + PathSet missing; + for (auto & path : outputs) + if (!store->isValidPath(path)) missing.insert(path); + + if (!missing.empty()) { + setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */ + copyPaths(ref<Store>(sshStore), store, missing); } + return; }); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 326202d295fb..a720afd6cdd4 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -332,11 +332,7 @@ RunPager::~RunPager() pid.wait(); } } catch (...) { - try { - pid.kill(true); - } catch (...) { - ignoreException(); - } + ignoreException(); } } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3e07a2aa2b60..25ad0d75b70a 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -97,7 +97,7 @@ void BinaryCacheStore::init() auto cacheInfo = getFile(cacheInfoFile); if (!cacheInfo) { - upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n"); + upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) { size_t colon = line.find(':'); @@ -224,7 +224,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str } } - upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str())); + upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json"); } else { @@ -250,10 +250,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : ""); if (repair || !fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, *narCompressed); + upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); } else stats.narWriteAverted++; @@ -264,7 +265,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str /* Atomically write the NAR info file.*/ if (secretKey) narInfo->sign(*secretKey); - upsertFile(narInfoFile, narInfo->to_string()); + upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo"); auto hashPart = storePathToHash(narInfo->path); @@ -382,4 +383,28 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor() return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); } +std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path) +{ + Path drvPath; + + if (isDerivation(path)) + drvPath = path; + else { + try { + auto info = queryPathInfo(path); + // FIXME: add a "Log" field to .narinfo + if (info->deriver == "") return nullptr; + drvPath = info->deriver; + } catch (InvalidPath &) { + return nullptr; + } + } + + auto logPath = "log/" + baseNameOf(drvPath); + + debug("fetching build log from binary cache ‘%s/%s’", getUri(), logPath); + + return getFile(logPath); +} + } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index a70d50d4949c..d42b1abd2455 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -31,7 +31,9 @@ public: virtual bool fileExists(const std::string & path) = 0; - virtual void upsertFile(const std::string & path, const std::string & data) = 0; + virtual void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) = 0; /* Return the contents of the specified file, or null if it doesn't exist. */ @@ -122,6 +124,8 @@ public: void addSignatures(const Path & storePath, const StringSet & sigs) override { notImpl(); } + std::shared_ptr<std::string> getBuildLog(const Path & path) override; + }; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index fd1f5dc3a4d4..fc840df81a56 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -642,7 +642,7 @@ HookInstance::~HookInstance() { try { toHook.writeSide = -1; - if (pid != -1) pid.kill(true); + if (pid != -1) pid.kill(); } catch (...) { ignoreException(); } @@ -1437,7 +1437,7 @@ void DerivationGoal::buildDone() to have terminated. In fact, the builder could also have simply have closed its end of the pipe, so just to be sure, kill it. */ - int status = hook ? hook->pid.kill(true) : pid.kill(true); + int status = hook ? hook->pid.kill() : pid.kill(); debug(format("builder process for ‘%1%’ finished") % drvPath); @@ -3048,9 +3048,6 @@ void DerivationGoal::registerOutputs() } -string drvsLogDir = "drvs"; - - Path DerivationGoal::openLogFile() { logSize = 0; @@ -3060,7 +3057,7 @@ Path DerivationGoal::openLogFile() string baseName = baseNameOf(drvPath); /* Create a log file. */ - Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % drvsLogDir % string(baseName, 0, 2)).str(); + Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % worker.store.drvsLogDir % string(baseName, 0, 2)).str(); createDirs(dir); Path logFileName = (format("%1%/%2%%3%") diff --git a/src/libstore/download.cc b/src/libstore/download.cc index ebea3800ac34..22bde086e6a2 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -5,6 +5,11 @@ #include "store-api.hh" #include "archive.hh" #include "s3.hh" +#include "compression.hh" + +#ifdef ENABLE_S3 +#include <aws/core/client/ClientConfiguration.h> +#endif #include <unistd.h> #include <fcntl.h> @@ -34,6 +39,16 @@ std::string resolveUri(const std::string & uri) return uri; } +ref<std::string> decodeContent(const std::string & encoding, ref<std::string> data) +{ + if (encoding == "") + return data; + else if (encoding == "br") + return decompress(encoding, *data); + else + throw Error("unsupported Content-Encoding ‘%s’", encoding); +} + struct CurlDownloader : public Downloader { CURLM * curlm = 0; @@ -67,6 +82,8 @@ struct CurlDownloader : public Downloader struct curl_slist * requestHeaders = 0; + std::string encoding; + DownloadItem(CurlDownloader & downloader, const DownloadRequest & request) : downloader(downloader), request(request) { @@ -124,6 +141,7 @@ struct CurlDownloader : public Downloader auto ss = tokenizeString<vector<string>>(line, " "); status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared<std::string>(); + encoding = ""; } else { auto i = line.find(':'); if (i != string::npos) { @@ -139,7 +157,8 @@ struct CurlDownloader : public Downloader debug(format("shutting down on 200 HTTP response with expected ETag")); return 0; } - } + } else if (name == "content-encoding") + encoding = trim(string(line, i + 1));; } } return realSize; @@ -265,7 +284,14 @@ struct CurlDownloader : public Downloader { result.cached = httpStatus == 304; done = true; - callSuccess(success, failure, const_cast<const DownloadResult &>(result)); + + try { + result.data = decodeContent(encoding, ref<std::string>(result.data)); + callSuccess(success, failure, const_cast<const DownloadResult &>(result)); + } catch (...) { + done = true; + callFailure(failure, std::current_exception()); + } } else { Error err = (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) ? NotFound : @@ -495,7 +521,7 @@ struct CurlDownloader : public Downloader // FIXME: do this on a worker thread sync2async<DownloadResult>(success, failure, [&]() -> DownloadResult { #ifdef ENABLE_S3 - S3Helper s3Helper; + S3Helper s3Helper(Aws::Region::US_EAST_1); // FIXME: make configurable auto slash = request.uri.find('/', 5); if (slash == std::string::npos) throw nix::Error("bad S3 URI ‘%s’", request.uri); @@ -645,7 +671,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, ""); + runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}); unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, false); } replaceSymlink(unpackedStorePath, unpackedLink); diff --git a/src/libstore/download.hh b/src/libstore/download.hh index bdb5011e7830..e2e16b361036 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -73,4 +73,7 @@ public: bool isUri(const string & s); +/* Decode data according to the Content-Encoding header. */ +ref<std::string> decodeContent(const std::string & encoding, ref<std::string> data); + } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index df537a51255a..012b3d5b8b98 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -179,7 +179,6 @@ void Settings::update() _get(envKeepDerivations, "env-keep-derivations"); _get(sshSubstituterHosts, "ssh-substituter-hosts"); _get(useSshSubstituter, "use-ssh-substituter"); - _get(logServers, "log-servers"); _get(enableImportNative, "allow-unsafe-native-code-during-evaluation"); _get(useCaseHack, "use-case-hack"); _get(preBuildHook, "pre-build-hook"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 7a9a9f6c0caa..462721681912 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -181,9 +181,6 @@ struct Settings { /* Whether to show a stack trace if Nix evaluation fails. */ bool showTrace; - /* A list of URL prefixes that can return Nix build logs. */ - Strings logServers; - /* Whether the importNative primop should be enabled */ bool enableImportNative; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 9d31f77c921f..37a7d6ace142 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -64,7 +64,9 @@ protected: } } - void upsertFile(const std::string & path, const std::string & data) override + void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) override { throw UploadToHTTP("uploading to an HTTP binary cache is not supported"); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 4a2ac42f31c3..0e838846c794 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -8,7 +8,7 @@ namespace nix { -static std::string uriScheme = "legacy-ssh://"; +static std::string uriScheme = "ssh://"; struct LegacySSHStore : public Store { @@ -45,7 +45,7 @@ struct LegacySSHStore : public Store ref<Connection> openConnection() { auto conn = make_ref<Connection>(); - conn->sshConn = master.startCommand("nix-store --serve"); + conn->sshConn = master.startCommand("nix-store --serve --write"); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); @@ -204,6 +204,41 @@ struct LegacySSHStore : public Store bool isTrusted() override { return true; } + void computeFSClosure(const PathSet & paths, + PathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false) override + { + if (flipDirection || includeDerivers) { + Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); + return; + } + + auto conn(connections->get()); + + conn->to + << cmdQueryClosure + << includeOutputs + << paths; + conn->to.flush(); + + auto res = readStorePaths<PathSet>(*this, conn->from); + + out.insert(res.begin(), res.end()); + } + + PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override + { + auto conn(connections->get()); + + conn->to + << cmdQueryValidPaths + << false // lock + << maybeSubstitute + << paths; + conn->to.flush(); + + return readStorePaths<PathSet>(*this, conn->from); + } }; static RegisterStoreImplementation regStore([]( diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 0f377989bd89..aff22f9fcc22 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -30,7 +30,9 @@ protected: bool fileExists(const std::string & path) override; - void upsertFile(const std::string & path, const std::string & data) override; + void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) override; void getFile(const std::string & path, std::function<void(std::shared_ptr<std::string>)> success, @@ -83,7 +85,9 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path) return pathExists(binaryCacheDir + "/" + path); } -void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data) +void LocalBinaryCacheStore::upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) { atomicWrite(binaryCacheDir + "/" + path, data); } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 4571a2211cd2..002ee4a65ce2 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -2,6 +2,8 @@ #include "fs-accessor.hh" #include "store-api.hh" #include "globals.hh" +#include "compression.hh" +#include "derivations.hh" namespace nix { @@ -84,4 +86,41 @@ void LocalFSStore::narFromPath(const Path & path, Sink & sink) dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink); } +const string LocalFSStore::drvsLogDir = "drvs"; + +std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_) +{ + auto path(path_); + + assertStorePath(path); + + if (!isDerivation(path)) { + try { + path = queryPathInfo(path)->deriver; + } catch (InvalidPath &) { + return nullptr; + } + if (path == "") return nullptr; + } + + string baseName = baseNameOf(path); + + for (int j = 0; j < 2; j++) { + + Path logPath = + j == 0 + ? (format("%1%/%2%/%3%/%4%") % logDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str() + : (format("%1%/%2%/%3%") % logDir % drvsLogDir % baseName).str(); + Path logBz2Path = logPath + ".bz2"; + + if (pathExists(logPath)) + return std::make_shared<std::string>(readFile(logPath)); + + else if (pathExists(logBz2Path)) + return decompress("bzip2", readFile(logBz2Path)); + } + + return nullptr; +} + } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 63f069c2ff18..dcfa000c4324 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -669,7 +669,7 @@ bool LocalStore::isValidPathUncached(const Path & path) } -PathSet LocalStore::queryValidPaths(const PathSet & paths) +PathSet LocalStore::queryValidPaths(const PathSet & paths, bool maybeSubstitute) { PathSet res; for (auto & i : paths) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 511209d8404a..28e9a31c9feb 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -21,9 +21,6 @@ namespace nix { const int nixSchemaVersion = 10; -extern string drvsLogDir; - - struct Derivation; @@ -102,7 +99,7 @@ public: bool isValidPathUncached(const Path & path) override; - PathSet queryValidPaths(const PathSet & paths) override; + PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override; PathSet queryAllValidPaths() override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1ac2d7b6e786..a1f2db5b0ec8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -187,7 +187,7 @@ bool RemoteStore::isValidPathUncached(const Path & path) } -PathSet RemoteStore::queryValidPaths(const PathSet & paths) +PathSet RemoteStore::queryValidPaths(const PathSet & paths, bool maybeSubstitute) { auto conn(connections->get()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 66540a2a2ec1..a08bd305639d 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -28,7 +28,7 @@ public: bool isValidPathUncached(const Path & path) override; - PathSet queryValidPaths(const PathSet & paths) override; + PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override; PathSet queryAllValidPaths() override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 800380c62d48..571bf7dfd801 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -5,6 +5,8 @@ #include "nar-info.hh" #include "nar-info-disk-cache.hh" #include "globals.hh" +#include "compression.hh" +#include "download.hh" #include <aws/core/Aws.h> #include <aws/core/client/ClientConfiguration.h> @@ -52,8 +54,8 @@ static void initAWS() }); } -S3Helper::S3Helper() - : config(makeConfig()) +S3Helper::S3Helper(const string & region) + : config(makeConfig(region)) , client(make_ref<Aws::S3::S3Client>(*config)) { } @@ -70,11 +72,11 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy } }; -ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig() +ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region) { initAWS(); auto res = make_ref<Aws::Client::ClientConfiguration>(); - res->region = Aws::Region::US_EAST_1; // FIXME: make configurable + res->region = region; res->requestTimeoutMs = 600 * 1000; res->retryStrategy = std::make_shared<RetryStrategy>(); res->caFile = settings.caFile; @@ -104,8 +106,10 @@ S3Helper::DownloadResult S3Helper::getObject( auto result = checkAws(fmt("AWS error fetching ‘%s’", key), client->GetObject(request)); - res.data = std::make_shared<std::string>( - dynamic_cast<std::stringstream &>(result.GetBody()).str()); + res.data = decodeContent( + result.GetContentEncoding(), + make_ref<std::string>( + dynamic_cast<std::stringstream &>(result.GetBody()).str())); } catch (S3Error & e) { if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; @@ -137,10 +141,16 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore S3Helper s3Helper; + std::string narinfoCompression, lsCompression, logCompression; + S3BinaryCacheStoreImpl( const Params & params, const std::string & bucketName) : S3BinaryCacheStore(params) , bucketName(bucketName) + , s3Helper(get(params, "aws-region", Aws::Region::US_EAST_1)) + , narinfoCompression(get(params, "narinfo-compression", "")) + , lsCompression(get(params, "ls-compression", "")) + , logCompression(get(params, "log-compression", "")) { diskCache = getNarInfoDiskCache(); } @@ -219,13 +229,20 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore return true; } - void upsertFile(const std::string & path, const std::string & data) override + void uploadFile(const std::string & path, const std::string & data, + const std::string & mimeType, + const std::string & contentEncoding) { auto request = Aws::S3::Model::PutObjectRequest() .WithBucket(bucketName) .WithKey(path); + request.SetContentType(mimeType); + + if (contentEncoding != "") + request.SetContentEncoding(contentEncoding); + auto stream = std::make_shared<istringstream_nocopy>(data); request.SetBody(stream); @@ -248,6 +265,19 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore stats.putTimeMs += duration; } + void upsertFile(const std::string & path, const std::string & data, + const std::string & mimeType) override + { + if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) + uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression); + else if (lsCompression != "" && hasSuffix(path, ".ls")) + uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); + else if (logCompression != "" && hasPrefix(path, "log/")) + uploadFile(path, *compress(logCompression, data), mimeType, logCompression); + else + uploadFile(path, data, mimeType, ""); + } + void getFile(const std::string & path, std::function<void(std::shared_ptr<std::string>)> success, std::function<void(std::exception_ptr exc)> failure) override diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index 5d5d3475c449..08a7fbf96e98 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -14,9 +14,9 @@ struct S3Helper ref<Aws::Client::ClientConfiguration> config; ref<Aws::S3::S3Client> client; - S3Helper(); + S3Helper(const std::string & region); - ref<Aws::Client::ClientConfiguration> makeConfig(); + ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region); struct DownloadResult { diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 20f020bdada9..2a81a8b1ebe5 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -8,7 +8,7 @@ namespace nix { -static std::string uriScheme = "ssh://"; +static std::string uriScheme = "ssh-ng://"; class SSHStore : public RemoteStore { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9c755965e45b..b1bf961e1bfb 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -377,7 +377,7 @@ void Store::queryPathInfo(const Path & storePath, } -PathSet Store::queryValidPaths(const PathSet & paths) +PathSet Store::queryValidPaths(const PathSet & paths, bool maybeSubstitute) { struct State { @@ -550,6 +550,8 @@ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, for (auto & path : storePaths) srcStore->computeFSClosure(path, closure); + // FIXME: use copyStorePaths() + PathSet valid = dstStore->queryValidPaths(closure); if (valid.size() == closure.size()) return; @@ -791,37 +793,25 @@ std::list<ref<Store>> getDefaultSubstituters() } -void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute) -{ - if (substitute) { - /* Filter out .drv files (we don't want to build anything). */ - PathSet paths2; - for (auto & path : storePaths) - if (!isDerivation(path)) paths2.insert(path); - unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; - to->queryMissing(PathSet(paths2.begin(), paths2.end()), - willBuild, willSubstitute, unknown, downloadSize, narSize); - /* FIXME: should use ensurePath(), but it only - does one path at a time. */ - if (!willSubstitute.empty()) - try { - to->buildPaths(willSubstitute); - } catch (Error & e) { - printMsg(lvlError, format("warning: %1%") % e.msg()); - } - } +void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute) +{ + PathSet valid = to->queryValidPaths(storePaths, substitute); + + PathSet missing; + for (auto & path : storePaths) + if (!valid.count(path)) missing.insert(path); std::string copiedLabel = "copied"; - logger->setExpected(copiedLabel, storePaths.size()); + logger->setExpected(copiedLabel, missing.size()); ThreadPool pool; processGraph<Path>(pool, - PathSet(storePaths.begin(), storePaths.end()), + PathSet(missing.begin(), missing.end()), [&](const Path & storePath) { + if (to->isValidPath(storePath)) return PathSet(); return from->queryPathInfo(storePath)->references; }, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 481d0b799068..98f2803f8136 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -324,8 +324,10 @@ protected: public: - /* Query which of the given paths is valid. */ - virtual PathSet queryValidPaths(const PathSet & paths); + /* Query which of the given paths is valid. Optionally, try to + substitute missing paths. */ + virtual PathSet queryValidPaths(const PathSet & paths, + bool maybeSubstitute = false); /* Query the set of all valid paths. Note that for some store backends, the name part of store paths may be omitted @@ -511,7 +513,7 @@ public: `storePath' is returned; that is, the closures under the `referrers' relation instead of the `references' relation is returned. */ - void computeFSClosure(const PathSet & paths, + virtual void computeFSClosure(const PathSet & paths, PathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); @@ -566,6 +568,11 @@ public: if they lack a signature. */ virtual bool isTrusted() { return false; } + /* Return the build log of the specified store path, if available, + or null otherwise. */ + virtual std::shared_ptr<std::string> getBuildLog(const Path & path) + { return nullptr; } + protected: Stats stats; @@ -579,6 +586,7 @@ public: const Path rootDir; const Path stateDir; const Path logDir; + const static string drvsLogDir; LocalFSStore(const Params & params); @@ -595,6 +603,8 @@ public: { return getRealStoreDir() + "/" + baseNameOf(storePath); } + + std::shared_ptr<std::string> getBuildLog(const Path & path) override; }; @@ -645,7 +655,7 @@ ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE")); ref<Store> openStore(const std::string & uri, const Store::Params & params); -void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false); +void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute = false); enum StoreType { tDaemon, diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index a3bbb5170d9f..8ffd55efb23c 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -89,6 +89,12 @@ static ref<std::string> decompressBzip2(const std::string & in) } } +static ref<std::string> decompressBrotli(const std::string & in) +{ + // FIXME: use libbrotli + return make_ref<std::string>(runProgram(BRO, true, {"-d"}, {in})); +} + ref<std::string> compress(const std::string & method, const std::string & in) { StringSink ssink; @@ -106,6 +112,8 @@ ref<std::string> decompress(const std::string & method, const std::string & in) return decompressXZ(in); else if (method == "bzip2") return decompressBzip2(in); + else if (method == "br") + return decompressBrotli(in); else throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); } @@ -139,7 +147,6 @@ struct XzSink : CompressionSink ~XzSink() { - assert(finished); lzma_end(&strm); } @@ -210,7 +217,6 @@ struct BzipSink : CompressionSink ~BzipSink() { - assert(finished); BZ2_bzCompressEnd(&strm); } @@ -261,6 +267,34 @@ struct BzipSink : CompressionSink } }; +struct BrotliSink : CompressionSink +{ + Sink & nextSink; + std::string data; + + BrotliSink(Sink & nextSink) : nextSink(nextSink) + { + } + + ~BrotliSink() + { + } + + // FIXME: use libbrotli + + void finish() override + { + flush(); + nextSink(runProgram(BRO, true, {}, data)); + } + + void write(const unsigned char * data, size_t len) override + { + checkInterrupt(); + this->data.append((const char *) data, len); + } +}; + ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink) { if (method == "none") @@ -269,6 +303,8 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next return make_ref<XzSink>(nextSink); else if (method == "bzip2") return make_ref<BzipSink>(nextSink); + else if (method == "br") + return make_ref<BrotliSink>(nextSink); else throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); } diff --git a/src/libutil/local.mk b/src/libutil/local.mk index cac5c8795db7..0721b21c2089 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -9,3 +9,5 @@ libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) libutil_LIBS = libformat + +libutil_CXXFLAGS = -DBRO=\"$(bro)\" diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 0a5f796e4eaa..99a91c8cc64a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,6 +1,7 @@ #include "util.hh" #include "affinity.hh" #include "sync.hh" +#include "finally.hh" #include <cctype> #include <cerrno> @@ -10,6 +11,7 @@ #include <iostream> #include <sstream> #include <thread> +#include <future> #include <sys/wait.h> #include <unistd.h> @@ -676,12 +678,11 @@ Pid::operator pid_t() } -int Pid::kill(bool quiet) +int Pid::kill() { assert(pid != -1); - if (!quiet) - printError(format("killing process %1%") % pid); + debug(format("killing process %1%") % pid); /* Send the requested signal to the child. If it has its own process group, send the signal to every process in the child @@ -837,23 +838,21 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss) string runProgram(Path program, bool searchPath, const Strings & args, - const string & input) + const std::experimental::optional<std::string> & input) { checkInterrupt(); /* Create a pipe. */ Pipe out, in; out.create(); - if (!input.empty()) in.create(); + if (input) in.create(); /* Fork. */ Pid pid = startProcess([&]() { if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); - if (!input.empty()) { - if (dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - } + if (input && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); Strings args_(args); args_.push_front(program); @@ -870,11 +869,27 @@ string runProgram(Path program, bool searchPath, const Strings & args, out.writeSide = -1; - /* FIXME: This can deadlock if the input is too long. */ - if (!input.empty()) { + std::thread writerThread; + + std::promise<void> promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) + writerThread.join(); + }); + + + if (input) { in.readSide = -1; - writeFull(in.writeSide.get(), input); - in.writeSide = -1; + writerThread = std::thread([&]() { + try { + writeFull(in.writeSide.get(), *input); + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide = -1; + }); } string result = drainFD(out.readSide.get()); @@ -885,6 +900,9 @@ string runProgram(Path program, bool searchPath, const Strings & args, throw ExecError(status, format("program ‘%1%’ %2%") % program % statusToString(status)); + /* Wait for the writer thread to finish. */ + if (input) promise.get_future().get(); + return result; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b74c1d41739a..f94c0ff1c5ee 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -14,6 +14,7 @@ #include <cstdio> #include <map> #include <sstream> +#include <experimental/optional> #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 @@ -202,7 +203,7 @@ public: ~Pid(); void operator =(pid_t pid); operator pid_t(); - int kill(bool quiet = false); + int kill(); int wait(); void setSeparatePG(bool separatePG); @@ -232,7 +233,8 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P /* Run a program and return its stdout in a string (i.e., like the shell backtick operator). */ string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings(), const string & input = ""); + const Strings & args = Strings(), + const std::experimental::optional<std::string> & input = {}); class ExecError : public Error { diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 4340443b5cc2..ed43bffbc8c8 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -47,13 +47,17 @@ int main(int argc, char ** argv) if (sshHost.empty()) throw UsageError("no host name specified"); - auto remoteUri = "legacy-ssh://" + sshHost + (gzip ? "?compress=true" : ""); + auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : ""); auto to = toMode ? openStore(remoteUri) : openStore(); auto from = toMode ? openStore() : openStore(remoteUri); + PathSet storePaths2; + for (auto & path : storePaths) + storePaths2.insert(from->followLinksToStorePath(path)); + PathSet closure; - from->computeFSClosure(storePaths, closure, false, includeOutputs); + from->computeFSClosure(storePaths2, closure, false, includeOutputs); - copyPaths(from, to, Paths(closure.begin(), closure.end()), useSubstitutes); + copyPaths(from, to, closure, useSubstitutes); }); } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index acf603025690..b3b2fcac7132 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -170,10 +170,10 @@ int main(int argc, char * * argv) Path unpacked = (Path) tmpDir + "/unpacked"; createDirs(unpacked); if (hasSuffix(baseNameOf(uri), ".zip")) - runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}, ""); + runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); else // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}, ""); + runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}); /* If the archive unpacks to a single file/directory, then use that as the top-level. */ diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk index 84ff15b241f3..ade0b233adf3 100644 --- a/src/nix-store/local.mk +++ b/src/nix-store/local.mk @@ -7,5 +7,3 @@ nix-store_SOURCES := $(wildcard $(d)/*.cc) nix-store_LIBS = libmain libstore libutil libformat nix-store_LDFLAGS = -lbz2 -pthread $(SODIUM_LIBS) - -nix-store_CXXFLAGS = -DCURL=\"$(curl)\" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 950c2a7c977f..024fa4168ebb 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,7 +9,6 @@ #include "util.hh" #include "worker-protocol.hh" #include "xmlgraph.hh" -#include "compression.hh" #include <iostream> #include <algorithm> @@ -482,58 +481,12 @@ static void opReadLog(Strings opFlags, Strings opArgs) RunPager pager; - // FIXME: move getting logs into Store. - auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store); - if (!store2) throw Error(format("store ‘%s’ does not support reading logs") % store->getUri()); - for (auto & i : opArgs) { - Path path = useDeriver(store->followLinksToStorePath(i)); - - string baseName = baseNameOf(path); - bool found = false; - - for (int j = 0; j < 2; j++) { - - Path logPath = - j == 0 - ? (format("%1%/%2%/%3%/%4%") % store2->logDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str() - : (format("%1%/%2%/%3%") % store2->logDir % drvsLogDir % baseName).str(); - Path logBz2Path = logPath + ".bz2"; - - if (pathExists(logPath)) { - /* !!! Make this run in O(1) memory. */ - string log = readFile(logPath); - writeFull(STDOUT_FILENO, log); - found = true; - break; - } - - else if (pathExists(logBz2Path)) { - std::cout << *decompress("bzip2", readFile(logBz2Path)); - found = true; - break; - } - } - - if (!found) { - for (auto & i : settings.logServers) { - string prefix = i; - if (!prefix.empty() && prefix.back() != '/') prefix += '/'; - string url = prefix + baseName; - try { - string log = runProgram(CURL, true, {"--fail", "--location", "--silent", "--", url}); - std::cout << "(using build log from " << url << ")" << std::endl; - std::cout << log; - found = true; - break; - } catch (ExecError & e) { - /* Ignore errors from curl. FIXME: actually, might be - nice to print a warning on HTTP status != 404. */ - } - } - } - - if (!found) throw Error(format("build log of derivation ‘%1%’ is not available") % path); + auto path = store->followLinksToStorePath(i); + auto log = store->getBuildLog(path); + if (!log) + throw Error("build log of derivation ‘%s’ is not available", path); + std::cout << *log; } } diff --git a/src/nix/command.cc b/src/nix/command.cc index 5a8288da912f..a1b2c120a5d9 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -79,9 +79,14 @@ StoreCommand::StoreCommand() mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri); } +ref<Store> StoreCommand::createStore() +{ + return openStore(storeUri); +} + void StoreCommand::run() { - run(openStore(storeUri)); + run(createStore()); } StorePathsCommand::StorePathsCommand() diff --git a/src/nix/command.hh b/src/nix/command.hh index a29cdcf7f50f..fa6c21abf8ad 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -33,6 +33,7 @@ struct StoreCommand : virtual Command std::string storeUri; StoreCommand(); void run() override; + virtual ref<Store> createStore(); virtual void run(ref<Store>) = 0; }; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 976b0d3e0b81..b2165cb8f85c 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -38,15 +38,19 @@ struct CmdCopy : StorePathsCommand }; } - void run(ref<Store> store, Paths storePaths) override + ref<Store> createStore() override + { + return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); + } + + void run(ref<Store> srcStore, Paths storePaths) override { if (srcUri.empty() && dstUri.empty()) throw UsageError("you must pass ‘--from’ and/or ‘--to’"); - ref<Store> srcStore = srcUri.empty() ? store : openStore(srcUri); - ref<Store> dstStore = dstUri.empty() ? store : openStore(dstUri); + ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(srcStore, dstStore, storePaths); + copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end())); } }; diff --git a/src/nix/log.cc b/src/nix/log.cc new file mode 100644 index 000000000000..d8a3830e91c8 --- /dev/null +++ b/src/nix/log.cc @@ -0,0 +1,57 @@ +#include "command.hh" +#include "common-args.hh" +#include "installables.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdLog : StoreCommand, MixInstallables +{ + CmdLog() + { + } + + std::string name() override + { + return "log"; + } + + std::string description() override + { + return "show the build log of the specified packages or paths"; + } + + void run(ref<Store> store) override + { + auto elems = evalInstallables(store); + + PathSet paths; + + for (auto & elem : elems) { + if (elem.isDrv) + paths.insert(elem.drvPath); + else + paths.insert(elem.outPaths.begin(), elem.outPaths.end()); + } + + auto subs = getDefaultSubstituters(); + + subs.push_front(store); + + for (auto & path : paths) { + bool found = false; + for (auto & sub : subs) { + auto log = sub->getBuildLog(path); + if (!log) continue; + std::cout << *log; + found = true; + break; + } + if (!found) + throw Error("build log of path ‘%s’ is not available", path); + } + } +}; + +static RegisterCommand r1(make_ref<CmdLog>()); |