diff options
-rw-r--r-- | doc/manual/expressions/builtins.xml | 72 | ||||
-rw-r--r-- | src/libstore/build.cc | 118 | ||||
-rw-r--r-- | src/libstore/download.cc | 11 | ||||
-rw-r--r-- | src/libstore/legacy-ssh-store.cc | 1 | ||||
-rw-r--r-- | src/libstore/local-store.hh | 2 | ||||
-rw-r--r-- | src/libstore/optimise-store.cc | 26 | ||||
-rw-r--r-- | src/libstore/ssh.cc | 11 | ||||
-rw-r--r-- | src/libstore/ssh.hh | 9 | ||||
-rw-r--r-- | src/libstore/store-api.cc | 53 | ||||
-rw-r--r-- | src/libutil/archive.cc | 2 | ||||
-rw-r--r-- | src/libutil/logging.cc | 10 | ||||
-rw-r--r-- | src/libutil/logging.hh | 117 | ||||
-rw-r--r-- | src/libutil/types.hh | 1 | ||||
-rw-r--r-- | src/libutil/util.hh | 14 | ||||
-rw-r--r-- | src/nix-daemon/nix-daemon.cc | 4 | ||||
-rw-r--r-- | src/nix/copy.cc | 10 | ||||
-rw-r--r-- | src/nix/installables.cc | 2 | ||||
-rw-r--r-- | src/nix/optimise-store.cc | 41 | ||||
-rw-r--r-- | src/nix/progress-bar.cc | 351 | ||||
-rw-r--r-- | src/nix/verify.cc | 29 |
20 files changed, 567 insertions, 317 deletions
diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index ac8ecedd0b1f..615314880aba 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -210,42 +210,6 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.match</function> - <replaceable>regex</replaceable> <replaceable>str</replaceable></term> - - <listitem><para>Returns a list if the <link - xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended - POSIX regular expression</link> <replaceable>regex</replaceable> - matches <replaceable>str</replaceable> precisely, otherwise returns - <literal>null</literal>. Each item in the list is a regex group. - -<programlisting> -builtins.match "ab" "abc" -</programlisting> - -Evaluates to <literal>null</literal>. - -<programlisting> -builtins.match "abc" "abc" -</programlisting> - -Evaluates to <literal>[ ]</literal>. - -<programlisting> -builtins.match "a(b)(c)" "abc" -</programlisting> - -Evaluates to <literal>[ "b" "c" ]</literal>. - -<programlisting> -builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " -</programlisting> - -Evaluates to <literal>[ "foo" ]</literal>. - - </para></listitem> - </varlistentry> - <varlistentry><term><function>builtins.elem</function> <replaceable>x</replaceable> <replaceable>xs</replaceable></term> @@ -726,6 +690,42 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ]</programlisting> </varlistentry> + <varlistentry><term><function>builtins.match</function> + <replaceable>regex</replaceable> <replaceable>str</replaceable></term> + + <listitem><para>Returns a list if the <link + xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended + POSIX regular expression</link> <replaceable>regex</replaceable> + matches <replaceable>str</replaceable> precisely, otherwise returns + <literal>null</literal>. Each item in the list is a regex group. + +<programlisting> +builtins.match "ab" "abc" +</programlisting> + +Evaluates to <literal>null</literal>. + +<programlisting> +builtins.match "abc" "abc" +</programlisting> + +Evaluates to <literal>[ ]</literal>. + +<programlisting> +builtins.match "a(b)(c)" "abc" +</programlisting> + +Evaluates to <literal>[ "b" "c" ]</literal>. + +<programlisting> +builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " +</programlisting> + +Evaluates to <literal>[ "foo" ]</literal>. + + </para></listitem> + </varlistentry> + <varlistentry><term><function>builtins.mul</function> <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9250a9b1778f..fc039a8e52b5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -9,6 +9,7 @@ #include "finally.hh" #include "compression.hh" #include "json.hh" +#include "nar-info.hh" #include <algorithm> #include <iostream> @@ -121,8 +122,6 @@ protected: /* Whether the goal is finished. */ ExitCode exitCode; - Activity act; - Goal(Worker & worker) : worker(worker) { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; @@ -244,6 +243,10 @@ private: public: + const Activity act; + const Activity actDerivations; + const Activity actSubstitutions; + /* Set if at least one derivation had a BuildError (i.e. permanent failure). */ bool permanentFailure; @@ -255,6 +258,20 @@ public: std::unique_ptr<HookInstance> hook; + uint64_t expectedBuilds = 0; + uint64_t doneBuilds = 0; + uint64_t failedBuilds = 0; + uint64_t runningBuilds = 0; + + uint64_t expectedSubstitutions = 0; + uint64_t doneSubstitutions = 0; + uint64_t failedSubstitutions = 0; + uint64_t runningSubstitutions = 0; + uint64_t expectedDownloadSize = 0; + uint64_t doneDownloadSize = 0; + uint64_t expectedNarSize = 0; + uint64_t doneNarSize = 0; + Worker(LocalStore & store); ~Worker(); @@ -313,6 +330,14 @@ public: bool pathContentsGood(const Path & path); void markContentsGood(const Path & path); + + void updateProgress() + { + actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); + actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); + act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize); + act.setExpected(actCopyPath, expectedNarSize + doneNarSize); + } }; @@ -810,6 +835,10 @@ private: const static Path homeDir; + std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds; + + std::unique_ptr<Activity> act; + public: DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -907,7 +936,6 @@ private: void amDone(ExitCode result) override { - logger->event(evBuildFinished, act, result == ecSuccess); Goal::amDone(result); } @@ -930,7 +958,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut name = (format("building of '%1%'") % drvPath).str(); trace("created"); - logger->event(evBuildCreated, act, drvPath); + mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); + worker.updateProgress(); } @@ -946,7 +975,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); trace("created"); - logger->event(evBuildCreated, act, drvPath); + mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); + worker.updateProgress(); /* Prevent the .chroot directory from being garbage-collected. (See isActiveTempFile() in gc.cc.) */ @@ -1355,6 +1385,12 @@ void DerivationGoal::tryToBuild() supported for local builds. */ bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); + auto started = [&]() { + act = std::make_unique<Activity>(*logger, actBuild, fmt("building '%s'", drvPath)); + mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); + worker.updateProgress(); + }; + /* Is the build hook willing to accept this job? */ if (!buildLocally) { switch (tryBuildHook()) { @@ -1363,6 +1399,7 @@ void DerivationGoal::tryToBuild() EOF from the hook. */ result.startTime = time(0); // inexact state = &DerivationGoal::buildDone; + started(); return; case rpPostpone: /* Not now; wait until at least one child finishes or @@ -1403,6 +1440,8 @@ void DerivationGoal::tryToBuild() /* This state will be reached when we get EOF on the child's log pipe. */ state = &DerivationGoal::buildDone; + + started(); } @@ -2128,8 +2167,6 @@ void DerivationGoal::startBuilder() } debug(msg); } - - logger->event(evBuildStarted, act); } @@ -3220,7 +3257,7 @@ void DerivationGoal::flushLine() logTail.push_back(currentLogLine); if (logTail.size() > settings.logLines) logTail.pop_front(); } - logger->event(evBuildOutput, act, currentLogLine); + act->result(resBuildLogLine, currentLogLine); currentLogLine = ""; currentLogLinePos = 0; } @@ -3263,6 +3300,19 @@ void DerivationGoal::done(BuildResult::Status status, const string & msg) worker.timedOut = true; if (result.status == BuildResult::PermanentFailure) worker.permanentFailure = true; + + mcExpectedBuilds.reset(); + mcRunningBuilds.reset(); + + if (result.success()) { + if (status == BuildResult::Built) + worker.doneBuilds++; + } else { + if (status != BuildResult::DependencyFailed) + worker.failedBuilds++; + } + + worker.updateProgress(); } @@ -3304,6 +3354,9 @@ private: storePath when doing a repair. */ Path destPath; + std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions, + maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; + typedef void (SubstitutionGoal::*GoalState)(); GoalState state; @@ -3338,7 +3391,6 @@ public: void amDone(ExitCode result) override { - logger->event(evSubstitutionFinished, act, result == ecSuccess); Goal::amDone(result); } }; @@ -3353,7 +3405,7 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, Repa state = &SubstitutionGoal::init; name = (format("substitution of '%1%'") % storePath).str(); trace("created"); - logger->event(evSubstitutionCreated, act, storePath); + maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions); } @@ -3411,6 +3463,12 @@ void SubstitutionGoal::tryNext() In that case the calling derivation should just do a build. */ amDone(hasSubstitute ? ecFailed : ecNoSubstituters); + + if (hasSubstitute) { + worker.failedSubstitutions++; + worker.updateProgress(); + } + return; } @@ -3430,6 +3488,18 @@ void SubstitutionGoal::tryNext() return; } + /* Update the total expected download size. */ + auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info); + + maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, narInfo->narSize); + + maintainExpectedDownload = + narInfo && narInfo->fileSize + ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize) + : nullptr; + + worker.updateProgress(); + hasSubstitute = true; /* Bail out early if this substituter lacks a valid @@ -3489,7 +3559,8 @@ void SubstitutionGoal::tryToRun() printInfo(format("fetching path '%1%'...") % storePath); - logger->event(evSubstitutionStarted, act); + maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); + worker.updateProgress(); outPipe.create(); @@ -3538,6 +3609,22 @@ void SubstitutionGoal::finished() printMsg(lvlChatty, format("substitution of path '%1%' succeeded") % storePath); + maintainRunningSubstitutions.reset(); + + maintainExpectedSubstitutions.reset(); + worker.doneSubstitutions++; + + if (maintainExpectedDownload) { + auto fileSize = maintainExpectedDownload->delta; + maintainExpectedDownload.reset(); + worker.doneDownloadSize += fileSize; + } + + worker.doneNarSize += maintainExpectedNar->delta; + maintainExpectedNar.reset(); + + worker.updateProgress(); + amDone(ecSuccess); } @@ -3560,7 +3647,10 @@ static bool working = false; Worker::Worker(LocalStore & store) - : store(store) + : act(*logger, actRealise) + , actDerivations(*logger, actBuilds) + , actSubstitutions(*logger, actCopyPaths) + , store(store) { /* Debugging: prevent recursive workers. */ if (working) abort(); @@ -3581,6 +3671,10 @@ Worker::~Worker() are in trouble, since goals may call childTerminated() etc. in their destructors). */ topGoals.clear(); + + assert(expectedSubstitutions == 0); + assert(expectedDownloadSize == 0); + assert(expectedNarSize == 0); } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index b731297a2086..71b720a3b59e 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -83,12 +83,12 @@ struct CurlDownloader : public Downloader std::string encoding; DownloadItem(CurlDownloader & downloader, const DownloadRequest & request) - : downloader(downloader), request(request) + : downloader(downloader) + , request(request) + , act(*logger, actDownload, fmt("downloading '%s'", request.uri)) { if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); - - logger->event(evDownloadCreated, act, request.uri); } ~DownloadItem() @@ -105,7 +105,6 @@ struct CurlDownloader : public Downloader } catch (...) { ignoreException(); } - logger->event(evDownloadDestroyed, act); } template<class T> @@ -168,7 +167,7 @@ struct CurlDownloader : public Downloader int progressCallback(double dltotal, double dlnow) { - logger->event(evDownloadProgress, act, dltotal, dlnow); + act.progress(dlnow, dltotal); return _isInterrupted; } @@ -267,7 +266,7 @@ struct CurlDownloader : public Downloader try { result.data = decodeContent(encoding, ref<std::string>(result.data)); callSuccess(success, failure, const_cast<const DownloadResult &>(result)); - logger->event(evDownloadSucceeded, act, result.data->size()); + act.progress(result.data->size(), result.data->size()); } catch (...) { done = true; callFailure(failure, std::current_exception()); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 65fe575d2536..855a7c99aed6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -135,7 +135,6 @@ struct LegacySSHStore : public Store if (readInt(conn->from) != 1) throw Error("failed to add path '%s' to remote host '%s', info.path, host"); - } void narFromPath(const Path & path, Sink & sink) override diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 551c6b506fb1..2af1bfbb3892 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -251,7 +251,7 @@ private: InodeHash loadInodeHash(); Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); - void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash); + void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash); // Internal versions that are not wrapped in retry_sqlite. bool isValidPath_(State & state, const Path & path); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index b736307b31b4..f1325ba5a1ce 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -89,7 +89,8 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa } -void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash) +void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, + const Path & path, InodeHash & inodeHash) { checkInterrupt(); @@ -114,7 +115,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa if (S_ISDIR(st.st_mode)) { Strings names = readDirectoryIgnoringInodes(path, inodeHash); for (auto & i : names) - optimisePath_(stats, path + "/" + i, inodeHash); + optimisePath_(act, stats, path + "/" + i, inodeHash); return; } @@ -244,19 +245,32 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa stats.filesLinked++; stats.bytesFreed += st.st_size; stats.blocksFreed += st.st_blocks; + + if (act) + act->result(resFileLinked, st.st_size, st.st_blocks); } void LocalStore::optimiseStore(OptimiseStats & stats) { + Activity act(*logger, actOptimiseStore); + PathSet paths = queryAllValidPaths(); InodeHash inodeHash = loadInodeHash(); + act.progress(0, paths.size()); + + uint64_t done = 0; + for (auto & i : paths) { addTempRoot(i); if (!isValidPath(i)) continue; /* path was GC'ed, probably */ - //Activity act(*logger, lvlChatty, format("hashing files in '%1%'") % i); - optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash); + { + Activity act(*logger, actUnknown, fmt("optimising path '%s'", i)); + optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash); + } + done++; + act.progress(done, paths.size()); } } @@ -271,7 +285,7 @@ void LocalStore::optimiseStore() optimiseStore(stats); - printError( + printInfo( format("%1% freed by hard-linking %2% files") % showBytes(stats.bytesFreed) % stats.filesLinked); @@ -282,7 +296,7 @@ void LocalStore::optimisePath(const Path & path) OptimiseStats stats; InodeHash inodeHash; - if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash); + if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash); } diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 776ffdb83431..7ff7a9bffc49 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -2,6 +2,17 @@ namespace nix { +SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD) + : host(host) + , keyFile(keyFile) + , useMaster(useMaster) + , compress(compress) + , logFD(logFD) +{ + if (host == "" || hasPrefix(host, "-")) + throw Error("invalid SSH host name '%s'", host); +} + void SSHMaster::addCommonSSHOpts(Strings & args) { for (auto & i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS"))) diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 18dea227ad1f..1268e6d00054 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -28,14 +28,7 @@ private: public: - SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1) - : host(host) - , keyFile(keyFile) - , useMaster(useMaster) - , compress(compress) - , logFD(logFD) - { - } + SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1); struct Connection { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index dd87a046e380..f520210615a3 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -565,9 +565,30 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { + Activity act(*logger, actCopyPath, fmt("copying path '%s'", storePath)); + auto info = srcStore->queryPathInfo(storePath); - StringSink sink; + uint64_t total = 0; + + auto progress = [&](size_t len) { + total += len; + act.progress(total, info->narSize); + }; + + struct MyStringSink : StringSink + { + typedef std::function<void(size_t)> Callback; + Callback callback; + MyStringSink(Callback callback) : callback(callback) { } + void operator () (const unsigned char * data, size_t len) override + { + StringSink::operator ()(data, len); + callback(len); + }; + }; + + MyStringSink sink(progress); srcStore->narFromPath({storePath}, sink); if (!info->narHash && !checkSigs) { @@ -600,23 +621,47 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa for (auto & path : storePaths) if (!valid.count(path)) missing.insert(path); + Activity act(*logger, actCopyPaths, fmt("copying %d paths", missing.size())); + + std::atomic<size_t> nrDone{0}; + std::atomic<uint64_t> bytesExpected{0}; + std::atomic<uint64_t> nrRunning{0}; + + auto showProgress = [&]() { + act.progress(nrDone, missing.size(), nrRunning); + }; + ThreadPool pool; processGraph<Path>(pool, PathSet(missing.begin(), missing.end()), [&](const Path & storePath) { - if (dstStore->isValidPath(storePath)) return PathSet(); - return srcStore->queryPathInfo(storePath)->references; + if (dstStore->isValidPath(storePath)) { + nrDone++; + showProgress(); + return PathSet(); + } + + auto info = srcStore->queryPathInfo(storePath); + + bytesExpected += info->narSize; + act.setExpected(actCopyPath, bytesExpected); + + return info->references; }, [&](const Path & storePath) { checkInterrupt(); if (!dstStore->isValidPath(storePath)) { - printInfo("copying '%s'...", storePath); + MaintainCount<decltype(nrRunning)> mc(nrRunning); + showProgress(); copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); } + + nrDone++; + showProgress(); }); } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 51b57a8f4e97..ea1deb924e67 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -56,6 +56,8 @@ static void dumpContents(const Path & path, size_t size, static void dump(const Path & path, Sink & sink, PathFilter & filter) { + checkInterrupt(); + struct stat st; if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path '%1%'") % path); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 43245f61c601..900f24e4cbdf 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -43,10 +43,6 @@ public: writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); } - - void event(const Event & ev) override - { - } }; Verbosity verbosity = lvlInfo; @@ -78,6 +74,10 @@ Logger * makeDefaultLogger() std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32}; -Activity::Activity() : id(nextId++) { }; +Activity::Activity(Logger & logger, ActivityType type, const std::string & s) + : logger(logger), id(nextId++) +{ + logger.startActivity(id, type, s); +} } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 9ef6e3ee30e4..1e0c735676b6 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -13,32 +13,52 @@ typedef enum { lvlVomit } Verbosity; -class Activity -{ -public: - typedef uint64_t t; - const t id; - Activity(); - Activity(const Activity & act) : id(act.id) { }; - Activity(uint64_t id) : id(id) { }; -}; +typedef enum { + actUnknown = 0, + actCopyPath = 100, + actDownload = 101, + actRealise = 102, + actCopyPaths = 103, + actBuilds = 104, + actBuild = 105, + actOptimiseStore = 106, + actVerifyPaths = 107, +} ActivityType; typedef enum { - evBuildCreated = 0, - evBuildStarted = 1, - evBuildOutput = 2, - evBuildFinished = 3, - evDownloadCreated = 4, - evDownloadDestroyed = 5, - evDownloadProgress = 6, - evDownloadSucceeded = 7, - evSubstitutionCreated = 8, - evSubstitutionStarted = 9, - evSubstitutionFinished = 10, -} EventType; - -struct Event + resFileLinked = 100, + resBuildLogLine = 101, + resUntrustedPath = 102, + resCorruptedPath = 103, +} ResultType; + +typedef uint64_t ActivityId; + +class Logger { + friend class Activity; + +public: + + virtual ~Logger() { } + + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + + void log(const FormatOrString & fs) + { + log(lvlInfo, fs); + } + + virtual void warn(const std::string & msg); + + virtual void startActivity(ActivityId act, ActivityType type, const std::string & s) { }; + + virtual void stopActivity(ActivityId act) { }; + + virtual void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) { }; + + virtual void setExpected(ActivityId act, ActivityType type, uint64_t expected) { }; + struct Field { // FIXME: use std::variant. @@ -48,56 +68,37 @@ struct Event Field(const std::string & s) : type(tString), s(s) { } Field(const char * s) : type(tString), s(s) { } Field(const uint64_t & i) : type(tInt), i(i) { } - Field(const Activity & act) : type(tInt), i(act.id) { } }; - typedef std::vector<Field> Fields; - - EventType type; - Fields fields; - - std::string getS(size_t n) const - { - assert(n < fields.size()); - assert(fields[n].type == Field::tString); - return fields[n].s; - } - - uint64_t getI(size_t n) const - { - assert(n < fields.size()); - assert(fields[n].type == Field::tInt); - return fields[n].i; - } + virtual void result(ActivityId act, ResultType type, const std::vector<Field> & fields) { }; }; -class Logger +struct Activity { - friend class Activity; + Logger & logger; -public: + const ActivityId id; - virtual ~Logger() { } + Activity(Logger & logger, ActivityType type, const std::string & s = ""); - virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + ~Activity() + { logger.stopActivity(id); } - void log(const FormatOrString & fs) - { - log(lvlInfo, fs); - } + void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const + { logger.progress(id, done, expected, running, failed); } - virtual void warn(const std::string & msg); + void setExpected(ActivityType type2, uint64_t expected) const + { logger.setExpected(id, type2, expected); } template<typename... Args> - void event(EventType type, const Args & ... args) + void result(ResultType type, const Args & ... args) { - Event ev; - ev.type = type; - nop{(ev.fields.emplace_back(Event::Field(args)), 1)...}; - event(ev); + std::vector<Logger::Field> fields; + nop{(fields.emplace_back(Logger::Field(args)), 1)...}; + logger.result(id, type, fields); } - virtual void event(const Event & ev) = 0; + friend class Logger; }; extern Logger * logger; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 9f32d31addbf..92bf469b5c6f 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -70,6 +70,7 @@ template<typename... Args> inline std::string fmt(const std::string & fs, Args... args) { boost::format f(fs); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); nop{boost::io::detail::feed(f, args)...}; return f.str(); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index f37f2c5d1be5..35909c8d5b31 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -462,4 +462,18 @@ struct ReceiveInterrupts { } }; + + +/* A RAII helper that increments a counter on construction and + decrements it on destruction. */ +template<typename T> +struct MaintainCount +{ + T & counter; + long delta; + MaintainCount(T & counter, long delta = 1) : counter(counter), delta(delta) { counter += delta; } + ~MaintainCount() { counter -= delta; } +}; + + } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 2b223c882647..7e6f3aa25b5e 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -81,10 +81,6 @@ class TunnelLogger : public Logger } else defaultLogger->log(lvl, fs); } - - void event(const Event & ev) override - { - } }; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 9f1a80d07bea..d9e6c949352b 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -12,10 +12,17 @@ struct CmdCopy : StorePathsCommand { std::string srcUri, dstUri; + CheckSigsFlag checkSigs = CheckSigs; + CmdCopy() { mkFlag(0, "from", "store-uri", "URI of the source Nix store", &srcUri); mkFlag(0, "to", "store-uri", "URI of the destination Nix store", &dstUri); + + mkFlag() + .longName("no-check-sigs") + .description("do not require that paths are signed by trusted keys") + .handler([&](Strings ss) { checkSigs = NoCheckSigs; }); } std::string name() override @@ -50,7 +57,8 @@ struct CmdCopy : StorePathsCommand ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end())); + copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()), + NoRepair, checkSigs); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 54c9f7841fe0..9d812fee47ec 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -229,7 +229,7 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables PathSet InstallablesCommand::toStorePaths(ref<Store> store, ToStorePathsMode mode) { - if (mode != DryRun) + if (mode != Build) settings.readOnlyMode = true; PathSet outPaths, buildables; diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc new file mode 100644 index 000000000000..725fb75a19ec --- /dev/null +++ b/src/nix/optimise-store.cc @@ -0,0 +1,41 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" + +#include <atomic> + +using namespace nix; + +struct CmdOptimiseStore : StoreCommand +{ + CmdOptimiseStore() + { + } + + std::string name() override + { + return "optimise-store"; + } + + std::string description() override + { + return "replace identical files in the store by hard links"; + } + + Examples examples() override + { + return { + Example{ + "To optimise the Nix store:", + "nix optimise-store" + }, + }; + } + + void run(ref<Store> store) override + { + store->optimiseStore(); + } +}; + +static RegisterCommand r1(make_ref<CmdOptimiseStore>()); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 2ecbea8eeea8..1616c420551a 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -8,8 +8,24 @@ #include <sys/ioctl.h> +#include <iostream> + namespace nix { +static std::string getS(const std::vector<Logger::Field> & fields, size_t n) +{ + assert(n < fields.size()); + assert(fields[n].type == Logger::Field::tString); + return fields[n].s; +} + +static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n) +{ + assert(n < fields.size()); + assert(fields[n].type == Logger::Field::tInt); + return fields[n].i; +} + class ProgressBar : public Logger { private: @@ -17,29 +33,32 @@ private: struct ActInfo { std::string s, s2; + ActivityType type = actUnknown; + uint64_t done = 0; + uint64_t expected = 0; + uint64_t running = 0; + uint64_t failed = 0; + std::map<ActivityType, uint64_t> expectedByType; }; - struct DownloadInfo + struct ActivitiesByType { - std::string uri; - uint64_t current = 0; + std::map<ActivityId, std::list<ActInfo>::iterator> its; + uint64_t done = 0; uint64_t expected = 0; - DownloadInfo(const std::string & uri) : uri(uri) { } + uint64_t failed = 0; }; struct State { - std::map<Activity::t, Path> builds; - std::set<Activity::t> runningBuilds; - uint64_t succeededBuilds = 0; - uint64_t failedBuilds = 0; - std::map<Activity::t, Path> substitutions; - std::set<Activity::t> runningSubstitutions; - uint64_t succeededSubstitutions = 0; - uint64_t downloadedBytes = 0; // finished downloads - std::map<Activity::t, DownloadInfo> downloads; std::list<ActInfo> activities; - std::map<Activity::t, std::list<ActInfo>::iterator> its; + std::map<ActivityId, std::list<ActInfo>::iterator> its; + + std::map<ActivityType, ActivitiesByType> activitiesByType; + + uint64_t filesLinked = 0, bytesLinked = 0; + + uint64_t corruptedPaths = 0, untrustedPaths = 0; }; Sync<State> state_; @@ -58,7 +77,10 @@ public: ~ProgressBar() { auto state(state_.lock()); + std::string status = getStatus(*state); writeToStderr("\r\e[K"); + if (status != "") + writeToStderr("[" + status + "]\n"); } void log(Verbosity lvl, const FormatOrString & fs) override @@ -73,30 +95,102 @@ public: update(state); } - void createActivity(State & state, Activity::t activity, const std::string & s) + void startActivity(ActivityId act, ActivityType type, const std::string & s) override { - state.activities.emplace_back(ActInfo{s}); - state.its.emplace(activity, std::prev(state.activities.end())); + auto state(state_.lock()); + + state->activities.emplace_back(ActInfo{s, "", type}); + auto i = std::prev(state->activities.end()); + state->its.emplace(act, i); + state->activitiesByType[type].its.emplace(act, i); + + update(*state); } - void deleteActivity(State & state, Activity::t activity) + void stopActivity(ActivityId act) override { - auto i = state.its.find(activity); - if (i != state.its.end()) { - state.activities.erase(i->second); - state.its.erase(i); + auto state(state_.lock()); + + auto i = state->its.find(act); + if (i != state->its.end()) { + auto & actByType = state->activitiesByType[i->second->type]; + actByType.done += i->second->done; + actByType.failed += i->second->failed; + + for (auto & j : i->second->expectedByType) + state->activitiesByType[j.first].expected -= j.second; + + actByType.its.erase(act); + state->activities.erase(i->second); + state->its.erase(i); } + + update(*state); + } + + void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) override + { + auto state(state_.lock()); + + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo & actInfo = *i->second; + actInfo.done = done; + actInfo.expected = expected; + actInfo.running = running; + actInfo.failed = failed; + + update(*state); + } + + void setExpected(ActivityId act, ActivityType type, uint64_t expected) override + { + auto state(state_.lock()); + + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo & actInfo = *i->second; + auto & j = actInfo.expectedByType[type]; + state->activitiesByType[type].expected -= j; + j = expected; + state->activitiesByType[type].expected += j; + + update(*state); } - void updateActivity(State & state, Activity::t activity, const std::string & s2) + void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override { - auto i = state.its.find(activity); - assert(i != state.its.end()); - ActInfo info = *i->second; - state.activities.erase(i->second); - info.s2 = s2; - state.activities.emplace_back(info); - i->second = std::prev(state.activities.end()); + auto state(state_.lock()); + + if (type == resFileLinked) { + state->filesLinked++; + state->bytesLinked += getI(fields, 0); + update(*state); + } + + else if (type == resBuildLogLine) { + auto s2 = trim(getS(fields, 0)); + if (!s2.empty()) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo info = *i->second; + state->activities.erase(i->second); + info.s2 = s2; + state->activities.emplace_back(info); + i->second = std::prev(state->activities.end()); + update(*state); + } + } + + else if (type == resUntrustedPath) { + state->untrustedPaths++; + update(*state); + } + + else if (type == resCorruptedPath) { + state->corruptedPaths++; + update(*state); + } } void update() @@ -119,10 +213,16 @@ public: if (!state.activities.empty()) { if (!status.empty()) line += " "; auto i = state.activities.rbegin(); - line += i->s; - if (!i->s2.empty()) { - line += ": "; - line += i->s2; + + while (i != state.activities.rend() && i->s.empty() && i->s2.empty()) + ++i; + + if (i != state.activities.rend()) { + line += i->s; + if (!i->s2.empty()) { + if (!i->s.empty()) line += ": "; + line += i->s2; + } } } @@ -132,154 +232,85 @@ public: std::string getStatus(State & state) { + auto MiB = 1024.0 * 1024.0; + std::string res; - if (state.failedBuilds) { - if (!res.empty()) res += ", "; - res += fmt(ANSI_RED "%d failed" ANSI_NORMAL, state.failedBuilds); - } + auto renderActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) { + auto & act = state.activitiesByType[type]; + uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed; + for (auto & j : act.its) { + done += j.second->done; + expected += j.second->expected; + running += j.second->running; + failed += j.second->failed; + } - if (!state.builds.empty() || state.succeededBuilds) - { - if (!res.empty()) res += ", "; - if (!state.runningBuilds.empty()) - res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningBuilds.size()); - res += fmt(ANSI_GREEN "%d/%d built" ANSI_NORMAL, - state.succeededBuilds, state.succeededBuilds + state.builds.size()); - } + expected = std::max(expected, act.expected); - if (!state.substitutions.empty() || state.succeededSubstitutions) { - if (!res.empty()) res += ", "; - if (!state.runningSubstitutions.empty()) - res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningSubstitutions.size()); - res += fmt(ANSI_GREEN "%d/%d fetched" ANSI_NORMAL, - state.succeededSubstitutions, - state.succeededSubstitutions + state.substitutions.size()); - } + std::string s; - if (!state.downloads.empty() || state.downloadedBytes) { - if (!res.empty()) res += ", "; - uint64_t expected = state.downloadedBytes, current = state.downloadedBytes; - for (auto & i : state.downloads) { - expected += i.second.expected; - current += i.second.current; - } - res += fmt("%1$.0f/%2$.0f KiB", current / 1024.0, expected / 1024.0); - } + if (running || done || expected || failed) { + if (running) + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + running / unit, done / unit, expected / unit); + else if (expected != done) + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + done / unit, expected / unit); + else + s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit); + s = fmt(itemFmt, s); - return res; - } + if (failed) + s += fmt(" (" ANSI_RED "%d failed" ANSI_NORMAL ")", failed / unit); + } - void event(const Event & ev) override - { - if (ev.type == evBuildCreated) { - auto state(state_.lock()); - state->builds[ev.getI(0)] = ev.getS(1); - update(*state); - } + return s; + }; - if (ev.type == evBuildStarted) { - auto state(state_.lock()); - Activity::t act = ev.getI(0); - state->runningBuilds.insert(act); - auto name = storePathToName(state->builds[act]); - if (hasSuffix(name, ".drv")) - name.resize(name.size() - 4); - createActivity(*state, act, fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name)); - update(*state); - } + auto showActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) { + auto s = renderActivity(type, itemFmt, numberFmt, unit); + if (s.empty()) return; + if (!res.empty()) res += ", "; + res += s; + }; - if (ev.type == evBuildFinished) { - auto state(state_.lock()); - Activity::t act = ev.getI(0); - if (ev.getI(1)) { - if (state->runningBuilds.count(act)) - state->succeededBuilds++; - } else - state->failedBuilds++; - state->runningBuilds.erase(act); - state->builds.erase(act); - deleteActivity(*state, act); - update(*state); - } + showActivity(actBuilds, "%s built"); - if (ev.type == evBuildOutput) { - auto state(state_.lock()); - Activity::t act = ev.getI(0); - assert(state->runningBuilds.count(act)); - updateActivity(*state, act, ev.getS(1)); - update(*state); - } + auto s1 = renderActivity(actCopyPaths, "%s copied"); + auto s2 = renderActivity(actCopyPath, "%s MiB", "%.1f", MiB); - if (ev.type == evSubstitutionCreated) { - auto state(state_.lock()); - state->substitutions[ev.getI(0)] = ev.getS(1); - update(*state); + if (!s1.empty() || !s2.empty()) { + if (!res.empty()) res += ", "; + if (s1.empty()) res += "0 copied"; else res += s1; + if (!s2.empty()) { res += " ("; res += s2; res += ')'; } } - if (ev.type == evSubstitutionStarted) { - auto state(state_.lock()); - Activity::t act = ev.getI(0); - state->runningSubstitutions.insert(act); - auto name = storePathToName(state->substitutions[act]); - createActivity(*state, act, fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL, name)); - update(*state); - } + showActivity(actDownload, "%s MiB DL", "%.1f", MiB); - if (ev.type == evSubstitutionFinished) { - auto state(state_.lock()); - Activity::t act = ev.getI(0); - if (ev.getI(1)) { - if (state->runningSubstitutions.count(act)) - state->succeededSubstitutions++; + { + auto s = renderActivity(actOptimiseStore, "%s paths optimised"); + if (s != "") { + s += fmt(", %.1f MiB / %d inodes freed", state.bytesLinked / MiB, state.filesLinked); + if (!res.empty()) res += ", "; + res += s; } - state->runningSubstitutions.erase(act); - state->substitutions.erase(act); - deleteActivity(*state, act); - update(*state); } - if (ev.type == evDownloadCreated) { - auto state(state_.lock()); - Activity::t act = ev.getI(0); - std::string uri = ev.getS(1); - state->downloads.emplace(act, DownloadInfo{uri}); - if (state->runningSubstitutions.empty()) // FIXME: hack - createActivity(*state, act, fmt("downloading " ANSI_BOLD "%s" ANSI_NORMAL "", uri)); - update(*state); - } + // FIXME: don't show "done" paths in green. + showActivity(actVerifyPaths, "%s paths verified"); - if (ev.type == evDownloadProgress) { - auto state(state_.lock()); - Activity::t act = ev.getI(0); - auto i = state->downloads.find(act); - assert(i != state->downloads.end()); - i->second.expected = ev.getI(1); - i->second.current = ev.getI(2); - update(*state); + if (state.corruptedPaths) { + if (!res.empty()) res += ", "; + res += fmt(ANSI_RED "%d corrupted" ANSI_NORMAL, state.corruptedPaths); } - if (ev.type == evDownloadSucceeded) { - auto state(state_.lock()); - Activity::t act = ev.getI(0); - auto i = state->downloads.find(act); - assert(i != state->downloads.end()); - state->downloadedBytes += ev.getI(1); - state->downloads.erase(i); - deleteActivity(*state, act); - update(*state); + if (state.untrustedPaths) { + if (!res.empty()) res += ", "; + res += fmt(ANSI_RED "%d untrusted" ANSI_NORMAL, state.untrustedPaths); } - if (ev.type == evDownloadDestroyed) { - auto state(state_.lock()); - Activity::t act = ev.getI(0); - auto i = state->downloads.find(act); - if (i != state->downloads.end()) { - state->downloads.erase(i); - deleteActivity(*state, act); - update(*state); - } - } + return res; } }; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 0d9739314b53..7156341396d3 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -61,16 +61,17 @@ struct CmdVerify : StorePathsCommand auto publicKeys = getDefaultPublicKeys(); + Activity act(*logger, actVerifyPaths); + std::atomic<size_t> done{0}; std::atomic<size_t> untrusted{0}; std::atomic<size_t> corrupted{0}; std::atomic<size_t> failed{0}; + std::atomic<size_t> active{0}; - std::string doneLabel("paths checked"); - std::string untrustedLabel("untrusted"); - std::string corruptedLabel("corrupted"); - std::string failedLabel("failed"); - //logger->setExpected(doneLabel, storePaths.size()); + auto update = [&]() { + act.progress(done, storePaths.size(), active, failed); + }; ThreadPool pool; @@ -78,7 +79,10 @@ struct CmdVerify : StorePathsCommand try { checkInterrupt(); - //Activity act(*logger, lvlInfo, format("checking '%s'") % storePath); + Activity act2(*logger, actUnknown, fmt("checking '%s'", storePath)); + + MaintainCount<std::atomic<size_t>> mcActive(active); + update(); auto info = store->queryPathInfo(storePath); @@ -90,8 +94,8 @@ struct CmdVerify : StorePathsCommand auto hash = sink.finish(); if (hash.first != info->narHash) { - //logger->incProgress(corruptedLabel); - corrupted = 1; + corrupted++; + act2.result(resCorruptedPath, info->path); printError( format("path '%s' was modified! expected hash '%s', got '%s'") % info->path % info->narHash.to_string() % hash.first.to_string()); @@ -142,21 +146,21 @@ struct CmdVerify : StorePathsCommand } if (!good) { - //logger->incProgress(untrustedLabel); untrusted++; + act2.result(resUntrustedPath, info->path); printError(format("path '%s' is untrusted") % info->path); } } - //logger->incProgress(doneLabel); done++; } catch (Error & e) { printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); - //logger->incProgress(failedLabel); failed++; } + + update(); }; for (auto & storePath : storePaths) @@ -164,9 +168,6 @@ struct CmdVerify : StorePathsCommand pool.process(); - printInfo(format("%d paths checked, %d untrusted, %d corrupted, %d failed") - % done % untrusted % corrupted % failed); - throw Exit( (corrupted ? 1 : 0) | (untrusted ? 2 : 0) | |