From c5e4404580164d3edd043e79cf72bac5bdaecb42 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Aug 2017 15:28:16 +0200 Subject: nix copy: Revive progress bar --- src/libstore/legacy-ssh-store.cc | 1 - src/libstore/store-api.cc | 26 ++++++++++++++++++++++- src/libutil/archive.cc | 2 ++ src/libutil/logging.cc | 11 ++++++++++ src/libutil/logging.hh | 15 +++++++++++++ src/nix/progress-bar.cc | 46 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 99 insertions(+), 2 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 65fe575d2536..855a7c99aed6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -135,7 +135,6 @@ struct LegacySSHStore : public Store if (readInt(conn->from) != 1) throw Error("failed to add path '%s' to remote host '%s', info.path, host"); - } void narFromPath(const Path & path, Sink & sink) override diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index dd87a046e380..88f27e806dce 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -565,8 +565,12 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref srcStore, ref dstStore, const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { + Activity act(actCopyPath, fmt("copying path '%s'", storePath)); + auto info = srcStore->queryPathInfo(storePath); + //act->progress(0, info->size()); + StringSink sink; srcStore->narFromPath({storePath}, sink); @@ -600,13 +604,28 @@ void copyPaths(ref srcStore, ref dstStore, const PathSet & storePa for (auto & path : storePaths) if (!valid.count(path)) missing.insert(path); + Activity act; + + logger->event(evCopyStarted, act); + + std::atomic nrCopied{0}; + std::atomic nrDone{storePaths.size() - missing.size()}; + + auto showProgress = [&]() { + logger->event(evCopyProgress, act, storePaths.size(), nrCopied, nrDone); + }; + ThreadPool pool; processGraph(pool, PathSet(missing.begin(), missing.end()), [&](const Path & storePath) { - if (dstStore->isValidPath(storePath)) return PathSet(); + if (dstStore->isValidPath(storePath)) { + nrDone++; + showProgress(); + return PathSet(); + } return srcStore->queryPathInfo(storePath)->references; }, @@ -616,7 +635,12 @@ void copyPaths(ref srcStore, ref dstStore, const PathSet & storePa if (!dstStore->isValidPath(storePath)) { printInfo("copying '%s'...", storePath); copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + nrCopied++; } + + nrDone++; + + showProgress(); }); } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 51b57a8f4e97..ea1deb924e67 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -56,6 +56,8 @@ static void dumpContents(const Path & path, size_t size, static void dump(const Path & path, Sink & sink, PathFilter & filter) { + checkInterrupt(); + struct stat st; if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path '%1%'") % path); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 43245f61c601..c11271e63007 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -80,4 +80,15 @@ std::atomic nextId{(uint64_t) getpid() << 32}; Activity::Activity() : id(nextId++) { }; +Activity::Activity(ActivityType type, std::string msg) + : Activity() +{ + logger->event(evStartActivity, id, msg); +} + +Activity::~Activity() +{ + logger->event(evStopActivity, id); +} + } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 9ef6e3ee30e4..aa407f60bb65 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -13,6 +13,10 @@ typedef enum { lvlVomit } Verbosity; +typedef enum { + actCopyPath = 100, +} ActivityType; + class Activity { public: @@ -21,6 +25,10 @@ public: Activity(); Activity(const Activity & act) : id(act.id) { }; Activity(uint64_t id) : id(id) { }; + Activity(ActivityType type, std::string msg = ""); + ~Activity(); + + //void progress(...); }; typedef enum { @@ -35,6 +43,13 @@ typedef enum { evSubstitutionCreated = 8, evSubstitutionStarted = 9, evSubstitutionFinished = 10, + + evCopyStarted = 100, + evCopyProgress = 101, + + evStartActivity = 1000, + evStopActivity = 1001, + } EventType; struct Event diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 2ecbea8eeea8..7561a00843aa 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -27,17 +27,29 @@ private: DownloadInfo(const std::string & uri) : uri(uri) { } }; + struct CopyInfo + { + uint64_t expected = 0; + uint64_t copied = 0; + uint64_t done = 0; + }; + struct State { std::map builds; std::set runningBuilds; uint64_t succeededBuilds = 0; uint64_t failedBuilds = 0; + std::map substitutions; std::set runningSubstitutions; uint64_t succeededSubstitutions = 0; + uint64_t downloadedBytes = 0; // finished downloads std::map downloads; + + std::map runningCopies; + std::list activities; std::map::iterator> its; }; @@ -167,11 +179,35 @@ public: res += fmt("%1$.0f/%2$.0f KiB", current / 1024.0, expected / 1024.0); } + if (!state.runningCopies.empty()) { + if (!res.empty()) res += ", "; + uint64_t copied = 0, expected = 0; + for (auto & i : state.runningCopies) { + copied += i.second.copied; + expected += i.second.expected - (i.second.done - i.second.copied); + } + res += fmt("%d/%d copied", copied, expected); + } + return res; } void event(const Event & ev) override { + if (ev.type == evStartActivity) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + createActivity(*state, act, ev.getS(1)); + update(*state); + } + + if (ev.type == evStopActivity) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + deleteActivity(*state, act); + update(*state); + } + if (ev.type == evBuildCreated) { auto state(state_.lock()); state->builds[ev.getI(0)] = ev.getS(1); @@ -280,6 +316,16 @@ public: update(*state); } } + + if (ev.type == evCopyProgress) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + auto & i = state->runningCopies[act]; + i.expected = ev.getI(1); + i.copied = ev.getI(2); + i.done = ev.getI(3); + update(*state); + } } }; -- cgit 1.4.1