diff options
Diffstat (limited to 'src')
38 files changed, 478 insertions, 407 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 7ffbdca7c0f4..8719959f0b69 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -201,7 +201,7 @@ connected: printError("somebody is hogging the upload lock for ‘%s’, continuing..."); alarm(0); signal(SIGALRM, old); - copyPaths(store, ref<Store>(sshStore), inputs, false, true); + copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs); uploadLock = -1; BasicDerivation drv(readDerivation(drvPath)); @@ -219,7 +219,7 @@ connected: if (!missing.empty()) { setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */ - copyPaths(ref<Store>(sshStore), store, missing, false, true); + copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs); } return; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 0748fbd3f3e1..8cc50e561354 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -33,7 +33,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) v.type = tBlackhole; //checkInterrupt(); expr->eval(*this, *env, v); - } catch (Error & e) { + } catch (...) { v.type = tThunk; v.thunk.env = env; v.thunk.expr = expr; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0cdce602d7b2..ca4c9a373a3a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -293,6 +293,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) , sWrong(symbols.create("wrong")) , sStructuredAttrs(symbols.create("__structuredAttrs")) , sBuilder(symbols.create("builder")) + , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 46d5a1cc866c..1e32db1e86bd 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -14,6 +14,7 @@ namespace nix { class Store; class EvalState; +enum RepairFlag : bool; typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); @@ -73,7 +74,7 @@ public: /* If set, force copying files to the Nix store even if they already exist there. */ - bool repair = false; + RepairFlag repair; /* If set, don't allow access to files outside of the Nix search path or to environment variables. */ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 99ffddaeb80c..b753d84e2e69 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -708,8 +708,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * HashType ht = parseHashType(outputHashAlgo); if (ht == htUnknown) throw EvalError(format("unknown hash algorithm ‘%1%’, at %2%") % outputHashAlgo % posDrvName); - Hash h = parseHash16or32(ht, *outputHash); - outputHash = printHash(h); + Hash h(*outputHash, ht); + outputHash = h.to_string(Base16, false); if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo; Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); @@ -1701,7 +1701,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, PathSet context; // discarded string s = state.forceString(*args[1], context, pos); - mkString(v, printHash(hashString(ht, s)), context); + mkString(v, hashString(ht, s).to_string(Base16, false), context); } @@ -1852,7 +1852,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (n == "url") url = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "sha256") - expectedHash = parseHash16or32(htSHA256, state.forceStringNoCtx(*attr.value, *attr.pos)); + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 46c5aa21b2eb..8147345c2e1c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -134,7 +134,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath) } void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { if (!repair && isValidPath(info.path)) return; @@ -239,7 +239,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str % duration); /* Atomically write the NAR file. */ - narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : @@ -328,7 +328,7 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, } Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) + bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { // FIXME: some cut&paste from LocalStore::addToStore(). @@ -349,13 +349,13 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, ValidPathInfo info; info.path = makeFixedOutputPath(recursive, h, name); - addToStore(info, sink.s, repair, false, 0); + addToStore(info, sink.s, repair, CheckSigs, nullptr); return info.path; } Path BinaryCacheStore::addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) + const PathSet & references, RepairFlag repair) { ValidPathInfo info; info.path = computeStorePathForText(name, s, references); @@ -364,7 +364,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToStore(info, sink.s, repair, false, 0); + addToStore(info, sink.s, repair, CheckSigs, nullptr); } return info.path; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 87d4aa43838e..f9c1c2cbe8a8 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -85,15 +85,15 @@ public: bool wantMassQuery() override { return wantMassQuery_; } void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; Path addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, - PathFilter & filter, bool repair) override; + PathFilter & filter, RepairFlag repair) override; Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) override; + const PathSet & references, RepairFlag repair) override; void narFromPath(const Path & path, Sink & sink) override; @@ -123,6 +123,8 @@ public: std::shared_ptr<std::string> getBuildLog(const Path & path) override; + int getPriority() override { return priority; } + }; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index bdec30151b08..60b0a531f423 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -262,7 +262,7 @@ public: GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false); + GoalPtr makeSubstitutionGoal(const Path & storePath, RepairFlag repair = NoRepair); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -1087,7 +1087,7 @@ void DerivationGoal::haveDerivation() them. */ if (settings.useSubstitutes && drv->substitutesAllowed()) for (auto & i : invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair)); + addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); @@ -1195,7 +1195,7 @@ void DerivationGoal::repairClosure() printError(format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath); Path drvPath2 = outputsToDrv[i]; if (drvPath2 == "") - addWaitee(worker.makeSubstitutionGoal(i, true)); + addWaitee(worker.makeSubstitutionGoal(i, Repair)); else addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); } @@ -2317,6 +2317,10 @@ void setupSeccomp() seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) throw SysError("unable to add 32-bit seccomp architecture"); + if (settings.thisSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) + throw SysError("unable to add X32 seccomp architecture"); + /* Prevent builders from creating setuid/setgid binaries. */ for (int perm : { S_ISUID, S_ISGID }) { if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, @@ -2340,6 +2344,9 @@ void setupSeccomp() seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) throw SysError("unable to add seccomp rule"); + if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) + throw SysError("unable to set 'no new privileges' seccomp attribute"); + if (seccomp_load(ctx) != 0) throw SysError("unable to load seccomp BPF program"); #endif @@ -2621,7 +2628,7 @@ void DerivationGoal::runChild() ; } #if __APPLE__ - else { + else if (getEnv("_NIX_TEST_NO_SANDBOX") == "") { /* This has to appear before import statements. */ std::string sandboxProfile = "(version 1)\n"; @@ -2736,13 +2743,12 @@ void DerivationGoal::runChild() args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); args.push_back(drv->builder); } -#else +#endif else { builder = drv->builder.c_str(); string builderBasename = baseNameOf(drv->builder); args.push_back(builderBasename); } -#endif for (auto & i : drv->args) args.push_back(rewriteStrings(i, inputRewrites)); @@ -3237,7 +3243,7 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) Path DerivationGoal::addHashRewrite(const Path & path) { string h1 = string(path, worker.store.storeDir.size() + 1, 32); - string h2 = string(printHash32(hashString(htSHA256, "rewrite:" + drvPath + ":" + path)), 0, 32); + string h2 = string(hashString(htSHA256, "rewrite:" + drvPath + ":" + path).to_string(Base32, false), 0, 32); Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33); deletePath(p); assert(path.size() == p.size()); @@ -3292,7 +3298,7 @@ private: std::promise<void> promise; /* Whether to try to repair a valid path. */ - bool repair; + RepairFlag repair; /* Location where we're downloading the substitute. Differs from storePath when doing a repair. */ @@ -3302,7 +3308,7 @@ private: GoalState state; public: - SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false); + SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair = NoRepair); ~SubstitutionGoal(); void timedOut() override { abort(); }; @@ -3338,7 +3344,7 @@ public: }; -SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool repair) +SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair) : Goal(worker) , hasSubstitute(false) , repair(repair) @@ -3601,7 +3607,7 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const Path & drv } -GoalPtr Worker::makeSubstitutionGoal(const Path & path, bool repair) +GoalPtr Worker::makeSubstitutionGoal(const Path & path, RepairFlag repair) { GoalPtr goal = substitutionGoals[path].lock(); if (!goal) { @@ -3954,7 +3960,7 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) if (isDerivation(i2.first)) goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode)); else - goals.insert(worker.makeSubstitutionGoal(i, buildMode)); + goals.insert(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); } worker.run(goals); @@ -4012,7 +4018,7 @@ void LocalStore::ensurePath(const Path & path) void LocalStore::repairPath(const Path & path) { Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, true); + GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); Goals goals = {goal}; worker.run(goals); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 0c6ceb9f6741..48c0837ffaaa 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -23,7 +23,7 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const if (hashType == htUnknown) throw Error(format("unknown hash algorithm ‘%1%’") % algo); - hash = parseHash(hashType, this->hash); + hash = Hash(this->hash, hashType); } @@ -71,7 +71,7 @@ bool BasicDerivation::canBuildLocally() const Path writeDerivation(ref<Store> store, - const Derivation & drv, const string & name, bool repair) + const Derivation & drv, const string & name, RepairFlag repair) { PathSet references; references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); @@ -354,7 +354,7 @@ Hash hashDerivationModulo(Store & store, Derivation drv) h = hashDerivationModulo(store, drv2); drvHashes[i.first] = h; } - inputs2[printHash(h)] = i.second; + inputs2[h.to_string(Base16, false)] = i.second; } drv.inputDrvs = inputs2; @@ -437,7 +437,7 @@ Sink & operator << (Sink & out, const BasicDerivation & drv) std::string hashPlaceholder(const std::string & outputName) { // FIXME: memoize? - return "/" + printHash32(hashString(htSHA256, "nix-output:" + outputName)); + return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 9717a81e469c..7b97730d3bf2 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -2,6 +2,7 @@ #include "types.hh" #include "hash.hh" +#include "store-api.hh" #include <map> @@ -85,7 +86,7 @@ class Store; /* Write a derivation to the Nix store, and return its path. */ Path writeDerivation(ref<Store> store, - const Derivation & drv, const string & name, bool repair = false); + const Derivation & drv, const string & name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ Derivation readDerivation(const Path & drvPath); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 33ab1f027829..15eb68c69ea4 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -581,7 +581,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa Path cacheDir = getCacheDir() + "/nix/tarballs"; createDirs(cacheDir); - string urlHash = printHash32(hashString(htSHA256, url)); + string urlHash = hashString(htSHA256, url).to_string(Base32, false); Path dataFile = cacheDir + "/" + urlHash + ".info"; Path fileLink = cacheDir + "/" + urlHash + "-file"; @@ -631,7 +631,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(false, hash); - store->addToStore(info, sink.s, false, true); + store->addToStore(info, sink.s, NoRepair, NoCheckSigs); storePath = info.path; } @@ -660,7 +660,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa AutoDelete autoDelete(tmpDir, true); // FIXME: this requires GNU tar for decompression. runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, false); + unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); } replaceSymlink(unpackedStorePath, unpackedLink); storePath = unpackedStorePath; diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 6e8bc692cdff..2cbcedc6fb00 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -56,12 +56,12 @@ void Store::exportPath(const Path & path, Sink & sink) Hash hash = hashAndWriteSink.currentHash(); if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error(format("hash of path ‘%1%’ has changed from ‘%2%’ to ‘%3%’!") % path - % printHash(info->narHash) % printHash(hash)); + % info->narHash.to_string() % hash.to_string()); hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0; } -Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, bool dontCheckSigs) +Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) { Paths res; while (true) { @@ -95,7 +95,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, if (readInt(source) == 1) readString(source); - addToStore(info, tee.source.data, false, dontCheckSigs, accessor); + addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); res.push_back(info.path); } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3cdbb114a79d..0cf9f87cac32 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -76,7 +76,7 @@ void LocalStore::syncWithGC() void LocalStore::addIndirectRoot(const Path & path) { - string hash = printHash32(hashString(htSHA1, path)); + string hash = hashString(htSHA1, path).to_string(Base32, false); Path realRoot = canonPath((format("%1%/%2%/auto/%3%") % stateDir % gcRootsDir % hash).str()); makeSymlink(realRoot, path); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index af37ec61d7a1..c8d67b07110b 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -321,6 +321,12 @@ public: Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", "String appended to the user agent in HTTP requests."}; + +#if __linux__ + Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges", + "Whether builders can acquire new privileges by calling programs with " + "setuid/setgid bits or with file capabilities."}; +#endif }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index e09932e3d182..a84f85c1b95a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store } void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override { debug("adding path ‘%s’ to remote host ‘%s’", info.path, host); @@ -168,11 +168,11 @@ struct LegacySSHStore : public Store Path addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, - PathFilter & filter, bool repair) override + PathFilter & filter, RepairFlag repair) override { unsupported(); } Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) override + const PathSet & references, RepairFlag repair) override { unsupported(); } BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, @@ -249,7 +249,8 @@ struct LegacySSHStore : public Store out.insert(res.begin(), res.end()); } - PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override + PathSet queryValidPaths(const PathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override { auto conn(connections->get()); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index aa985ee53d97..7c41dfca7f31 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -30,6 +30,10 @@ #include <sys/xattr.h> #endif +#ifdef __CYGWIN__ +#include <windows.h> +#endif + #include <sqlite3.h> @@ -281,6 +285,16 @@ void LocalStore::openDB(State & state, bool create) SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open Nix database ‘%1%’") % dbPath); +#ifdef __CYGWIN__ + /* The cygwin version of sqlite3 has a patch which calls + SetDllDirectory("/usr/bin") on init. It was intended to fix extension + loading, which we don't use, and the effect of SetDllDirectory is + inherited by child processes, and causes libraries to be loaded from + /usr/bin instead of $PATH. This breaks quite a few things (e.g. + checkPhase on openssh), so we set it back to default behaviour. */ + SetDllDirectoryW(L""); +#endif + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) throwSQLiteError(db, "setting timeout"); @@ -400,6 +414,16 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe { checkInterrupt(); +#if __APPLE__ + /* Remove flags, in particular UF_IMMUTABLE which would prevent + the file from being garbage-collected. FIXME: Use + setattrlist() to remove other attributes as well. */ + if (lchflags(path.c_str(), 0)) { + if (errno != ENOTSUP) + throw SysError(format("clearing flags of path ‘%1%’") % path); + } +#endif + struct stat st; if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path ‘%1%’") % path); @@ -548,7 +572,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (info.path) - ("sha256:" + printHash(info.narHash)) + (info.narHash.to_string(Base16)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver, info.deriver != "") (info.narSize, info.narSize != 0) @@ -590,20 +614,6 @@ uint64_t LocalStore::addValidPath(State & state, } -Hash parseHashField(const Path & path, const string & s) -{ - string::size_type colon = s.find(':'); - if (colon == string::npos) - throw Error(format("corrupt hash ‘%1%’ in valid-path entry for ‘%2%’") - % s % path); - HashType ht = parseHashType(string(s, 0, colon)); - if (ht == htUnknown) - throw Error(format("unknown hash type ‘%1%’ in valid-path entry for ‘%2%’") - % string(s, 0, colon) % path); - return parseHash(ht, string(s, colon + 1)); -} - - void LocalStore::queryPathInfoUncached(const Path & path, std::function<void(std::shared_ptr<ValidPathInfo>)> success, std::function<void(std::exception_ptr exc)> failure) @@ -626,7 +636,11 @@ void LocalStore::queryPathInfoUncached(const Path & path, info->id = useQueryPathInfo.getInt(0); - info->narHash = parseHashField(path, useQueryPathInfo.getStr(1)); + try { + info->narHash = Hash(useQueryPathInfo.getStr(1)); + } catch (BadHash & e) { + throw Error("in valid-path entry for ‘%s’: %s", path, e.what()); + } info->registrationTime = useQueryPathInfo.getInt(2); @@ -661,7 +675,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - ("sha256:" + printHash(info.narHash)) + (info.narHash.to_string(Base16)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) @@ -694,7 +708,7 @@ bool LocalStore::isValidPathUncached(const Path & path) } -PathSet LocalStore::queryValidPaths(const PathSet & paths, bool maybeSubstitute) +PathSet LocalStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) { PathSet res; for (auto & i : paths) @@ -937,7 +951,7 @@ void LocalStore::invalidatePath(State & state, const Path & path) void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { assert(info.narHash); @@ -950,7 +964,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & throw Error("size mismatch importing path ‘%s’; expected %s, got %s", info.path, info.narSize, nar->size()); - if (requireSigs && !dontCheckSigs && !info.checkSignatures(*this, publicKeys)) + if (requireSigs && checkSigs && !info.checkSignatures(*this, publicKeys)) throw Error("cannot add path ‘%s’ because it lacks a valid signature", info.path); addTempRoot(info.path); @@ -988,7 +1002,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & Path LocalStore::addToStoreFromDump(const string & dump, const string & name, - bool recursive, HashType hashAlgo, bool repair) + bool recursive, HashType hashAlgo, RepairFlag repair) { Hash h = hashString(hashAlgo, dump); @@ -1046,7 +1060,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, Path LocalStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) + bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); @@ -1064,7 +1078,7 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, Path LocalStore::addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) + const PathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); auto dstPath = makeTextPath(name, hash, references); @@ -1146,7 +1160,7 @@ void LocalStore::invalidatePathChecked(const Path & path) } -bool LocalStore::verifyStore(bool checkContents, bool repair) +bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { printError(format("reading the Nix store...")); @@ -1187,7 +1201,7 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) if (info->narHash != nullHash && info->narHash != current.first) { printError(format("path ‘%1%’ was modified! " "expected hash ‘%2%’, got ‘%3%’") - % i % printHash(info->narHash) % printHash(current.first)); + % i % info->narHash.to_string() % current.first.to_string()); if (repair) repairPath(i); else errors = true; } else { @@ -1231,7 +1245,7 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) void LocalStore::verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, bool repair, bool & errors) + PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f2c40e96464b..551c6b506fb1 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -98,7 +98,8 @@ public: bool isValidPathUncached(const Path & path) override; - PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override; + PathSet queryValidPaths(const PathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; PathSet queryAllValidPaths() override; @@ -122,22 +123,22 @@ public: SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; Path addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, - PathFilter & filter, bool repair) override; + PathFilter & filter, RepairFlag repair) override; /* Like addToStore(), but the contents of the path are contained in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == false). */ Path addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, bool repair = false); + bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair); Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) override; + const PathSet & references, RepairFlag repair) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; @@ -174,7 +175,7 @@ public: /* Optimise a single store path. */ void optimisePath(const Path & path); - bool verifyStore(bool checkContents, bool repair) override; + bool verifyStore(bool checkContents, RepairFlag repair) override; /* Register the validity of a path, i.e., that `path' exists, that the paths referenced by it exists, and in the case of an output @@ -212,7 +213,7 @@ private: void invalidatePathChecked(const Path & path); void verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, bool repair, bool & errors); + PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors); void updatePathInfo(State & state, const ValidPathInfo & info); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 180a936edb85..6e155e877803 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,9 +203,9 @@ public: narInfo->url = queryNAR.getStr(3); narInfo->compression = queryNAR.getStr(4); if (!queryNAR.isNull(5)) - narInfo->fileHash = parseHash(queryNAR.getStr(5)); + narInfo->fileHash = Hash(queryNAR.getStr(5)); narInfo->fileSize = queryNAR.getInt(6); - narInfo->narHash = parseHash(queryNAR.getStr(7)); + narInfo->narHash = Hash(queryNAR.getStr(7)); narInfo->narSize = queryNAR.getInt(8); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " ")) narInfo->references.insert(cache.storeDir + "/" + r); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d1042c6de25e..660f6a42a19d 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -11,7 +11,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto parseHashField = [&](const string & s) { try { - return parseHash(s); + return Hash(s); } catch (BadHash &) { corrupt(); return Hash(); // never reached @@ -90,10 +90,10 @@ std::string NarInfo::to_string() const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash.type == htSHA256); - res += "FileHash: sha256:" + printHash32(fileHash) + "\n"; + res += "FileHash: " + fileHash.to_string(Base32) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: sha256:" + printHash32(narHash) + "\n"; + res += "NarHash: " + narHash.to_string(Base32) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 9e651ebeaf72..8e8002a30db5 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -149,10 +149,10 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug(format("‘%1%’ has hash ‘%2%’") % path % printHash(hash)); + debug(format("‘%1%’ has hash ‘%2%’") % path % hash.to_string()); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + printHash32(hash); + Path linkPath = linksDir + "/" + hash.to_string(Base32, false); retry: if (!pathExists(linkPath)) { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index be8819bbc004..ab726e79534a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -185,7 +185,7 @@ bool RemoteStore::isValidPathUncached(const Path & path) } -PathSet RemoteStore::queryValidPaths(const PathSet & paths, bool maybeSubstitute) +PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) { auto conn(connections->get()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { @@ -294,7 +294,7 @@ void RemoteStore::queryPathInfoUncached(const Path & path, info->path = path; info->deriver = readString(conn->from); if (info->deriver != "") assertStorePath(info->deriver); - info->narHash = parseHash(htSHA256, readString(conn->from)); + info->narHash = Hash(readString(conn->from), htSHA256); info->references = readStorePaths<PathSet>(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { @@ -357,7 +357,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart) void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { auto conn(connections->get()); @@ -387,10 +387,10 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> else { conn->to << wopAddToStoreNar - << info.path << info.deriver << printHash(info.narHash) + << info.path << info.deriver << info.narHash.to_string(Base16, false) << info.references << info.registrationTime << info.narSize << info.ultimate << info.sigs << info.ca - << repair << dontCheckSigs; + << repair << !checkSigs; conn->to(*nar); conn->processStderr(); } @@ -398,7 +398,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> Path RemoteStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) + bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); @@ -434,7 +434,7 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, Path RemoteStore::addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) + const PathSet & references, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); @@ -570,7 +570,7 @@ void RemoteStore::optimiseStore() } -bool RemoteStore::verifyStore(bool checkContents, bool repair) +bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) { auto conn(connections->get()); conn->to << wopVerifyStore << checkContents << repair; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index ed430e4cabb6..e370e4797d24 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -31,7 +31,8 @@ public: bool isValidPathUncached(const Path & path) override; - PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override; + PathSet queryValidPaths(const PathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; PathSet queryAllValidPaths() override; @@ -55,15 +56,15 @@ public: SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; Path addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false) override; + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false) override; + const PathSet & references, RepairFlag repair) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; @@ -84,7 +85,7 @@ public: void optimiseStore() override; - bool verifyStore(bool checkContents, bool repair) override; + bool verifyStore(bool checkContents, RepairFlag repair) override; void addSignatures(const Path & storePath, const StringSet & sigs) override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index fb36dbc7be74..145a8191c55c 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -12,6 +12,8 @@ #include <aws/core/Aws.h> #include <aws/core/client/ClientConfiguration.h> #include <aws/core/client/DefaultRetryStrategy.h> +#include <aws/core/utils/logging/FormattedLogSystem.h> +#include <aws/core/utils/logging/LogMacros.h> #include <aws/s3/S3Client.h> #include <aws/s3/model/CreateBucketRequest.h> #include <aws/s3/model/GetBucketLocationRequest.h> @@ -41,6 +43,16 @@ R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome) return outcome.GetResultWithOwnership(); } +class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem +{ + using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem; + + void ProcessFormattedStatement(Aws::String && statement) override + { + debug("AWS: %s", chomp(statement)); + } +}; + static void initAWS() { static std::once_flag flag; @@ -51,25 +63,36 @@ static void initAWS() shared.cc), so don't let aws-sdk-cpp override it. */ options.cryptoOptions.initAndCleanupOpenSSL = false; + if (verbosity >= lvlDebug) { + options.loggingOptions.logLevel = + verbosity == lvlDebug + ? Aws::Utils::Logging::LogLevel::Debug + : Aws::Utils::Logging::LogLevel::Trace; + options.loggingOptions.logger_create_fn = [options]() { + return std::make_shared<AwsLogger>(options.loggingOptions.logLevel); + }; + } + Aws::InitAPI(options); }); } S3Helper::S3Helper(const string & region) : config(makeConfig(region)) - , client(make_ref<Aws::S3::S3Client>(*config)) + , client(make_ref<Aws::S3::S3Client>(*config, true, false)) { } /* Log AWS retries. */ class RetryStrategy : public Aws::Client::DefaultRetryStrategy { - long CalculateDelayBeforeNextRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override + bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override { - auto res = Aws::Client::DefaultRetryStrategy::CalculateDelayBeforeNextRetry(error, attemptedRetries); - printError("AWS error '%s' (%s), will retry in %d ms", - error.GetExceptionName(), error.GetMessage(), res); - return res; + auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); + if (retry) + printError("AWS error '%s' (%s), will retry in %d ms", + error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); + return retry; } }; @@ -164,14 +187,20 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET) throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage()); + printInfo("creating S3 bucket ‘%s’...", bucketName); + + // Stupid S3 bucket locations. + auto bucketConfig = Aws::S3::Model::CreateBucketConfiguration(); + if (s3Helper.config->region != "us-east-1") + bucketConfig.SetLocationConstraint( + Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName( + s3Helper.config->region)); + checkAws(format("AWS error creating bucket ‘%s’") % bucketName, s3Helper.client->CreateBucket( Aws::S3::Model::CreateBucketRequest() .WithBucket(bucketName) - .WithCreateBucketConfiguration( - Aws::S3::Model::CreateBucketConfiguration() - /* .WithLocationConstraint( - Aws::S3::Model::BucketLocationConstraint::US) */ ))); + .WithCreateBucketConfiguration(bucketConfig))); } BinaryCacheStore::init(); @@ -210,8 +239,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore if (!res.IsSuccess()) { auto & error = res.GetError(); - if (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME - && error.GetMessage().find("404") != std::string::npos) + if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND + || error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY + || (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME + && error.GetMessage().find("404") != std::string::npos)) return false; throw Error(format("AWS error fetching ‘%s’: %s") % path % error.GetMessage()); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 76ed9942256b..108e2d4ce9b0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -176,13 +176,12 @@ Path Store::makeStorePath(const string & type, const Hash & hash, const string & name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":sha256:" + printHash(hash) + ":" - + storeDir + ":" + name; + string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name; checkStoreName(name); return storeDir + "/" - + printHash32(compressHash(hashString(htSHA256, s), 20)) + + compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) + "-" + name; } @@ -202,7 +201,7 @@ Path Store::makeFixedOutputPath(bool recursive, ? makeStorePath("source", hash, name) : makeStorePath("output:out", hashString(htSHA256, "fixed:out:" + (recursive ? (string) "r:" : "") + - printHashType(hash.type) + ":" + printHash(hash) + ":"), + hash.to_string(Base16) + ":"), name); } @@ -378,7 +377,7 @@ void Store::queryPathInfo(const Path & storePath, } -PathSet Store::queryValidPaths(const PathSet & paths, bool maybeSubstitute) +PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) { struct State { @@ -438,7 +437,7 @@ string Store::makeValidityRegistration(const PathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += printHash(info->narHash) + "\n"; + s += info->narHash.to_string(Base16, false) + "\n"; s += (format("%1%\n") % info->narSize).str(); } @@ -537,14 +536,14 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const Path & storePath, bool repair, bool dontCheckSigs) + const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { auto info = srcStore->queryPathInfo(storePath); StringSink sink; srcStore->narFromPath({storePath}, sink); - if (!info->narHash && dontCheckSigs) { + if (!info->narHash && !checkSigs) { auto info2 = make_ref<ValidPathInfo>(*info); info2->narHash = hashString(htSHA256, *sink.s); if (!info->narSize) info2->narSize = sink.s->size(); @@ -561,33 +560,47 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, assert(info->narHash); - dstStore->addToStore(*info, sink.s, repair, dontCheckSigs); + dstStore->addToStore(*info, sink.s, repair, checkSigs); } -void copyClosure(ref<Store> srcStore, ref<Store> dstStore, - const PathSet & storePaths, bool repair, bool dontCheckSigs) +void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths, + RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { - PathSet closure; + PathSet valid = dstStore->queryValidPaths(storePaths, substitute); + + PathSet missing; for (auto & path : storePaths) - srcStore->computeFSClosure(path, closure); + if (!valid.count(path)) missing.insert(path); - // FIXME: use copyStorePaths() + ThreadPool pool; - PathSet valid = dstStore->queryValidPaths(closure); + processGraph<Path>(pool, + PathSet(missing.begin(), missing.end()), - if (valid.size() == closure.size()) return; + [&](const Path & storePath) { + if (dstStore->isValidPath(storePath)) return PathSet(); + return srcStore->queryPathInfo(storePath)->references; + }, - Paths sorted = srcStore->topoSortPaths(closure); + [&](const Path & storePath) { + checkInterrupt(); - Paths missing; - for (auto i = sorted.rbegin(); i != sorted.rend(); ++i) - if (!valid.count(*i)) missing.push_back(*i); + if (!dstStore->isValidPath(storePath)) { + printError("copying ‘%s’...", storePath); + copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + } + }); +} - printMsg(lvlDebug, format("copying %1% missing paths") % missing.size()); - for (auto & i : missing) - copyStorePath(srcStore, dstStore, i, repair, dontCheckSigs); +void copyClosure(ref<Store> srcStore, ref<Store> dstStore, + const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, + SubstituteFlag substitute) +{ + PathSet closure; + srcStore->computeFSClosure({storePaths}, closure); + copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); } @@ -599,7 +612,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) if (hashGiven) { string s; getline(str, s); - info.narHash = parseHash(htSHA256, s); + info.narHash = Hash(s, htSHA256); getline(str, s); if (!string2Int(s, info.narSize)) throw Error("number expected"); } @@ -634,7 +647,7 @@ std::string ValidPathInfo::fingerprint() const % path); return "1;" + path + ";" - + printHashType(narHash.type) + ":" + printHash32(narHash) + ";" + + narHash.to_string(Base32) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", references); } @@ -653,7 +666,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const }; if (hasPrefix(ca, "text:")) { - auto hash = parseHash(std::string(ca, 5)); + Hash hash(std::string(ca, 5)); if (store.makeTextPath(storePathToName(path), hash, references) == path) return true; else @@ -662,7 +675,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const else if (hasPrefix(ca, "fixed:")) { bool recursive = ca.compare(6, 2, "r:") == 0; - auto hash = parseHash(std::string(ca, recursive ? 8 : 6)); + Hash hash(std::string(ca, recursive ? 8 : 6)); if (store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) return true; else @@ -782,74 +795,31 @@ static RegisterStoreImplementation regStore([]( std::list<ref<Store>> getDefaultSubstituters() { - struct State { - bool done = false; + static auto stores([]() { std::list<ref<Store>> stores; - }; - static Sync<State> state_; - - auto state(state_.lock()); - - if (state->done) return state->stores; - - StringSet done; - - auto addStore = [&](const std::string & uri) { - if (done.count(uri)) return; - done.insert(uri); - state->stores.push_back(openStore(uri)); - }; - for (auto uri : settings.substituters.get()) - addStore(uri); + StringSet done; - for (auto uri : settings.extraSubstituters.get()) - addStore(uri); + auto addStore = [&](const std::string & uri) { + if (done.count(uri)) return; + done.insert(uri); + stores.push_back(openStore(uri)); + }; - state->done = true; + for (auto uri : settings.substituters.get()) + addStore(uri); - return state->stores; -} - - -void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, - bool substitute, bool dontCheckSigs) -{ - PathSet valid = to->queryValidPaths(storePaths, substitute); - - PathSet missing; - for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(path); - - std::string copiedLabel = "copied"; - - //logger->setExpected(copiedLabel, missing.size()); - - ThreadPool pool; + for (auto uri : settings.extraSubstituters.get()) + addStore(uri); - processGraph<Path>(pool, - PathSet(missing.begin(), missing.end()), - - [&](const Path & storePath) { - if (to->isValidPath(storePath)) return PathSet(); - return from->queryPathInfo(storePath)->references; - }, - - [&](const Path & storePath) { - checkInterrupt(); - - if (!to->isValidPath(storePath)) { - //Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); - - copyStorePath(from, to, storePath, false, dontCheckSigs); - - //logger->incProgress(copiedLabel); - } else - ; - //logger->incExpected(copiedLabel, -1); + stores.sort([](ref<Store> & a, ref<Store> & b) { + return a->getPriority() < b->getPriority(); }); - pool.process(); + return stores; + } ()); + + return stores; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 929c95a0f2f8..cada37653e6f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -32,6 +32,11 @@ class Store; class JSONPlaceholder; +enum RepairFlag : bool { NoRepair = false, Repair = true }; +enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; +enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; + + /* Size of the hash part of store paths, in base-32 characters. */ const size_t storePathHashLen = 32; // i.e. 160 bits @@ -332,7 +337,7 @@ public: /* Query which of the given paths is valid. Optionally, try to substitute missing paths. */ virtual PathSet queryValidPaths(const PathSet & paths, - bool maybeSubstitute = false); + SubstituteFlag maybeSubstitute = NoSubstitute); /* Query the set of all valid paths. Note that for some store backends, the name part of store paths may be omitted @@ -392,7 +397,7 @@ public: /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair = false, bool dontCheckSigs = false, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, std::shared_ptr<FSAccessor> accessor = 0) = 0; /* Copy the contents of a path to the store and register the @@ -401,12 +406,12 @@ public: libutil/archive.hh). */ virtual Path addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false) = 0; + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ virtual Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false) = 0; + const PathSet & references, RepairFlag repair = NoRepair) = 0; /* Write a NAR dump of a store path. */ virtual void narFromPath(const Path & path, Sink & sink) = 0; @@ -496,7 +501,7 @@ public: /* Check the integrity of the Nix store. Returns true if errors remain. */ - virtual bool verifyStore(bool checkContents, bool repair) { return false; }; + virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; /* Return an object to access files in the Nix store. */ virtual ref<FSAccessor> getFSAccessor() = 0; @@ -548,7 +553,7 @@ public: preloaded into the specified FS accessor to speed up subsequent access. */ Paths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, - bool dontCheckSigs = false); + CheckSigsFlag checkSigs = CheckSigs); struct Stats { @@ -585,6 +590,11 @@ public: a notion of connection. Otherwise this is a no-op. */ virtual void connect() { }; + /* Get the priority of the store, used to order substituters. In + particular, binary caches can specify a priority field in their + "nix-cache-info" file. Lower value means higher priority. */ + virtual int getPriority() { return 0; } + protected: Stats stats; @@ -650,12 +660,26 @@ void checkStoreName(const string & name); /* Copy a path from one store to another. */ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const Path & storePath, bool repair = false, bool dontCheckSigs = false); + const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + + +/* Copy store paths from one store to another. The paths may be copied + in parallel. They are copied in a topologically sorted order + (i.e. if A is a reference of B, then A is copied before B), but + the set of store paths is not automatically closed; use + copyClosure() for that. */ +void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); /* Copy the closure of the specified paths from one store to another. */ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, - const PathSet & storePaths, bool repair = false, bool dontCheckSigs = false); + const PathSet & storePaths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); /* Remove the temporary roots file for this process. Any temporary @@ -694,9 +718,6 @@ ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"), const Store::Params & extraParams = Store::Params()); -void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, - bool substitute = false, bool dontCheckSigs = false); - enum StoreType { tDaemon, tLocal, diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index fa1bb5d97183..6b45ac859d59 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -16,17 +16,8 @@ namespace nix { -Hash::Hash() +void Hash::init() { - type = htUnknown; - hashSize = 0; - memset(hash, 0, maxHashSize); -} - - -Hash::Hash(HashType type) -{ - this->type = type; if (type == htMD5) hashSize = md5HashSize; else if (type == htSHA1) hashSize = sha1HashSize; else if (type == htSHA256) hashSize = sha256HashSize; @@ -62,16 +53,10 @@ bool Hash::operator < (const Hash & h) const } -std::string Hash::to_string(bool base32) const -{ - return printHashType(type) + ":" + (base32 ? printHash32(*this) : printHash(*this)); -} - - const string base16Chars = "0123456789abcdef"; -string printHash(const Hash & hash) +static string printHash16(const Hash & hash) { char buf[hash.hashSize * 2]; for (unsigned int i = 0; i < hash.hashSize; i++) { @@ -82,42 +67,11 @@ string printHash(const Hash & hash) } -Hash parseHash(const string & s) -{ - string::size_type colon = s.find(':'); - if (colon == string::npos) - throw BadHash(format("invalid hash ‘%s’") % s); - string hts = string(s, 0, colon); - HashType ht = parseHashType(hts); - if (ht == htUnknown) - throw BadHash(format("unknown hash type ‘%s’") % hts); - return parseHash16or32(ht, string(s, colon + 1)); -} - - -Hash parseHash(HashType ht, const string & s) -{ - Hash hash(ht); - if (s.length() != hash.hashSize * 2) - throw BadHash(format("invalid hash ‘%1%’") % s); - for (unsigned int i = 0; i < hash.hashSize; i++) { - string s2(s, i * 2, 2); - if (!isxdigit(s2[0]) || !isxdigit(s2[1])) - throw BadHash(format("invalid hash ‘%1%’") % s); - istringstream_nocopy str(s2); - int n; - str >> std::hex >> n; - hash.hash[i] = n; - } - return hash; -} - - // omitted: E O U T const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; -string printHash32(const Hash & hash) +static string printHash32(const Hash & hash) { assert(hash.hashSize); size_t len = hash.base32Len(); @@ -142,66 +96,103 @@ string printHash32(const Hash & hash) string printHash16or32(const Hash & hash) { - return hash.type == htMD5 ? printHash(hash) : printHash32(hash); + return hash.to_string(hash.type == htMD5 ? Base16 : Base32); } -Hash parseHash32(HashType ht, const string & s) +std::string Hash::to_string(Base base, bool includeType) const { - Hash hash(ht); - size_t len = hash.base32Len(); - assert(s.size() == len); - - for (unsigned int n = 0; n < len; ++n) { - char c = s[len - n - 1]; - unsigned char digit; - for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ - if (base32Chars[digit] == c) break; - if (digit >= 32) - throw BadHash(format("invalid base-32 hash ‘%1%’") % s); - unsigned int b = n * 5; - unsigned int i = b / 8; - unsigned int j = b % 8; - hash.hash[i] |= digit << j; - - if (i < hash.hashSize - 1) { - hash.hash[i + 1] |= digit >> (8 - j); - } else { - if (digit >> (8 - j)) - throw BadHash(format("invalid base-32 hash ‘%1%’") % s); - } + std::string s; + if (includeType) { + s += printHashType(type); + s += ':'; } - - return hash; + switch (base) { + case Base16: + s += printHash16(*this); + break; + case Base32: + s += printHash32(*this); + break; + case Base64: + s += base64Encode(std::string((const char *) hash, hashSize)); + break; + } + return s; } -Hash parseHash16or32(HashType ht, const string & s) +Hash::Hash(const std::string & s, HashType type) + : type(type) { - Hash hash(ht); - if (s.size() == hash.hashSize * 2) - /* hexadecimal representation */ - hash = parseHash(ht, s); - else if (s.size() == hash.base32Len()) - /* base-32 representation */ - hash = parseHash32(ht, s); - else - throw BadHash(format("hash ‘%1%’ has wrong length for hash type ‘%2%’") - % s % printHashType(ht)); - return hash; -} + auto colon = s.find(':'); + + size_t pos = 0; + + if (colon == string::npos) { + if (type == htUnknown) + throw BadHash("hash ‘%s’ does not include a type", s); + } else { + string hts = string(s, 0, colon); + this->type = parseHashType(hts); + if (this->type == htUnknown) + throw BadHash("unknown hash type ‘%s’", hts); + if (type != htUnknown && type != this->type) + throw BadHash("hash ‘%s’ should have type ‘%s’", s, printHashType(type)); + pos = colon + 1; + } + init(); -bool isHash(const string & s) -{ - if (s.length() != 32) return false; - for (int i = 0; i < 32; i++) { - char c = s[i]; - if (!((c >= '0' && c <= '9') || - (c >= 'a' && c <= 'f'))) - return false; + size_t size = s.size() - pos; + + if (size == base16Len()) { + + auto parseHexDigit = [&](char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + throw BadHash("invalid base-16 hash ‘%s’", s); + }; + + for (unsigned int i = 0; i < hashSize; i++) { + hash[i] = + parseHexDigit(s[pos + i * 2]) << 4 + | parseHexDigit(s[pos + i * 2 + 1]); + } } - return true; + + else if (size == base32Len()) { + + for (unsigned int n = 0; n < size; ++n) { + char c = s[pos + size - n - 1]; + unsigned char digit; + for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ + if (base32Chars[digit] == c) break; + if (digit >= 32) + throw BadHash("invalid base-32 hash ‘%s’", s); + unsigned int b = n * 5; + unsigned int i = b / 8; + unsigned int j = b % 8; + hash[i] |= digit << j; + + if (i < hashSize - 1) { + hash[i + 1] |= digit >> (8 - j); + } else { + if (digit >> (8 - j)) + throw BadHash("invalid base-32 hash ‘%s’", s); + } + } + } + + else if (size == base64Len()) { + auto d = base64Decode(std::string(s, pos)); + assert(d.size() == hashSize); + memcpy(hash, d.data(), hashSize); + } + + else + throw BadHash("hash ‘%s’ has wrong length for hash type ‘%s’", s, printHashType(type)); } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 02e213fc7b35..b8b432256c97 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -20,20 +20,30 @@ const int sha512HashSize = 64; extern const string base32Chars; +enum Base : int { Base64, Base32, Base16 }; + struct Hash { static const unsigned int maxHashSize = 64; - unsigned int hashSize; - unsigned char hash[maxHashSize]; + unsigned int hashSize = 0; + unsigned char hash[maxHashSize] = {}; - HashType type; + HashType type = htUnknown; /* Create an unset hash object. */ - Hash(); + Hash() { }; /* Create a zero-filled hash object. */ - Hash(HashType type); + Hash(HashType type) : type(type) { init(); }; + + /* Initialize the hash from a string representation, in the format + "[<type>:]<base16|base32|base64>". If the ‘type’ argument is + htUnknown, then the hash type must be specified in the + string. */ + Hash(const std::string & s, HashType type = htUnknown); + + void init(); /* Check whether a hash is set. */ operator bool () const { return type != htUnknown; } @@ -59,33 +69,22 @@ struct Hash return (hashSize * 8 - 1) / 5 + 1; } - std::string to_string(bool base32 = true) const; -}; - - -/* Convert a hash to a hexadecimal representation. */ -string printHash(const Hash & hash); - -Hash parseHash(const string & s); + /* Returns the length of a base-64 representation of this hash. */ + size_t base64Len() const + { + return ((4 * hashSize / 3) + 3) & ~3; + } -/* Parse a hexadecimal representation of a hash code. */ -Hash parseHash(HashType ht, const string & s); + /* Return a string representation of the hash, in base-16, base-32 + or base-64. By default, this is prefixed by the hash type + (e.g. "sha256:"). */ + std::string to_string(Base base = Base32, bool includeType = true) const; +}; -/* Convert a hash to a base-32 representation. */ -string printHash32(const Hash & hash); /* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ string printHash16or32(const Hash & hash); -/* Parse a base-32 representation of a hash code. */ -Hash parseHash32(HashType ht, const string & s); - -/* Parse a base-16 or base-32 representation of a hash code. */ -Hash parseHash16or32(HashType ht, const string & s); - -/* Verify that the given string is a valid hash code. */ -bool isHash(const string & s); - /* Compute the hash of the given string. */ Hash hashString(HashType ht, const string & s); diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index b64dc52d473e..361a9d33a732 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -70,50 +70,69 @@ void processGraph( struct Graph { std::set<T> left; std::map<T, std::set<T>> refs, rrefs; - std::function<void(T)> wrap; }; - ref<Sync<Graph>> graph_ = make_ref<Sync<Graph>>(); + Sync<Graph> graph_(Graph{nodes, {}, {}}); - auto wrapWork = [&pool, graph_, processNode](const T & node) { + std::function<void(const T &)> worker; + + worker = [&](const T & node) { + + { + auto graph(graph_.lock()); + auto i = graph->refs.find(node); + if (i == graph->refs.end()) + goto getRefs; + goto doWork; + } + + getRefs: + { + auto refs = getEdges(node); + refs.erase(node); + + { + auto graph(graph_.lock()); + for (auto & ref : refs) + if (graph->left.count(ref)) { + graph->refs[node].insert(ref); + graph->rrefs[ref].insert(node); + } + if (graph->refs[node].empty()) + goto doWork; + } + } + + return; + + doWork: processNode(node); - /* Enqueue work for all nodes that were waiting on this one. */ + /* Enqueue work for all nodes that were waiting on this one + and have no unprocessed dependencies. */ { - auto graph(graph_->lock()); - graph->left.erase(node); + auto graph(graph_.lock()); for (auto & rref : graph->rrefs[node]) { auto & refs(graph->refs[rref]); auto i = refs.find(node); assert(i != refs.end()); refs.erase(i); if (refs.empty()) - pool.enqueue(std::bind(graph->wrap, rref)); + pool.enqueue(std::bind(worker, rref)); } + graph->left.erase(node); + graph->refs.erase(node); + graph->rrefs.erase(node); } }; - { - auto graph(graph_->lock()); - graph->left = nodes; - graph->wrap = wrapWork; - } - - /* Build the dependency graph; enqueue all nodes with no - dependencies. */ - for (auto & node : nodes) { - auto refs = getEdges(node); - { - auto graph(graph_->lock()); - for (auto & ref : refs) - if (ref != node && graph->left.count(ref)) { - graph->refs[node].insert(ref); - graph->rrefs[ref].insert(node); - } - if (graph->refs[node].empty()) - pool.enqueue(std::bind(graph->wrap, node)); - } - } + for (auto & node : nodes) + pool.enqueue(std::bind(worker, std::ref(node))); + + pool.process(); + + if (!graph_.lock()->left.empty()) + throw Error("graph processing incomplete (cyclic reference?)"); } } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 42d68fdfdd70..dc80dd6a583e 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -390,6 +390,8 @@ int main(int argc, char ** argv) maybePrintExecError(e); } + if (dryRun) return; + // Set the environment. auto env = getEnv(); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index dc324abcb3ba..0c69bd413569 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -12,7 +12,7 @@ int main(int argc, char ** argv) auto toMode = true; auto includeOutputs = false; auto dryRun = false; - auto useSubstitutes = false; + auto useSubstitutes = NoSubstitute; std::string sshHost; PathSet storePaths; @@ -36,7 +36,7 @@ int main(int argc, char ** argv) else if (*arg == "--dry-run") dryRun = true; else if (*arg == "--use-substitutes" || *arg == "-s") - useSubstitutes = true; + useSubstitutes = Substitute; else if (sshHost.empty()) sshHost = *arg; else @@ -58,6 +58,6 @@ int main(int argc, char ** argv) PathSet closure; from->computeFSClosure(storePaths2, closure, false, includeOutputs); - copyPaths(from, to, closure, useSubstitutes, true); + copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); }); } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 44127635ded8..b029b92db15c 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -216,7 +216,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe startWork(); auto hash = store->queryPathInfo(path)->narHash; stopWork(); - to << printHash(hash); + to << hash.to_string(Base16, false); break; } @@ -304,7 +304,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe string s = readString(from); PathSet refs = readStorePaths<PathSet>(*store, from); startWork(); - Path path = store->addTextToStore(suffix, s, refs, false); + Path path = store->addTextToStore(suffix, s, refs, NoRepair); stopWork(); to << path; break; @@ -324,7 +324,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe case wopImportPaths: { startWork(); TunnelSource source(from); - Paths paths = store->importPaths(source, 0, trusted); + Paths paths = store->importPaths(source, nullptr, + trusted ? NoCheckSigs : CheckSigs); stopWork(); to << paths; break; @@ -549,7 +550,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - to << info->deriver << printHash(info->narHash) << info->references + to << info->deriver << info->narHash.to_string(Base16, false) << info->references << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate @@ -576,7 +577,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe startWork(); if (repair && !trusted) throw Error("you are not privileged to repair paths"); - bool errors = store->verifyStore(checkContents, repair); + bool errors = store->verifyStore(checkContents, (RepairFlag) repair); stopWork(); to << errors; break; @@ -609,7 +610,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe from >> info.deriver; if (!info.deriver.empty()) store->assertStorePath(info.deriver); - info.narHash = parseHash(htSHA256, readString(from)); + info.narHash = Hash(readString(from), htSHA256); info.references = readStorePaths<PathSet>(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); @@ -623,7 +624,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe parseDump(tee, tee.source); startWork(); - store->addToStore(info, tee.source.data, repair, dontCheckSigs, nullptr); + store->addToStore(info, tee.source.data, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); stopWork(); break; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 464bcee4a848..10100d6a6019 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1310,7 +1310,7 @@ int main(int argc, char * * argv) Strings opFlags, opArgs, searchPath; std::map<string, string> autoArgs_; Operation op = 0; - bool repair = false; + RepairFlag repair = NoRepair; string file; Globals globals; @@ -1372,7 +1372,7 @@ int main(int argc, char * * argv) else if (*arg == "--prebuilt-only" || *arg == "-b") globals.prebuiltOnly = true; else if (*arg == "--repair") - repair = true; + repair = Repair; else if (*arg != "" && arg->at(0) == '-') { opFlags.push_back(*arg); /* FIXME: hacky */ diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 25f0b1bd692e..a5d12c1466f7 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -108,7 +108,7 @@ int main(int argc, char * * argv) Strings attrPaths; bool wantsReadWrite = false; std::map<string, string> autoArgs_; - bool repair = false; + RepairFlag repair = NoRepair; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") @@ -146,7 +146,7 @@ int main(int argc, char * * argv) else if (*arg == "--strict") strict = true; else if (*arg == "--repair") - repair = true; + repair = Repair; else if (*arg == "--dry-run") settings.readOnlyMode = true; else if (*arg != "" && arg->at(0) == '-') diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index b3b2fcac7132..47e66eaa6513 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -145,7 +145,7 @@ int main(int argc, char * * argv) Hash hash, expectedHash(ht); Path storePath; if (args.size() == 2) { - expectedHash = parseHash16or32(ht, args[1]); + expectedHash = Hash(args[1], ht); storePath = store->makeFixedOutputPath(unpack, expectedHash, name); if (store->isValidPath(storePath)) hash = expectedHash; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 950222812e2b..6cea57a76714 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -212,7 +212,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string name = *i++; cout << format("%1%\n") % - store->makeFixedOutputPath(recursive, parseHash16or32(hashAlgo, hash), name); + store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name); } @@ -380,9 +380,9 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); - cout << format("sha256:%1%\n") % printHash32(info->narHash); + cout << fmt("%s\n", info->narHash.to_string(Base32)); } else if (query == qSize) - cout << format("%1%\n") % info->narSize; + cout << fmt("%d\n", info->narSize); } } break; @@ -677,7 +677,7 @@ static void opImport(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - Paths paths = store->importPaths(source, nullptr, true); + Paths paths = store->importPaths(source, nullptr, NoCheckSigs); for (auto & i : paths) cout << format("%1%\n") % i << std::flush; @@ -702,11 +702,11 @@ static void opVerify(Strings opFlags, Strings opArgs) throw UsageError("no arguments expected"); bool checkContents = false; - bool repair = false; + RepairFlag repair = NoRepair; for (auto & i : opFlags) if (i == "--check-contents") checkContents = true; - else if (i == "--repair") repair = true; + else if (i == "--repair") repair = Repair; else throw UsageError(format("unknown flag ‘%1%’") % i); if (store->verifyStore(checkContents, repair)) { @@ -734,7 +734,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError( format("path ‘%1%’ was modified! expected hash ‘%2%’, got ‘%3%’") - % path % printHash(info->narHash) % printHash(current.first)); + % path % info->narHash.to_string() % current.first.to_string()); status = 1; } } @@ -871,7 +871,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); - store->importPaths(in, 0, true); // FIXME: should we skip sig checking? + store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking? out << 1; // indicate success break; } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 5dd891e8add3..98de88971127 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -9,15 +9,16 @@ struct CmdHash : Command { enum Mode { mFile, mPath }; Mode mode; - bool base32 = false; + Base base = Base16; bool truncate = false; HashType ht = htSHA512; Strings paths; CmdHash(Mode mode) : mode(mode) { - mkFlag(0, "base32", "print hash in base-32", &base32); - mkFlag(0, "base16", "print hash in base-16", &base32, false); + mkFlag(0, "base64", "print hash in base-64", &base, Base64); + mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); + mkFlag(0, "base16", "print hash in base-16", &base, Base16); mkHashTypeFlag("type", &ht); expectArgs("paths", &paths); } @@ -40,7 +41,7 @@ struct CmdHash : Command Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); std::cout << format("%1%\n") % - (base32 ? printHash32(h) : printHash(h)); + h.to_string(base, false); } } }; @@ -50,11 +51,11 @@ static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath)); struct CmdToBase : Command { - bool toBase32; + Base base; HashType ht = htSHA512; Strings args; - CmdToBase(bool toBase32) : toBase32(toBase32) + CmdToBase(Base base) : base(base) { mkHashTypeFlag("type", &ht); expectArgs("strings", &args); @@ -62,28 +63,29 @@ struct CmdToBase : Command std::string name() override { - return toBase32 ? "to-base32" : "to-base16"; + return + base == Base16 ? "to-base16" : + base == Base32 ? "to-base32" : + "to-base64"; } std::string description() override { - return toBase32 - ? "convert a hash to base-32 representation" - : "convert a hash to base-16 representation"; + return fmt("convert a hash to base-%d representation", + base == Base16 ? 16 : + base == Base32 ? 32 : 64); } void run() override { - for (auto s : args) { - Hash h = parseHash16or32(ht, s); - std::cout << format("%1%\n") % - (toBase32 ? printHash32(h) : printHash(h)); - } + for (auto s : args) + std::cout << fmt("%s\n", Hash(s, ht).to_string(base, false)); } }; -static RegisterCommand r3(make_ref<CmdToBase>(false)); -static RegisterCommand r4(make_ref<CmdToBase>(true)); +static RegisterCommand r3(make_ref<CmdToBase>(Base16)); +static RegisterCommand r4(make_ref<CmdToBase>(Base32)); +static RegisterCommand r5(make_ref<CmdToBase>(Base64)); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) @@ -121,14 +123,14 @@ static int compatNixHash(int argc, char * * argv) if (op == opHash) { CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); cmd.ht = ht; - cmd.base32 = base32; + cmd.base = base32 ? Base32 : Base16; cmd.truncate = truncate; cmd.paths = ss; cmd.run(); } else { - CmdToBase cmd(op == opTo32); + CmdToBase cmd(op == opTo32 ? Base32 : Base16); cmd.args = ss; cmd.ht = ht; cmd.run(); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index f23308b9bc30..9982ff75f4f4 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -189,7 +189,10 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables for (auto & s : ss) { - if (s.find("/") != std::string::npos) { + if (s.compare(0, 1, "(") == 0) + result.push_back(std::make_shared<InstallableExpr>(*this, s)); + + else if (s.find("/") != std::string::npos) { auto path = store->toStorePath(store->followLinksToStore(s)); @@ -201,9 +204,6 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables } } - else if (s.compare(0, 1, "(") == 0) - result.push_back(std::make_shared<InstallableExpr>(*this, s)); - else if (s == "" || std::regex_match(s, attrPathRegex)) result.push_back(std::make_shared<InstallableAttrPath>(*this, s)); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 18533e6066cd..973f60a74ffe 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -94,7 +94,7 @@ struct CmdVerify : StorePathsCommand corrupted = 1; printError( format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’") - % info->path % printHash(info->narHash) % printHash(hash.first)); + % info->path % info->narHash.to_string() % hash.first.to_string()); } } |