diff options
Diffstat (limited to '')
174 files changed, 32044 insertions, 34606 deletions
diff --git a/third_party/nix/src/build-remote/build-remote.cc b/third_party/nix/src/build-remote/build-remote.cc index 279ae62f69cc..42a93bae8a0d 100644 --- a/third_party/nix/src/build-remote/build-remote.cc +++ b/third_party/nix/src/build-remote/build-remote.cc @@ -1,266 +1,275 @@ +#include <algorithm> #include <cstdlib> #include <cstring> -#include <algorithm> -#include <set> +#include <iomanip> #include <memory> +#include <set> #include <tuple> -#include <iomanip> #if __APPLE__ #include <sys/time.h> #endif +#include "derivations.hh" +#include "globals.hh" +#include "legacy.hh" +#include "local-store.hh" #include "machines.hh" -#include "shared.hh" #include "pathlocks.hh" -#include "globals.hh" #include "serialise.hh" +#include "shared.hh" #include "store-api.hh" -#include "derivations.hh" -#include "local-store.hh" -#include "legacy.hh" using namespace nix; using std::cin; -static void handleAlarm(int sig) { -} +static void handleAlarm(int sig) {} -std::string escapeUri(std::string uri) -{ - std::replace(uri.begin(), uri.end(), '/', '_'); - return uri; +std::string escapeUri(std::string uri) { + std::replace(uri.begin(), uri.end(), '/', '_'); + return uri; } static string currentLoad; -static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) -{ - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); +static AutoCloseFD openSlotLock(const Machine& m, unsigned long long slot) { + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), + true); } static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) { - for (auto & feature : requiredFeatures) - if (!settings.systemFeatures.get().count(feature)) return false; - return true; + for (auto& feature : requiredFeatures) + if (!settings.systemFeatures.get().count(feature)) return false; + return true; } -static int _main(int argc, char * * argv) -{ - { - logger = makeJSONLogger(*logger); +static int _main(int argc, char** argv) { + { + logger = makeJSONLogger(*logger); - /* Ensure we don't get any SSH passphrase or host key popups. */ - unsetenv("DISPLAY"); - unsetenv("SSH_ASKPASS"); + /* Ensure we don't get any SSH passphrase or host key popups. */ + unsetenv("DISPLAY"); + unsetenv("SSH_ASKPASS"); - if (argc != 2) - throw UsageError("called without required arguments"); + if (argc != 2) throw UsageError("called without required arguments"); - verbosity = (Verbosity) std::stoll(argv[1]); + verbosity = (Verbosity)std::stoll(argv[1]); - FdSource source(STDIN_FILENO); + FdSource source(STDIN_FILENO); - /* Read the parent's settings. */ - while (readInt(source)) { - auto name = readString(source); - auto value = readString(source); - settings.set(name, value); - } + /* Read the parent's settings. */ + while (readInt(source)) { + auto name = readString(source); + auto value = readString(source); + settings.set(name, value); + } - settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work + settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work - initPlugins(); + initPlugins(); - auto store = openStore().cast<LocalStore>(); + auto store = openStore().cast<LocalStore>(); - /* It would be more appropriate to use $XDG_RUNTIME_DIR, since - that gets cleared on reboot, but it wouldn't work on macOS. */ - currentLoad = store->stateDir + "/current-load"; + /* It would be more appropriate to use $XDG_RUNTIME_DIR, since + that gets cleared on reboot, but it wouldn't work on macOS. */ + currentLoad = store->stateDir + "/current-load"; - std::shared_ptr<Store> sshStore; - AutoCloseFD bestSlotLock; + std::shared_ptr<Store> sshStore; + AutoCloseFD bestSlotLock; - auto machines = getMachines(); - debug("got %d remote builders", machines.size()); + auto machines = getMachines(); + debug("got %d remote builders", machines.size()); - if (machines.empty()) { - std::cerr << "# decline-permanently\n"; - return 0; - } + if (machines.empty()) { + std::cerr << "# decline-permanently\n"; + return 0; + } - string drvPath; - string storeUri; - - while (true) { - - try { - auto s = readString(source); - if (s != "try") return 0; - } catch (EndOfFile &) { return 0; } - - auto amWilling = readInt(source); - auto neededSystem = readString(source); - source >> drvPath; - auto requiredFeatures = readStrings<std::set<std::string>>(source); - - auto canBuildLocally = amWilling - && ( neededSystem == settings.thisSystem - || settings.extraPlatforms.get().count(neededSystem) > 0) - && allSupportedLocally(requiredFeatures); - - /* Error ignored here, will be caught later */ - mkdir(currentLoad.c_str(), 0777); - - while (true) { - bestSlotLock = -1; - AutoCloseFD lock = openLockFile(currentLoad + "/main-lock", true); - lockFile(lock.get(), ltWrite, true); - - bool rightType = false; - - Machine * bestMachine = nullptr; - unsigned long long bestLoad = 0; - for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); - - if (m.enabled && std::find(m.systemTypes.begin(), - m.systemTypes.end(), - neededSystem) != m.systemTypes.end() && - m.allSupported(requiredFeatures) && - m.mandatoryMet(requiredFeatures)) { - rightType = true; - AutoCloseFD free; - unsigned long long load = 0; - for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) { - auto slotLock = openSlotLock(m, slot); - if (lockFile(slotLock.get(), ltWrite, false)) { - if (!free) { - free = std::move(slotLock); - } - } else { - ++load; - } - } - if (!free) { - continue; - } - bool best = false; - if (!bestSlotLock) { - best = true; - } else if (load / m.speedFactor < bestLoad / bestMachine->speedFactor) { - best = true; - } else if (load / m.speedFactor == bestLoad / bestMachine->speedFactor) { - if (m.speedFactor > bestMachine->speedFactor) { - best = true; - } else if (m.speedFactor == bestMachine->speedFactor) { - if (load < bestLoad) { - best = true; - } - } - } - if (best) { - bestLoad = load; - bestSlotLock = std::move(free); - bestMachine = &m; - } - } - } + string drvPath; + string storeUri; - if (!bestSlotLock) { - if (rightType && !canBuildLocally) - std::cerr << "# postpone\n"; - else - std::cerr << "# decline\n"; - break; + while (true) { + try { + auto s = readString(source); + if (s != "try") return 0; + } catch (EndOfFile&) { + return 0; + } + + auto amWilling = readInt(source); + auto neededSystem = readString(source); + source >> drvPath; + auto requiredFeatures = readStrings<std::set<std::string>>(source); + + auto canBuildLocally = + amWilling && + (neededSystem == settings.thisSystem || + settings.extraPlatforms.get().count(neededSystem) > 0) && + allSupportedLocally(requiredFeatures); + + /* Error ignored here, will be caught later */ + mkdir(currentLoad.c_str(), 0777); + + while (true) { + bestSlotLock = -1; + AutoCloseFD lock = openLockFile(currentLoad + "/main-lock", true); + lockFile(lock.get(), ltWrite, true); + + bool rightType = false; + + Machine* bestMachine = nullptr; + unsigned long long bestLoad = 0; + for (auto& m : machines) { + debug("considering building on remote machine '%s'", m.storeUri); + + if (m.enabled && + std::find(m.systemTypes.begin(), m.systemTypes.end(), + neededSystem) != m.systemTypes.end() && + m.allSupported(requiredFeatures) && + m.mandatoryMet(requiredFeatures)) { + rightType = true; + AutoCloseFD free; + unsigned long long load = 0; + for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) { + auto slotLock = openSlotLock(m, slot); + if (lockFile(slotLock.get(), ltWrite, false)) { + if (!free) { + free = std::move(slotLock); } + } else { + ++load; + } + } + if (!free) { + continue; + } + bool best = false; + if (!bestSlotLock) { + best = true; + } else if (load / m.speedFactor < + bestLoad / bestMachine->speedFactor) { + best = true; + } else if (load / m.speedFactor == + bestLoad / bestMachine->speedFactor) { + if (m.speedFactor > bestMachine->speedFactor) { + best = true; + } else if (m.speedFactor == bestMachine->speedFactor) { + if (load < bestLoad) { + best = true; + } + } + } + if (best) { + bestLoad = load; + bestSlotLock = std::move(free); + bestMachine = &m; + } + } + } + + if (!bestSlotLock) { + if (rightType && !canBuildLocally) + std::cerr << "# postpone\n"; + else + std::cerr << "# decline\n"; + break; + } #if __APPLE__ - futimes(bestSlotLock.get(), NULL); + futimes(bestSlotLock.get(), NULL); #else - futimens(bestSlotLock.get(), NULL); + futimens(bestSlotLock.get(), NULL); #endif - lock = -1; - - try { - - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); - - Store::Params storeParams; - if (hasPrefix(bestMachine->storeUri, "ssh://")) { - storeParams["max-connections"] ="1"; - storeParams["log-fd"] = "4"; - if (bestMachine->sshKey != "") - storeParams["ssh-key"] = bestMachine->sshKey; - } - - sshStore = openStore(bestMachine->storeUri, storeParams); - sshStore->connect(); - storeUri = bestMachine->storeUri; - - } catch (std::exception & e) { - auto msg = chomp(drainFD(5, false)); - printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), - (msg.empty() ? "" : ": " + msg)); - bestMachine->enabled = false; - continue; - } - - goto connected; - } + lock = -1; + + try { + Activity act(*logger, lvlTalkative, actUnknown, + fmt("connecting to '%s'", bestMachine->storeUri)); + + Store::Params storeParams; + if (hasPrefix(bestMachine->storeUri, "ssh://")) { + storeParams["max-connections"] = "1"; + storeParams["log-fd"] = "4"; + if (bestMachine->sshKey != "") + storeParams["ssh-key"] = bestMachine->sshKey; + } + + sshStore = openStore(bestMachine->storeUri, storeParams); + sshStore->connect(); + storeUri = bestMachine->storeUri; + + } catch (std::exception& e) { + auto msg = chomp(drainFD(5, false)); + printError("cannot build on '%s': %s%s", bestMachine->storeUri, + e.what(), (msg.empty() ? "" : ": " + msg)); + bestMachine->enabled = false; + continue; } -connected: - close(5); - - std::cerr << "# accept\n" << storeUri << "\n"; + goto connected; + } + } - auto inputs = readStrings<PathSet>(source); - auto outputs = readStrings<PathSet>(source); + connected: + close(5); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + std::cerr << "# accept\n" << storeUri << "\n"; - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); + auto inputs = readStrings<PathSet>(source); + auto outputs = readStrings<PathSet>(source); - auto old = signal(SIGALRM, handleAlarm); - alarm(15 * 60); - if (!lockFile(uploadLock.get(), ltWrite, true)) - printError("somebody is hogging the upload lock for '%s', continuing..."); - alarm(0); - signal(SIGALRM, old); - } + AutoCloseFD uploadLock = openLockFile( + currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); - auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; + { + Activity act(*logger, lvlTalkative, actUnknown, + fmt("waiting for the upload lock to '%s'", storeUri)); + + auto old = signal(SIGALRM, handleAlarm); + alarm(15 * 60); + if (!lockFile(uploadLock.get(), ltWrite, true)) + printError( + "somebody is hogging the upload lock for '%s', continuing..."); + alarm(0); + signal(SIGALRM, old); + } - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); - copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute); - } + auto substitute = + settings.buildersUseSubstitutes ? Substitute : NoSubstitute; - uploadLock = -1; + { + Activity act(*logger, lvlTalkative, actUnknown, + fmt("copying dependencies to '%s'", storeUri)); + copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, + substitute); + } - BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath))); - drv.inputSrcs = inputs; + uploadLock = -1; - auto result = sshStore->buildDerivation(drvPath, drv); + BasicDerivation drv( + readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath))); + drv.inputSrcs = inputs; - if (!result.success()) - throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, result.errorMsg); + auto result = sshStore->buildDerivation(drvPath, drv); - PathSet missing; - for (auto & path : outputs) - if (!store->isValidPath(path)) missing.insert(path); + if (!result.success()) + throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, + result.errorMsg); - if (!missing.empty()) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); - store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */ - copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); - } + PathSet missing; + for (auto& path : outputs) + if (!store->isValidPath(path)) missing.insert(path); - return 0; + if (!missing.empty()) { + Activity act(*logger, lvlTalkative, actUnknown, + fmt("copying outputs from '%s'", storeUri)); + store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */ + copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, + NoSubstitute); } + + return 0; + } } static RegisterLegacyCommand s1("build-remote", _main); diff --git a/third_party/nix/src/libexpr/attr-path.cc b/third_party/nix/src/libexpr/attr-path.cc index b0f80db32a88..4eb44ec3357d 100644 --- a/third_party/nix/src/libexpr/attr-path.cc +++ b/third_party/nix/src/libexpr/attr-path.cc @@ -2,95 +2,92 @@ #include "eval-inline.hh" #include "util.hh" - namespace nix { - -static Strings parseAttrPath(const string & s) -{ - Strings res; - string cur; - string::const_iterator i = s.begin(); - while (i != s.end()) { - if (*i == '.') { - res.push_back(cur); - cur.clear(); - } else if (*i == '"') { - ++i; - while (1) { - if (i == s.end()) - throw Error(format("missing closing quote in selection path '%1%'") % s); - if (*i == '"') break; - cur.push_back(*i++); - } - } else - cur.push_back(*i); - ++i; - } - if (!cur.empty()) res.push_back(cur); - return res; +static Strings parseAttrPath(const string& s) { + Strings res; + string cur; + string::const_iterator i = s.begin(); + while (i != s.end()) { + if (*i == '.') { + res.push_back(cur); + cur.clear(); + } else if (*i == '"') { + ++i; + while (1) { + if (i == s.end()) + throw Error(format("missing closing quote in selection path '%1%'") % + s); + if (*i == '"') break; + cur.push_back(*i++); + } + } else + cur.push_back(*i); + ++i; + } + if (!cur.empty()) res.push_back(cur); + return res; } +Value* findAlongAttrPath(EvalState& state, const string& attrPath, + Bindings& autoArgs, Value& vIn) { + Strings tokens = parseAttrPath(attrPath); + + Error attrError = + Error(format("attribute selection path '%1%' does not match expression") % + attrPath); + + Value* v = &vIn; + + for (auto& attr : tokens) { + /* Is i an index (integer) or a normal attribute name? */ + enum { apAttr, apIndex } apType = apAttr; + unsigned int attrIndex; + if (string2Int(attr, attrIndex)) apType = apIndex; + + /* Evaluate the expression. */ + Value* vNew = state.allocValue(); + state.autoCallFunction(autoArgs, *v, *vNew); + v = vNew; + state.forceValue(*v); + + /* It should evaluate to either a set or an expression, + according to what is specified in the attrPath. */ + + if (apType == apAttr) { + if (v->type != tAttrs) + throw TypeError(format("the expression selected by the selection path " + "'%1%' should be a set but is %2%") % + attrPath % showType(*v)); + + if (attr.empty()) + throw Error(format("empty attribute name in selection path '%1%'") % + attrPath); + + Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); + if (a == v->attrs->end()) + throw Error( + format("attribute '%1%' in selection path '%2%' not found") % attr % + attrPath); + v = &*a->value; + } -Value * findAlongAttrPath(EvalState & state, const string & attrPath, - Bindings & autoArgs, Value & vIn) -{ - Strings tokens = parseAttrPath(attrPath); - - Error attrError = - Error(format("attribute selection path '%1%' does not match expression") % attrPath); - - Value * v = &vIn; - - for (auto & attr : tokens) { - - /* Is i an index (integer) or a normal attribute name? */ - enum { apAttr, apIndex } apType = apAttr; - unsigned int attrIndex; - if (string2Int(attr, attrIndex)) apType = apIndex; - - /* Evaluate the expression. */ - Value * vNew = state.allocValue(); - state.autoCallFunction(autoArgs, *v, *vNew); - v = vNew; - state.forceValue(*v); - - /* It should evaluate to either a set or an expression, - according to what is specified in the attrPath. */ - - if (apType == apAttr) { - - if (v->type != tAttrs) - throw TypeError( - format("the expression selected by the selection path '%1%' should be a set but is %2%") - % attrPath % showType(*v)); - - if (attr.empty()) - throw Error(format("empty attribute name in selection path '%1%'") % attrPath); - - Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); - if (a == v->attrs->end()) - throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath); - v = &*a->value; - } - - else if (apType == apIndex) { - - if (!v->isList()) - throw TypeError( - format("the expression selected by the selection path '%1%' should be a list but is %2%") - % attrPath % showType(*v)); - - if (attrIndex >= v->listSize()) - throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath); + else if (apType == apIndex) { + if (!v->isList()) + throw TypeError(format("the expression selected by the selection path " + "'%1%' should be a list but is %2%") % + attrPath % showType(*v)); - v = v->listElems()[attrIndex]; - } + if (attrIndex >= v->listSize()) + throw Error( + format("list index %1% in selection path '%2%' is out of range") % + attrIndex % attrPath); + v = v->listElems()[attrIndex]; } + } - return v; + return v; } - -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/attr-path.hh b/third_party/nix/src/libexpr/attr-path.hh index 46a341950939..9c38cdde049f 100644 --- a/third_party/nix/src/libexpr/attr-path.hh +++ b/third_party/nix/src/libexpr/attr-path.hh @@ -1,13 +1,12 @@ #pragma once -#include "eval.hh" - -#include <string> #include <map> +#include <string> +#include "eval.hh" namespace nix { -Value * findAlongAttrPath(EvalState & state, const string & attrPath, - Bindings & autoArgs, Value & vIn); +Value* findAlongAttrPath(EvalState& state, const string& attrPath, + Bindings& autoArgs, Value& vIn); } diff --git a/third_party/nix/src/libexpr/attr-set.cc b/third_party/nix/src/libexpr/attr-set.cc index 0785897d2513..cdca3953e653 100644 --- a/third_party/nix/src/libexpr/attr-set.cc +++ b/third_party/nix/src/libexpr/attr-set.cc @@ -1,52 +1,40 @@ #include "attr-set.hh" -#include "eval-inline.hh" - #include <algorithm> - +#include "eval-inline.hh" namespace nix { - /* Allocate a new array of attributes for an attribute set with a specific capacity. The space is implicitly reserved after the Bindings structure. */ -Bindings * EvalState::allocBindings(size_t capacity) -{ - if (capacity > std::numeric_limits<Bindings::size_t>::max()) - throw Error("attribute set of size %d is too big", capacity); - return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity); +Bindings* EvalState::allocBindings(size_t capacity) { + if (capacity > std::numeric_limits<Bindings::size_t>::max()) + throw Error("attribute set of size %d is too big", capacity); + return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) + Bindings((Bindings::size_t)capacity); } - -void EvalState::mkAttrs(Value & v, size_t capacity) -{ - if (capacity == 0) { - v = vEmptySet; - return; - } - clearValue(v); - v.type = tAttrs; - v.attrs = allocBindings(capacity); - nrAttrsets++; - nrAttrsInAttrsets += capacity; +void EvalState::mkAttrs(Value& v, size_t capacity) { + if (capacity == 0) { + v = vEmptySet; + return; + } + clearValue(v); + v.type = tAttrs; + v.attrs = allocBindings(capacity); + nrAttrsets++; + nrAttrsInAttrsets += capacity; } - /* Create a new attribute named 'name' on an existing attribute set stored in 'vAttrs' and return the newly allocated Value which is associated with this attribute. */ -Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) -{ - Value * v = allocValue(); - vAttrs.attrs->push_back(Attr(name, v)); - return v; +Value* EvalState::allocAttr(Value& vAttrs, const Symbol& name) { + Value* v = allocValue(); + vAttrs.attrs->push_back(Attr(name, v)); + return v; } +void Bindings::sort() { std::sort(begin(), end()); } -void Bindings::sort() -{ - std::sort(begin(), end()); -} - - -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/attr-set.hh b/third_party/nix/src/libexpr/attr-set.hh index 3119a1848af2..f56261ad6a5a 100644 --- a/third_party/nix/src/libexpr/attr-set.hh +++ b/third_party/nix/src/libexpr/attr-set.hh @@ -1,95 +1,80 @@ #pragma once +#include <algorithm> #include "nixexpr.hh" #include "symbol-table.hh" -#include <algorithm> - namespace nix { - class EvalState; struct Value; /* Map one attribute name to its value. */ -struct Attr -{ - Symbol name; - Value * value; - Pos * pos; - Attr(Symbol name, Value * value, Pos * pos = &noPos) - : name(name), value(value), pos(pos) { }; - Attr() : pos(&noPos) { }; - bool operator < (const Attr & a) const - { - return name < a.name; - } +struct Attr { + Symbol name; + Value* value; + Pos* pos; + Attr(Symbol name, Value* value, Pos* pos = &noPos) + : name(name), value(value), pos(pos){}; + Attr() : pos(&noPos){}; + bool operator<(const Attr& a) const { return name < a.name; } }; /* Bindings contains all the attributes of an attribute set. It is defined by its size and its capacity, the capacity being the number of Attr elements allocated after this structure, while the size corresponds to the number of elements already inserted in this structure. */ -class Bindings -{ -public: - typedef uint32_t size_t; - -private: - size_t size_, capacity_; - Attr attrs[0]; - - Bindings(size_t capacity) : size_(0), capacity_(capacity) { } - Bindings(const Bindings & bindings) = delete; - -public: - size_t size() const { return size_; } - - bool empty() const { return !size_; } - - typedef Attr * iterator; - - void push_back(const Attr & attr) - { - assert(size_ < capacity_); - attrs[size_++] = attr; - } - - iterator find(const Symbol & name) - { - Attr key(name, 0); - iterator i = std::lower_bound(begin(), end(), key); - if (i != end() && i->name == name) return i; - return end(); - } - - iterator begin() { return &attrs[0]; } - iterator end() { return &attrs[size_]; } - - Attr & operator[](size_t pos) - { - return attrs[pos]; - } - - void sort(); - - size_t capacity() { return capacity_; } - - /* Returns the attributes in lexicographically sorted order. */ - std::vector<const Attr *> lexicographicOrder() const - { - std::vector<const Attr *> res; - res.reserve(size_); - for (size_t n = 0; n < size_; n++) - res.emplace_back(&attrs[n]); - std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) { - return (const string &) a->name < (const string &) b->name; - }); - return res; - } - - friend class EvalState; -}; +class Bindings { + public: + typedef uint32_t size_t; + + private: + size_t size_, capacity_; + Attr attrs[0]; + + Bindings(size_t capacity) : size_(0), capacity_(capacity) {} + Bindings(const Bindings& bindings) = delete; + + public: + size_t size() const { return size_; } + + bool empty() const { return !size_; } + typedef Attr* iterator; + + void push_back(const Attr& attr) { + assert(size_ < capacity_); + attrs[size_++] = attr; + } + + iterator find(const Symbol& name) { + Attr key(name, 0); + iterator i = std::lower_bound(begin(), end(), key); + if (i != end() && i->name == name) return i; + return end(); + } + + iterator begin() { return &attrs[0]; } + iterator end() { return &attrs[size_]; } + + Attr& operator[](size_t pos) { return attrs[pos]; } + + void sort(); + + size_t capacity() { return capacity_; } + + /* Returns the attributes in lexicographically sorted order. */ + std::vector<const Attr*> lexicographicOrder() const { + std::vector<const Attr*> res; + res.reserve(size_); + for (size_t n = 0; n < size_; n++) res.emplace_back(&attrs[n]); + std::sort(res.begin(), res.end(), [](const Attr* a, const Attr* b) { + return (const string&)a->name < (const string&)b->name; + }); + return res; + } + + friend class EvalState; +}; -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/common-eval-args.cc b/third_party/nix/src/libexpr/common-eval-args.cc index 13950ab8d169..1e2fe891f544 100644 --- a/third_party/nix/src/libexpr/common-eval-args.cc +++ b/third_party/nix/src/libexpr/common-eval-args.cc @@ -1,59 +1,61 @@ #include "common-eval-args.hh" -#include "shared.hh" #include "download.hh" -#include "util.hh" #include "eval.hh" +#include "shared.hh" +#include "util.hh" namespace nix { -MixEvalArgs::MixEvalArgs() -{ - mkFlag() - .longName("arg") - .description("argument to be passed to Nix functions") - .labels({"name", "expr"}) - .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; }); +MixEvalArgs::MixEvalArgs() { + mkFlag() + .longName("arg") + .description("argument to be passed to Nix functions") + .labels({"name", "expr"}) + .handler( + [&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; }); - mkFlag() - .longName("argstr") - .description("string-valued argument to be passed to Nix functions") - .labels({"name", "string"}) - .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; }); + mkFlag() + .longName("argstr") + .description("string-valued argument to be passed to Nix functions") + .labels({"name", "string"}) + .handler( + [&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; }); - mkFlag() - .shortName('I') - .longName("include") - .description("add a path to the list of locations used to look up <...> file names") - .label("path") - .handler([&](std::string s) { searchPath.push_back(s); }); + mkFlag() + .shortName('I') + .longName("include") + .description( + "add a path to the list of locations used to look up <...> file " + "names") + .label("path") + .handler([&](std::string s) { searchPath.push_back(s); }); } -Bindings * MixEvalArgs::getAutoArgs(EvalState & state) -{ - Bindings * res = state.allocBindings(autoArgs.size()); - for (auto & i : autoArgs) { - Value * v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath("."))); - else - mkString(*v, string(i.second, 1)); - res->push_back(Attr(state.symbols.create(i.first), v)); - } - res->sort(); - return res; +Bindings* MixEvalArgs::getAutoArgs(EvalState& state) { + Bindings* res = state.allocBindings(autoArgs.size()); + for (auto& i : autoArgs) { + Value* v = state.allocValue(); + if (i.second[0] == 'E') + state.mkThunk_( + *v, state.parseExprFromString(string(i.second, 1), absPath("."))); + else + mkString(*v, string(i.second, 1)); + res->push_back(Attr(state.symbols.create(i.first), v)); + } + res->sort(); + return res; } -Path lookupFileArg(EvalState & state, string s) -{ - if (isUri(s)) { - CachedDownloadRequest request(s); - request.unpack = true; - return getDownloader()->downloadCached(state.store, request).path; - } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { - Path p = s.substr(1, s.size() - 2); - return state.findFile(p); - } else - return absPath(s); +Path lookupFileArg(EvalState& state, string s) { + if (isUri(s)) { + CachedDownloadRequest request(s); + request.unpack = true; + return getDownloader()->downloadCached(state.store, request).path; + } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + Path p = s.substr(1, s.size() - 2); + return state.findFile(p); + } else + return absPath(s); } -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/common-eval-args.hh b/third_party/nix/src/libexpr/common-eval-args.hh index be7fda783783..9663d40c148c 100644 --- a/third_party/nix/src/libexpr/common-eval-args.hh +++ b/third_party/nix/src/libexpr/common-eval-args.hh @@ -8,19 +8,17 @@ class Store; class EvalState; class Bindings; -struct MixEvalArgs : virtual Args -{ - MixEvalArgs(); +struct MixEvalArgs : virtual Args { + MixEvalArgs(); - Bindings * getAutoArgs(EvalState & state); + Bindings* getAutoArgs(EvalState& state); - Strings searchPath; + Strings searchPath; -private: - - std::map<std::string, std::string> autoArgs; + private: + std::map<std::string, std::string> autoArgs; }; -Path lookupFileArg(EvalState & state, string s); +Path lookupFileArg(EvalState& state, string s); -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/eval-inline.hh b/third_party/nix/src/libexpr/eval-inline.hh index c27116e3b448..37105e7e398d 100644 --- a/third_party/nix/src/libexpr/eval-inline.hh +++ b/third_party/nix/src/libexpr/eval-inline.hh @@ -2,94 +2,81 @@ #include "eval.hh" -#define LocalNoInline(f) static f __attribute__((noinline)); f -#define LocalNoInlineNoReturn(f) static f __attribute__((noinline, noreturn)); f +#define LocalNoInline(f) \ + static f __attribute__((noinline)); \ + f +#define LocalNoInlineNoReturn(f) \ + static f __attribute__((noinline, noreturn)); \ + f namespace nix { -LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) -{ - throw EvalError(format(s) % pos); +LocalNoInlineNoReturn(void throwEvalError(const char* s, const Pos& pos)) { + throw EvalError(format(s) % pos); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) -{ - throw TypeError(format(s) % showType(v)); +LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v)) { + throw TypeError(format(s) % showType(v)); } - -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos)) -{ - throw TypeError(format(s) % showType(v) % pos); +LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v, + const Pos& pos)) { + throw TypeError(format(s) % showType(v) % pos); } - -void EvalState::forceValue(Value & v, const Pos & pos) -{ - if (v.type == tThunk) { - Env * env = v.thunk.env; - Expr * expr = v.thunk.expr; - try { - v.type = tBlackhole; - //checkInterrupt(); - expr->eval(*this, *env, v); - } catch (...) { - v.type = tThunk; - v.thunk.env = env; - v.thunk.expr = expr; - throw; - } +void EvalState::forceValue(Value& v, const Pos& pos) { + if (v.type == tThunk) { + Env* env = v.thunk.env; + Expr* expr = v.thunk.expr; + try { + v.type = tBlackhole; + // checkInterrupt(); + expr->eval(*this, *env, v); + } catch (...) { + v.type = tThunk; + v.thunk.env = env; + v.thunk.expr = expr; + throw; } - else if (v.type == tApp) - callFunction(*v.app.left, *v.app.right, v, noPos); - else if (v.type == tBlackhole) - throwEvalError("infinite recursion encountered, at %1%", pos); + } else if (v.type == tApp) + callFunction(*v.app.left, *v.app.right, v, noPos); + else if (v.type == tBlackhole) + throwEvalError("infinite recursion encountered, at %1%", pos); } - -inline void EvalState::forceAttrs(Value & v) -{ - forceValue(v); - if (v.type != tAttrs) - throwTypeError("value is %1% while a set was expected", v); +inline void EvalState::forceAttrs(Value& v) { + forceValue(v); + if (v.type != tAttrs) + throwTypeError("value is %1% while a set was expected", v); } - -inline void EvalState::forceAttrs(Value & v, const Pos & pos) -{ - forceValue(v); - if (v.type != tAttrs) - throwTypeError("value is %1% while a set was expected, at %2%", v, pos); +inline void EvalState::forceAttrs(Value& v, const Pos& pos) { + forceValue(v); + if (v.type != tAttrs) + throwTypeError("value is %1% while a set was expected, at %2%", v, pos); } - -inline void EvalState::forceList(Value & v) -{ - forceValue(v); - if (!v.isList()) - throwTypeError("value is %1% while a list was expected", v); +inline void EvalState::forceList(Value& v) { + forceValue(v); + if (!v.isList()) throwTypeError("value is %1% while a list was expected", v); } - -inline void EvalState::forceList(Value & v, const Pos & pos) -{ - forceValue(v); - if (!v.isList()) - throwTypeError("value is %1% while a list was expected, at %2%", v, pos); +inline void EvalState::forceList(Value& v, const Pos& pos) { + forceValue(v); + if (!v.isList()) + throwTypeError("value is %1% while a list was expected, at %2%", v, pos); } /* Note: Various places expect the allocated memory to be zeroed. */ -inline void * allocBytes(size_t n) -{ - void * p; +inline void* allocBytes(size_t n) { + void* p; #if HAVE_BOEHMGC - p = GC_MALLOC(n); + p = GC_MALLOC(n); #else - p = calloc(n, 1); + p = calloc(n, 1); #endif - if (!p) throw std::bad_alloc(); - return p; + if (!p) throw std::bad_alloc(); + return p; } - -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/eval.cc b/third_party/nix/src/libexpr/eval.cc index 3426afb6cf6e..14b600b06ec4 100644 --- a/third_party/nix/src/libexpr/eval.cc +++ b/third_party/nix/src/libexpr/eval.cc @@ -1,24 +1,21 @@ #include "eval.hh" -#include "hash.hh" -#include "util.hh" -#include "store-api.hh" -#include "derivations.hh" -#include "globals.hh" -#include "eval-inline.hh" -#include "download.hh" -#include "json.hh" -#include "function-trace.hh" - +#include <sys/resource.h> +#include <sys/time.h> +#include <unistd.h> #include <algorithm> #include <chrono> #include <cstring> -#include <unistd.h> -#include <sys/time.h> -#include <sys/resource.h> -#include <iostream> #include <fstream> - -#include <sys/resource.h> +#include <iostream> +#include "derivations.hh" +#include "download.hh" +#include "eval-inline.hh" +#include "function-trace.hh" +#include "globals.hh" +#include "hash.hh" +#include "json.hh" +#include "store-api.hh" +#include "util.hh" #if HAVE_BOEHMGC @@ -29,1963 +26,1823 @@ namespace nix { - -static char * dupString(const char * s) -{ - char * t; +static char* dupString(const char* s) { + char* t; #if HAVE_BOEHMGC - t = GC_STRDUP(s); + t = GC_STRDUP(s); #else - t = strdup(s); + t = strdup(s); #endif - if (!t) throw std::bad_alloc(); - return t; + if (!t) throw std::bad_alloc(); + return t; } +static void printValue(std::ostream& str, std::set<const Value*>& active, + const Value& v) { + checkInterrupt(); -static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) -{ - checkInterrupt(); + if (active.find(&v) != active.end()) { + str << "<CYCLE>"; + return; + } + active.insert(&v); - if (active.find(&v) != active.end()) { - str << "<CYCLE>"; - return; - } - active.insert(&v); - - switch (v.type) { + switch (v.type) { case tInt: - str << v.integer; - break; + str << v.integer; + break; case tBool: - str << (v.boolean ? "true" : "false"); - break; + str << (v.boolean ? "true" : "false"); + break; case tString: - str << "\""; - for (const char * i = v.string.s; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else str << *i; - str << "\""; - break; + str << "\""; + for (const char* i = v.string.s; *i; i++) + if (*i == '\"' || *i == '\\') + str << "\\" << *i; + else if (*i == '\n') + str << "\\n"; + else if (*i == '\r') + str << "\\r"; + else if (*i == '\t') + str << "\\t"; + else + str << *i; + str << "\""; + break; case tPath: - str << v.path; // !!! escaping? - break; + str << v.path; // !!! escaping? + break; case tNull: - str << "null"; - break; + str << "null"; + break; case tAttrs: { - str << "{ "; - for (auto & i : v.attrs->lexicographicOrder()) { - str << i->name << " = "; - printValue(str, active, *i->value); - str << "; "; - } - str << "}"; - break; + str << "{ "; + for (auto& i : v.attrs->lexicographicOrder()) { + str << i->name << " = "; + printValue(str, active, *i->value); + str << "; "; + } + str << "}"; + break; } case tList1: case tList2: case tListN: - str << "[ "; - for (unsigned int n = 0; n < v.listSize(); ++n) { - printValue(str, active, *v.listElems()[n]); - str << " "; - } - str << "]"; - break; + str << "[ "; + for (unsigned int n = 0; n < v.listSize(); ++n) { + printValue(str, active, *v.listElems()[n]); + str << " "; + } + str << "]"; + break; case tThunk: case tApp: - str << "<CODE>"; - break; + str << "<CODE>"; + break; case tLambda: - str << "<LAMBDA>"; - break; + str << "<LAMBDA>"; + break; case tPrimOp: - str << "<PRIMOP>"; - break; + str << "<PRIMOP>"; + break; case tPrimOpApp: - str << "<PRIMOP-APP>"; - break; + str << "<PRIMOP-APP>"; + break; case tExternal: - str << *v.external; - break; + str << *v.external; + break; case tFloat: - str << v.fpoint; - break; + str << v.fpoint; + break; default: - throw Error("invalid value"); - } + throw Error("invalid value"); + } - active.erase(&v); + active.erase(&v); } - -std::ostream & operator << (std::ostream & str, const Value & v) -{ - std::set<const Value *> active; - printValue(str, active, v); - return str; +std::ostream& operator<<(std::ostream& str, const Value& v) { + std::set<const Value*> active; + printValue(str, active, v); + return str; } - -const Value *getPrimOp(const Value &v) { - const Value * primOp = &v; - while (primOp->type == tPrimOpApp) { - primOp = primOp->primOpApp.left; - } - assert(primOp->type == tPrimOp); - return primOp; +const Value* getPrimOp(const Value& v) { + const Value* primOp = &v; + while (primOp->type == tPrimOpApp) { + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + return primOp; } - -string showType(const Value & v) -{ - switch (v.type) { - case tInt: return "an integer"; - case tBool: return "a boolean"; - case tString: return v.string.context ? "a string with context" : "a string"; - case tPath: return "a path"; - case tNull: return "null"; - case tAttrs: return "a set"; - case tList1: case tList2: case tListN: return "a list"; - case tThunk: return "a thunk"; - case tApp: return "a function application"; - case tLambda: return "a function"; - case tBlackhole: return "a black hole"; - case tPrimOp: - return fmt("the built-in function '%s'", string(v.primOp->name)); - case tPrimOpApp: - return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); - case tExternal: return v.external->showType(); - case tFloat: return "a float"; - } - abort(); +string showType(const Value& v) { + switch (v.type) { + case tInt: + return "an integer"; + case tBool: + return "a boolean"; + case tString: + return v.string.context ? "a string with context" : "a string"; + case tPath: + return "a path"; + case tNull: + return "null"; + case tAttrs: + return "a set"; + case tList1: + case tList2: + case tListN: + return "a list"; + case tThunk: + return "a thunk"; + case tApp: + return "a function application"; + case tLambda: + return "a function"; + case tBlackhole: + return "a black hole"; + case tPrimOp: + return fmt("the built-in function '%s'", string(v.primOp->name)); + case tPrimOpApp: + return fmt("the partially applied built-in function '%s'", + string(getPrimOp(v)->primOp->name)); + case tExternal: + return v.external->showType(); + case tFloat: + return "a float"; + } + abort(); } - #if HAVE_BOEHMGC /* Called when the Boehm GC runs out of memory. */ -static void * oomHandler(size_t requested) -{ - /* Convert this to a proper C++ exception. */ - throw std::bad_alloc(); +static void* oomHandler(size_t requested) { + /* Convert this to a proper C++ exception. */ + throw std::bad_alloc(); } #endif - -static Symbol getName(const AttrName & name, EvalState & state, Env & env) -{ - if (name.symbol.set()) { - return name.symbol; - } else { - Value nameValue; - name.expr->eval(state, env, nameValue); - state.forceStringNoCtx(nameValue); - return state.symbols.create(nameValue.string.s); - } +static Symbol getName(const AttrName& name, EvalState& state, Env& env) { + if (name.symbol.set()) { + return name.symbol; + } else { + Value nameValue; + name.expr->eval(state, env, nameValue); + state.forceStringNoCtx(nameValue); + return state.symbols.create(nameValue.string.s); + } } - static bool gcInitialised = false; -void initGC() -{ - if (gcInitialised) return; +void initGC() { + if (gcInitialised) return; #if HAVE_BOEHMGC - /* Initialise the Boehm garbage collector. */ - - /* Don't look for interior pointers. This reduces the odds of - misdetection a bit. */ - GC_set_all_interior_pointers(0); - - /* We don't have any roots in data segments, so don't scan from - there. */ - GC_set_no_dls(1); - - GC_INIT(); - - GC_set_oom_fn(oomHandler); - - /* Set the initial heap size to something fairly big (25% of - physical RAM, up to a maximum of 384 MiB) so that in most cases - we don't need to garbage collect at all. (Collection has a - fairly significant overhead.) The heap size can be overridden - through libgc's GC_INITIAL_HEAP_SIZE environment variable. We - should probably also provide a nix.conf setting for this. Note - that GC_expand_hp() causes a lot of virtual, but not physical - (resident) memory to be allocated. This might be a problem on - systems that don't overcommit. */ - if (!getenv("GC_INITIAL_HEAP_SIZE")) { - size_t size = 32 * 1024 * 1024; + /* Initialise the Boehm garbage collector. */ + + /* Don't look for interior pointers. This reduces the odds of + misdetection a bit. */ + GC_set_all_interior_pointers(0); + + /* We don't have any roots in data segments, so don't scan from + there. */ + GC_set_no_dls(1); + + GC_INIT(); + + GC_set_oom_fn(oomHandler); + + /* Set the initial heap size to something fairly big (25% of + physical RAM, up to a maximum of 384 MiB) so that in most cases + we don't need to garbage collect at all. (Collection has a + fairly significant overhead.) The heap size can be overridden + through libgc's GC_INITIAL_HEAP_SIZE environment variable. We + should probably also provide a nix.conf setting for this. Note + that GC_expand_hp() causes a lot of virtual, but not physical + (resident) memory to be allocated. This might be a problem on + systems that don't overcommit. */ + if (!getenv("GC_INITIAL_HEAP_SIZE")) { + size_t size = 32 * 1024 * 1024; #if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) - size_t maxSize = 384 * 1024 * 1024; - long pageSize = sysconf(_SC_PAGESIZE); - long pages = sysconf(_SC_PHYS_PAGES); - if (pageSize != -1) - size = (pageSize * pages) / 4; // 25% of RAM - if (size > maxSize) size = maxSize; + size_t maxSize = 384 * 1024 * 1024; + long pageSize = sysconf(_SC_PAGESIZE); + long pages = sysconf(_SC_PHYS_PAGES); + if (pageSize != -1) size = (pageSize * pages) / 4; // 25% of RAM + if (size > maxSize) size = maxSize; #endif - debug(format("setting initial heap size to %1% bytes") % size); - GC_expand_hp(size); - } + debug(format("setting initial heap size to %1% bytes") % size); + GC_expand_hp(size); + } #endif - gcInitialised = true; + gcInitialised = true; } - /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const string & s) -{ - Strings res; - - auto p = s.begin(); +static Strings parseNixPath(const string& s) { + Strings res; - while (p != s.end()) { - auto start = p; - auto start2 = p; + auto p = s.begin(); - while (p != s.end() && *p != ':') { - if (*p == '=') start2 = p + 1; - ++p; - } - - if (p == s.end()) { - if (p != start) res.push_back(std::string(start, p)); - break; - } + while (p != s.end()) { + auto start = p; + auto start2 = p; - if (*p == ':') { - if (isUri(std::string(start2, s.end()))) { - ++p; - while (p != s.end() && *p != ':') ++p; - } - res.push_back(std::string(start, p)); - if (p == s.end()) break; - } - - ++p; + while (p != s.end() && *p != ':') { + if (*p == '=') start2 = p + 1; + ++p; } - return res; -} - - -EvalState::EvalState(const Strings & _searchPath, ref<Store> store) - : sWith(symbols.create("<with>")) - , sOutPath(symbols.create("outPath")) - , sDrvPath(symbols.create("drvPath")) - , sType(symbols.create("type")) - , sMeta(symbols.create("meta")) - , sName(symbols.create("name")) - , sValue(symbols.create("value")) - , sSystem(symbols.create("system")) - , sOverrides(symbols.create("__overrides")) - , sOutputs(symbols.create("outputs")) - , sOutputName(symbols.create("outputName")) - , sIgnoreNulls(symbols.create("__ignoreNulls")) - , sFile(symbols.create("file")) - , sLine(symbols.create("line")) - , sColumn(symbols.create("column")) - , sFunctor(symbols.create("__functor")) - , sToString(symbols.create("__toString")) - , sRight(symbols.create("right")) - , sWrong(symbols.create("wrong")) - , sStructuredAttrs(symbols.create("__structuredAttrs")) - , sBuilder(symbols.create("builder")) - , sArgs(symbols.create("args")) - , sOutputHash(symbols.create("outputHash")) - , sOutputHashAlgo(symbols.create("outputHashAlgo")) - , sOutputHashMode(symbols.create("outputHashMode")) - , repair(NoRepair) - , store(store) - , baseEnv(allocEnv(128)) - , staticBaseEnv(false, 0) -{ - countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; - - assert(gcInitialised); - - static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); - - /* Initialise the Nix expression search path. */ - if (!evalSettings.pureEval) { - Strings paths = parseNixPath(getEnv("NIX_PATH", "")); - for (auto & i : _searchPath) addToSearchPath(i); - for (auto & i : paths) addToSearchPath(i); + if (p == s.end()) { + if (p != start) res.push_back(std::string(start, p)); + break; } - addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); - - if (evalSettings.restrictEval || evalSettings.pureEval) { - allowedPaths = PathSet(); - - for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i); - if (!r.first) continue; - - auto path = r.second; - if (store->isInStore(r.second)) { - PathSet closure; - store->computeFSClosure(store->toStorePath(r.second), closure); - for (auto & path : closure) - allowedPaths->insert(path); - } else - allowedPaths->insert(r.second); - } + if (*p == ':') { + if (isUri(std::string(start2, s.end()))) { + ++p; + while (p != s.end() && *p != ':') ++p; + } + res.push_back(std::string(start, p)); + if (p == s.end()) break; } - clearValue(vEmptySet); - vEmptySet.type = tAttrs; - vEmptySet.attrs = allocBindings(0); - - createBaseEnv(); -} + ++p; + } + + return res; +} + +EvalState::EvalState(const Strings& _searchPath, ref<Store> store) + : sWith(symbols.create("<with>")), + sOutPath(symbols.create("outPath")), + sDrvPath(symbols.create("drvPath")), + sType(symbols.create("type")), + sMeta(symbols.create("meta")), + sName(symbols.create("name")), + sValue(symbols.create("value")), + sSystem(symbols.create("system")), + sOverrides(symbols.create("__overrides")), + sOutputs(symbols.create("outputs")), + sOutputName(symbols.create("outputName")), + sIgnoreNulls(symbols.create("__ignoreNulls")), + sFile(symbols.create("file")), + sLine(symbols.create("line")), + sColumn(symbols.create("column")), + sFunctor(symbols.create("__functor")), + sToString(symbols.create("__toString")), + sRight(symbols.create("right")), + sWrong(symbols.create("wrong")), + sStructuredAttrs(symbols.create("__structuredAttrs")), + sBuilder(symbols.create("builder")), + sArgs(symbols.create("args")), + sOutputHash(symbols.create("outputHash")), + sOutputHashAlgo(symbols.create("outputHashAlgo")), + sOutputHashMode(symbols.create("outputHashMode")), + repair(NoRepair), + store(store), + baseEnv(allocEnv(128)), + staticBaseEnv(false, 0) { + countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; + + assert(gcInitialised); + + static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); + + /* Initialise the Nix expression search path. */ + if (!evalSettings.pureEval) { + Strings paths = parseNixPath(getEnv("NIX_PATH", "")); + for (auto& i : _searchPath) addToSearchPath(i); + for (auto& i : paths) addToSearchPath(i); + } + addToSearchPath("nix=" + + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); + + if (evalSettings.restrictEval || evalSettings.pureEval) { + allowedPaths = PathSet(); + + for (auto& i : searchPath) { + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + + auto path = r.second; + + if (store->isInStore(r.second)) { + PathSet closure; + store->computeFSClosure(store->toStorePath(r.second), closure); + for (auto& path : closure) allowedPaths->insert(path); + } else + allowedPaths->insert(r.second); + } + } + clearValue(vEmptySet); + vEmptySet.type = tAttrs; + vEmptySet.attrs = allocBindings(0); -EvalState::~EvalState() -{ + createBaseEnv(); } +EvalState::~EvalState() {} -Path EvalState::checkSourcePath(const Path & path_) -{ - if (!allowedPaths) return path_; - - auto i = resolvedPaths.find(path_); - if (i != resolvedPaths.end()) - return i->second; - - bool found = false; +Path EvalState::checkSourcePath(const Path& path_) { + if (!allowedPaths) return path_; - /* First canonicalize the path without symlinks, so we make sure an - * attacker can't append ../../... to a path that would be in allowedPaths - * and thus leak symlink targets. - */ - Path abspath = canonPath(path_); - - for (auto & i : *allowedPaths) { - if (isDirOrInDir(abspath, i)) { - found = true; - break; - } - } + auto i = resolvedPaths.find(path_); + if (i != resolvedPaths.end()) return i->second; - if (!found) - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath); + bool found = false; - /* Resolve symlinks. */ - debug(format("checking access to '%s'") % abspath); - Path path = canonPath(abspath, true); + /* First canonicalize the path without symlinks, so we make sure an + * attacker can't append ../../... to a path that would be in allowedPaths + * and thus leak symlink targets. + */ + Path abspath = canonPath(path_); - for (auto & i : *allowedPaths) { - if (isDirOrInDir(path, i)) { - resolvedPaths[path_] = path; - return path; - } + for (auto& i : *allowedPaths) { + if (isDirOrInDir(abspath, i)) { + found = true; + break; } + } - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path); -} - + if (!found) + throw RestrictedPathError( + "access to path '%1%' is forbidden in restricted mode", abspath); -void EvalState::checkURI(const std::string & uri) -{ - if (!evalSettings.restrictEval) return; - - /* 'uri' should be equal to a prefix, or in a subdirectory of a - prefix. Thus, the prefix https://github.co does not permit - access to https://github.com. Note: this allows 'http://' and - 'https://' as prefixes for any http/https URI. */ - for (auto & prefix : evalSettings.allowedUris.get()) - if (uri == prefix || - (uri.size() > prefix.size() - && prefix.size() > 0 - && hasPrefix(uri, prefix) - && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) - return; - - /* If the URI is a path, then check it against allowedPaths as - well. */ - if (hasPrefix(uri, "/")) { - checkSourcePath(uri); - return; - } + /* Resolve symlinks. */ + debug(format("checking access to '%s'") % abspath); + Path path = canonPath(abspath, true); - if (hasPrefix(uri, "file://")) { - checkSourcePath(std::string(uri, 7)); - return; + for (auto& i : *allowedPaths) { + if (isDirOrInDir(path, i)) { + resolvedPaths[path_] = path; + return path; } - - throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri); -} - - -Path EvalState::toRealPath(const Path & path, const PathSet & context) -{ - // FIXME: check whether 'path' is in 'context'. - return - !context.empty() && store->isInStore(path) - ? store->toRealPath(path) - : path; + } + + throw RestrictedPathError( + "access to path '%1%' is forbidden in restricted mode", path); +} + +void EvalState::checkURI(const std::string& uri) { + if (!evalSettings.restrictEval) return; + + /* 'uri' should be equal to a prefix, or in a subdirectory of a + prefix. Thus, the prefix https://github.co does not permit + access to https://github.com. Note: this allows 'http://' and + 'https://' as prefixes for any http/https URI. */ + for (auto& prefix : evalSettings.allowedUris.get()) + if (uri == prefix || + (uri.size() > prefix.size() && prefix.size() > 0 && + hasPrefix(uri, prefix) && + (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) + return; + + /* If the URI is a path, then check it against allowedPaths as + well. */ + if (hasPrefix(uri, "/")) { + checkSourcePath(uri); + return; + } + + if (hasPrefix(uri, "file://")) { + checkSourcePath(std::string(uri, 7)); + return; + } + + throw RestrictedPathError( + "access to URI '%s' is forbidden in restricted mode", uri); +} + +Path EvalState::toRealPath(const Path& path, const PathSet& context) { + // FIXME: check whether 'path' is in 'context'. + return !context.empty() && store->isInStore(path) ? store->toRealPath(path) + : path; }; - -Value * EvalState::addConstant(const string & name, Value & v) -{ - Value * v2 = allocValue(); - *v2 = v; - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; - baseEnv.values[baseEnvDispl++] = v2; - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); - return v2; +Value* EvalState::addConstant(const string& name, Value& v) { + Value* v2 = allocValue(); + *v2 = v; + staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v2; + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + return v2; } - -Value * EvalState::addPrimOp(const string & name, - size_t arity, PrimOpFun primOp) -{ - if (arity == 0) { - Value v; - primOp(*this, noPos, nullptr, v); - return addConstant(name, v); - } - Value * v = allocValue(); - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - Symbol sym = symbols.create(name2); - v->type = tPrimOp; - v->primOp = new PrimOp(primOp, arity, sym); - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; - baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->attrs->push_back(Attr(sym, v)); - return v; +Value* EvalState::addPrimOp(const string& name, size_t arity, + PrimOpFun primOp) { + if (arity == 0) { + Value v; + primOp(*this, noPos, nullptr, v); + return addConstant(name, v); + } + Value* v = allocValue(); + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + Symbol sym = symbols.create(name2); + v->type = tPrimOp; + v->primOp = new PrimOp(primOp, arity, sym); + staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(sym, v)); + return v; } - -Value & EvalState::getBuiltin(const string & name) -{ - return *baseEnv.values[0]->attrs->find(symbols.create(name))->value; +Value& EvalState::getBuiltin(const string& name) { + return *baseEnv.values[0]->attrs->find(symbols.create(name))->value; } - /* Every "format" object (even temporary) takes up a few hundred bytes of stack space, which is a real killer in the recursive evaluator. So here are some helper functions for throwing exceptions. */ -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) -{ - throw EvalError(format(s) % s2); +LocalNoInlineNoReturn(void throwEvalError(const char* s, const string& s2)) { + throw EvalError(format(s) % s2); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos)) -{ - throw EvalError(format(s) % s2 % pos); +LocalNoInlineNoReturn(void throwEvalError(const char* s, const string& s2, + const Pos& pos)) { + throw EvalError(format(s) % s2 % pos); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3)) -{ - throw EvalError(format(s) % s2 % s3); +LocalNoInlineNoReturn(void throwEvalError(const char* s, const string& s2, + const string& s3)) { + throw EvalError(format(s) % s2 % s3); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos)) -{ - throw EvalError(format(s) % s2 % s3 % pos); +LocalNoInlineNoReturn(void throwEvalError(const char* s, const string& s2, + const string& s3, const Pos& pos)) { + throw EvalError(format(s) % s2 % s3 % pos); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2)) -{ - throw EvalError(format(s) % sym % p1 % p2); +LocalNoInlineNoReturn(void throwEvalError(const char* s, const Symbol& sym, + const Pos& p1, const Pos& p2)) { + throw EvalError(format(s) % sym % p1 % p2); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos)) -{ - throw TypeError(format(s) % pos); +LocalNoInlineNoReturn(void throwTypeError(const char* s, const Pos& pos)) { + throw TypeError(format(s) % pos); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) -{ - throw TypeError(format(s) % s1); +LocalNoInlineNoReturn(void throwTypeError(const char* s, const string& s1)) { + throw TypeError(format(s) % s1); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos)) -{ - throw TypeError(format(s) % fun.showNamePos() % s2 % pos); +LocalNoInlineNoReturn(void throwTypeError(const char* s, const ExprLambda& fun, + const Symbol& s2, const Pos& pos)) { + throw TypeError(format(s) % fun.showNamePos() % s2 % pos); } -LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos)) -{ - throw AssertionError(format(s) % s1 % pos); +LocalNoInlineNoReturn(void throwAssertionError(const char* s, const string& s1, + const Pos& pos)) { + throw AssertionError(format(s) % s1 % pos); } -LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) -{ - throw UndefinedVarError(format(s) % s1 % pos); +LocalNoInlineNoReturn(void throwUndefinedVarError(const char* s, + const string& s1, + const Pos& pos)) { + throw UndefinedVarError(format(s) % s1 % pos); } -LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2)) -{ - e.addPrefix(format(s) % s2); +LocalNoInline(void addErrorPrefix(Error& e, const char* s, const string& s2)) { + e.addPrefix(format(s) % s2); } -LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun, const Pos & pos)) -{ - e.addPrefix(format(s) % fun.showNamePos() % pos); +LocalNoInline(void addErrorPrefix(Error& e, const char* s, + const ExprLambda& fun, const Pos& pos)) { + e.addPrefix(format(s) % fun.showNamePos() % pos); } -LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos)) -{ - e.addPrefix(format(s) % s2 % pos); +LocalNoInline(void addErrorPrefix(Error& e, const char* s, const string& s2, + const Pos& pos)) { + e.addPrefix(format(s) % s2 % pos); } +void mkString(Value& v, const char* s) { mkStringNoCopy(v, dupString(s)); } -void mkString(Value & v, const char * s) -{ - mkStringNoCopy(v, dupString(s)); +Value& mkString(Value& v, const string& s, const PathSet& context) { + mkString(v, s.c_str()); + if (!context.empty()) { + size_t n = 0; + v.string.context = + (const char**)allocBytes((context.size() + 1) * sizeof(char*)); + for (auto& i : context) v.string.context[n++] = dupString(i.c_str()); + v.string.context[n] = 0; + } + return v; } +void mkPath(Value& v, const char* s) { mkPathNoCopy(v, dupString(s)); } -Value & mkString(Value & v, const string & s, const PathSet & context) -{ - mkString(v, s.c_str()); - if (!context.empty()) { - size_t n = 0; - v.string.context = (const char * *) - allocBytes((context.size() + 1) * sizeof(char *)); - for (auto & i : context) - v.string.context[n++] = dupString(i.c_str()); - v.string.context[n] = 0; - } - return v; -} - - -void mkPath(Value & v, const char * s) -{ - mkPathNoCopy(v, dupString(s)); -} - +inline Value* EvalState::lookupVar(Env* env, const ExprVar& var, bool noEval) { + for (size_t l = var.level; l; --l, env = env->up) + ; -inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) -{ - for (size_t l = var.level; l; --l, env = env->up) ; + if (!var.fromWith) return env->values[var.displ]; - if (!var.fromWith) return env->values[var.displ]; - - while (1) { - if (env->type == Env::HasWithExpr) { - if (noEval) return 0; - Value * v = allocValue(); - evalAttrs(*env->up, (Expr *) env->values[0], *v); - env->values[0] = v; - env->type = Env::HasWithAttrs; - } - Bindings::iterator j = env->values[0]->attrs->find(var.name); - if (j != env->values[0]->attrs->end()) { - if (countCalls && j->pos) attrSelects[*j->pos]++; - return j->value; - } - if (!env->prevWith) - throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos); - for (size_t l = env->prevWith; l; --l, env = env->up) ; + while (1) { + if (env->type == Env::HasWithExpr) { + if (noEval) return 0; + Value* v = allocValue(); + evalAttrs(*env->up, (Expr*)env->values[0], *v); + env->values[0] = v; + env->type = Env::HasWithAttrs; } + Bindings::iterator j = env->values[0]->attrs->find(var.name); + if (j != env->values[0]->attrs->end()) { + if (countCalls && j->pos) attrSelects[*j->pos]++; + return j->value; + } + if (!env->prevWith) + throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, + var.pos); + for (size_t l = env->prevWith; l; --l, env = env->up) + ; + } } - std::atomic<uint64_t> nrValuesFreed{0}; -void finalizeValue(void * obj, void * data) -{ - nrValuesFreed++; -} +void finalizeValue(void* obj, void* data) { nrValuesFreed++; } -Value * EvalState::allocValue() -{ - nrValues++; - auto v = (Value *) allocBytes(sizeof(Value)); - //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr); - return v; +Value* EvalState::allocValue() { + nrValues++; + auto v = (Value*)allocBytes(sizeof(Value)); + // GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, + // nullptr); + return v; } +Env& EvalState::allocEnv(size_t size) { + if (size > std::numeric_limits<decltype(Env::size)>::max()) + throw Error("environment size %d is too big", size); -Env & EvalState::allocEnv(size_t size) -{ - if (size > std::numeric_limits<decltype(Env::size)>::max()) - throw Error("environment size %d is too big", size); - - nrEnvs++; - nrValuesInEnvs += size; - Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); - env->size = (decltype(Env::size)) size; - env->type = Env::Plain; + nrEnvs++; + nrValuesInEnvs += size; + Env* env = (Env*)allocBytes(sizeof(Env) + size * sizeof(Value*)); + env->size = (decltype(Env::size))size; + env->type = Env::Plain; - /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ + /* We assume that env->values has been cleared by the allocator; maybeThunk() + * and lookupVar fromWith expect this. */ - return *env; + return *env; } - -void EvalState::mkList(Value & v, size_t size) -{ - clearValue(v); - if (size == 1) - v.type = tList1; - else if (size == 2) - v.type = tList2; - else { - v.type = tListN; - v.bigList.size = size; - v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0; - } - nrListElems += size; +void EvalState::mkList(Value& v, size_t size) { + clearValue(v); + if (size == 1) + v.type = tList1; + else if (size == 2) + v.type = tList2; + else { + v.type = tListN; + v.bigList.size = size; + v.bigList.elems = size ? (Value**)allocBytes(size * sizeof(Value*)) : 0; + } + nrListElems += size; } - unsigned long nrThunks = 0; -static inline void mkThunk(Value & v, Env & env, Expr * expr) -{ - v.type = tThunk; - v.thunk.env = &env; - v.thunk.expr = expr; - nrThunks++; +static inline void mkThunk(Value& v, Env& env, Expr* expr) { + v.type = tThunk; + v.thunk.env = &env; + v.thunk.expr = expr; + nrThunks++; } +void EvalState::mkThunk_(Value& v, Expr* expr) { mkThunk(v, baseEnv, expr); } -void EvalState::mkThunk_(Value & v, Expr * expr) -{ - mkThunk(v, baseEnv, expr); +void EvalState::mkPos(Value& v, Pos* pos) { + if (pos && pos->file.set()) { + mkAttrs(v, 3); + mkString(*allocAttr(v, sFile), pos->file); + mkInt(*allocAttr(v, sLine), pos->line); + mkInt(*allocAttr(v, sColumn), pos->column); + v.attrs->sort(); + } else + mkNull(v); } - -void EvalState::mkPos(Value & v, Pos * pos) -{ - if (pos && pos->file.set()) { - mkAttrs(v, 3); - mkString(*allocAttr(v, sFile), pos->file); - mkInt(*allocAttr(v, sLine), pos->line); - mkInt(*allocAttr(v, sColumn), pos->column); - v.attrs->sort(); - } else - mkNull(v); -} - - /* Create a thunk for the delayed computation of the given expression in the given environment. But if the expression is a variable, then look it up right away. This significantly reduces the number of thunks allocated. */ -Value * Expr::maybeThunk(EvalState & state, Env & env) -{ - Value * v = state.allocValue(); - mkThunk(*v, env, this); - return v; +Value* Expr::maybeThunk(EvalState& state, Env& env) { + Value* v = state.allocValue(); + mkThunk(*v, env, this); + return v; } - unsigned long nrAvoided = 0; -Value * ExprVar::maybeThunk(EvalState & state, Env & env) -{ - Value * v = state.lookupVar(&env, *this, true); - /* The value might not be initialised in the environment yet. - In that case, ignore it. */ - if (v) { nrAvoided++; return v; } - return Expr::maybeThunk(state, env); -} - - -Value * ExprString::maybeThunk(EvalState & state, Env & env) -{ +Value* ExprVar::maybeThunk(EvalState& state, Env& env) { + Value* v = state.lookupVar(&env, *this, true); + /* The value might not be initialised in the environment yet. + In that case, ignore it. */ + if (v) { nrAvoided++; - return &v; + return v; + } + return Expr::maybeThunk(state, env); } -Value * ExprInt::maybeThunk(EvalState & state, Env & env) -{ - nrAvoided++; - return &v; +Value* ExprString::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; } -Value * ExprFloat::maybeThunk(EvalState & state, Env & env) -{ - nrAvoided++; - return &v; +Value* ExprInt::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; } -Value * ExprPath::maybeThunk(EvalState & state, Env & env) -{ - nrAvoided++; - return &v; +Value* ExprFloat::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; } - -void EvalState::evalFile(const Path & path_, Value & v) -{ - auto path = checkSourcePath(path_); - - FileEvalCache::iterator i; - if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { - v = i->second; - return; - } - - Path path2 = resolveExprPath(path); - if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) { - v = i->second; - return; - } - - printTalkative("evaluating file '%1%'", path2); - Expr * e = nullptr; - - auto j = fileParseCache.find(path2); - if (j != fileParseCache.end()) - e = j->second; - - if (!e) - e = parseExprFromFile(checkSourcePath(path2)); - - fileParseCache[path2] = e; - - try { - eval(e, v); - } catch (Error & e) { - addErrorPrefix(e, "while evaluating the file '%1%':\n", path2); - throw; - } - - fileEvalCache[path2] = v; - if (path != path2) fileEvalCache[path] = v; +Value* ExprPath::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; } +void EvalState::evalFile(const Path& path_, Value& v) { + auto path = checkSourcePath(path_); -void EvalState::resetFileCache() -{ - fileEvalCache.clear(); - fileParseCache.clear(); -} + FileEvalCache::iterator i; + if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { + v = i->second; + return; + } + Path path2 = resolveExprPath(path); + if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) { + v = i->second; + return; + } -void EvalState::eval(Expr * e, Value & v) -{ - e->eval(*this, baseEnv, v); -} + printTalkative("evaluating file '%1%'", path2); + Expr* e = nullptr; + auto j = fileParseCache.find(path2); + if (j != fileParseCache.end()) e = j->second; -inline bool EvalState::evalBool(Env & env, Expr * e) -{ - Value v; - e->eval(*this, env, v); - if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected", v); - return v.boolean; -} - - -inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) -{ - Value v; - e->eval(*this, env, v); - if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); - return v.boolean; -} - + if (!e) e = parseExprFromFile(checkSourcePath(path2)); -inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) -{ - e->eval(*this, env, v); - if (v.type != tAttrs) - throwTypeError("value is %1% while a set was expected", v); -} + fileParseCache[path2] = e; + try { + eval(e, v); + } catch (Error& e) { + addErrorPrefix(e, "while evaluating the file '%1%':\n", path2); + throw; + } -void Expr::eval(EvalState & state, Env & env, Value & v) -{ - abort(); + fileEvalCache[path2] = v; + if (path != path2) fileEvalCache[path] = v; } - -void ExprInt::eval(EvalState & state, Env & env, Value & v) -{ - v = this->v; +void EvalState::resetFileCache() { + fileEvalCache.clear(); + fileParseCache.clear(); } +void EvalState::eval(Expr* e, Value& v) { e->eval(*this, baseEnv, v); } -void ExprFloat::eval(EvalState & state, Env & env, Value & v) -{ - v = this->v; +inline bool EvalState::evalBool(Env& env, Expr* e) { + Value v; + e->eval(*this, env, v); + if (v.type != tBool) + throwTypeError("value is %1% while a Boolean was expected", v); + return v.boolean; } -void ExprString::eval(EvalState & state, Env & env, Value & v) -{ - v = this->v; +inline bool EvalState::evalBool(Env& env, Expr* e, const Pos& pos) { + Value v; + e->eval(*this, env, v); + if (v.type != tBool) + throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + return v.boolean; } - -void ExprPath::eval(EvalState & state, Env & env, Value & v) -{ - v = this->v; +inline void EvalState::evalAttrs(Env& env, Expr* e, Value& v) { + e->eval(*this, env, v); + if (v.type != tAttrs) + throwTypeError("value is %1% while a set was expected", v); } +void Expr::eval(EvalState& state, Env& env, Value& v) { abort(); } -void ExprAttrs::eval(EvalState & state, Env & env, Value & v) -{ - state.mkAttrs(v, attrs.size() + dynamicAttrs.size()); - Env *dynamicEnv = &env; +void ExprInt::eval(EvalState& state, Env& env, Value& v) { v = this->v; } - if (recursive) { - /* Create a new environment that contains the attributes in - this `rec'. */ - Env & env2(state.allocEnv(attrs.size())); - env2.up = &env; - dynamicEnv = &env2; +void ExprFloat::eval(EvalState& state, Env& env, Value& v) { v = this->v; } - AttrDefs::iterator overrides = attrs.find(state.sOverrides); - bool hasOverrides = overrides != attrs.end(); +void ExprString::eval(EvalState& state, Env& env, Value& v) { v = this->v; } - /* The recursive attributes are evaluated in the new - environment, while the inherited attributes are evaluated - in the original environment. */ - size_t displ = 0; - for (auto & i : attrs) { - Value * vAttr; - if (hasOverrides && !i.second.inherited) { - vAttr = state.allocValue(); - mkThunk(*vAttr, env2, i.second.e); - } else - vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); - env2.values[displ++] = vAttr; - v.attrs->push_back(Attr(i.first, vAttr, &i.second.pos)); - } +void ExprPath::eval(EvalState& state, Env& env, Value& v) { v = this->v; } - /* If the rec contains an attribute called `__overrides', then - evaluate it, and add the attributes in that set to the rec. - This allows overriding of recursive attributes, which is - otherwise not possible. (You can use the // operator to - replace an attribute, but other attributes in the rec will - still reference the original value, because that value has - been substituted into the bodies of the other attributes. - Hence we need __overrides.) */ - if (hasOverrides) { - Value * vOverrides = (*v.attrs)[overrides->second.displ].value; - state.forceAttrs(*vOverrides); - Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size()); - for (auto & i : *v.attrs) - newBnds->push_back(i); - for (auto & i : *vOverrides->attrs) { - AttrDefs::iterator j = attrs.find(i.name); - if (j != attrs.end()) { - (*newBnds)[j->second.displ] = i; - env2.values[j->second.displ] = i.value; - } else - newBnds->push_back(i); - } - newBnds->sort(); - v.attrs = newBnds; - } - } +void ExprAttrs::eval(EvalState& state, Env& env, Value& v) { + state.mkAttrs(v, attrs.size() + dynamicAttrs.size()); + Env* dynamicEnv = &env; - else - for (auto & i : attrs) - v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), &i.second.pos)); - - /* Dynamic attrs apply *after* rec and __overrides. */ - for (auto & i : dynamicAttrs) { - Value nameVal; - i.nameExpr->eval(state, *dynamicEnv, nameVal); - state.forceValue(nameVal, i.pos); - if (nameVal.type == tNull) - continue; - state.forceStringNoCtx(nameVal); - Symbol nameSym = state.symbols.create(nameVal.string.s); - Bindings::iterator j = v.attrs->find(nameSym); - if (j != v.attrs->end()) - throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos); - - i.valueExpr->setName(nameSym); - /* Keep sorted order so find can catch duplicates */ - v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos)); - v.attrs->sort(); // FIXME: inefficient - } -} - - -void ExprLet::eval(EvalState & state, Env & env, Value & v) -{ - /* Create a new environment that contains the attributes in this - `let'. */ - Env & env2(state.allocEnv(attrs->attrs.size())); + if (recursive) { + /* Create a new environment that contains the attributes in + this `rec'. */ + Env& env2(state.allocEnv(attrs.size())); env2.up = &env; + dynamicEnv = &env2; - /* The recursive attributes are evaluated in the new environment, - while the inherited attributes are evaluated in the original - environment. */ - size_t displ = 0; - for (auto & i : attrs->attrs) - env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); - - body->eval(state, env2, v); -} - - -void ExprList::eval(EvalState & state, Env & env, Value & v) -{ - state.mkList(v, elems.size()); - for (size_t n = 0; n < elems.size(); ++n) - v.listElems()[n] = elems[n]->maybeThunk(state, env); -} - - -void ExprVar::eval(EvalState & state, Env & env, Value & v) -{ - Value * v2 = state.lookupVar(&env, *this, false); - state.forceValue(*v2, pos); - v = *v2; -} - + AttrDefs::iterator overrides = attrs.find(state.sOverrides); + bool hasOverrides = overrides != attrs.end(); -static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath) -{ - std::ostringstream out; - bool first = true; - for (auto & i : attrPath) { - if (!first) out << '.'; else first = false; - try { - out << getName(i, state, env); - } catch (Error & e) { - assert(!i.symbol.set()); - out << "\"${" << *i.expr << "}\""; - } + /* The recursive attributes are evaluated in the new + environment, while the inherited attributes are evaluated + in the original environment. */ + size_t displ = 0; + for (auto& i : attrs) { + Value* vAttr; + if (hasOverrides && !i.second.inherited) { + vAttr = state.allocValue(); + mkThunk(*vAttr, env2, i.second.e); + } else + vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); + env2.values[displ++] = vAttr; + v.attrs->push_back(Attr(i.first, vAttr, &i.second.pos)); } - return out.str(); -} - - -unsigned long nrLookups = 0; - -void ExprSelect::eval(EvalState & state, Env & env, Value & v) -{ - Value vTmp; - Pos * pos2 = 0; - Value * vAttrs = &vTmp; - - e->eval(state, env, vTmp); + /* If the rec contains an attribute called `__overrides', then + evaluate it, and add the attributes in that set to the rec. + This allows overriding of recursive attributes, which is + otherwise not possible. (You can use the // operator to + replace an attribute, but other attributes in the rec will + still reference the original value, because that value has + been substituted into the bodies of the other attributes. + Hence we need __overrides.) */ + if (hasOverrides) { + Value* vOverrides = (*v.attrs)[overrides->second.displ].value; + state.forceAttrs(*vOverrides); + Bindings* newBnds = + state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size()); + for (auto& i : *v.attrs) newBnds->push_back(i); + for (auto& i : *vOverrides->attrs) { + AttrDefs::iterator j = attrs.find(i.name); + if (j != attrs.end()) { + (*newBnds)[j->second.displ] = i; + env2.values[j->second.displ] = i.value; + } else + newBnds->push_back(i); + } + newBnds->sort(); + v.attrs = newBnds; + } + } + + else + for (auto& i : attrs) + v.attrs->push_back( + Attr(i.first, i.second.e->maybeThunk(state, env), &i.second.pos)); + + /* Dynamic attrs apply *after* rec and __overrides. */ + for (auto& i : dynamicAttrs) { + Value nameVal; + i.nameExpr->eval(state, *dynamicEnv, nameVal); + state.forceValue(nameVal, i.pos); + if (nameVal.type == tNull) continue; + state.forceStringNoCtx(nameVal); + Symbol nameSym = state.symbols.create(nameVal.string.s); + Bindings::iterator j = v.attrs->find(nameSym); + if (j != v.attrs->end()) + throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", + nameSym, i.pos, *j->pos); + + i.valueExpr->setName(nameSym); + /* Keep sorted order so find can catch duplicates */ + v.attrs->push_back( + Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos)); + v.attrs->sort(); // FIXME: inefficient + } +} + +void ExprLet::eval(EvalState& state, Env& env, Value& v) { + /* Create a new environment that contains the attributes in this + `let'. */ + Env& env2(state.allocEnv(attrs->attrs.size())); + env2.up = &env; + + /* The recursive attributes are evaluated in the new environment, + while the inherited attributes are evaluated in the original + environment. */ + size_t displ = 0; + for (auto& i : attrs->attrs) + env2.values[displ++] = + i.second.e->maybeThunk(state, i.second.inherited ? env : env2); + + body->eval(state, env2, v); +} + +void ExprList::eval(EvalState& state, Env& env, Value& v) { + state.mkList(v, elems.size()); + for (size_t n = 0; n < elems.size(); ++n) + v.listElems()[n] = elems[n]->maybeThunk(state, env); +} + +void ExprVar::eval(EvalState& state, Env& env, Value& v) { + Value* v2 = state.lookupVar(&env, *this, false); + state.forceValue(*v2, pos); + v = *v2; +} + +static string showAttrPath(EvalState& state, Env& env, + const AttrPath& attrPath) { + std::ostringstream out; + bool first = true; + for (auto& i : attrPath) { + if (!first) + out << '.'; + else + first = false; try { - - for (auto & i : attrPath) { - nrLookups++; - Bindings::iterator j; - Symbol name = getName(i, state, env); - if (def) { - state.forceValue(*vAttrs, pos); - if (vAttrs->type != tAttrs || - (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - { - def->eval(state, env, v); - return; - } - } else { - state.forceAttrs(*vAttrs, pos); - if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - throwEvalError("attribute '%1%' missing, at %2%", name, pos); - } - vAttrs = j->value; - pos2 = j->pos; - if (state.countCalls && pos2) state.attrSelects[*pos2]++; - } - - state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) ); - - } catch (Error & e) { - if (pos2 && pos2->file != state.sDerivationNix) - addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", - showAttrPath(state, env, attrPath), *pos2); - throw; + out << getName(i, state, env); + } catch (Error& e) { + assert(!i.symbol.set()); + out << "\"${" << *i.expr << "}\""; } - - v = *vAttrs; + } + return out.str(); } +unsigned long nrLookups = 0; -void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) -{ - Value vTmp; - Value * vAttrs = &vTmp; +void ExprSelect::eval(EvalState& state, Env& env, Value& v) { + Value vTmp; + Pos* pos2 = 0; + Value* vAttrs = &vTmp; - e->eval(state, env, vTmp); + e->eval(state, env, vTmp); - for (auto & i : attrPath) { - state.forceValue(*vAttrs); - Bindings::iterator j; - Symbol name = getName(i, state, env); + try { + for (auto& i : attrPath) { + nrLookups++; + Bindings::iterator j; + Symbol name = getName(i, state, env); + if (def) { + state.forceValue(*vAttrs, pos); if (vAttrs->type != tAttrs || - (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - { - mkBool(v, false); - return; - } else { - vAttrs = j->value; + (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + def->eval(state, env, v); + return; } + } else { + state.forceAttrs(*vAttrs, pos); + if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) + throwEvalError("attribute '%1%' missing, at %2%", name, pos); + } + vAttrs = j->value; + pos2 = j->pos; + if (state.countCalls && pos2) state.attrSelects[*pos2]++; } - mkBool(v, true); -} + state.forceValue(*vAttrs, (pos2 != NULL ? *pos2 : this->pos)); + } catch (Error& e) { + if (pos2 && pos2->file != state.sDerivationNix) + addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", + showAttrPath(state, env, attrPath), *pos2); + throw; + } -void ExprLambda::eval(EvalState & state, Env & env, Value & v) -{ - v.type = tLambda; - v.lambda.env = &env; - v.lambda.fun = this; + v = *vAttrs; } +void ExprOpHasAttr::eval(EvalState& state, Env& env, Value& v) { + Value vTmp; + Value* vAttrs = &vTmp; -void ExprApp::eval(EvalState & state, Env & env, Value & v) -{ - /* FIXME: vFun prevents GCC from doing tail call optimisation. */ - Value vFun; - e1->eval(state, env, vFun); - state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos); -} - + e->eval(state, env, vTmp); -void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) -{ - /* Figure out the number of arguments still needed. */ - size_t argsDone = 0; - Value * primOp = &fun; - while (primOp->type == tPrimOpApp) { - argsDone++; - primOp = primOp->primOpApp.left; - } - assert(primOp->type == tPrimOp); - auto arity = primOp->primOp->arity; - auto argsLeft = arity - argsDone; - - if (argsLeft == 1) { - /* We have all the arguments, so call the primop. */ - - /* Put all the arguments in an array. */ - Value * vArgs[arity]; - auto n = arity - 1; - vArgs[n--] = &arg; - for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left) - vArgs[n--] = arg->primOpApp.right; - - /* And call the primop. */ - nrPrimOpCalls++; - if (countCalls) primOpCalls[primOp->primOp->name]++; - primOp->primOp->fun(*this, pos, vArgs, v); + for (auto& i : attrPath) { + state.forceValue(*vAttrs); + Bindings::iterator j; + Symbol name = getName(i, state, env); + if (vAttrs->type != tAttrs || + (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + mkBool(v, false); + return; } else { - Value * fun2 = allocValue(); - *fun2 = fun; - v.type = tPrimOpApp; - v.primOpApp.left = fun2; - v.primOpApp.right = &arg; + vAttrs = j->value; } -} - -void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) -{ - auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr; - - forceValue(fun, pos); - - if (fun.type == tPrimOp || fun.type == tPrimOpApp) { - callPrimOp(fun, arg, v, pos); - return; + } + + mkBool(v, true); +} + +void ExprLambda::eval(EvalState& state, Env& env, Value& v) { + v.type = tLambda; + v.lambda.env = &env; + v.lambda.fun = this; +} + +void ExprApp::eval(EvalState& state, Env& env, Value& v) { + /* FIXME: vFun prevents GCC from doing tail call optimisation. */ + Value vFun; + e1->eval(state, env, vFun); + state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos); +} + +void EvalState::callPrimOp(Value& fun, Value& arg, Value& v, const Pos& pos) { + /* Figure out the number of arguments still needed. */ + size_t argsDone = 0; + Value* primOp = &fun; + while (primOp->type == tPrimOpApp) { + argsDone++; + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + auto arity = primOp->primOp->arity; + auto argsLeft = arity - argsDone; + + if (argsLeft == 1) { + /* We have all the arguments, so call the primop. */ + + /* Put all the arguments in an array. */ + Value* vArgs[arity]; + auto n = arity - 1; + vArgs[n--] = &arg; + for (Value* arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left) + vArgs[n--] = arg->primOpApp.right; + + /* And call the primop. */ + nrPrimOpCalls++; + if (countCalls) primOpCalls[primOp->primOp->name]++; + primOp->primOp->fun(*this, pos, vArgs, v); + } else { + Value* fun2 = allocValue(); + *fun2 = fun; + v.type = tPrimOpApp; + v.primOpApp.left = fun2; + v.primOpApp.right = &arg; + } +} + +void EvalState::callFunction(Value& fun, Value& arg, Value& v, const Pos& pos) { + auto trace = evalSettings.traceFunctionCalls + ? std::make_unique<FunctionCallTrace>(pos) + : nullptr; + + forceValue(fun, pos); + + if (fun.type == tPrimOp || fun.type == tPrimOpApp) { + callPrimOp(fun, arg, v, pos); + return; + } + + if (fun.type == tAttrs) { + auto found = fun.attrs->find(sFunctor); + if (found != fun.attrs->end()) { + /* fun may be allocated on the stack of the calling function, + * but for functors we may keep a reference, so heap-allocate + * a copy and use that instead. + */ + auto& fun2 = *allocValue(); + fun2 = fun; + /* !!! Should we use the attr pos here? */ + Value v2; + callFunction(*found->value, fun2, v2, pos); + return callFunction(v2, arg, v, pos); } - - if (fun.type == tAttrs) { - auto found = fun.attrs->find(sFunctor); - if (found != fun.attrs->end()) { - /* fun may be allocated on the stack of the calling function, - * but for functors we may keep a reference, so heap-allocate - * a copy and use that instead. - */ - auto & fun2 = *allocValue(); - fun2 = fun; - /* !!! Should we use the attr pos here? */ - Value v2; - callFunction(*found->value, fun2, v2, pos); - return callFunction(v2, arg, v, pos); + } + + if (fun.type != tLambda) + throwTypeError( + "attempt to call something which is not a function but %1%, at %2%", + fun, pos); + + ExprLambda& lambda(*fun.lambda.fun); + + auto size = (lambda.arg.empty() ? 0 : 1) + + (lambda.matchAttrs ? lambda.formals->formals.size() : 0); + Env& env2(allocEnv(size)); + env2.up = fun.lambda.env; + + size_t displ = 0; + + if (!lambda.matchAttrs) + env2.values[displ++] = &arg; + + else { + forceAttrs(arg, pos); + + if (!lambda.arg.empty()) env2.values[displ++] = &arg; + + /* For each formal argument, get the actual argument. If + there is no matching actual argument but the formal + argument has a default, use the default. */ + size_t attrsUsed = 0; + for (auto& i : lambda.formals->formals) { + Bindings::iterator j = arg.attrs->find(i.name); + if (j == arg.attrs->end()) { + if (!i.def) + throwTypeError("%1% called without required argument '%2%', at %3%", + lambda, i.name, pos); + env2.values[displ++] = i.def->maybeThunk(*this, env2); + } else { + attrsUsed++; + env2.values[displ++] = j->value; } } - if (fun.type != tLambda) - throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos); - - ExprLambda & lambda(*fun.lambda.fun); - - auto size = - (lambda.arg.empty() ? 0 : 1) + - (lambda.matchAttrs ? lambda.formals->formals.size() : 0); - Env & env2(allocEnv(size)); - env2.up = fun.lambda.env; - - size_t displ = 0; - - if (!lambda.matchAttrs) - env2.values[displ++] = &arg; - - else { - forceAttrs(arg, pos); - - if (!lambda.arg.empty()) - env2.values[displ++] = &arg; - - /* For each formal argument, get the actual argument. If - there is no matching actual argument but the formal - argument has a default, use the default. */ - size_t attrsUsed = 0; - for (auto & i : lambda.formals->formals) { - Bindings::iterator j = arg.attrs->find(i.name); - if (j == arg.attrs->end()) { - if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%", - lambda, i.name, pos); - env2.values[displ++] = i.def->maybeThunk(*this, env2); - } else { - attrsUsed++; - env2.values[displ++] = j->value; - } - } - - /* Check that each actual argument is listed as a formal - argument (unless the attribute match specifies a `...'). */ - if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { - /* Nope, so show the first unexpected argument to the - user. */ - for (auto & i : *arg.attrs) - if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) - throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos); - abort(); // can't happen - } + /* Check that each actual argument is listed as a formal + argument (unless the attribute match specifies a `...'). */ + if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { + /* Nope, so show the first unexpected argument to the + user. */ + for (auto& i : *arg.attrs) + if (lambda.formals->argNames.find(i.name) == + lambda.formals->argNames.end()) + throwTypeError("%1% called with unexpected argument '%2%', at %3%", + lambda, i.name, pos); + abort(); // can't happen } - - nrFunctionCalls++; - if (countCalls) incrFunctionCall(&lambda); - - /* Evaluate the body. This is conditional on showTrace, because - catching exceptions makes this function not tail-recursive. */ - if (settings.showTrace) - try { - lambda.body->eval(*this, env2, v); - } catch (Error & e) { - addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, pos); - throw; - } - else - fun.lambda.fun->body->eval(*this, env2, v); + } + + nrFunctionCalls++; + if (countCalls) incrFunctionCall(&lambda); + + /* Evaluate the body. This is conditional on showTrace, because + catching exceptions makes this function not tail-recursive. */ + if (settings.showTrace) try { + lambda.body->eval(*this, env2, v); + } catch (Error& e) { + addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, + pos); + throw; + } + else + fun.lambda.fun->body->eval(*this, env2, v); } - // Lifted out of callFunction() because it creates a temporary that // prevents tail-call optimisation. -void EvalState::incrFunctionCall(ExprLambda * fun) -{ - functionCalls[fun]++; -} - - -void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) -{ - forceValue(fun); - - if (fun.type == tAttrs) { - auto found = fun.attrs->find(sFunctor); - if (found != fun.attrs->end()) { - Value * v = allocValue(); - callFunction(*found->value, fun, *v, noPos); - forceValue(*v); - return autoCallFunction(args, *v, res); - } +void EvalState::incrFunctionCall(ExprLambda* fun) { functionCalls[fun]++; } + +void EvalState::autoCallFunction(Bindings& args, Value& fun, Value& res) { + forceValue(fun); + + if (fun.type == tAttrs) { + auto found = fun.attrs->find(sFunctor); + if (found != fun.attrs->end()) { + Value* v = allocValue(); + callFunction(*found->value, fun, *v, noPos); + forceValue(*v); + return autoCallFunction(args, *v, res); } + } - if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { - res = fun; - return; - } + if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { + res = fun; + return; + } - Value * actualArgs = allocValue(); - mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size()); + Value* actualArgs = allocValue(); + mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size()); - for (auto & i : fun.lambda.fun->formals->formals) { - Bindings::iterator j = args.find(i.name); - if (j != args.end()) - actualArgs->attrs->push_back(*j); - else if (!i.def) - throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name); - } + for (auto& i : fun.lambda.fun->formals->formals) { + Bindings::iterator j = args.find(i.name); + if (j != args.end()) + actualArgs->attrs->push_back(*j); + else if (!i.def) + throwTypeError( + "cannot auto-call a function that has an argument without a default " + "value ('%1%')", + i.name); + } - actualArgs->attrs->sort(); + actualArgs->attrs->sort(); - callFunction(fun, *actualArgs, res, noPos); + callFunction(fun, *actualArgs, res, noPos); } +void ExprWith::eval(EvalState& state, Env& env, Value& v) { + Env& env2(state.allocEnv(1)); + env2.up = &env; + env2.prevWith = prevWith; + env2.type = Env::HasWithExpr; + env2.values[0] = (Value*)attrs; -void ExprWith::eval(EvalState & state, Env & env, Value & v) -{ - Env & env2(state.allocEnv(1)); - env2.up = &env; - env2.prevWith = prevWith; - env2.type = Env::HasWithExpr; - env2.values[0] = (Value *) attrs; - - body->eval(state, env2, v); + body->eval(state, env2, v); } - -void ExprIf::eval(EvalState & state, Env & env, Value & v) -{ - (state.evalBool(env, cond) ? then : else_)->eval(state, env, v); +void ExprIf::eval(EvalState& state, Env& env, Value& v) { + (state.evalBool(env, cond) ? then : else_)->eval(state, env, v); } - -void ExprAssert::eval(EvalState & state, Env & env, Value & v) -{ - if (!state.evalBool(env, cond, pos)) { - std::ostringstream out; - cond->show(out); - throwAssertionError("assertion %1% failed at %2%", out.str(), pos); - } - body->eval(state, env, v); +void ExprAssert::eval(EvalState& state, Env& env, Value& v) { + if (!state.evalBool(env, cond, pos)) { + std::ostringstream out; + cond->show(out); + throwAssertionError("assertion %1% failed at %2%", out.str(), pos); + } + body->eval(state, env, v); } - -void ExprOpNot::eval(EvalState & state, Env & env, Value & v) -{ - mkBool(v, !state.evalBool(env, e)); +void ExprOpNot::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, !state.evalBool(env, e)); } - -void ExprOpEq::eval(EvalState & state, Env & env, Value & v) -{ - Value v1; e1->eval(state, env, v1); - Value v2; e2->eval(state, env, v2); - mkBool(v, state.eqValues(v1, v2)); +void ExprOpEq::eval(EvalState& state, Env& env, Value& v) { + Value v1; + e1->eval(state, env, v1); + Value v2; + e2->eval(state, env, v2); + mkBool(v, state.eqValues(v1, v2)); } - -void ExprOpNEq::eval(EvalState & state, Env & env, Value & v) -{ - Value v1; e1->eval(state, env, v1); - Value v2; e2->eval(state, env, v2); - mkBool(v, !state.eqValues(v1, v2)); +void ExprOpNEq::eval(EvalState& state, Env& env, Value& v) { + Value v1; + e1->eval(state, env, v1); + Value v2; + e2->eval(state, env, v2); + mkBool(v, !state.eqValues(v1, v2)); } - -void ExprOpAnd::eval(EvalState & state, Env & env, Value & v) -{ - mkBool(v, state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); +void ExprOpAnd::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); } - -void ExprOpOr::eval(EvalState & state, Env & env, Value & v) -{ - mkBool(v, state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); +void ExprOpOr::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); } - -void ExprOpImpl::eval(EvalState & state, Env & env, Value & v) -{ - mkBool(v, !state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); +void ExprOpImpl::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, !state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); } +void ExprOpUpdate::eval(EvalState& state, Env& env, Value& v) { + Value v1, v2; + state.evalAttrs(env, e1, v1); + state.evalAttrs(env, e2, v2); -void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) -{ - Value v1, v2; - state.evalAttrs(env, e1, v1); - state.evalAttrs(env, e2, v2); - - state.nrOpUpdates++; + state.nrOpUpdates++; - if (v1.attrs->size() == 0) { v = v2; return; } - if (v2.attrs->size() == 0) { v = v1; return; } + if (v1.attrs->size() == 0) { + v = v2; + return; + } + if (v2.attrs->size() == 0) { + v = v1; + return; + } - state.mkAttrs(v, v1.attrs->size() + v2.attrs->size()); + state.mkAttrs(v, v1.attrs->size() + v2.attrs->size()); - /* Merge the sets, preferring values from the second set. Make - sure to keep the resulting vector in sorted order. */ - Bindings::iterator i = v1.attrs->begin(); - Bindings::iterator j = v2.attrs->begin(); + /* Merge the sets, preferring values from the second set. Make + sure to keep the resulting vector in sorted order. */ + Bindings::iterator i = v1.attrs->begin(); + Bindings::iterator j = v2.attrs->begin(); - while (i != v1.attrs->end() && j != v2.attrs->end()) { - if (i->name == j->name) { - v.attrs->push_back(*j); - ++i; ++j; - } - else if (i->name < j->name) - v.attrs->push_back(*i++); - else - v.attrs->push_back(*j++); - } + while (i != v1.attrs->end() && j != v2.attrs->end()) { + if (i->name == j->name) { + v.attrs->push_back(*j); + ++i; + ++j; + } else if (i->name < j->name) + v.attrs->push_back(*i++); + else + v.attrs->push_back(*j++); + } - while (i != v1.attrs->end()) v.attrs->push_back(*i++); - while (j != v2.attrs->end()) v.attrs->push_back(*j++); + while (i != v1.attrs->end()) v.attrs->push_back(*i++); + while (j != v2.attrs->end()) v.attrs->push_back(*j++); - state.nrOpUpdateValuesCopied += v.attrs->size(); + state.nrOpUpdateValuesCopied += v.attrs->size(); } - -void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) -{ - Value v1; e1->eval(state, env, v1); - Value v2; e2->eval(state, env, v2); - Value * lists[2] = { &v1, &v2 }; - state.concatLists(v, 2, lists, pos); +void ExprOpConcatLists::eval(EvalState& state, Env& env, Value& v) { + Value v1; + e1->eval(state, env, v1); + Value v2; + e2->eval(state, env, v2); + Value* lists[2] = {&v1, &v2}; + state.concatLists(v, 2, lists, pos); } +void EvalState::concatLists(Value& v, size_t nrLists, Value** lists, + const Pos& pos) { + nrListConcats++; -void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos) -{ - nrListConcats++; - - Value * nonEmpty = 0; - size_t len = 0; - for (size_t n = 0; n < nrLists; ++n) { - forceList(*lists[n], pos); - auto l = lists[n]->listSize(); - len += l; - if (l) nonEmpty = lists[n]; - } + Value* nonEmpty = 0; + size_t len = 0; + for (size_t n = 0; n < nrLists; ++n) { + forceList(*lists[n], pos); + auto l = lists[n]->listSize(); + len += l; + if (l) nonEmpty = lists[n]; + } - if (nonEmpty && len == nonEmpty->listSize()) { - v = *nonEmpty; - return; - } + if (nonEmpty && len == nonEmpty->listSize()) { + v = *nonEmpty; + return; + } - mkList(v, len); - auto out = v.listElems(); - for (size_t n = 0, pos = 0; n < nrLists; ++n) { - auto l = lists[n]->listSize(); - if (l) - memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *)); - pos += l; - } + mkList(v, len); + auto out = v.listElems(); + for (size_t n = 0, pos = 0; n < nrLists; ++n) { + auto l = lists[n]->listSize(); + if (l) memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value*)); + pos += l; + } } +void ExprConcatStrings::eval(EvalState& state, Env& env, Value& v) { + PathSet context; + std::ostringstream s; + NixInt n = 0; + NixFloat nf = 0; -void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) -{ - PathSet context; - std::ostringstream s; - NixInt n = 0; - NixFloat nf = 0; - - bool first = !forceString; - ValueType firstType = tString; + bool first = !forceString; + ValueType firstType = tString; - for (auto & i : *es) { - Value vTmp; - i->eval(state, env, vTmp); - - /* If the first element is a path, then the result will also - be a path, we don't copy anything (yet - that's done later, - since paths are copied when they are used in a derivation), - and none of the strings are allowed to have contexts. */ - if (first) { - firstType = vTmp.type; - first = false; - } - - if (firstType == tInt) { - if (vTmp.type == tInt) { - n += vTmp.integer; - } else if (vTmp.type == tFloat) { - // Upgrade the type from int to float; - firstType = tFloat; - nf = n; - nf += vTmp.fpoint; - } else - throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos); - } else if (firstType == tFloat) { - if (vTmp.type == tInt) { - nf += vTmp.integer; - } else if (vTmp.type == tFloat) { - nf += vTmp.fpoint; - } else - throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos); - } else - s << state.coerceToString(pos, vTmp, context, false, firstType == tString); + for (auto& i : *es) { + Value vTmp; + i->eval(state, env, vTmp); + + /* If the first element is a path, then the result will also + be a path, we don't copy anything (yet - that's done later, + since paths are copied when they are used in a derivation), + and none of the strings are allowed to have contexts. */ + if (first) { + firstType = vTmp.type; + first = false; } - if (firstType == tInt) - mkInt(v, n); - else if (firstType == tFloat) - mkFloat(v, nf); - else if (firstType == tPath) { - if (!context.empty()) - throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos); - auto path = canonPath(s.str()); - mkPath(v, path.c_str()); + if (firstType == tInt) { + if (vTmp.type == tInt) { + n += vTmp.integer; + } else if (vTmp.type == tFloat) { + // Upgrade the type from int to float; + firstType = tFloat; + nf = n; + nf += vTmp.fpoint; + } else + throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), + pos); + } else if (firstType == tFloat) { + if (vTmp.type == tInt) { + nf += vTmp.integer; + } else if (vTmp.type == tFloat) { + nf += vTmp.fpoint; + } else + throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), + pos); } else - mkString(v, s.str(), context); -} + s << state.coerceToString(pos, vTmp, context, false, + firstType == tString); + } - -void ExprPos::eval(EvalState & state, Env & env, Value & v) -{ - state.mkPos(v, &pos); + if (firstType == tInt) + mkInt(v, n); + else if (firstType == tFloat) + mkFloat(v, nf); + else if (firstType == tPath) { + if (!context.empty()) + throwEvalError( + "a string that refers to a store path cannot be appended to a path, " + "at %1%", + pos); + auto path = canonPath(s.str()); + mkPath(v, path.c_str()); + } else + mkString(v, s.str(), context); } +void ExprPos::eval(EvalState& state, Env& env, Value& v) { + state.mkPos(v, &pos); +} -void EvalState::forceValueDeep(Value & v) -{ - std::set<const Value *> seen; +void EvalState::forceValueDeep(Value& v) { + std::set<const Value*> seen; - std::function<void(Value & v)> recurse; + std::function<void(Value & v)> recurse; - recurse = [&](Value & v) { - if (seen.find(&v) != seen.end()) return; - seen.insert(&v); + recurse = [&](Value& v) { + if (seen.find(&v) != seen.end()) return; + seen.insert(&v); - forceValue(v); + forceValue(v); - if (v.type == tAttrs) { - for (auto & i : *v.attrs) - try { - recurse(*i.value); - } catch (Error & e) { - addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos); - throw; - } + if (v.type == tAttrs) { + for (auto& i : *v.attrs) try { + recurse(*i.value); + } catch (Error& e) { + addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", + i.name, *i.pos); + throw; } + } - else if (v.isList()) { - for (size_t n = 0; n < v.listSize(); ++n) - recurse(*v.listElems()[n]); - } - }; + else if (v.isList()) { + for (size_t n = 0; n < v.listSize(); ++n) recurse(*v.listElems()[n]); + } + }; - recurse(v); + recurse(v); } +NixInt EvalState::forceInt(Value& v, const Pos& pos) { + forceValue(v, pos); + if (v.type != tInt) + throwTypeError("value is %1% while an integer was expected, at %2%", v, + pos); + return v.integer; +} -NixInt EvalState::forceInt(Value & v, const Pos & pos) -{ - forceValue(v, pos); - if (v.type != tInt) - throwTypeError("value is %1% while an integer was expected, at %2%", v, pos); +NixFloat EvalState::forceFloat(Value& v, const Pos& pos) { + forceValue(v, pos); + if (v.type == tInt) return v.integer; + else if (v.type != tFloat) + throwTypeError("value is %1% while a float was expected, at %2%", v, pos); + return v.fpoint; } - -NixFloat EvalState::forceFloat(Value & v, const Pos & pos) -{ - forceValue(v, pos); - if (v.type == tInt) - return v.integer; - else if (v.type != tFloat) - throwTypeError("value is %1% while a float was expected, at %2%", v, pos); - return v.fpoint; +bool EvalState::forceBool(Value& v, const Pos& pos) { + forceValue(v); + if (v.type != tBool) + throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + return v.boolean; } - -bool EvalState::forceBool(Value & v, const Pos & pos) -{ - forceValue(v); - if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); - return v.boolean; +bool EvalState::isFunctor(Value& fun) { + return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); } - -bool EvalState::isFunctor(Value & fun) -{ - return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); -} - - -void EvalState::forceFunction(Value & v, const Pos & pos) -{ - forceValue(v); - if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) - throwTypeError("value is %1% while a function was expected, at %2%", v, pos); +void EvalState::forceFunction(Value& v, const Pos& pos) { + forceValue(v); + if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && + !isFunctor(v)) + throwTypeError("value is %1% while a function was expected, at %2%", v, + pos); } - -string EvalState::forceString(Value & v, const Pos & pos) -{ - forceValue(v, pos); - if (v.type != tString) { - if (pos) - throwTypeError("value is %1% while a string was expected, at %2%", v, pos); - else - throwTypeError("value is %1% while a string was expected", v); - } - return string(v.string.s); +string EvalState::forceString(Value& v, const Pos& pos) { + forceValue(v, pos); + if (v.type != tString) { + if (pos) + throwTypeError("value is %1% while a string was expected, at %2%", v, + pos); + else + throwTypeError("value is %1% while a string was expected", v); + } + return string(v.string.s); } - -void copyContext(const Value & v, PathSet & context) -{ - if (v.string.context) - for (const char * * p = v.string.context; *p; ++p) - context.insert(*p); +void copyContext(const Value& v, PathSet& context) { + if (v.string.context) + for (const char** p = v.string.context; *p; ++p) context.insert(*p); } - -string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) -{ - string s = forceString(v, pos); - copyContext(v, context); - return s; +string EvalState::forceString(Value& v, PathSet& context, const Pos& pos) { + string s = forceString(v, pos); + copyContext(v, context); + return s; } - -string EvalState::forceStringNoCtx(Value & v, const Pos & pos) -{ - string s = forceString(v, pos); - if (v.string.context) { - if (pos) - throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%", - v.string.s, v.string.context[0], pos); - else - throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", - v.string.s, v.string.context[0]); - } - return s; +string EvalState::forceStringNoCtx(Value& v, const Pos& pos) { + string s = forceString(v, pos); + if (v.string.context) { + if (pos) + throwEvalError( + "the string '%1%' is not allowed to refer to a store path (such as " + "'%2%'), at %3%", + v.string.s, v.string.context[0], pos); + else + throwEvalError( + "the string '%1%' is not allowed to refer to a store path (such as " + "'%2%')", + v.string.s, v.string.context[0]); + } + return s; } - -bool EvalState::isDerivation(Value & v) -{ - if (v.type != tAttrs) return false; - Bindings::iterator i = v.attrs->find(sType); - if (i == v.attrs->end()) return false; - forceValue(*i->value); - if (i->value->type != tString) return false; - return strcmp(i->value->string.s, "derivation") == 0; +bool EvalState::isDerivation(Value& v) { + if (v.type != tAttrs) return false; + Bindings::iterator i = v.attrs->find(sType); + if (i == v.attrs->end()) return false; + forceValue(*i->value); + if (i->value->type != tString) return false; + return strcmp(i->value->string.s, "derivation") == 0; } +std::optional<string> EvalState::tryAttrsToString(const Pos& pos, Value& v, + PathSet& context, + bool coerceMore, + bool copyToStore) { + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToString(pos, v1, context, coerceMore, copyToStore); + } -std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v, - PathSet & context, bool coerceMore, bool copyToStore) -{ - auto i = v.attrs->find(sToString); - if (i != v.attrs->end()) { - Value v1; - callFunction(*i->value, v, v1, pos); - return coerceToString(pos, v1, context, coerceMore, copyToStore); - } - - return {}; + return {}; } -string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, - bool coerceMore, bool copyToStore) -{ - forceValue(v); - - string s; +string EvalState::coerceToString(const Pos& pos, Value& v, PathSet& context, + bool coerceMore, bool copyToStore) { + forceValue(v); - if (v.type == tString) { - copyContext(v, context); - return v.string.s; - } - - if (v.type == tPath) { - Path path(canonPath(v.path)); - return copyToStore ? copyPathToStore(context, path) : path; - } + string s; - if (v.type == tAttrs) { - auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); - if (maybeString) { - return *maybeString; - } - auto i = v.attrs->find(sOutPath); - if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos); - return coerceToString(pos, *i->value, context, coerceMore, copyToStore); + if (v.type == tString) { + copyContext(v, context); + return v.string.s; + } + + if (v.type == tPath) { + Path path(canonPath(v.path)); + return copyToStore ? copyPathToStore(context, path) : path; + } + + if (v.type == tAttrs) { + auto maybeString = + tryAttrsToString(pos, v, context, coerceMore, copyToStore); + if (maybeString) { + return *maybeString; } - - if (v.type == tExternal) - return v.external->coerceToString(pos, context, coerceMore, copyToStore); - - if (coerceMore) { - - /* Note that `false' is represented as an empty string for - shell scripting convenience, just like `null'. */ - if (v.type == tBool && v.boolean) return "1"; - if (v.type == tBool && !v.boolean) return ""; - if (v.type == tInt) return std::to_string(v.integer); - if (v.type == tFloat) return std::to_string(v.fpoint); - if (v.type == tNull) return ""; - - if (v.isList()) { - string result; - for (size_t n = 0; n < v.listSize(); ++n) { - result += coerceToString(pos, *v.listElems()[n], - context, coerceMore, copyToStore); - if (n < v.listSize() - 1 - /* !!! not quite correct */ - && (!v.listElems()[n]->isList() || v.listElems()[n]->listSize() != 0)) - result += " "; - } - return result; - } + auto i = v.attrs->find(sOutPath); + if (i == v.attrs->end()) + throwTypeError("cannot coerce a set to a string, at %1%", pos); + return coerceToString(pos, *i->value, context, coerceMore, copyToStore); + } + + if (v.type == tExternal) + return v.external->coerceToString(pos, context, coerceMore, copyToStore); + + if (coerceMore) { + /* Note that `false' is represented as an empty string for + shell scripting convenience, just like `null'. */ + if (v.type == tBool && v.boolean) return "1"; + if (v.type == tBool && !v.boolean) return ""; + if (v.type == tInt) return std::to_string(v.integer); + if (v.type == tFloat) return std::to_string(v.fpoint); + if (v.type == tNull) return ""; + + if (v.isList()) { + string result; + for (size_t n = 0; n < v.listSize(); ++n) { + result += coerceToString(pos, *v.listElems()[n], context, coerceMore, + copyToStore); + if (n < v.listSize() - 1 + /* !!! not quite correct */ + && + (!v.listElems()[n]->isList() || v.listElems()[n]->listSize() != 0)) + result += " "; + } + return result; } + } - throwTypeError("cannot coerce %1% to a string, at %2%", v, pos); + throwTypeError("cannot coerce %1% to a string, at %2%", v, pos); } +string EvalState::copyPathToStore(PathSet& context, const Path& path) { + if (nix::isDerivation(path)) + throwEvalError("file names are not allowed to end in '%1%'", drvExtension); -string EvalState::copyPathToStore(PathSet & context, const Path & path) -{ - if (nix::isDerivation(path)) - throwEvalError("file names are not allowed to end in '%1%'", drvExtension); + Path dstPath; + if (srcToStore[path] != "") + dstPath = srcToStore[path]; + else { + dstPath = + settings.readOnlyMode + ? store + ->computeStorePathForPath(baseNameOf(path), + checkSourcePath(path)) + .first + : store->addToStore(baseNameOf(path), checkSourcePath(path), true, + htSHA256, defaultPathFilter, repair); + srcToStore[path] = dstPath; + printMsg(lvlChatty, + format("copied source '%1%' -> '%2%'") % path % dstPath); + } - Path dstPath; - if (srcToStore[path] != "") - dstPath = srcToStore[path]; - else { - dstPath = settings.readOnlyMode - ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first - : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); - srcToStore[path] = dstPath; - printMsg(lvlChatty, format("copied source '%1%' -> '%2%'") - % path % dstPath); - } - - context.insert(dstPath); - return dstPath; + context.insert(dstPath); + return dstPath; } - -Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context) -{ - string path = coerceToString(pos, v, context, false, false); - if (path == "" || path[0] != '/') - throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos); - return path; +Path EvalState::coerceToPath(const Pos& pos, Value& v, PathSet& context) { + string path = coerceToString(pos, v, context, false, false); + if (path == "" || path[0] != '/') + throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", + path, pos); + return path; } +bool EvalState::eqValues(Value& v1, Value& v2) { + forceValue(v1); + forceValue(v2); -bool EvalState::eqValues(Value & v1, Value & v2) -{ - forceValue(v1); - forceValue(v2); + /* !!! Hack to support some old broken code that relies on pointer + equality tests between sets. (Specifically, builderDefs calls + uniqList on a list of sets.) Will remove this eventually. */ + if (&v1 == &v2) return true; - /* !!! Hack to support some old broken code that relies on pointer - equality tests between sets. (Specifically, builderDefs calls - uniqList on a list of sets.) Will remove this eventually. */ - if (&v1 == &v2) return true; + // Special case type-compatibility between float and int + if (v1.type == tInt && v2.type == tFloat) return v1.integer == v2.fpoint; + if (v1.type == tFloat && v2.type == tInt) return v1.fpoint == v2.integer; - // Special case type-compatibility between float and int - if (v1.type == tInt && v2.type == tFloat) - return v1.integer == v2.fpoint; - if (v1.type == tFloat && v2.type == tInt) - return v1.fpoint == v2.integer; + // All other types are not compatible with each other. + if (v1.type != v2.type) return false; - // All other types are not compatible with each other. - if (v1.type != v2.type) return false; - - switch (v1.type) { - - case tInt: - return v1.integer == v2.integer; + switch (v1.type) { + case tInt: + return v1.integer == v2.integer; - case tBool: - return v1.boolean == v2.boolean; + case tBool: + return v1.boolean == v2.boolean; - case tString: - return strcmp(v1.string.s, v2.string.s) == 0; + case tString: + return strcmp(v1.string.s, v2.string.s) == 0; - case tPath: - return strcmp(v1.path, v2.path) == 0; + case tPath: + return strcmp(v1.path, v2.path) == 0; - case tNull: - return true; + case tNull: + return true; - case tList1: - case tList2: - case tListN: - if (v1.listSize() != v2.listSize()) return false; - for (size_t n = 0; n < v1.listSize(); ++n) - if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; - return true; + case tList1: + case tList2: + case tListN: + if (v1.listSize() != v2.listSize()) return false; + for (size_t n = 0; n < v1.listSize(); ++n) + if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; + return true; - case tAttrs: { - /* If both sets denote a derivation (type = "derivation"), - then compare their outPaths. */ - if (isDerivation(v1) && isDerivation(v2)) { - Bindings::iterator i = v1.attrs->find(sOutPath); - Bindings::iterator j = v2.attrs->find(sOutPath); - if (i != v1.attrs->end() && j != v2.attrs->end()) - return eqValues(*i->value, *j->value); - } + case tAttrs: { + /* If both sets denote a derivation (type = "derivation"), + then compare their outPaths. */ + if (isDerivation(v1) && isDerivation(v2)) { + Bindings::iterator i = v1.attrs->find(sOutPath); + Bindings::iterator j = v2.attrs->find(sOutPath); + if (i != v1.attrs->end() && j != v2.attrs->end()) + return eqValues(*i->value, *j->value); + } - if (v1.attrs->size() != v2.attrs->size()) return false; + if (v1.attrs->size() != v2.attrs->size()) return false; - /* Otherwise, compare the attributes one by one. */ - Bindings::iterator i, j; - for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); ++i, ++j) - if (i->name != j->name || !eqValues(*i->value, *j->value)) - return false; + /* Otherwise, compare the attributes one by one. */ + Bindings::iterator i, j; + for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); + ++i, ++j) + if (i->name != j->name || !eqValues(*i->value, *j->value)) return false; - return true; - } + return true; + } - /* Functions are incomparable. */ - case tLambda: - case tPrimOp: - case tPrimOpApp: - return false; + /* Functions are incomparable. */ + case tLambda: + case tPrimOp: + case tPrimOpApp: + return false; - case tExternal: - return *v1.external == *v2.external; + case tExternal: + return *v1.external == *v2.external; - case tFloat: - return v1.fpoint == v2.fpoint; + case tFloat: + return v1.fpoint == v2.fpoint; - default: - throwEvalError("cannot compare %1% with %2%", showType(v1), showType(v2)); - } + default: + throwEvalError("cannot compare %1% with %2%", showType(v1), showType(v2)); + } } -void EvalState::printStats() -{ - bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0"; +void EvalState::printStats() { + bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0"; - struct rusage buf; - getrusage(RUSAGE_SELF, &buf); - float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000); + struct rusage buf; + getrusage(RUSAGE_SELF, &buf); + float cpuTime = buf.ru_utime.tv_sec + ((float)buf.ru_utime.tv_usec / 1000000); - uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *); - uint64_t bLists = nrListElems * sizeof(Value *); - uint64_t bValues = nrValues * sizeof(Value); - uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr); + uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value*); + uint64_t bLists = nrListElems * sizeof(Value*); + uint64_t bValues = nrValues * sizeof(Value); + uint64_t bAttrsets = + nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr); #if HAVE_BOEHMGC - GC_word heapSize, totalBytes; - GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); + GC_word heapSize, totalBytes; + GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); #endif - if (showStats) { - auto outPath = getEnv("NIX_SHOW_STATS_PATH","-"); - std::fstream fs; - if (outPath != "-") - fs.open(outPath, std::fstream::out); - JSONObject topObj(outPath == "-" ? std::cerr : fs, true); - topObj.attr("cpuTime",cpuTime); - { - auto envs = topObj.object("envs"); - envs.attr("number", nrEnvs); - envs.attr("elements", nrValuesInEnvs); - envs.attr("bytes", bEnvs); - } - { - auto lists = topObj.object("list"); - lists.attr("elements", nrListElems); - lists.attr("bytes", bLists); - lists.attr("concats", nrListConcats); - } - { - auto values = topObj.object("values"); - values.attr("number", nrValues); - values.attr("bytes", bValues); - } - { - auto syms = topObj.object("symbols"); - syms.attr("number", symbols.size()); - syms.attr("bytes", symbols.totalSize()); - } - { - auto sets = topObj.object("sets"); - sets.attr("number", nrAttrsets); - sets.attr("bytes", bAttrsets); - sets.attr("elements", nrAttrsInAttrsets); - } - { - auto sizes = topObj.object("sizes"); - sizes.attr("Env", sizeof(Env)); - sizes.attr("Value", sizeof(Value)); - sizes.attr("Bindings", sizeof(Bindings)); - sizes.attr("Attr", sizeof(Attr)); - } - topObj.attr("nrOpUpdates", nrOpUpdates); - topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied); - topObj.attr("nrThunks", nrThunks); - topObj.attr("nrAvoided", nrAvoided); - topObj.attr("nrLookups", nrLookups); - topObj.attr("nrPrimOpCalls", nrPrimOpCalls); - topObj.attr("nrFunctionCalls", nrFunctionCalls); + if (showStats) { + auto outPath = getEnv("NIX_SHOW_STATS_PATH", "-"); + std::fstream fs; + if (outPath != "-") fs.open(outPath, std::fstream::out); + JSONObject topObj(outPath == "-" ? std::cerr : fs, true); + topObj.attr("cpuTime", cpuTime); + { + auto envs = topObj.object("envs"); + envs.attr("number", nrEnvs); + envs.attr("elements", nrValuesInEnvs); + envs.attr("bytes", bEnvs); + } + { + auto lists = topObj.object("list"); + lists.attr("elements", nrListElems); + lists.attr("bytes", bLists); + lists.attr("concats", nrListConcats); + } + { + auto values = topObj.object("values"); + values.attr("number", nrValues); + values.attr("bytes", bValues); + } + { + auto syms = topObj.object("symbols"); + syms.attr("number", symbols.size()); + syms.attr("bytes", symbols.totalSize()); + } + { + auto sets = topObj.object("sets"); + sets.attr("number", nrAttrsets); + sets.attr("bytes", bAttrsets); + sets.attr("elements", nrAttrsInAttrsets); + } + { + auto sizes = topObj.object("sizes"); + sizes.attr("Env", sizeof(Env)); + sizes.attr("Value", sizeof(Value)); + sizes.attr("Bindings", sizeof(Bindings)); + sizes.attr("Attr", sizeof(Attr)); + } + topObj.attr("nrOpUpdates", nrOpUpdates); + topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied); + topObj.attr("nrThunks", nrThunks); + topObj.attr("nrAvoided", nrAvoided); + topObj.attr("nrLookups", nrLookups); + topObj.attr("nrPrimOpCalls", nrPrimOpCalls); + topObj.attr("nrFunctionCalls", nrFunctionCalls); #if HAVE_BOEHMGC - { - auto gc = topObj.object("gc"); - gc.attr("heapSize", heapSize); - gc.attr("totalBytes", totalBytes); - } + { + auto gc = topObj.object("gc"); + gc.attr("heapSize", heapSize); + gc.attr("totalBytes", totalBytes); + } #endif - if (countCalls) { - { - auto obj = topObj.object("primops"); - for (auto & i : primOpCalls) - obj.attr(i.first, i.second); - } - { - auto list = topObj.list("functions"); - for (auto & i : functionCalls) { - auto obj = list.object(); - if (i.first->name.set()) - obj.attr("name", (const string &) i.first->name); - else - obj.attr("name", nullptr); - if (i.first->pos) { - obj.attr("file", (const string &) i.first->pos.file); - obj.attr("line", i.first->pos.line); - obj.attr("column", i.first->pos.column); - } - obj.attr("count", i.second); - } - } - { - auto list = topObj.list("attributes"); - for (auto & i : attrSelects) { - auto obj = list.object(); - if (i.first) { - obj.attr("file", (const string &) i.first.file); - obj.attr("line", i.first.line); - obj.attr("column", i.first.column); - } - obj.attr("count", i.second); - } - } + if (countCalls) { + { + auto obj = topObj.object("primops"); + for (auto& i : primOpCalls) obj.attr(i.first, i.second); + } + { + auto list = topObj.list("functions"); + for (auto& i : functionCalls) { + auto obj = list.object(); + if (i.first->name.set()) + obj.attr("name", (const string&)i.first->name); + else + obj.attr("name", nullptr); + if (i.first->pos) { + obj.attr("file", (const string&)i.first->pos.file); + obj.attr("line", i.first->pos.line); + obj.attr("column", i.first->pos.column); + } + obj.attr("count", i.second); } - - if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") { - auto list = topObj.list("symbols"); - symbols.dump([&](const std::string & s) { list.elem(s); }); + } + { + auto list = topObj.list("attributes"); + for (auto& i : attrSelects) { + auto obj = list.object(); + if (i.first) { + obj.attr("file", (const string&)i.first.file); + obj.attr("line", i.first.line); + obj.attr("column", i.first.column); + } + obj.attr("count", i.second); } + } + } + + if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") { + auto list = topObj.list("symbols"); + symbols.dump([&](const std::string& s) { list.elem(s); }); } + } } +size_t valueSize(Value& v) { + std::set<const void*> seen; -size_t valueSize(Value & v) -{ - std::set<const void *> seen; - - auto doString = [&](const char * s) -> size_t { - if (seen.find(s) != seen.end()) return 0; - seen.insert(s); - return strlen(s) + 1; - }; - - std::function<size_t(Value & v)> doValue; - std::function<size_t(Env & v)> doEnv; - - doValue = [&](Value & v) -> size_t { - if (seen.find(&v) != seen.end()) return 0; - seen.insert(&v); - - size_t sz = sizeof(Value); - - switch (v.type) { - case tString: - sz += doString(v.string.s); - if (v.string.context) - for (const char * * p = v.string.context; *p; ++p) - sz += doString(*p); - break; - case tPath: - sz += doString(v.path); - break; - case tAttrs: - if (seen.find(v.attrs) == seen.end()) { - seen.insert(v.attrs); - sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity(); - for (auto & i : *v.attrs) - sz += doValue(*i.value); - } - break; - case tList1: - case tList2: - case tListN: - if (seen.find(v.listElems()) == seen.end()) { - seen.insert(v.listElems()); - sz += v.listSize() * sizeof(Value *); - for (size_t n = 0; n < v.listSize(); ++n) - sz += doValue(*v.listElems()[n]); - } - break; - case tThunk: - sz += doEnv(*v.thunk.env); - break; - case tApp: - sz += doValue(*v.app.left); - sz += doValue(*v.app.right); - break; - case tLambda: - sz += doEnv(*v.lambda.env); - break; - case tPrimOpApp: - sz += doValue(*v.primOpApp.left); - sz += doValue(*v.primOpApp.right); - break; - case tExternal: - if (seen.find(v.external) != seen.end()) break; - seen.insert(v.external); - sz += v.external->valueSize(seen); - break; - default: - ; - } + auto doString = [&](const char* s) -> size_t { + if (seen.find(s) != seen.end()) return 0; + seen.insert(s); + return strlen(s) + 1; + }; - return sz; - }; + std::function<size_t(Value & v)> doValue; + std::function<size_t(Env & v)> doEnv; - doEnv = [&](Env & env) -> size_t { - if (seen.find(&env) != seen.end()) return 0; - seen.insert(&env); + doValue = [&](Value& v) -> size_t { + if (seen.find(&v) != seen.end()) return 0; + seen.insert(&v); - size_t sz = sizeof(Env) + sizeof(Value *) * env.size; + size_t sz = sizeof(Value); - if (env.type != Env::HasWithExpr) - for (size_t i = 0; i < env.size; ++i) - if (env.values[i]) - sz += doValue(*env.values[i]); + switch (v.type) { + case tString: + sz += doString(v.string.s); + if (v.string.context) + for (const char** p = v.string.context; *p; ++p) sz += doString(*p); + break; + case tPath: + sz += doString(v.path); + break; + case tAttrs: + if (seen.find(v.attrs) == seen.end()) { + seen.insert(v.attrs); + sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity(); + for (auto& i : *v.attrs) sz += doValue(*i.value); + } + break; + case tList1: + case tList2: + case tListN: + if (seen.find(v.listElems()) == seen.end()) { + seen.insert(v.listElems()); + sz += v.listSize() * sizeof(Value*); + for (size_t n = 0; n < v.listSize(); ++n) + sz += doValue(*v.listElems()[n]); + } + break; + case tThunk: + sz += doEnv(*v.thunk.env); + break; + case tApp: + sz += doValue(*v.app.left); + sz += doValue(*v.app.right); + break; + case tLambda: + sz += doEnv(*v.lambda.env); + break; + case tPrimOpApp: + sz += doValue(*v.primOpApp.left); + sz += doValue(*v.primOpApp.right); + break; + case tExternal: + if (seen.find(v.external) != seen.end()) break; + seen.insert(v.external); + sz += v.external->valueSize(seen); + break; + default:; + } - if (env.up) sz += doEnv(*env.up); + return sz; + }; - return sz; - }; + doEnv = [&](Env& env) -> size_t { + if (seen.find(&env) != seen.end()) return 0; + seen.insert(&env); - return doValue(v); -} + size_t sz = sizeof(Env) + sizeof(Value*) * env.size; + if (env.type != Env::HasWithExpr) + for (size_t i = 0; i < env.size; ++i) + if (env.values[i]) sz += doValue(*env.values[i]); -string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const -{ - throw TypeError(format("cannot coerce %1% to a string, at %2%") % - showType() % pos); -} + if (env.up) sz += doEnv(*env.up); + return sz; + }; -bool ExternalValueBase::operator==(const ExternalValueBase & b) const -{ - return false; + return doValue(v); } +string ExternalValueBase::coerceToString(const Pos& pos, PathSet& context, + bool copyMore, + bool copyToStore) const { + throw TypeError(format("cannot coerce %1% to a string, at %2%") % showType() % + pos); +} -std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { - return v.print(str); +bool ExternalValueBase::operator==(const ExternalValueBase& b) const { + return false; } +std::ostream& operator<<(std::ostream& str, const ExternalValueBase& v) { + return v.print(str); +} EvalSettings evalSettings; static GlobalConfig::Register r1(&evalSettings); - -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/eval.hh b/third_party/nix/src/libexpr/eval.hh index 8126e4ea50ff..74777e48fa95 100644 --- a/third_party/nix/src/libexpr/eval.hh +++ b/third_party/nix/src/libexpr/eval.hh @@ -1,363 +1,357 @@ #pragma once -#include "attr-set.hh" -#include "value.hh" -#include "nixexpr.hh" -#include "symbol-table.hh" -#include "hash.hh" -#include "config.hh" - #include <map> #include <optional> #include <unordered_map> - +#include "attr-set.hh" +#include "config.hh" +#include "hash.hh" +#include "nixexpr.hh" +#include "symbol-table.hh" +#include "value.hh" namespace nix { - class Store; class EvalState; enum RepairFlag : bool; +typedef void (*PrimOpFun)(EvalState& state, const Pos& pos, Value** args, + Value& v); -typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); - - -struct PrimOp -{ - PrimOpFun fun; - size_t arity; - Symbol name; - PrimOp(PrimOpFun fun, size_t arity, Symbol name) - : fun(fun), arity(arity), name(name) { } +struct PrimOp { + PrimOpFun fun; + size_t arity; + Symbol name; + PrimOp(PrimOpFun fun, size_t arity, Symbol name) + : fun(fun), arity(arity), name(name) {} }; - -struct Env -{ - Env * up; - unsigned short size; // used by ‘valueSize’ - unsigned short prevWith:14; // nr of levels up to next `with' environment - enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; - Value * values[0]; +struct Env { + Env* up; + unsigned short size; // used by ‘valueSize’ + unsigned short prevWith : 14; // nr of levels up to next `with' environment + enum { Plain = 0, HasWithExpr, HasWithAttrs } type : 2; + Value* values[0]; }; +Value& mkString(Value& v, const string& s, const PathSet& context = PathSet()); -Value & mkString(Value & v, const string & s, const PathSet & context = PathSet()); - -void copyContext(const Value & v, PathSet & context); - +void copyContext(const Value& v, PathSet& context); /* Cache for calls to addToStore(); maps source paths to the store paths. */ typedef std::map<Path, Path> SrcToStore; - -std::ostream & operator << (std::ostream & str, const Value & v); - +std::ostream& operator<<(std::ostream& str, const Value& v); typedef std::pair<std::string, std::string> SearchPathElem; typedef std::list<SearchPathElem> SearchPath; - /* Initialise the Boehm GC, if applicable. */ void initGC(); +class EvalState { + public: + SymbolTable symbols; -class EvalState -{ -public: - SymbolTable symbols; - - const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, - sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, - sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, - sOutputHash, sOutputHashAlgo, sOutputHashMode; - Symbol sDerivationNix; + const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, + sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, + sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sOutputHash, sOutputHashAlgo, sOutputHashMode; + Symbol sDerivationNix; - /* If set, force copying files to the Nix store even if they - already exist there. */ - RepairFlag repair; + /* If set, force copying files to the Nix store even if they + already exist there. */ + RepairFlag repair; - /* The allowed filesystem paths in restricted or pure evaluation - mode. */ - std::optional<PathSet> allowedPaths; + /* The allowed filesystem paths in restricted or pure evaluation + mode. */ + std::optional<PathSet> allowedPaths; - Value vEmptySet; + Value vEmptySet; - const ref<Store> store; + const ref<Store> store; -private: - SrcToStore srcToStore; + private: + SrcToStore srcToStore; - /* A cache from path names to parse trees. */ + /* A cache from path names to parse trees. */ #if HAVE_BOEHMGC - typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache; + typedef std::map<Path, Expr*, std::less<Path>, + traceable_allocator<std::pair<const Path, Expr*>>> + FileParseCache; #else - typedef std::map<Path, Expr *> FileParseCache; + typedef std::map<Path, Expr*> FileParseCache; #endif - FileParseCache fileParseCache; + FileParseCache fileParseCache; - /* A cache from path names to values. */ + /* A cache from path names to values. */ #if HAVE_BOEHMGC - typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache; + typedef std::map<Path, Value, std::less<Path>, + traceable_allocator<std::pair<const Path, Value>>> + FileEvalCache; #else - typedef std::map<Path, Value> FileEvalCache; + typedef std::map<Path, Value> FileEvalCache; #endif - FileEvalCache fileEvalCache; + FileEvalCache fileEvalCache; + + SearchPath searchPath; + + std::map<std::string, std::pair<bool, std::string>> searchPathResolved; + + /* Cache used by checkSourcePath(). */ + std::unordered_map<Path, Path> resolvedPaths; + + public: + EvalState(const Strings& _searchPath, ref<Store> store); + ~EvalState(); + + void addToSearchPath(const string& s); + + SearchPath getSearchPath() { return searchPath; } + + Path checkSourcePath(const Path& path); + + void checkURI(const std::string& uri); + + /* When using a diverted store and 'path' is in the Nix store, map + 'path' to the diverted location (e.g. /nix/store/foo is mapped + to /home/alice/my-nix/nix/store/foo). However, this is only + done if the context is not empty, since otherwise we're + probably trying to read from the actual /nix/store. This is + intended to distinguish between import-from-derivation and + sources stored in the actual /nix/store. */ + Path toRealPath(const Path& path, const PathSet& context); - SearchPath searchPath; + /* Parse a Nix expression from the specified file. */ + Expr* parseExprFromFile(const Path& path); + Expr* parseExprFromFile(const Path& path, StaticEnv& staticEnv); - std::map<std::string, std::pair<bool, std::string>> searchPathResolved; + /* Parse a Nix expression from the specified string. */ + Expr* parseExprFromString(const string& s, const Path& basePath, + StaticEnv& staticEnv); + Expr* parseExprFromString(const string& s, const Path& basePath); - /* Cache used by checkSourcePath(). */ - std::unordered_map<Path, Path> resolvedPaths; + Expr* parseStdin(); -public: + /* Evaluate an expression read from the given file to normal + form. */ + void evalFile(const Path& path, Value& v); - EvalState(const Strings & _searchPath, ref<Store> store); - ~EvalState(); + void resetFileCache(); - void addToSearchPath(const string & s); + /* Look up a file in the search path. */ + Path findFile(const string& path); + Path findFile(SearchPath& searchPath, const string& path, + const Pos& pos = noPos); - SearchPath getSearchPath() { return searchPath; } + /* If the specified search path element is a URI, download it. */ + std::pair<bool, std::string> resolveSearchPathElem( + const SearchPathElem& elem); - Path checkSourcePath(const Path & path); + /* Evaluate an expression to normal form, storing the result in + value `v'. */ + void eval(Expr* e, Value& v); - void checkURI(const std::string & uri); + /* Evaluation the expression, then verify that it has the expected + type. */ + inline bool evalBool(Env& env, Expr* e); + inline bool evalBool(Env& env, Expr* e, const Pos& pos); + inline void evalAttrs(Env& env, Expr* e, Value& v); - /* When using a diverted store and 'path' is in the Nix store, map - 'path' to the diverted location (e.g. /nix/store/foo is mapped - to /home/alice/my-nix/nix/store/foo). However, this is only - done if the context is not empty, since otherwise we're - probably trying to read from the actual /nix/store. This is - intended to distinguish between import-from-derivation and - sources stored in the actual /nix/store. */ - Path toRealPath(const Path & path, const PathSet & context); + /* If `v' is a thunk, enter it and overwrite `v' with the result + of the evaluation of the thunk. If `v' is a delayed function + application, call the function and overwrite `v' with the + result. Otherwise, this is a no-op. */ + inline void forceValue(Value& v, const Pos& pos = noPos); - /* Parse a Nix expression from the specified file. */ - Expr * parseExprFromFile(const Path & path); - Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv); + /* Force a value, then recursively force list elements and + attributes. */ + void forceValueDeep(Value& v); - /* Parse a Nix expression from the specified string. */ - Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv); - Expr * parseExprFromString(const string & s, const Path & basePath); + /* Force `v', and then verify that it has the expected type. */ + NixInt forceInt(Value& v, const Pos& pos); + NixFloat forceFloat(Value& v, const Pos& pos); + bool forceBool(Value& v, const Pos& pos); + inline void forceAttrs(Value& v); + inline void forceAttrs(Value& v, const Pos& pos); + inline void forceList(Value& v); + inline void forceList(Value& v, const Pos& pos); + void forceFunction(Value& v, const Pos& pos); // either lambda or primop + string forceString(Value& v, const Pos& pos = noPos); + string forceString(Value& v, PathSet& context, const Pos& pos = noPos); + string forceStringNoCtx(Value& v, const Pos& pos = noPos); - Expr * parseStdin(); + /* Return true iff the value `v' denotes a derivation (i.e. a + set with attribute `type = "derivation"'). */ + bool isDerivation(Value& v); - /* Evaluate an expression read from the given file to normal - form. */ - void evalFile(const Path & path, Value & v); + std::optional<string> tryAttrsToString(const Pos& pos, Value& v, + PathSet& context, + bool coerceMore = false, + bool copyToStore = true); - void resetFileCache(); + /* String coercion. Converts strings, paths and derivations to a + string. If `coerceMore' is set, also converts nulls, integers, + booleans and lists to a string. If `copyToStore' is set, + referenced paths are copied to the Nix store as a side effect. */ + string coerceToString(const Pos& pos, Value& v, PathSet& context, + bool coerceMore = false, bool copyToStore = true); - /* Look up a file in the search path. */ - Path findFile(const string & path); - Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos); + string copyPathToStore(PathSet& context, const Path& path); - /* If the specified search path element is a URI, download it. */ - std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem); + /* Path coercion. Converts strings, paths and derivations to a + path. The result is guaranteed to be a canonicalised, absolute + path. Nothing is copied to the store. */ + Path coerceToPath(const Pos& pos, Value& v, PathSet& context); - /* Evaluate an expression to normal form, storing the result in - value `v'. */ - void eval(Expr * e, Value & v); + public: + /* The base environment, containing the builtin functions and + values. */ + Env& baseEnv; - /* Evaluation the expression, then verify that it has the expected - type. */ - inline bool evalBool(Env & env, Expr * e); - inline bool evalBool(Env & env, Expr * e, const Pos & pos); - inline void evalAttrs(Env & env, Expr * e, Value & v); + /* The same, but used during parsing to resolve variables. */ + StaticEnv staticBaseEnv; // !!! should be private + + private: + unsigned int baseEnvDispl = 0; + + void createBaseEnv(); - /* If `v' is a thunk, enter it and overwrite `v' with the result - of the evaluation of the thunk. If `v' is a delayed function - application, call the function and overwrite `v' with the - result. Otherwise, this is a no-op. */ - inline void forceValue(Value & v, const Pos & pos = noPos); + Value* addConstant(const string& name, Value& v); - /* Force a value, then recursively force list elements and - attributes. */ - void forceValueDeep(Value & v); + Value* addPrimOp(const string& name, size_t arity, PrimOpFun primOp); - /* Force `v', and then verify that it has the expected type. */ - NixInt forceInt(Value & v, const Pos & pos); - NixFloat forceFloat(Value & v, const Pos & pos); - bool forceBool(Value & v, const Pos & pos); - inline void forceAttrs(Value & v); - inline void forceAttrs(Value & v, const Pos & pos); - inline void forceList(Value & v); - inline void forceList(Value & v, const Pos & pos); - void forceFunction(Value & v, const Pos & pos); // either lambda or primop - string forceString(Value & v, const Pos & pos = noPos); - string forceString(Value & v, PathSet & context, const Pos & pos = noPos); - string forceStringNoCtx(Value & v, const Pos & pos = noPos); + public: + Value& getBuiltin(const string& name); - /* Return true iff the value `v' denotes a derivation (i.e. a - set with attribute `type = "derivation"'). */ - bool isDerivation(Value & v); + private: + inline Value* lookupVar(Env* env, const ExprVar& var, bool noEval); - std::optional<string> tryAttrsToString(const Pos & pos, Value & v, - PathSet & context, bool coerceMore = false, bool copyToStore = true); + friend struct ExprVar; + friend struct ExprAttrs; + friend struct ExprLet; - /* String coercion. Converts strings, paths and derivations to a - string. If `coerceMore' is set, also converts nulls, integers, - booleans and lists to a string. If `copyToStore' is set, - referenced paths are copied to the Nix store as a side effect. */ - string coerceToString(const Pos & pos, Value & v, PathSet & context, - bool coerceMore = false, bool copyToStore = true); + Expr* parse(const char* text, const Path& path, const Path& basePath, + StaticEnv& staticEnv); - string copyPathToStore(PathSet & context, const Path & path); + public: + /* Do a deep equality test between two values. That is, list + elements and attributes are compared recursively. */ + bool eqValues(Value& v1, Value& v2); - /* Path coercion. Converts strings, paths and derivations to a - path. The result is guaranteed to be a canonicalised, absolute - path. Nothing is copied to the store. */ - Path coerceToPath(const Pos & pos, Value & v, PathSet & context); + bool isFunctor(Value& fun); -public: + void callFunction(Value& fun, Value& arg, Value& v, const Pos& pos); + void callPrimOp(Value& fun, Value& arg, Value& v, const Pos& pos); - /* The base environment, containing the builtin functions and - values. */ - Env & baseEnv; + /* Automatically call a function for which each argument has a + default value or has a binding in the `args' map. */ + void autoCallFunction(Bindings& args, Value& fun, Value& res); - /* The same, but used during parsing to resolve variables. */ - StaticEnv staticBaseEnv; // !!! should be private + /* Allocation primitives. */ + Value* allocValue(); + Env& allocEnv(size_t size); -private: + Value* allocAttr(Value& vAttrs, const Symbol& name); - unsigned int baseEnvDispl = 0; + Bindings* allocBindings(size_t capacity); - void createBaseEnv(); + void mkList(Value& v, size_t length); + void mkAttrs(Value& v, size_t capacity); + void mkThunk_(Value& v, Expr* expr); + void mkPos(Value& v, Pos* pos); - Value * addConstant(const string & name, Value & v); + void concatLists(Value& v, size_t nrLists, Value** lists, const Pos& pos); - Value * addPrimOp(const string & name, - size_t arity, PrimOpFun primOp); + /* Print statistics. */ + void printStats(); -public: + void realiseContext(const PathSet& context); - Value & getBuiltin(const string & name); + private: + unsigned long nrEnvs = 0; + unsigned long nrValuesInEnvs = 0; + unsigned long nrValues = 0; + unsigned long nrListElems = 0; + unsigned long nrAttrsets = 0; + unsigned long nrAttrsInAttrsets = 0; + unsigned long nrOpUpdates = 0; + unsigned long nrOpUpdateValuesCopied = 0; + unsigned long nrListConcats = 0; + unsigned long nrPrimOpCalls = 0; + unsigned long nrFunctionCalls = 0; -private: + bool countCalls; - inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval); + typedef std::map<Symbol, size_t> PrimOpCalls; + PrimOpCalls primOpCalls; - friend struct ExprVar; - friend struct ExprAttrs; - friend struct ExprLet; + typedef std::map<ExprLambda*, size_t> FunctionCalls; + FunctionCalls functionCalls; - Expr * parse(const char * text, const Path & path, - const Path & basePath, StaticEnv & staticEnv); + void incrFunctionCall(ExprLambda* fun); -public: + typedef std::map<Pos, size_t> AttrSelects; + AttrSelects attrSelects; - /* Do a deep equality test between two values. That is, list - elements and attributes are compared recursively. */ - bool eqValues(Value & v1, Value & v2); - - bool isFunctor(Value & fun); - - void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos); - void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos); - - /* Automatically call a function for which each argument has a - default value or has a binding in the `args' map. */ - void autoCallFunction(Bindings & args, Value & fun, Value & res); - - /* Allocation primitives. */ - Value * allocValue(); - Env & allocEnv(size_t size); - - Value * allocAttr(Value & vAttrs, const Symbol & name); - - Bindings * allocBindings(size_t capacity); - - void mkList(Value & v, size_t length); - void mkAttrs(Value & v, size_t capacity); - void mkThunk_(Value & v, Expr * expr); - void mkPos(Value & v, Pos * pos); - - void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos); - - /* Print statistics. */ - void printStats(); - - void realiseContext(const PathSet & context); - -private: - - unsigned long nrEnvs = 0; - unsigned long nrValuesInEnvs = 0; - unsigned long nrValues = 0; - unsigned long nrListElems = 0; - unsigned long nrAttrsets = 0; - unsigned long nrAttrsInAttrsets = 0; - unsigned long nrOpUpdates = 0; - unsigned long nrOpUpdateValuesCopied = 0; - unsigned long nrListConcats = 0; - unsigned long nrPrimOpCalls = 0; - unsigned long nrFunctionCalls = 0; - - bool countCalls; - - typedef std::map<Symbol, size_t> PrimOpCalls; - PrimOpCalls primOpCalls; - - typedef std::map<ExprLambda *, size_t> FunctionCalls; - FunctionCalls functionCalls; - - void incrFunctionCall(ExprLambda * fun); - - typedef std::map<Pos, size_t> AttrSelects; - AttrSelects attrSelects; - - friend struct ExprOpUpdate; - friend struct ExprOpConcatLists; - friend struct ExprSelect; - friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); + friend struct ExprOpUpdate; + friend struct ExprOpConcatLists; + friend struct ExprSelect; + friend void prim_getAttr(EvalState& state, const Pos& pos, Value** args, + Value& v); }; - /* Return a string representing the type of the value `v'. */ -string showType(const Value & v); +string showType(const Value& v); /* Decode a context string ‘!<name>!<path>’ into a pair <path, name>. */ -std::pair<string, string> decodeContext(const string & s); +std::pair<string, string> decodeContext(const string& s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); -struct InvalidPathError : EvalError -{ - Path path; - InvalidPathError(const Path & path); +struct InvalidPathError : EvalError { + Path path; + InvalidPathError(const Path& path); #ifdef EXCEPTION_NEEDS_THROW_SPEC - ~InvalidPathError() throw () { }; + ~InvalidPathError() throw(){}; #endif }; -struct EvalSettings : Config -{ - Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", - "Whether builtin functions that allow executing native code should be enabled."}; - - Setting<bool> restrictEval{this, false, "restrict-eval", - "Whether to restrict file system access to paths in $NIX_PATH, " - "and network access to the URI prefixes listed in 'allowed-uris'."}; - - Setting<bool> pureEval{this, false, "pure-eval", - "Whether to restrict file system and network access to files specified by cryptographic hash."}; - - Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation", - "Whether the evaluator allows importing the result of a derivation."}; - - Setting<Strings> allowedUris{this, {}, "allowed-uris", - "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; - - Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", - "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; +struct EvalSettings : Config { + Setting<bool> enableNativeCode{this, false, + "allow-unsafe-native-code-during-evaluation", + "Whether builtin functions that allow " + "executing native code should be enabled."}; + + Setting<bool> restrictEval{ + this, false, "restrict-eval", + "Whether to restrict file system access to paths in $NIX_PATH, " + "and network access to the URI prefixes listed in 'allowed-uris'."}; + + Setting<bool> pureEval{this, false, "pure-eval", + "Whether to restrict file system and network access " + "to files specified by cryptographic hash."}; + + Setting<bool> enableImportFromDerivation{ + this, true, "allow-import-from-derivation", + "Whether the evaluator allows importing the result of a derivation."}; + + Setting<Strings> allowedUris{ + this, + {}, + "allowed-uris", + "Prefixes of URIs that builtin functions such as fetchurl and fetchGit " + "are allowed to fetch."}; + + Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", + "Emit log messages for each function entry " + "and exit at the 'vomit' log level (-vvvv)"}; }; extern EvalSettings evalSettings; -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/function-trace.cc b/third_party/nix/src/libexpr/function-trace.cc index af1486f78973..2bc0d943f339 100644 --- a/third_party/nix/src/libexpr/function-trace.cc +++ b/third_party/nix/src/libexpr/function-trace.cc @@ -2,16 +2,16 @@ namespace nix { -FunctionCallTrace::FunctionCallTrace(const Pos & pos) : pos(pos) { - auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); - auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); - printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count()); +FunctionCallTrace::FunctionCallTrace(const Pos& pos) : pos(pos) { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count()); } FunctionCallTrace::~FunctionCallTrace() { - auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); - auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); - printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count()); + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count()); } -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/function-trace.hh b/third_party/nix/src/libexpr/function-trace.hh index 472f2045ed65..bd27b8aebc7c 100644 --- a/third_party/nix/src/libexpr/function-trace.hh +++ b/third_party/nix/src/libexpr/function-trace.hh @@ -1,15 +1,13 @@ #pragma once -#include "eval.hh" - #include <chrono> +#include "eval.hh" namespace nix { -struct FunctionCallTrace -{ - const Pos & pos; - FunctionCallTrace(const Pos & pos); - ~FunctionCallTrace(); +struct FunctionCallTrace { + const Pos& pos; + FunctionCallTrace(const Pos& pos); + ~FunctionCallTrace(); }; -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/get-drvs.cc b/third_party/nix/src/libexpr/get-drvs.cc index 21a4d7917fce..cdbcb7d76275 100644 --- a/third_party/nix/src/libexpr/get-drvs.cc +++ b/third_party/nix/src/libexpr/get-drvs.cc @@ -1,380 +1,351 @@ #include "get-drvs.hh" -#include "util.hh" -#include "eval-inline.hh" -#include "derivations.hh" - #include <cstring> #include <regex> - +#include "derivations.hh" +#include "eval-inline.hh" +#include "util.hh" namespace nix { +DrvInfo::DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs) + : state(&state), attrs(attrs), attrPath(attrPath) {} -DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) - : state(&state), attrs(attrs), attrPath(attrPath) -{ -} - - -DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs) - : state(&state), attrs(nullptr), attrPath("") -{ - auto spec = parseDrvPathWithOutputs(drvPathWithOutputs); +DrvInfo::DrvInfo(EvalState& state, ref<Store> store, + const std::string& drvPathWithOutputs) + : state(&state), attrs(nullptr), attrPath("") { + auto spec = parseDrvPathWithOutputs(drvPathWithOutputs); - drvPath = spec.first; + drvPath = spec.first; - auto drv = store->derivationFromPath(drvPath); + auto drv = store->derivationFromPath(drvPath); - name = storePathToName(drvPath); + name = storePathToName(drvPath); - if (spec.second.size() > 1) - throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs); + if (spec.second.size() > 1) + throw Error( + "building more than one derivation output is not supported, in '%s'", + drvPathWithOutputs); - outputName = - spec.second.empty() - ? get(drv.env, "outputName", "out") - : *spec.second.begin(); + outputName = spec.second.empty() ? get(drv.env, "outputName", "out") + : *spec.second.begin(); - auto i = drv.outputs.find(outputName); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have output '%s'", drvPath, outputName); + auto i = drv.outputs.find(outputName); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", drvPath, + outputName); - outPath = i->second.path; + outPath = i->second.path; } - -string DrvInfo::queryName() const -{ - if (name == "" && attrs) { - auto i = attrs->find(state->sName); - if (i == attrs->end()) throw TypeError("derivation name missing"); - name = state->forceStringNoCtx(*i->value); - } - return name; +string DrvInfo::queryName() const { + if (name == "" && attrs) { + auto i = attrs->find(state->sName); + if (i == attrs->end()) throw TypeError("derivation name missing"); + name = state->forceStringNoCtx(*i->value); + } + return name; } - -string DrvInfo::querySystem() const -{ - if (system == "" && attrs) { - auto i = attrs->find(state->sSystem); - system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos); - } - return system; +string DrvInfo::querySystem() const { + if (system == "" && attrs) { + auto i = attrs->find(state->sSystem); + system = i == attrs->end() ? "unknown" + : state->forceStringNoCtx(*i->value, *i->pos); + } + return system; } - -string DrvInfo::queryDrvPath() const -{ - if (drvPath == "" && attrs) { - Bindings::iterator i = attrs->find(state->sDrvPath); - PathSet context; - drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; - } - return drvPath; +string DrvInfo::queryDrvPath() const { + if (drvPath == "" && attrs) { + Bindings::iterator i = attrs->find(state->sDrvPath); + PathSet context; + drvPath = i != attrs->end() + ? state->coerceToPath(*i->pos, *i->value, context) + : ""; + } + return drvPath; } - -string DrvInfo::queryOutPath() const -{ - if (outPath == "" && attrs) { - Bindings::iterator i = attrs->find(state->sOutPath); - PathSet context; - outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; - } - return outPath; +string DrvInfo::queryOutPath() const { + if (outPath == "" && attrs) { + Bindings::iterator i = attrs->find(state->sOutPath); + PathSet context; + outPath = i != attrs->end() + ? state->coerceToPath(*i->pos, *i->value, context) + : ""; + } + return outPath; } - -DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) -{ - if (outputs.empty()) { - /* Get the ‘outputs’ list. */ - Bindings::iterator i; - if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { - state->forceList(*i->value, *i->pos); - - /* For each output... */ - for (unsigned int j = 0; j < i->value->listSize(); ++j) { - /* Evaluate the corresponding set. */ - string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos); - Bindings::iterator out = attrs->find(state->symbols.create(name)); - if (out == attrs->end()) continue; // FIXME: throw error? - state->forceAttrs(*out->value); - - /* And evaluate its ‘outPath’ attribute. */ - Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); - if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? - PathSet context; - outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context); - } - } else - outputs["out"] = queryOutPath(); - } - if (!onlyOutputsToInstall || !attrs) - return outputs; - - /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ - const Value * outTI = queryMeta("outputsToInstall"); - if (!outTI) return outputs; - const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); - /* ^ this shows during `nix-env -i` right under the bad derivation */ - if (!outTI->isList()) throw errMsg; - Outputs result; - for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { - if ((*i)->type != tString) throw errMsg; - auto out = outputs.find((*i)->string.s); - if (out == outputs.end()) throw errMsg; - result.insert(*out); - } - return result; +DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) { + if (outputs.empty()) { + /* Get the ‘outputs’ list. */ + Bindings::iterator i; + if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { + state->forceList(*i->value, *i->pos); + + /* For each output... */ + for (unsigned int j = 0; j < i->value->listSize(); ++j) { + /* Evaluate the corresponding set. */ + string name = + state->forceStringNoCtx(*i->value->listElems()[j], *i->pos); + Bindings::iterator out = attrs->find(state->symbols.create(name)); + if (out == attrs->end()) continue; // FIXME: throw error? + state->forceAttrs(*out->value); + + /* And evaluate its ‘outPath’ attribute. */ + Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); + if (outPath == out->value->attrs->end()) + continue; // FIXME: throw error? + PathSet context; + outputs[name] = + state->coerceToPath(*outPath->pos, *outPath->value, context); + } + } else + outputs["out"] = queryOutPath(); + } + if (!onlyOutputsToInstall || !attrs) return outputs; + + /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ + const Value* outTI = queryMeta("outputsToInstall"); + if (!outTI) return outputs; + const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); + /* ^ this shows during `nix-env -i` right under the bad derivation */ + if (!outTI->isList()) throw errMsg; + Outputs result; + for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); + ++i) { + if ((*i)->type != tString) throw errMsg; + auto out = outputs.find((*i)->string.s); + if (out == outputs.end()) throw errMsg; + result.insert(*out); + } + return result; } - -string DrvInfo::queryOutputName() const -{ - if (outputName == "" && attrs) { - Bindings::iterator i = attrs->find(state->sOutputName); - outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : ""; - } - return outputName; +string DrvInfo::queryOutputName() const { + if (outputName == "" && attrs) { + Bindings::iterator i = attrs->find(state->sOutputName); + outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : ""; + } + return outputName; } - -Bindings * DrvInfo::getMeta() -{ - if (meta) return meta; - if (!attrs) return 0; - Bindings::iterator a = attrs->find(state->sMeta); - if (a == attrs->end()) return 0; - state->forceAttrs(*a->value, *a->pos); - meta = a->value->attrs; - return meta; +Bindings* DrvInfo::getMeta() { + if (meta) return meta; + if (!attrs) return 0; + Bindings::iterator a = attrs->find(state->sMeta); + if (a == attrs->end()) return 0; + state->forceAttrs(*a->value, *a->pos); + meta = a->value->attrs; + return meta; } - -StringSet DrvInfo::queryMetaNames() -{ - StringSet res; - if (!getMeta()) return res; - for (auto & i : *meta) - res.insert(i.name); - return res; +StringSet DrvInfo::queryMetaNames() { + StringSet res; + if (!getMeta()) return res; + for (auto& i : *meta) res.insert(i.name); + return res; } - -bool DrvInfo::checkMeta(Value & v) -{ - state->forceValue(v); - if (v.isList()) { - for (unsigned int n = 0; n < v.listSize(); ++n) - if (!checkMeta(*v.listElems()[n])) return false; - return true; - } - else if (v.type == tAttrs) { - Bindings::iterator i = v.attrs->find(state->sOutPath); - if (i != v.attrs->end()) return false; - for (auto & i : *v.attrs) - if (!checkMeta(*i.value)) return false; - return true; - } - else return v.type == tInt || v.type == tBool || v.type == tString || - v.type == tFloat; +bool DrvInfo::checkMeta(Value& v) { + state->forceValue(v); + if (v.isList()) { + for (unsigned int n = 0; n < v.listSize(); ++n) + if (!checkMeta(*v.listElems()[n])) return false; + return true; + } else if (v.type == tAttrs) { + Bindings::iterator i = v.attrs->find(state->sOutPath); + if (i != v.attrs->end()) return false; + for (auto& i : *v.attrs) + if (!checkMeta(*i.value)) return false; + return true; + } else + return v.type == tInt || v.type == tBool || v.type == tString || + v.type == tFloat; } - -Value * DrvInfo::queryMeta(const string & name) -{ - if (!getMeta()) return 0; - Bindings::iterator a = meta->find(state->symbols.create(name)); - if (a == meta->end() || !checkMeta(*a->value)) return 0; - return a->value; +Value* DrvInfo::queryMeta(const string& name) { + if (!getMeta()) return 0; + Bindings::iterator a = meta->find(state->symbols.create(name)); + if (a == meta->end() || !checkMeta(*a->value)) return 0; + return a->value; } - -string DrvInfo::queryMetaString(const string & name) -{ - Value * v = queryMeta(name); - if (!v || v->type != tString) return ""; - return v->string.s; +string DrvInfo::queryMetaString(const string& name) { + Value* v = queryMeta(name); + if (!v || v->type != tString) return ""; + return v->string.s; } - -NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) -{ - Value * v = queryMeta(name); - if (!v) return def; - if (v->type == tInt) return v->integer; - if (v->type == tString) { - /* Backwards compatibility with before we had support for - integer meta fields. */ - NixInt n; - if (string2Int(v->string.s, n)) return n; - } - return def; +NixInt DrvInfo::queryMetaInt(const string& name, NixInt def) { + Value* v = queryMeta(name); + if (!v) return def; + if (v->type == tInt) return v->integer; + if (v->type == tString) { + /* Backwards compatibility with before we had support for + integer meta fields. */ + NixInt n; + if (string2Int(v->string.s, n)) return n; + } + return def; } -NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) -{ - Value * v = queryMeta(name); - if (!v) return def; - if (v->type == tFloat) return v->fpoint; - if (v->type == tString) { - /* Backwards compatibility with before we had support for - float meta fields. */ - NixFloat n; - if (string2Float(v->string.s, n)) return n; - } - return def; +NixFloat DrvInfo::queryMetaFloat(const string& name, NixFloat def) { + Value* v = queryMeta(name); + if (!v) return def; + if (v->type == tFloat) return v->fpoint; + if (v->type == tString) { + /* Backwards compatibility with before we had support for + float meta fields. */ + NixFloat n; + if (string2Float(v->string.s, n)) return n; + } + return def; } - -bool DrvInfo::queryMetaBool(const string & name, bool def) -{ - Value * v = queryMeta(name); - if (!v) return def; - if (v->type == tBool) return v->boolean; - if (v->type == tString) { - /* Backwards compatibility with before we had support for - Boolean meta fields. */ - if (strcmp(v->string.s, "true") == 0) return true; - if (strcmp(v->string.s, "false") == 0) return false; - } - return def; +bool DrvInfo::queryMetaBool(const string& name, bool def) { + Value* v = queryMeta(name); + if (!v) return def; + if (v->type == tBool) return v->boolean; + if (v->type == tString) { + /* Backwards compatibility with before we had support for + Boolean meta fields. */ + if (strcmp(v->string.s, "true") == 0) return true; + if (strcmp(v->string.s, "false") == 0) return false; + } + return def; } - -void DrvInfo::setMeta(const string & name, Value * v) -{ - getMeta(); - Bindings * old = meta; - meta = state->allocBindings(1 + (old ? old->size() : 0)); - Symbol sym = state->symbols.create(name); - if (old) - for (auto i : *old) - if (i.name != sym) - meta->push_back(i); - if (v) meta->push_back(Attr(sym, v)); - meta->sort(); +void DrvInfo::setMeta(const string& name, Value* v) { + getMeta(); + Bindings* old = meta; + meta = state->allocBindings(1 + (old ? old->size() : 0)); + Symbol sym = state->symbols.create(name); + if (old) + for (auto i : *old) + if (i.name != sym) meta->push_back(i); + if (v) meta->push_back(Attr(sym, v)); + meta->sort(); } - /* Cache for already considered attrsets. */ -typedef set<Bindings *> Done; - +typedef set<Bindings*> Done; /* Evaluate value `v'. If it evaluates to a set of type `derivation', then put information about it in `drvs' (unless it's already in `done'). The result boolean indicates whether it makes sense for the caller to recursively search for derivations in `v'. */ -static bool getDerivation(EvalState & state, Value & v, - const string & attrPath, DrvInfos & drvs, Done & done, - bool ignoreAssertionFailures) -{ - try { - state.forceValue(v); - if (!state.isDerivation(v)) return true; +static bool getDerivation(EvalState& state, Value& v, const string& attrPath, + DrvInfos& drvs, Done& done, + bool ignoreAssertionFailures) { + try { + state.forceValue(v); + if (!state.isDerivation(v)) return true; - /* Remove spurious duplicates (e.g., a set like `rec { x = - derivation {...}; y = x;}'. */ - if (done.find(v.attrs) != done.end()) return false; - done.insert(v.attrs); + /* Remove spurious duplicates (e.g., a set like `rec { x = + derivation {...}; y = x;}'. */ + if (done.find(v.attrs) != done.end()) return false; + done.insert(v.attrs); - DrvInfo drv(state, attrPath, v.attrs); + DrvInfo drv(state, attrPath, v.attrs); - drv.queryName(); + drv.queryName(); - drvs.push_back(drv); + drvs.push_back(drv); - return false; + return false; - } catch (AssertionError & e) { - if (ignoreAssertionFailures) return false; - throw; - } + } catch (AssertionError& e) { + if (ignoreAssertionFailures) return false; + throw; + } } - -std::optional<DrvInfo> getDerivation(EvalState & state, Value & v, - bool ignoreAssertionFailures) -{ - Done done; - DrvInfos drvs; - getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); - if (drvs.size() != 1) return {}; - return std::move(drvs.front()); +std::optional<DrvInfo> getDerivation(EvalState& state, Value& v, + bool ignoreAssertionFailures) { + Done done; + DrvInfos drvs; + getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); + if (drvs.size() != 1) return {}; + return std::move(drvs.front()); } - -static string addToPath(const string & s1, const string & s2) -{ - return s1.empty() ? s2 : s1 + "." + s2; +static string addToPath(const string& s1, const string& s2) { + return s1.empty() ? s2 : s1 + "." + s2; } - static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); - -static void getDerivations(EvalState & state, Value & vIn, - const string & pathPrefix, Bindings & autoArgs, - DrvInfos & drvs, Done & done, - bool ignoreAssertionFailures) -{ - Value v; - state.autoCallFunction(autoArgs, vIn, v); - - /* Process the expression. */ - if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; - - else if (v.type == tAttrs) { - - /* !!! undocumented hackery to support combining channels in - nix-env.cc. */ - bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end(); - - /* Consider the attributes in sorted order to get more - deterministic behaviour in nix-env operations (e.g. when - there are names clashes between derivations, the derivation - bound to the attribute with the "lower" name should take - precedence). */ - for (auto & i : v.attrs->lexicographicOrder()) { - debug("evaluating attribute '%1%'", i->name); - if (!std::regex_match(std::string(i->name), attrRegex)) - continue; - string pathPrefix2 = addToPath(pathPrefix, i->name); - if (combineChannels) - getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { - /* If the value of this attribute is itself a set, - should we recurse into it? => Only if it has a - `recurseForDerivations = true' attribute. */ - if (i->value->type == tAttrs) { - Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations")); - if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) - getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - } - } +static void getDerivations(EvalState& state, Value& vIn, + const string& pathPrefix, Bindings& autoArgs, + DrvInfos& drvs, Done& done, + bool ignoreAssertionFailures) { + Value v; + state.autoCallFunction(autoArgs, vIn, v); + + /* Process the expression. */ + if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) + ; + + else if (v.type == tAttrs) { + /* !!! undocumented hackery to support combining channels in + nix-env.cc. */ + bool combineChannels = + v.attrs->find(state.symbols.create("_combineChannels")) != + v.attrs->end(); + + /* Consider the attributes in sorted order to get more + deterministic behaviour in nix-env operations (e.g. when + there are names clashes between derivations, the derivation + bound to the attribute with the "lower" name should take + precedence). */ + for (auto& i : v.attrs->lexicographicOrder()) { + debug("evaluating attribute '%1%'", i->name); + if (!std::regex_match(std::string(i->name), attrRegex)) continue; + string pathPrefix2 = addToPath(pathPrefix, i->name); + if (combineChannels) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, + ignoreAssertionFailures); + else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, + ignoreAssertionFailures)) { + /* If the value of this attribute is itself a set, + should we recurse into it? => Only if it has a + `recurseForDerivations = true' attribute. */ + if (i->value->type == tAttrs) { + Bindings::iterator j = i->value->attrs->find( + state.symbols.create("recurseForDerivations")); + if (j != i->value->attrs->end() && + state.forceBool(*j->value, *j->pos)) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, + ignoreAssertionFailures); } + } } - - else if (v.isList()) { - for (unsigned int n = 0; n < v.listSize(); ++n) { - string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); - if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) - getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - } + } + + else if (v.isList()) { + for (unsigned int n = 0; n < v.listSize(); ++n) { + string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); + if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, + ignoreAssertionFailures)) + getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, + done, ignoreAssertionFailures); } + } - else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)"); + else + throw TypeError( + "expression does not evaluate to a derivation (or a set or list of " + "those)"); } - -void getDerivations(EvalState & state, Value & v, const string & pathPrefix, - Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures) -{ - Done done; - getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures); +void getDerivations(EvalState& state, Value& v, const string& pathPrefix, + Bindings& autoArgs, DrvInfos& drvs, + bool ignoreAssertionFailures) { + Done done; + getDerivations(state, v, pathPrefix, autoArgs, drvs, done, + ignoreAssertionFailures); } - -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/get-drvs.hh b/third_party/nix/src/libexpr/get-drvs.hh index d7860fc6a4bc..0dca12cafc4e 100644 --- a/third_party/nix/src/libexpr/get-drvs.hh +++ b/third_party/nix/src/libexpr/get-drvs.hh @@ -1,89 +1,84 @@ #pragma once -#include "eval.hh" - -#include <string> #include <map> - +#include <string> +#include "eval.hh" namespace nix { +struct DrvInfo { + public: + typedef std::map<string, Path> Outputs; -struct DrvInfo -{ -public: - typedef std::map<string, Path> Outputs; - -private: - EvalState * state; + private: + EvalState* state; - mutable string name; - mutable string system; - mutable string drvPath; - mutable string outPath; - mutable string outputName; - Outputs outputs; + mutable string name; + mutable string system; + mutable string drvPath; + mutable string outPath; + mutable string outputName; + Outputs outputs; - bool failed = false; // set if we get an AssertionError + bool failed = false; // set if we get an AssertionError - Bindings * attrs = nullptr, * meta = nullptr; + Bindings *attrs = nullptr, *meta = nullptr; - Bindings * getMeta(); + Bindings* getMeta(); - bool checkMeta(Value & v); + bool checkMeta(Value& v); -public: - string attrPath; /* path towards the derivation */ + public: + string attrPath; /* path towards the derivation */ - DrvInfo(EvalState & state) : state(&state) { }; - DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs); - DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs); + DrvInfo(EvalState& state) : state(&state){}; + DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs); + DrvInfo(EvalState& state, ref<Store> store, + const std::string& drvPathWithOutputs); - string queryName() const; - string querySystem() const; - string queryDrvPath() const; - string queryOutPath() const; - string queryOutputName() const; - /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ - Outputs queryOutputs(bool onlyOutputsToInstall = false); + string queryName() const; + string querySystem() const; + string queryDrvPath() const; + string queryOutPath() const; + string queryOutputName() const; + /** Return the list of outputs. The "outputs to install" are determined by + * `meta.outputsToInstall`. */ + Outputs queryOutputs(bool onlyOutputsToInstall = false); - StringSet queryMetaNames(); - Value * queryMeta(const string & name); - string queryMetaString(const string & name); - NixInt queryMetaInt(const string & name, NixInt def); - NixFloat queryMetaFloat(const string & name, NixFloat def); - bool queryMetaBool(const string & name, bool def); - void setMeta(const string & name, Value * v); + StringSet queryMetaNames(); + Value* queryMeta(const string& name); + string queryMetaString(const string& name); + NixInt queryMetaInt(const string& name, NixInt def); + NixFloat queryMetaFloat(const string& name, NixFloat def); + bool queryMetaBool(const string& name, bool def); + void setMeta(const string& name, Value* v); - /* - MetaInfo queryMetaInfo(EvalState & state) const; - MetaValue queryMetaInfo(EvalState & state, const string & name) const; - */ + /* + MetaInfo queryMetaInfo(EvalState & state) const; + MetaValue queryMetaInfo(EvalState & state, const string & name) const; + */ - void setName(const string & s) { name = s; } - void setDrvPath(const string & s) { drvPath = s; } - void setOutPath(const string & s) { outPath = s; } + void setName(const string& s) { name = s; } + void setDrvPath(const string& s) { drvPath = s; } + void setOutPath(const string& s) { outPath = s; } - void setFailed() { failed = true; }; - bool hasFailed() { return failed; }; + void setFailed() { failed = true; }; + bool hasFailed() { return failed; }; }; - #if HAVE_BOEHMGC typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos; #else typedef list<DrvInfo> DrvInfos; #endif - /* If value `v' denotes a derivation, return a DrvInfo object describing it. Otherwise return nothing. */ -std::optional<DrvInfo> getDerivation(EvalState & state, - Value & v, bool ignoreAssertionFailures); - -void getDerivations(EvalState & state, Value & v, const string & pathPrefix, - Bindings & autoArgs, DrvInfos & drvs, - bool ignoreAssertionFailures); +std::optional<DrvInfo> getDerivation(EvalState& state, Value& v, + bool ignoreAssertionFailures); +void getDerivations(EvalState& state, Value& v, const string& pathPrefix, + Bindings& autoArgs, DrvInfos& drvs, + bool ignoreAssertionFailures); -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/json-to-value.cc b/third_party/nix/src/libexpr/json-to-value.cc index 8bae986f97fc..8615d9797fa8 100644 --- a/third_party/nix/src/libexpr/json-to-value.cc +++ b/third_party/nix/src/libexpr/json-to-value.cc @@ -1,149 +1,153 @@ #include "json-to-value.hh" - #include <cstring> namespace nix { - -static void skipWhitespace(const char * & s) -{ - while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; +static void skipWhitespace(const char*& s) { + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; } - -static string parseJSONString(const char * & s) -{ - string res; - if (*s++ != '"') throw JSONParseError("expected JSON string"); - while (*s != '"') { - if (!*s) throw JSONParseError("got end-of-string in JSON string"); - if (*s == '\\') { - s++; - if (*s == '"') res += '"'; - else if (*s == '\\') res += '\\'; - else if (*s == '/') res += '/'; - else if (*s == '/') res += '/'; - else if (*s == 'b') res += '\b'; - else if (*s == 'f') res += '\f'; - else if (*s == 'n') res += '\n'; - else if (*s == 'r') res += '\r'; - else if (*s == 't') res += '\t'; - else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported"); - else throw JSONParseError("invalid escaped character in JSON string"); - s++; - } else - res += *s++; - } - s++; - return res; +static string parseJSONString(const char*& s) { + string res; + if (*s++ != '"') throw JSONParseError("expected JSON string"); + while (*s != '"') { + if (!*s) throw JSONParseError("got end-of-string in JSON string"); + if (*s == '\\') { + s++; + if (*s == '"') + res += '"'; + else if (*s == '\\') + res += '\\'; + else if (*s == '/') + res += '/'; + else if (*s == '/') + res += '/'; + else if (*s == 'b') + res += '\b'; + else if (*s == 'f') + res += '\f'; + else if (*s == 'n') + res += '\n'; + else if (*s == 'r') + res += '\r'; + else if (*s == 't') + res += '\t'; + else if (*s == 'u') + throw JSONParseError( + "\\u characters in JSON strings are currently not supported"); + else + throw JSONParseError("invalid escaped character in JSON string"); + s++; + } else + res += *s++; + } + s++; + return res; } +static void parseJSON(EvalState& state, const char*& s, Value& v) { + skipWhitespace(s); -static void parseJSON(EvalState & state, const char * & s, Value & v) -{ - skipWhitespace(s); + if (!*s) throw JSONParseError("expected JSON value"); - if (!*s) throw JSONParseError("expected JSON value"); - - if (*s == '[') { - s++; - ValueVector values; - values.reserve(128); - skipWhitespace(s); - while (1) { - if (values.empty() && *s == ']') break; - Value * v2 = state.allocValue(); - parseJSON(state, s, *v2); - values.push_back(v2); - skipWhitespace(s); - if (*s == ']') break; - if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element"); - s++; - } - s++; - state.mkList(v, values.size()); - for (size_t n = 0; n < values.size(); ++n) - v.listElems()[n] = values[n]; + if (*s == '[') { + s++; + ValueVector values; + values.reserve(128); + skipWhitespace(s); + while (1) { + if (values.empty() && *s == ']') break; + Value* v2 = state.allocValue(); + parseJSON(state, s, *v2); + values.push_back(v2); + skipWhitespace(s); + if (*s == ']') break; + if (*s != ',') + throw JSONParseError("expected ',' or ']' after JSON array element"); + s++; } + s++; + state.mkList(v, values.size()); + for (size_t n = 0; n < values.size(); ++n) v.listElems()[n] = values[n]; + } - else if (*s == '{') { - s++; - ValueMap attrs; - while (1) { - skipWhitespace(s); - if (attrs.empty() && *s == '}') break; - string name = parseJSONString(s); - skipWhitespace(s); - if (*s != ':') throw JSONParseError("expected ':' in JSON object"); - s++; - Value * v2 = state.allocValue(); - parseJSON(state, s, *v2); - attrs[state.symbols.create(name)] = v2; - skipWhitespace(s); - if (*s == '}') break; - if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member"); - s++; - } - state.mkAttrs(v, attrs.size()); - for (auto & i : attrs) - v.attrs->push_back(Attr(i.first, i.second)); - v.attrs->sort(); - s++; + else if (*s == '{') { + s++; + ValueMap attrs; + while (1) { + skipWhitespace(s); + if (attrs.empty() && *s == '}') break; + string name = parseJSONString(s); + skipWhitespace(s); + if (*s != ':') throw JSONParseError("expected ':' in JSON object"); + s++; + Value* v2 = state.allocValue(); + parseJSON(state, s, *v2); + attrs[state.symbols.create(name)] = v2; + skipWhitespace(s); + if (*s == '}') break; + if (*s != ',') + throw JSONParseError("expected ',' or '}' after JSON member"); + s++; } + state.mkAttrs(v, attrs.size()); + for (auto& i : attrs) v.attrs->push_back(Attr(i.first, i.second)); + v.attrs->sort(); + s++; + } - else if (*s == '"') { - mkString(v, parseJSONString(s)); - } + else if (*s == '"') { + mkString(v, parseJSONString(s)); + } - else if (isdigit(*s) || *s == '-' || *s == '.' ) { - // Buffer into a string first, then use built-in C++ conversions - std::string tmp_number; - ValueType number_type = tInt; - - while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') { - if (*s == '.' || *s == 'e' || *s == 'E') - number_type = tFloat; - tmp_number += *s++; - } - - try { - if (number_type == tFloat) - mkFloat(v, stod(tmp_number)); - else - mkInt(v, stol(tmp_number)); - } catch (std::invalid_argument & e) { - throw JSONParseError("invalid JSON number"); - } catch (std::out_of_range & e) { - throw JSONParseError("out-of-range JSON number"); - } - } + else if (isdigit(*s) || *s == '-' || *s == '.') { + // Buffer into a string first, then use built-in C++ conversions + std::string tmp_number; + ValueType number_type = tInt; - else if (strncmp(s, "true", 4) == 0) { - s += 4; - mkBool(v, true); + while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') { + if (*s == '.' || *s == 'e' || *s == 'E') number_type = tFloat; + tmp_number += *s++; } - else if (strncmp(s, "false", 5) == 0) { - s += 5; - mkBool(v, false); + try { + if (number_type == tFloat) + mkFloat(v, stod(tmp_number)); + else + mkInt(v, stol(tmp_number)); + } catch (std::invalid_argument& e) { + throw JSONParseError("invalid JSON number"); + } catch (std::out_of_range& e) { + throw JSONParseError("out-of-range JSON number"); } + } - else if (strncmp(s, "null", 4) == 0) { - s += 4; - mkNull(v); - } + else if (strncmp(s, "true", 4) == 0) { + s += 4; + mkBool(v, true); + } - else throw JSONParseError("unrecognised JSON value"); -} + else if (strncmp(s, "false", 5) == 0) { + s += 5; + mkBool(v, false); + } + else if (strncmp(s, "null", 4) == 0) { + s += 4; + mkNull(v); + } -void parseJSON(EvalState & state, const string & s_, Value & v) -{ - const char * s = s_.c_str(); - parseJSON(state, s, v); - skipWhitespace(s); - if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s); + else + throw JSONParseError("unrecognised JSON value"); } - +void parseJSON(EvalState& state, const string& s_, Value& v) { + const char* s = s_.c_str(); + parseJSON(state, s, v); + skipWhitespace(s); + if (*s) + throw JSONParseError( + format("expected end-of-string while parsing JSON value: %1%") % s); } + +} // namespace nix diff --git a/third_party/nix/src/libexpr/json-to-value.hh b/third_party/nix/src/libexpr/json-to-value.hh index 33f35b16ce89..eb555c643fab 100644 --- a/third_party/nix/src/libexpr/json-to-value.hh +++ b/third_party/nix/src/libexpr/json-to-value.hh @@ -1,13 +1,12 @@ #pragma once -#include "eval.hh" - #include <string> +#include "eval.hh" namespace nix { MakeError(JSONParseError, EvalError) -void parseJSON(EvalState & state, const string & s, Value & v); + void parseJSON(EvalState& state, const string& s, Value& v); } diff --git a/third_party/nix/src/libexpr/names.cc b/third_party/nix/src/libexpr/names.cc index 382088c78872..c3dd2a9bd1ed 100644 --- a/third_party/nix/src/libexpr/names.cc +++ b/third_party/nix/src/libexpr/names.cc @@ -1,107 +1,98 @@ #include "names.hh" #include "util.hh" - namespace nix { - -DrvName::DrvName() -{ - name = ""; -} - +DrvName::DrvName() { name = ""; } /* Parse a derivation name. The `name' part of a derivation name is everything up to but not including the first dash *not* followed by a letter. The `version' part is the rest (excluding the separating dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', '2.0.48'). */ -DrvName::DrvName(const string & s) : hits(0) -{ - name = fullName = s; - for (unsigned int i = 0; i < s.size(); ++i) { - /* !!! isalpha/isdigit are affected by the locale. */ - if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) { - name = string(s, 0, i); - version = string(s, i + 1); - break; - } +DrvName::DrvName(const string& s) : hits(0) { + name = fullName = s; + for (unsigned int i = 0; i < s.size(); ++i) { + /* !!! isalpha/isdigit are affected by the locale. */ + if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) { + name = string(s, 0, i); + version = string(s, i + 1); + break; } + } } - -bool DrvName::matches(DrvName & n) -{ - if (name != "*") { - if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended)); - if (!std::regex_match(n.name, *regex)) return false; - } - if (version != "" && version != n.version) return false; - return true; +bool DrvName::matches(DrvName& n) { + if (name != "*") { + if (!regex) + regex = std::unique_ptr<std::regex>( + new std::regex(name, std::regex::extended)); + if (!std::regex_match(n.name, *regex)) return false; + } + if (version != "" && version != n.version) return false; + return true; } +string nextComponent(string::const_iterator& p, + const string::const_iterator end) { + /* Skip any dots and dashes (component separators). */ + while (p != end && (*p == '.' || *p == '-')) ++p; -string nextComponent(string::const_iterator & p, - const string::const_iterator end) -{ - /* Skip any dots and dashes (component separators). */ - while (p != end && (*p == '.' || *p == '-')) ++p; + if (p == end) return ""; - if (p == end) return ""; + /* If the first character is a digit, consume the longest sequence + of digits. Otherwise, consume the longest sequence of + non-digit, non-separator characters. */ + string s; + if (isdigit(*p)) + while (p != end && isdigit(*p)) s += *p++; + else + while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) s += *p++; - /* If the first character is a digit, consume the longest sequence - of digits. Otherwise, consume the longest sequence of - non-digit, non-separator characters. */ - string s; - if (isdigit(*p)) - while (p != end && isdigit(*p)) s += *p++; - else - while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) - s += *p++; - - return s; + return s; } +static bool componentsLT(const string& c1, const string& c2) { + int n1, n2; + bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); -static bool componentsLT(const string & c1, const string & c2) -{ - int n1, n2; - bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); - - if (c1Num && c2Num) return n1 < n2; - else if (c1 == "" && c2Num) return true; - else if (c1 == "pre" && c2 != "pre") return true; - else if (c2 == "pre") return false; - /* Assume that `2.3a' < `2.3.1'. */ - else if (c2Num) return true; - else if (c1Num) return false; - else return c1 < c2; + if (c1Num && c2Num) + return n1 < n2; + else if (c1 == "" && c2Num) + return true; + else if (c1 == "pre" && c2 != "pre") + return true; + else if (c2 == "pre") + return false; + /* Assume that `2.3a' < `2.3.1'. */ + else if (c2Num) + return true; + else if (c1Num) + return false; + else + return c1 < c2; } +int compareVersions(const string& v1, const string& v2) { + string::const_iterator p1 = v1.begin(); + string::const_iterator p2 = v2.begin(); -int compareVersions(const string & v1, const string & v2) -{ - string::const_iterator p1 = v1.begin(); - string::const_iterator p2 = v2.begin(); - - while (p1 != v1.end() || p2 != v2.end()) { - string c1 = nextComponent(p1, v1.end()); - string c2 = nextComponent(p2, v2.end()); - if (componentsLT(c1, c2)) return -1; - else if (componentsLT(c2, c1)) return 1; - } + while (p1 != v1.end() || p2 != v2.end()) { + string c1 = nextComponent(p1, v1.end()); + string c2 = nextComponent(p2, v2.end()); + if (componentsLT(c1, c2)) + return -1; + else if (componentsLT(c2, c1)) + return 1; + } - return 0; + return 0; } - -DrvNames drvNamesFromArgs(const Strings & opArgs) -{ - DrvNames result; - for (auto & i : opArgs) - result.push_back(DrvName(i)); - return result; +DrvNames drvNamesFromArgs(const Strings& opArgs) { + DrvNames result; + for (auto& i : opArgs) result.push_back(DrvName(i)); + return result; } - -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/names.hh b/third_party/nix/src/libexpr/names.hh index 13c3093e77b0..219a595d15e2 100644 --- a/third_party/nix/src/libexpr/names.hh +++ b/third_party/nix/src/libexpr/names.hh @@ -1,32 +1,30 @@ #pragma once #include <memory> - -#include "types.hh" #include <regex> +#include "types.hh" namespace nix { -struct DrvName -{ - string fullName; - string name; - string version; - unsigned int hits; +struct DrvName { + string fullName; + string name; + string version; + unsigned int hits; - DrvName(); - DrvName(const string & s); - bool matches(DrvName & n); + DrvName(); + DrvName(const string& s); + bool matches(DrvName& n); -private: - std::unique_ptr<std::regex> regex; + private: + std::unique_ptr<std::regex> regex; }; typedef list<DrvName> DrvNames; -string nextComponent(string::const_iterator & p, - const string::const_iterator end); -int compareVersions(const string & v1, const string & v2); -DrvNames drvNamesFromArgs(const Strings & opArgs); +string nextComponent(string::const_iterator& p, + const string::const_iterator end); +int compareVersions(const string& v1, const string& v2); +DrvNames drvNamesFromArgs(const Strings& opArgs); -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/nixexpr.cc b/third_party/nix/src/libexpr/nixexpr.cc index 63cbef1ddf84..4147cbbf532b 100644 --- a/third_party/nix/src/libexpr/nixexpr.cc +++ b/third_party/nix/src/libexpr/nixexpr.cc @@ -1,438 +1,361 @@ #include "nixexpr.hh" +#include <cstdlib> #include "derivations.hh" #include "util.hh" -#include <cstdlib> - - namespace nix { - /* Displaying abstract syntax trees. */ -std::ostream & operator << (std::ostream & str, const Expr & e) -{ - e.show(str); - return str; -} - -static void showString(std::ostream & str, const string & s) -{ - str << '"'; - for (auto c : (string) s) - if (c == '"' || c == '\\' || c == '$') str << "\\" << c; - else if (c == '\n') str << "\\n"; - else if (c == '\r') str << "\\r"; - else if (c == '\t') str << "\\t"; - else str << c; - str << '"'; -} - -static void showId(std::ostream & str, const string & s) -{ - if (s.empty()) - str << "\"\""; - else if (s == "if") // FIXME: handle other keywords - str << '"' << s << '"'; - else { - char c = s[0]; - if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { - showString(str, s); - return; - } - for (auto c : s) - if (!((c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - c == '_' || c == '\'' || c == '-')) { - showString(str, s); - return; - } - str << s; +std::ostream& operator<<(std::ostream& str, const Expr& e) { + e.show(str); + return str; +} + +static void showString(std::ostream& str, const string& s) { + str << '"'; + for (auto c : (string)s) + if (c == '"' || c == '\\' || c == '$') + str << "\\" << c; + else if (c == '\n') + str << "\\n"; + else if (c == '\r') + str << "\\r"; + else if (c == '\t') + str << "\\t"; + else + str << c; + str << '"'; +} + +static void showId(std::ostream& str, const string& s) { + if (s.empty()) + str << "\"\""; + else if (s == "if") // FIXME: handle other keywords + str << '"' << s << '"'; + else { + char c = s[0]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + showString(str, s); + return; } + for (auto c : s) + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '\'' || c == '-')) { + showString(str, s); + return; + } + str << s; + } } -std::ostream & operator << (std::ostream & str, const Symbol & sym) -{ - showId(str, *sym.s); - return str; +std::ostream& operator<<(std::ostream& str, const Symbol& sym) { + showId(str, *sym.s); + return str; } -void Expr::show(std::ostream & str) const -{ - abort(); -} +void Expr::show(std::ostream& str) const { abort(); } -void ExprInt::show(std::ostream & str) const -{ - str << n; -} +void ExprInt::show(std::ostream& str) const { str << n; } -void ExprFloat::show(std::ostream & str) const -{ - str << nf; -} +void ExprFloat::show(std::ostream& str) const { str << nf; } -void ExprString::show(std::ostream & str) const -{ - showString(str, s); -} +void ExprString::show(std::ostream& str) const { showString(str, s); } -void ExprPath::show(std::ostream & str) const -{ - str << s; -} +void ExprPath::show(std::ostream& str) const { str << s; } -void ExprVar::show(std::ostream & str) const -{ - str << name; -} +void ExprVar::show(std::ostream& str) const { str << name; } -void ExprSelect::show(std::ostream & str) const -{ - str << "(" << *e << ")." << showAttrPath(attrPath); - if (def) str << " or (" << *def << ")"; +void ExprSelect::show(std::ostream& str) const { + str << "(" << *e << ")." << showAttrPath(attrPath); + if (def) str << " or (" << *def << ")"; } -void ExprOpHasAttr::show(std::ostream & str) const -{ - str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")"; +void ExprOpHasAttr::show(std::ostream& str) const { + str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")"; } -void ExprAttrs::show(std::ostream & str) const -{ - if (recursive) str << "rec "; - str << "{ "; - for (auto & i : attrs) - if (i.second.inherited) - str << "inherit " << i.first << " " << "; "; - else - str << i.first << " = " << *i.second.e << "; "; - for (auto & i : dynamicAttrs) - str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; "; - str << "}"; +void ExprAttrs::show(std::ostream& str) const { + if (recursive) str << "rec "; + str << "{ "; + for (auto& i : attrs) + if (i.second.inherited) + str << "inherit " << i.first << " " + << "; "; + else + str << i.first << " = " << *i.second.e << "; "; + for (auto& i : dynamicAttrs) + str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; "; + str << "}"; } -void ExprList::show(std::ostream & str) const -{ - str << "[ "; - for (auto & i : elems) - str << "(" << *i << ") "; - str << "]"; +void ExprList::show(std::ostream& str) const { + str << "[ "; + for (auto& i : elems) str << "(" << *i << ") "; + str << "]"; } -void ExprLambda::show(std::ostream & str) const -{ - str << "("; - if (matchAttrs) { - str << "{ "; - bool first = true; - for (auto & i : formals->formals) { - if (first) first = false; else str << ", "; - str << i.name; - if (i.def) str << " ? " << *i.def; - } - if (formals->ellipsis) { - if (!first) str << ", "; - str << "..."; - } - str << " }"; - if (!arg.empty()) str << " @ "; +void ExprLambda::show(std::ostream& str) const { + str << "("; + if (matchAttrs) { + str << "{ "; + bool first = true; + for (auto& i : formals->formals) { + if (first) + first = false; + else + str << ", "; + str << i.name; + if (i.def) str << " ? " << *i.def; } - if (!arg.empty()) str << arg; - str << ": " << *body << ")"; -} - -void ExprLet::show(std::ostream & str) const -{ - str << "(let "; - for (auto & i : attrs->attrs) - if (i.second.inherited) { - str << "inherit " << i.first << "; "; - } - else - str << i.first << " = " << *i.second.e << "; "; - str << "in " << *body << ")"; -} - -void ExprWith::show(std::ostream & str) const -{ - str << "(with " << *attrs << "; " << *body << ")"; -} - -void ExprIf::show(std::ostream & str) const -{ - str << "(if " << *cond << " then " << *then << " else " << *else_ << ")"; + if (formals->ellipsis) { + if (!first) str << ", "; + str << "..."; + } + str << " }"; + if (!arg.empty()) str << " @ "; + } + if (!arg.empty()) str << arg; + str << ": " << *body << ")"; } -void ExprAssert::show(std::ostream & str) const -{ - str << "assert " << *cond << "; " << *body; +void ExprLet::show(std::ostream& str) const { + str << "(let "; + for (auto& i : attrs->attrs) + if (i.second.inherited) { + str << "inherit " << i.first << "; "; + } else + str << i.first << " = " << *i.second.e << "; "; + str << "in " << *body << ")"; } -void ExprOpNot::show(std::ostream & str) const -{ - str << "(! " << *e << ")"; +void ExprWith::show(std::ostream& str) const { + str << "(with " << *attrs << "; " << *body << ")"; } -void ExprConcatStrings::show(std::ostream & str) const -{ - bool first = true; - str << "("; - for (auto & i : *es) { - if (first) first = false; else str << " + "; - str << *i; - } - str << ")"; +void ExprIf::show(std::ostream& str) const { + str << "(if " << *cond << " then " << *then << " else " << *else_ << ")"; } -void ExprPos::show(std::ostream & str) const -{ - str << "__curPos"; +void ExprAssert::show(std::ostream& str) const { + str << "assert " << *cond << "; " << *body; } +void ExprOpNot::show(std::ostream& str) const { str << "(! " << *e << ")"; } -std::ostream & operator << (std::ostream & str, const Pos & pos) -{ - if (!pos) - str << "undefined position"; +void ExprConcatStrings::show(std::ostream& str) const { + bool first = true; + str << "("; + for (auto& i : *es) { + if (first) + first = false; else - str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str(); - return str; -} - - -string showAttrPath(const AttrPath & attrPath) -{ - std::ostringstream out; - bool first = true; - for (auto & i : attrPath) { - if (!first) out << '.'; else first = false; - if (i.symbol.set()) - out << i.symbol; - else - out << "\"${" << *i.expr << "}\""; - } - return out.str(); + str << " + "; + str << *i; + } + str << ")"; +} + +void ExprPos::show(std::ostream& str) const { str << "__curPos"; } + +std::ostream& operator<<(std::ostream& str, const Pos& pos) { + if (!pos) + str << "undefined position"; + else + str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string)pos.file % + pos.line % pos.column) + .str(); + return str; +} + +string showAttrPath(const AttrPath& attrPath) { + std::ostringstream out; + bool first = true; + for (auto& i : attrPath) { + if (!first) + out << '.'; + else + first = false; + if (i.symbol.set()) + out << i.symbol; + else + out << "\"${" << *i.expr << "}\""; + } + return out.str(); } - Pos noPos; - /* Computing levels/displacements for variables. */ -void Expr::bindVars(const StaticEnv & env) -{ - abort(); -} - -void ExprInt::bindVars(const StaticEnv & env) -{ -} - -void ExprFloat::bindVars(const StaticEnv & env) -{ -} - -void ExprString::bindVars(const StaticEnv & env) -{ -} - -void ExprPath::bindVars(const StaticEnv & env) -{ -} - -void ExprVar::bindVars(const StaticEnv & env) -{ - /* Check whether the variable appears in the environment. If so, - set its level and displacement. */ - const StaticEnv * curEnv; - unsigned int level; - int withLevel = -1; - for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) { - if (curEnv->isWith) { - if (withLevel == -1) withLevel = level; - } else { - StaticEnv::Vars::const_iterator i = curEnv->vars.find(name); - if (i != curEnv->vars.end()) { - fromWith = false; - this->level = level; - displ = i->second; - return; - } - } +void Expr::bindVars(const StaticEnv& env) { abort(); } + +void ExprInt::bindVars(const StaticEnv& env) {} + +void ExprFloat::bindVars(const StaticEnv& env) {} + +void ExprString::bindVars(const StaticEnv& env) {} + +void ExprPath::bindVars(const StaticEnv& env) {} + +void ExprVar::bindVars(const StaticEnv& env) { + /* Check whether the variable appears in the environment. If so, + set its level and displacement. */ + const StaticEnv* curEnv; + unsigned int level; + int withLevel = -1; + for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) { + if (curEnv->isWith) { + if (withLevel == -1) withLevel = level; + } else { + StaticEnv::Vars::const_iterator i = curEnv->vars.find(name); + if (i != curEnv->vars.end()) { + fromWith = false; + this->level = level; + displ = i->second; + return; + } } + } - /* Otherwise, the variable must be obtained from the nearest - enclosing `with'. If there is no `with', then we can issue an - "undefined variable" error now. */ - if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos); + /* Otherwise, the variable must be obtained from the nearest + enclosing `with'. If there is no `with', then we can issue an + "undefined variable" error now. */ + if (withLevel == -1) + throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % + pos); - fromWith = true; - this->level = withLevel; + fromWith = true; + this->level = withLevel; } -void ExprSelect::bindVars(const StaticEnv & env) -{ - e->bindVars(env); - if (def) def->bindVars(env); - for (auto & i : attrPath) - if (!i.symbol.set()) - i.expr->bindVars(env); +void ExprSelect::bindVars(const StaticEnv& env) { + e->bindVars(env); + if (def) def->bindVars(env); + for (auto& i : attrPath) + if (!i.symbol.set()) i.expr->bindVars(env); } -void ExprOpHasAttr::bindVars(const StaticEnv & env) -{ - e->bindVars(env); - for (auto & i : attrPath) - if (!i.symbol.set()) - i.expr->bindVars(env); +void ExprOpHasAttr::bindVars(const StaticEnv& env) { + e->bindVars(env); + for (auto& i : attrPath) + if (!i.symbol.set()) i.expr->bindVars(env); } -void ExprAttrs::bindVars(const StaticEnv & env) -{ - const StaticEnv * dynamicEnv = &env; - StaticEnv newEnv(false, &env); +void ExprAttrs::bindVars(const StaticEnv& env) { + const StaticEnv* dynamicEnv = &env; + StaticEnv newEnv(false, &env); - if (recursive) { - dynamicEnv = &newEnv; + if (recursive) { + dynamicEnv = &newEnv; - unsigned int displ = 0; - for (auto & i : attrs) - newEnv.vars[i.first] = i.second.displ = displ++; + unsigned int displ = 0; + for (auto& i : attrs) newEnv.vars[i.first] = i.second.displ = displ++; - for (auto & i : attrs) - i.second.e->bindVars(i.second.inherited ? env : newEnv); - } + for (auto& i : attrs) + i.second.e->bindVars(i.second.inherited ? env : newEnv); + } - else - for (auto & i : attrs) - i.second.e->bindVars(env); + else + for (auto& i : attrs) i.second.e->bindVars(env); - for (auto & i : dynamicAttrs) { - i.nameExpr->bindVars(*dynamicEnv); - i.valueExpr->bindVars(*dynamicEnv); - } + for (auto& i : dynamicAttrs) { + i.nameExpr->bindVars(*dynamicEnv); + i.valueExpr->bindVars(*dynamicEnv); + } } -void ExprList::bindVars(const StaticEnv & env) -{ - for (auto & i : elems) - i->bindVars(env); +void ExprList::bindVars(const StaticEnv& env) { + for (auto& i : elems) i->bindVars(env); } -void ExprLambda::bindVars(const StaticEnv & env) -{ - StaticEnv newEnv(false, &env); +void ExprLambda::bindVars(const StaticEnv& env) { + StaticEnv newEnv(false, &env); - unsigned int displ = 0; + unsigned int displ = 0; - if (!arg.empty()) newEnv.vars[arg] = displ++; + if (!arg.empty()) newEnv.vars[arg] = displ++; - if (matchAttrs) { - for (auto & i : formals->formals) - newEnv.vars[i.name] = displ++; + if (matchAttrs) { + for (auto& i : formals->formals) newEnv.vars[i.name] = displ++; - for (auto & i : formals->formals) - if (i.def) i.def->bindVars(newEnv); - } + for (auto& i : formals->formals) + if (i.def) i.def->bindVars(newEnv); + } - body->bindVars(newEnv); + body->bindVars(newEnv); } -void ExprLet::bindVars(const StaticEnv & env) -{ - StaticEnv newEnv(false, &env); +void ExprLet::bindVars(const StaticEnv& env) { + StaticEnv newEnv(false, &env); - unsigned int displ = 0; - for (auto & i : attrs->attrs) - newEnv.vars[i.first] = i.second.displ = displ++; + unsigned int displ = 0; + for (auto& i : attrs->attrs) newEnv.vars[i.first] = i.second.displ = displ++; - for (auto & i : attrs->attrs) - i.second.e->bindVars(i.second.inherited ? env : newEnv); + for (auto& i : attrs->attrs) + i.second.e->bindVars(i.second.inherited ? env : newEnv); - body->bindVars(newEnv); + body->bindVars(newEnv); } -void ExprWith::bindVars(const StaticEnv & env) -{ - /* Does this `with' have an enclosing `with'? If so, record its - level so that `lookupVar' can look up variables in the previous - `with' if this one doesn't contain the desired attribute. */ - const StaticEnv * curEnv; - unsigned int level; - prevWith = 0; - for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++) - if (curEnv->isWith) { - prevWith = level; - break; - } - - attrs->bindVars(env); - StaticEnv newEnv(true, &env); - body->bindVars(newEnv); -} +void ExprWith::bindVars(const StaticEnv& env) { + /* Does this `with' have an enclosing `with'? If so, record its + level so that `lookupVar' can look up variables in the previous + `with' if this one doesn't contain the desired attribute. */ + const StaticEnv* curEnv; + unsigned int level; + prevWith = 0; + for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++) + if (curEnv->isWith) { + prevWith = level; + break; + } -void ExprIf::bindVars(const StaticEnv & env) -{ - cond->bindVars(env); - then->bindVars(env); - else_->bindVars(env); + attrs->bindVars(env); + StaticEnv newEnv(true, &env); + body->bindVars(newEnv); } -void ExprAssert::bindVars(const StaticEnv & env) -{ - cond->bindVars(env); - body->bindVars(env); +void ExprIf::bindVars(const StaticEnv& env) { + cond->bindVars(env); + then->bindVars(env); + else_->bindVars(env); } -void ExprOpNot::bindVars(const StaticEnv & env) -{ - e->bindVars(env); +void ExprAssert::bindVars(const StaticEnv& env) { + cond->bindVars(env); + body->bindVars(env); } -void ExprConcatStrings::bindVars(const StaticEnv & env) -{ - for (auto & i : *es) - i->bindVars(env); -} +void ExprOpNot::bindVars(const StaticEnv& env) { e->bindVars(env); } -void ExprPos::bindVars(const StaticEnv & env) -{ +void ExprConcatStrings::bindVars(const StaticEnv& env) { + for (auto& i : *es) i->bindVars(env); } +void ExprPos::bindVars(const StaticEnv& env) {} /* Storing function names. */ -void Expr::setName(Symbol & name) -{ -} - +void Expr::setName(Symbol& name) {} -void ExprLambda::setName(Symbol & name) -{ - this->name = name; - body->setName(name); +void ExprLambda::setName(Symbol& name) { + this->name = name; + body->setName(name); } - -string ExprLambda::showNamePos() const -{ - return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str(); +string ExprLambda::showNamePos() const { + return (format("%1% at %2%") % + (name.set() ? "'" + (string)name + "'" : "anonymous function") % pos) + .str(); } - - /* Symbol table. */ -size_t SymbolTable::totalSize() const -{ - size_t n = 0; - for (auto & i : symbols) - n += i.size(); - return n; +size_t SymbolTable::totalSize() const { + size_t n = 0; + for (auto& i : symbols) n += i.size(); + return n; } - -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/nixexpr.hh b/third_party/nix/src/libexpr/nixexpr.hh index 665a42987dc1..efc4398c2371 100644 --- a/third_party/nix/src/libexpr/nixexpr.hh +++ b/third_party/nix/src/libexpr/nixexpr.hh @@ -1,342 +1,309 @@ #pragma once -#include "value.hh" -#include "symbol-table.hh" - #include <map> - +#include "symbol-table.hh" +#include "value.hh" namespace nix { - -MakeError(EvalError, Error) -MakeError(ParseError, Error) -MakeError(AssertionError, EvalError) -MakeError(ThrownError, AssertionError) -MakeError(Abort, EvalError) -MakeError(TypeError, EvalError) -MakeError(UndefinedVarError, Error) -MakeError(RestrictedPathError, Error) - - -/* Position objects. */ - -struct Pos -{ - Symbol file; - unsigned int line, column; - Pos() : line(0), column(0) { }; - Pos(const Symbol & file, unsigned int line, unsigned int column) - : file(file), line(line), column(column) { }; - operator bool() const - { - return line != 0; - } - bool operator < (const Pos & p2) const - { - if (!line) return p2.line; - if (!p2.line) return false; - int d = ((string) file).compare((string) p2.file); - if (d < 0) return true; - if (d > 0) return false; - if (line < p2.line) return true; - if (line > p2.line) return false; - return column < p2.column; - } +MakeError(EvalError, Error) MakeError(ParseError, Error) + MakeError(AssertionError, EvalError) MakeError(ThrownError, AssertionError) + MakeError(Abort, EvalError) MakeError(TypeError, EvalError) + MakeError(UndefinedVarError, Error) + MakeError(RestrictedPathError, Error) + + /* Position objects. */ + + struct Pos { + Symbol file; + unsigned int line, column; + Pos() : line(0), column(0){}; + Pos(const Symbol& file, unsigned int line, unsigned int column) + : file(file), line(line), column(column){}; + operator bool() const { return line != 0; } + bool operator<(const Pos& p2) const { + if (!line) return p2.line; + if (!p2.line) return false; + int d = ((string)file).compare((string)p2.file); + if (d < 0) return true; + if (d > 0) return false; + if (line < p2.line) return true; + if (line > p2.line) return false; + return column < p2.column; + } }; extern Pos noPos; -std::ostream & operator << (std::ostream & str, const Pos & pos); - +std::ostream& operator<<(std::ostream& str, const Pos& pos); struct Env; struct Value; class EvalState; struct StaticEnv; - /* An attribute path is a sequence of attribute names. */ -struct AttrName -{ - Symbol symbol; - Expr * expr; - AttrName(const Symbol & s) : symbol(s) {}; - AttrName(Expr * e) : expr(e) {}; +struct AttrName { + Symbol symbol; + Expr* expr; + AttrName(const Symbol& s) : symbol(s){}; + AttrName(Expr* e) : expr(e){}; }; typedef std::vector<AttrName> AttrPath; -string showAttrPath(const AttrPath & attrPath); - +string showAttrPath(const AttrPath& attrPath); /* Abstract syntax of Nix expressions. */ -struct Expr -{ - virtual ~Expr() { }; - virtual void show(std::ostream & str) const; - virtual void bindVars(const StaticEnv & env); - virtual void eval(EvalState & state, Env & env, Value & v); - virtual Value * maybeThunk(EvalState & state, Env & env); - virtual void setName(Symbol & name); +struct Expr { + virtual ~Expr(){}; + virtual void show(std::ostream& str) const; + virtual void bindVars(const StaticEnv& env); + virtual void eval(EvalState& state, Env& env, Value& v); + virtual Value* maybeThunk(EvalState& state, Env& env); + virtual void setName(Symbol& name); }; -std::ostream & operator << (std::ostream & str, const Expr & e); +std::ostream& operator<<(std::ostream& str, const Expr& e); -#define COMMON_METHODS \ - void show(std::ostream & str) const; \ - void eval(EvalState & state, Env & env, Value & v); \ - void bindVars(const StaticEnv & env); +#define COMMON_METHODS \ + void show(std::ostream& str) const; \ + void eval(EvalState& state, Env& env, Value& v); \ + void bindVars(const StaticEnv& env); -struct ExprInt : Expr -{ - NixInt n; - Value v; - ExprInt(NixInt n) : n(n) { mkInt(v, n); }; - COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); +struct ExprInt : Expr { + NixInt n; + Value v; + ExprInt(NixInt n) : n(n) { mkInt(v, n); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); }; -struct ExprFloat : Expr -{ - NixFloat nf; - Value v; - ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); }; - COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); +struct ExprFloat : Expr { + NixFloat nf; + Value v; + ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); }; -struct ExprString : Expr -{ - Symbol s; - Value v; - ExprString(const Symbol & s) : s(s) { mkString(v, s); }; - COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); +struct ExprString : Expr { + Symbol s; + Value v; + ExprString(const Symbol& s) : s(s) { mkString(v, s); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); }; /* Temporary class used during parsing of indented strings. */ -struct ExprIndStr : Expr -{ - string s; - ExprIndStr(const string & s) : s(s) { }; +struct ExprIndStr : Expr { + string s; + ExprIndStr(const string& s) : s(s){}; }; -struct ExprPath : Expr -{ - string s; - Value v; - ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; - COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); +struct ExprPath : Expr { + string s; + Value v; + ExprPath(const string& s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); }; -struct ExprVar : Expr -{ - Pos pos; - Symbol name; - - /* Whether the variable comes from an environment (e.g. a rec, let - or function argument) or from a "with". */ - bool fromWith; - - /* In the former case, the value is obtained by going `level' - levels up from the current environment and getting the - `displ'th value in that environment. In the latter case, the - value is obtained by getting the attribute named `name' from - the set stored in the environment that is `level' levels up - from the current one.*/ - unsigned int level; - unsigned int displ; - - ExprVar(const Symbol & name) : name(name) { }; - ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { }; - COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); +struct ExprVar : Expr { + Pos pos; + Symbol name; + + /* Whether the variable comes from an environment (e.g. a rec, let + or function argument) or from a "with". */ + bool fromWith; + + /* In the former case, the value is obtained by going `level' + levels up from the current environment and getting the + `displ'th value in that environment. In the latter case, the + value is obtained by getting the attribute named `name' from + the set stored in the environment that is `level' levels up + from the current one.*/ + unsigned int level; + unsigned int displ; + + ExprVar(const Symbol& name) : name(name){}; + ExprVar(const Pos& pos, const Symbol& name) : pos(pos), name(name){}; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); }; -struct ExprSelect : Expr -{ - Pos pos; - Expr * e, * def; - AttrPath attrPath; - ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; - ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; - COMMON_METHODS +struct ExprSelect : Expr { + Pos pos; + Expr *e, *def; + AttrPath attrPath; + ExprSelect(const Pos& pos, Expr* e, const AttrPath& attrPath, Expr* def) + : pos(pos), e(e), def(def), attrPath(attrPath){}; + ExprSelect(const Pos& pos, Expr* e, const Symbol& name) + : pos(pos), e(e), def(0) { + attrPath.push_back(AttrName(name)); + }; + COMMON_METHODS }; -struct ExprOpHasAttr : Expr -{ - Expr * e; - AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { }; - COMMON_METHODS +struct ExprOpHasAttr : Expr { + Expr* e; + AttrPath attrPath; + ExprOpHasAttr(Expr* e, const AttrPath& attrPath) : e(e), attrPath(attrPath){}; + COMMON_METHODS }; -struct ExprAttrs : Expr -{ - bool recursive; - struct AttrDef { - bool inherited; - Expr * e; - Pos pos; - unsigned int displ; // displacement - AttrDef(Expr * e, const Pos & pos, bool inherited=false) - : inherited(inherited), e(e), pos(pos) { }; - AttrDef() { }; - }; - typedef std::map<Symbol, AttrDef> AttrDefs; - AttrDefs attrs; - struct DynamicAttrDef { - Expr * nameExpr, * valueExpr; - Pos pos; - DynamicAttrDef(Expr * nameExpr, Expr * valueExpr, const Pos & pos) - : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos) { }; - }; - typedef std::vector<DynamicAttrDef> DynamicAttrDefs; - DynamicAttrDefs dynamicAttrs; - ExprAttrs() : recursive(false) { }; - COMMON_METHODS +struct ExprAttrs : Expr { + bool recursive; + struct AttrDef { + bool inherited; + Expr* e; + Pos pos; + unsigned int displ; // displacement + AttrDef(Expr* e, const Pos& pos, bool inherited = false) + : inherited(inherited), e(e), pos(pos){}; + AttrDef(){}; + }; + typedef std::map<Symbol, AttrDef> AttrDefs; + AttrDefs attrs; + struct DynamicAttrDef { + Expr *nameExpr, *valueExpr; + Pos pos; + DynamicAttrDef(Expr* nameExpr, Expr* valueExpr, const Pos& pos) + : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos){}; + }; + typedef std::vector<DynamicAttrDef> DynamicAttrDefs; + DynamicAttrDefs dynamicAttrs; + ExprAttrs() : recursive(false){}; + COMMON_METHODS }; -struct ExprList : Expr -{ - std::vector<Expr *> elems; - ExprList() { }; - COMMON_METHODS +struct ExprList : Expr { + std::vector<Expr*> elems; + ExprList(){}; + COMMON_METHODS }; -struct Formal -{ - Symbol name; - Expr * def; - Formal(const Symbol & name, Expr * def) : name(name), def(def) { }; +struct Formal { + Symbol name; + Expr* def; + Formal(const Symbol& name, Expr* def) : name(name), def(def){}; }; -struct Formals -{ - typedef std::list<Formal> Formals_; - Formals_ formals; - std::set<Symbol> argNames; // used during parsing - bool ellipsis; +struct Formals { + typedef std::list<Formal> Formals_; + Formals_ formals; + std::set<Symbol> argNames; // used during parsing + bool ellipsis; }; -struct ExprLambda : Expr -{ - Pos pos; - Symbol name; - Symbol arg; - bool matchAttrs; - Formals * formals; - Expr * body; - ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body) - : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) - { - if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) - throw ParseError(format("duplicate formal function argument '%1%' at %2%") - % arg % pos); - }; - void setName(Symbol & name); - string showNamePos() const; - COMMON_METHODS +struct ExprLambda : Expr { + Pos pos; + Symbol name; + Symbol arg; + bool matchAttrs; + Formals* formals; + Expr* body; + ExprLambda(const Pos& pos, const Symbol& arg, bool matchAttrs, + Formals* formals, Expr* body) + : pos(pos), + arg(arg), + matchAttrs(matchAttrs), + formals(formals), + body(body) { + if (!arg.empty() && formals && + formals->argNames.find(arg) != formals->argNames.end()) + throw ParseError( + format("duplicate formal function argument '%1%' at %2%") % arg % + pos); + }; + void setName(Symbol& name); + string showNamePos() const; + COMMON_METHODS }; -struct ExprLet : Expr -{ - ExprAttrs * attrs; - Expr * body; - ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { }; - COMMON_METHODS +struct ExprLet : Expr { + ExprAttrs* attrs; + Expr* body; + ExprLet(ExprAttrs* attrs, Expr* body) : attrs(attrs), body(body){}; + COMMON_METHODS }; -struct ExprWith : Expr -{ - Pos pos; - Expr * attrs, * body; - size_t prevWith; - ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; - COMMON_METHODS +struct ExprWith : Expr { + Pos pos; + Expr *attrs, *body; + size_t prevWith; + ExprWith(const Pos& pos, Expr* attrs, Expr* body) + : pos(pos), attrs(attrs), body(body){}; + COMMON_METHODS }; -struct ExprIf : Expr -{ - Expr * cond, * then, * else_; - ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { }; - COMMON_METHODS +struct ExprIf : Expr { + Expr *cond, *then, *else_; + ExprIf(Expr* cond, Expr* then, Expr* else_) + : cond(cond), then(then), else_(else_){}; + COMMON_METHODS }; -struct ExprAssert : Expr -{ - Pos pos; - Expr * cond, * body; - ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { }; - COMMON_METHODS +struct ExprAssert : Expr { + Pos pos; + Expr *cond, *body; + ExprAssert(const Pos& pos, Expr* cond, Expr* body) + : pos(pos), cond(cond), body(body){}; + COMMON_METHODS }; -struct ExprOpNot : Expr -{ - Expr * e; - ExprOpNot(Expr * e) : e(e) { }; - COMMON_METHODS +struct ExprOpNot : Expr { + Expr* e; + ExprOpNot(Expr* e) : e(e){}; + COMMON_METHODS }; -#define MakeBinOp(name, s) \ - struct name : Expr \ - { \ - Pos pos; \ - Expr * e1, * e2; \ - name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \ - name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \ - void show(std::ostream & str) const \ - { \ - str << "(" << *e1 << " " s " " << *e2 << ")"; \ - } \ - void bindVars(const StaticEnv & env) \ - { \ - e1->bindVars(env); e2->bindVars(env); \ - } \ - void eval(EvalState & state, Env & env, Value & v); \ - }; - -MakeBinOp(ExprApp, "") -MakeBinOp(ExprOpEq, "==") -MakeBinOp(ExprOpNEq, "!=") -MakeBinOp(ExprOpAnd, "&&") -MakeBinOp(ExprOpOr, "||") -MakeBinOp(ExprOpImpl, "->") -MakeBinOp(ExprOpUpdate, "//") -MakeBinOp(ExprOpConcatLists, "++") - -struct ExprConcatStrings : Expr -{ - Pos pos; - bool forceString; - vector<Expr *> * es; - ExprConcatStrings(const Pos & pos, bool forceString, vector<Expr *> * es) - : pos(pos), forceString(forceString), es(es) { }; - COMMON_METHODS +#define MakeBinOp(name, s) \ + struct name : Expr { \ + Pos pos; \ + Expr *e1, *e2; \ + name(Expr* e1, Expr* e2) : e1(e1), e2(e2){}; \ + name(const Pos& pos, Expr* e1, Expr* e2) : pos(pos), e1(e1), e2(e2){}; \ + void show(std::ostream& str) const { \ + str << "(" << *e1 << " " s " " << *e2 << ")"; \ + } \ + void bindVars(const StaticEnv& env) { \ + e1->bindVars(env); \ + e2->bindVars(env); \ + } \ + void eval(EvalState& state, Env& env, Value& v); \ + }; + +MakeBinOp(ExprApp, "") MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=") + MakeBinOp(ExprOpAnd, "&&") MakeBinOp(ExprOpOr, "||") + MakeBinOp(ExprOpImpl, "->") MakeBinOp(ExprOpUpdate, "//") + MakeBinOp(ExprOpConcatLists, "++") + + struct ExprConcatStrings : Expr { + Pos pos; + bool forceString; + vector<Expr*>* es; + ExprConcatStrings(const Pos& pos, bool forceString, vector<Expr*>* es) + : pos(pos), forceString(forceString), es(es){}; + COMMON_METHODS }; -struct ExprPos : Expr -{ - Pos pos; - ExprPos(const Pos & pos) : pos(pos) { }; - COMMON_METHODS +struct ExprPos : Expr { + Pos pos; + ExprPos(const Pos& pos) : pos(pos){}; + COMMON_METHODS }; - /* Static environments are used to map variable names onto (level, displacement) pairs used to obtain the value of the variable at runtime. */ -struct StaticEnv -{ - bool isWith; - const StaticEnv * up; - typedef std::map<Symbol, unsigned int> Vars; - Vars vars; - StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { }; +struct StaticEnv { + bool isWith; + const StaticEnv* up; + typedef std::map<Symbol, unsigned int> Vars; + Vars vars; + StaticEnv(bool isWith, const StaticEnv* up) : isWith(isWith), up(up){}; }; - -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops.cc b/third_party/nix/src/libexpr/primops.cc index d4c60f870ed2..9d6d4eaba5ee 100644 --- a/third_party/nix/src/libexpr/primops.cc +++ b/third_party/nix/src/libexpr/primops.cc @@ -1,3 +1,11 @@ +#include "primops.hh" +#include <dlfcn.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <algorithm> +#include <cstring> +#include <regex> #include "archive.hh" #include "derivations.hh" #include "download.hh" @@ -5,520 +13,514 @@ #include "eval.hh" #include "globals.hh" #include "json-to-value.hh" +#include "json.hh" #include "names.hh" #include "store-api.hh" #include "util.hh" -#include "json.hh" #include "value-to-json.hh" #include "value-to-xml.hh" -#include "primops.hh" - -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> - -#include <algorithm> -#include <cstring> -#include <regex> -#include <dlfcn.h> - namespace nix { - /************************************************************* * Miscellaneous *************************************************************/ - /* Decode a context string ‘!<name>!<path>’ into a pair <path, name>. */ -std::pair<string, string> decodeContext(const string & s) -{ - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1)); - } else - return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), ""); -} - - -InvalidPathError::InvalidPathError(const Path & path) : - EvalError(format("path '%1%' is not valid") % path), path(path) {} - -void EvalState::realiseContext(const PathSet & context) -{ - PathSet drvs; - - for (auto & i : context) { - std::pair<string, string> decoded = decodeContext(i); - Path ctx = decoded.first; - assert(store->isStorePath(ctx)); - if (!store->isValidPath(ctx)) - throw InvalidPathError(ctx); - if (!decoded.second.empty() && nix::isDerivation(ctx)) { - drvs.insert(decoded.first + "!" + decoded.second); - - /* Add the output of this derivation to the allowed - paths. */ - if (allowedPaths) { - auto drv = store->derivationFromPath(decoded.first); - DerivationOutputs::iterator i = drv.outputs.find(decoded.second); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second); - allowedPaths->insert(i->second.path); - } - } +std::pair<string, string> decodeContext(const string& s) { + if (s.at(0) == '!') { + size_t index = s.find("!", 1); + return std::pair<string, string>(string(s, index + 1), + string(s, 1, index - 1)); + } else + return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), ""); +} + +InvalidPathError::InvalidPathError(const Path& path) + : EvalError(format("path '%1%' is not valid") % path), path(path) {} + +void EvalState::realiseContext(const PathSet& context) { + PathSet drvs; + + for (auto& i : context) { + std::pair<string, string> decoded = decodeContext(i); + Path ctx = decoded.first; + assert(store->isStorePath(ctx)); + if (!store->isValidPath(ctx)) throw InvalidPathError(ctx); + if (!decoded.second.empty() && nix::isDerivation(ctx)) { + drvs.insert(decoded.first + "!" + decoded.second); + + /* Add the output of this derivation to the allowed + paths. */ + if (allowedPaths) { + auto drv = store->derivationFromPath(decoded.first); + DerivationOutputs::iterator i = drv.outputs.find(decoded.second); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have an output named '%s'", + decoded.first, decoded.second); + allowedPaths->insert(i->second.path); + } } + } - if (drvs.empty()) return; + if (drvs.empty()) return; - if (!evalSettings.enableImportFromDerivation) - throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin())); + if (!evalSettings.enableImportFromDerivation) + throw EvalError(format("attempted to realize '%1%' during evaluation but " + "'allow-import-from-derivation' is false") % + *(drvs.begin())); - /* For performance, prefetch all substitute info. */ - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(drvs); + /* For performance, prefetch all substitute info. */ + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, + narSize); + store->buildPaths(drvs); } - /* Load and evaluate an expression from path specified by the argument. */ -static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); - - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); +static void prim_scopedImport(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot import '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + + Path realPath = state.checkSourcePath(state.toRealPath(path, context)); + + if (state.store->isStorePath(path) && state.store->isValidPath(path) && + isDerivation(path)) { + Derivation drv = readDerivation(realPath); + Value& w = *state.allocValue(); + state.mkAttrs(w, 3 + drv.outputs.size()); + Value* v2 = state.allocAttr(w, state.sDrvPath); + mkString(*v2, path, {"=" + path}); + v2 = state.allocAttr(w, state.sName); + mkString(*v2, drv.env["name"]); + Value* outputsVal = state.allocAttr(w, state.symbols.create("outputs")); + state.mkList(*outputsVal, drv.outputs.size()); + unsigned int outputs_index = 0; + + for (const auto& o : drv.outputs) { + v2 = state.allocAttr(w, state.symbols.create(o.first)); + mkString(*v2, o.second.path, {"!" + o.first + "!" + path}); + outputsVal->listElems()[outputs_index] = state.allocValue(); + mkString(*(outputsVal->listElems()[outputs_index++]), o.first); } + w.attrs->sort(); + Value fun; + state.evalFile( + settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", + fun); + state.forceFunction(fun, pos); + mkApp(v, fun, w); + state.forceAttrs(v, pos); + } else { + state.forceAttrs(*args[0]); + if (args[0]->attrs->empty()) + state.evalFile(realPath, v); + else { + Env* env = &state.allocEnv(args[0]->attrs->size()); + env->up = &state.baseEnv; - Path realPath = state.checkSourcePath(state.toRealPath(path, context)); - - if (state.store->isStorePath(path) && state.store->isValidPath(path) && isDerivation(path)) { - Derivation drv = readDerivation(realPath); - Value & w = *state.allocValue(); - state.mkAttrs(w, 3 + drv.outputs.size()); - Value * v2 = state.allocAttr(w, state.sDrvPath); - mkString(*v2, path, {"=" + path}); - v2 = state.allocAttr(w, state.sName); - mkString(*v2, drv.env["name"]); - Value * outputsVal = - state.allocAttr(w, state.symbols.create("outputs")); - state.mkList(*outputsVal, drv.outputs.size()); - unsigned int outputs_index = 0; - - for (const auto & o : drv.outputs) { - v2 = state.allocAttr(w, state.symbols.create(o.first)); - mkString(*v2, o.second.path, {"!" + o.first + "!" + path}); - outputsVal->listElems()[outputs_index] = state.allocValue(); - mkString(*(outputsVal->listElems()[outputs_index++]), o.first); - } - w.attrs->sort(); - Value fun; - state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun); - state.forceFunction(fun, pos); - mkApp(v, fun, w); - state.forceAttrs(v, pos); - } else { - state.forceAttrs(*args[0]); - if (args[0]->attrs->empty()) - state.evalFile(realPath, v); - else { - Env * env = &state.allocEnv(args[0]->attrs->size()); - env->up = &state.baseEnv; - - StaticEnv staticEnv(false, &state.staticBaseEnv); + StaticEnv staticEnv(false, &state.staticBaseEnv); - unsigned int displ = 0; - for (auto & attr : *args[0]->attrs) { - staticEnv.vars[attr.name] = displ; - env->values[displ++] = attr.value; - } + unsigned int displ = 0; + for (auto& attr : *args[0]->attrs) { + staticEnv.vars[attr.name] = displ; + env->values[displ++] = attr.value; + } - printTalkative("evaluating file '%1%'", realPath); - Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); + printTalkative("evaluating file '%1%'", realPath); + Expr* e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); - e->eval(state, *env, v); - } + e->eval(state, *env, v); } + } } - /* Want reasonable symbol names, so extern C */ /* !!! Should we pass the Pos or the file name too? */ -extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); +extern "C" typedef void (*ValueInitializer)(EvalState& state, Value& v); /* Load a ValueInitializer from a DSO and return whatever it initializes */ -void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); - - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); - } - - path = state.checkSourcePath(path); - - string sym = state.forceStringNoCtx(*args[1], pos); - - void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); - if (!handle) - throw EvalError(format("could not open '%1%': %2%") % path % dlerror()); - - dlerror(); - ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); - if(!func) { - char *message = dlerror(); - if (message) - throw EvalError(format("could not load symbol '%1%' from '%2%': %3%") % sym % path % message); - else - throw EvalError(format("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected") - % sym % path); - } +void prim_importNative(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot import '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + + path = state.checkSourcePath(path); + + string sym = state.forceStringNoCtx(*args[1], pos); + + void* handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw EvalError(format("could not open '%1%': %2%") % path % dlerror()); + + dlerror(); + ValueInitializer func = (ValueInitializer)dlsym(handle, sym.c_str()); + if (!func) { + char* message = dlerror(); + if (message) + throw EvalError(format("could not load symbol '%1%' from '%2%': %3%") % + sym % path % message); + else + throw EvalError(format("symbol '%1%' from '%2%' resolved to NULL when a " + "function pointer was expected") % + sym % path); + } - (func)(state, v); + (func)(state, v); - /* We don't dlclose because v may be a primop referencing a function in the shared object file */ + /* We don't dlclose because v may be a primop referencing a function in the + * shared object file */ } - /* Execute a program and parse its output */ -void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceList(*args[0], pos); - auto elems = args[0]->listElems(); - auto count = args[0]->listSize(); - if (count == 0) { - throw EvalError(format("at least one argument to 'exec' required, at %1%") % pos); - } - PathSet context; - auto program = state.coerceToString(pos, *elems[0], context, false, false); - Strings commandArgs; - for (unsigned int i = 1; i < args[0]->listSize(); ++i) { - commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false)); - } - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError(format("cannot execute '%1%', since path '%2%' is not valid, at %3%") - % program % e.path % pos); - } - - auto output = runProgram(program, true, commandArgs); - Expr * parsed; - try { - parsed = state.parseExprFromString(output, pos.file); - } catch (Error & e) { - e.addPrefix(format("While parsing the output from '%1%', at %2%\n") % program % pos); - throw; - } - try { - state.eval(parsed, v); - } catch (Error & e) { - e.addPrefix(format("While evaluating the output from '%1%', at %2%\n") % program % pos); - throw; - } +void prim_exec(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceList(*args[0], pos); + auto elems = args[0]->listElems(); + auto count = args[0]->listSize(); + if (count == 0) { + throw EvalError(format("at least one argument to 'exec' required, at %1%") % + pos); + } + PathSet context; + auto program = state.coerceToString(pos, *elems[0], context, false, false); + Strings commandArgs; + for (unsigned int i = 1; i < args[0]->listSize(); ++i) { + commandArgs.emplace_back( + state.coerceToString(pos, *elems[i], context, false, false)); + } + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot execute '%1%', since path '%2%' is not valid, at %3%") % + program % e.path % pos); + } + + auto output = runProgram(program, true, commandArgs); + Expr* parsed; + try { + parsed = state.parseExprFromString(output, pos.file); + } catch (Error& e) { + e.addPrefix(format("While parsing the output from '%1%', at %2%\n") % + program % pos); + throw; + } + try { + state.eval(parsed, v); + } catch (Error& e) { + e.addPrefix(format("While evaluating the output from '%1%', at %2%\n") % + program % pos); + throw; + } } - /* Return a string representing the type of the expression. */ -static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - string t; - switch (args[0]->type) { - case tInt: t = "int"; break; - case tBool: t = "bool"; break; - case tString: t = "string"; break; - case tPath: t = "path"; break; - case tNull: t = "null"; break; - case tAttrs: t = "set"; break; - case tList1: case tList2: case tListN: t = "list"; break; - case tLambda: - case tPrimOp: - case tPrimOpApp: - t = "lambda"; - break; - case tExternal: - t = args[0]->external->typeOf(); - break; - case tFloat: t = "float"; break; - default: abort(); - } - mkString(v, state.symbols.create(t)); +static void prim_typeOf(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + string t; + switch (args[0]->type) { + case tInt: + t = "int"; + break; + case tBool: + t = "bool"; + break; + case tString: + t = "string"; + break; + case tPath: + t = "path"; + break; + case tNull: + t = "null"; + break; + case tAttrs: + t = "set"; + break; + case tList1: + case tList2: + case tListN: + t = "list"; + break; + case tLambda: + case tPrimOp: + case tPrimOpApp: + t = "lambda"; + break; + case tExternal: + t = args[0]->external->typeOf(); + break; + case tFloat: + t = "float"; + break; + default: + abort(); + } + mkString(v, state.symbols.create(t)); } - /* Determine whether the argument is the null value. */ -static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tNull); +static void prim_isNull(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tNull); } - /* Determine whether the argument is a function. */ -static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - bool res; - switch (args[0]->type) { - case tLambda: - case tPrimOp: - case tPrimOpApp: - res = true; - break; - default: - res = false; - break; - } - mkBool(v, res); +static void prim_isFunction(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + bool res; + switch (args[0]->type) { + case tLambda: + case tPrimOp: + case tPrimOpApp: + res = true; + break; + default: + res = false; + break; + } + mkBool(v, res); } - /* Determine whether the argument is an integer. */ -static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tInt); +static void prim_isInt(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tInt); } /* Determine whether the argument is a float. */ -static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tFloat); +static void prim_isFloat(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tFloat); } /* Determine whether the argument is a string. */ -static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tString); +static void prim_isString(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tString); } - /* Determine whether the argument is a Boolean. */ -static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tBool); +static void prim_isBool(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tBool); } /* Determine whether the argument is a path. */ -static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tPath); -} - -struct CompareValues -{ - bool operator () (const Value * v1, const Value * v2) const - { - if (v1->type == tFloat && v2->type == tInt) - return v1->fpoint < v2->integer; - if (v1->type == tInt && v2->type == tFloat) - return v1->integer < v2->fpoint; - if (v1->type != v2->type) - throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); - switch (v1->type) { - case tInt: - return v1->integer < v2->integer; - case tFloat: - return v1->fpoint < v2->fpoint; - case tString: - return strcmp(v1->string.s, v2->string.s) < 0; - case tPath: - return strcmp(v1->path, v2->path) < 0; - default: - throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); - } +static void prim_isPath(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tPath); +} + +struct CompareValues { + bool operator()(const Value* v1, const Value* v2) const { + if (v1->type == tFloat && v2->type == tInt) return v1->fpoint < v2->integer; + if (v1->type == tInt && v2->type == tFloat) return v1->integer < v2->fpoint; + if (v1->type != v2->type) + throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % + showType(*v2)); + switch (v1->type) { + case tInt: + return v1->integer < v2->integer; + case tFloat: + return v1->fpoint < v2->fpoint; + case tString: + return strcmp(v1->string.s, v2->string.s) < 0; + case tPath: + return strcmp(v1->path, v2->path) < 0; + default: + throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % + showType(*v2)); } + } }; - #if HAVE_BOEHMGC -typedef list<Value *, gc_allocator<Value *> > ValueList; +typedef list<Value*, gc_allocator<Value*>> ValueList; #else -typedef list<Value *> ValueList; +typedef list<Value*> ValueList; #endif - -static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceAttrs(*args[0], pos); - - /* Get the start set. */ - Bindings::iterator startSet = - args[0]->attrs->find(state.symbols.create("startSet")); - if (startSet == args[0]->attrs->end()) - throw EvalError(format("attribute 'startSet' required, at %1%") % pos); - state.forceList(*startSet->value, pos); - - ValueList workSet; - for (unsigned int n = 0; n < startSet->value->listSize(); ++n) - workSet.push_back(startSet->value->listElems()[n]); - - /* Get the operator. */ - Bindings::iterator op = - args[0]->attrs->find(state.symbols.create("operator")); - if (op == args[0]->attrs->end()) - throw EvalError(format("attribute 'operator' required, at %1%") % pos); - state.forceValue(*op->value); - - /* Construct the closure by applying the operator to element of - `workSet', adding the result to `workSet', continuing until - no new elements are found. */ - ValueList res; - // `doneKeys' doesn't need to be a GC root, because its values are - // reachable from res. - set<Value *, CompareValues> doneKeys; - while (!workSet.empty()) { - Value * e = *(workSet.begin()); - workSet.pop_front(); - - state.forceAttrs(*e, pos); - - Bindings::iterator key = - e->attrs->find(state.symbols.create("key")); - if (key == e->attrs->end()) - throw EvalError(format("attribute 'key' required, at %1%") % pos); - state.forceValue(*key->value); - - if (doneKeys.find(key->value) != doneKeys.end()) continue; - doneKeys.insert(key->value); - res.push_back(e); - - /* Call the `operator' function with `e' as argument. */ - Value call; - mkApp(call, *op->value, *e); - state.forceList(call, pos); - - /* Add the values returned by the operator to the work set. */ - for (unsigned int n = 0; n < call.listSize(); ++n) { - state.forceValue(*call.listElems()[n]); - workSet.push_back(call.listElems()[n]); - } +static void prim_genericClosure(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + + /* Get the start set. */ + Bindings::iterator startSet = + args[0]->attrs->find(state.symbols.create("startSet")); + if (startSet == args[0]->attrs->end()) + throw EvalError(format("attribute 'startSet' required, at %1%") % pos); + state.forceList(*startSet->value, pos); + + ValueList workSet; + for (unsigned int n = 0; n < startSet->value->listSize(); ++n) + workSet.push_back(startSet->value->listElems()[n]); + + /* Get the operator. */ + Bindings::iterator op = + args[0]->attrs->find(state.symbols.create("operator")); + if (op == args[0]->attrs->end()) + throw EvalError(format("attribute 'operator' required, at %1%") % pos); + state.forceValue(*op->value); + + /* Construct the closure by applying the operator to element of + `workSet', adding the result to `workSet', continuing until + no new elements are found. */ + ValueList res; + // `doneKeys' doesn't need to be a GC root, because its values are + // reachable from res. + set<Value*, CompareValues> doneKeys; + while (!workSet.empty()) { + Value* e = *(workSet.begin()); + workSet.pop_front(); + + state.forceAttrs(*e, pos); + + Bindings::iterator key = e->attrs->find(state.symbols.create("key")); + if (key == e->attrs->end()) + throw EvalError(format("attribute 'key' required, at %1%") % pos); + state.forceValue(*key->value); + + if (doneKeys.find(key->value) != doneKeys.end()) continue; + doneKeys.insert(key->value); + res.push_back(e); + + /* Call the `operator' function with `e' as argument. */ + Value call; + mkApp(call, *op->value, *e); + state.forceList(call, pos); + + /* Add the values returned by the operator to the work set. */ + for (unsigned int n = 0; n < call.listSize(); ++n) { + state.forceValue(*call.listElems()[n]); + workSet.push_back(call.listElems()[n]); } + } - /* Create the result list. */ - state.mkList(v, res.size()); - unsigned int n = 0; - for (auto & i : res) - v.listElems()[n++] = i; + /* Create the result list. */ + state.mkList(v, res.size()); + unsigned int n = 0; + for (auto& i : res) v.listElems()[n++] = i; } - -static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - throw Abort(format("evaluation aborted with the following error message: '%1%'") % s); +static void prim_abort(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + throw Abort( + format("evaluation aborted with the following error message: '%1%'") % s); } - -static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - throw ThrownError(s); +static void prim_throw(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + throw ThrownError(s); } - -static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - try { - state.forceValue(*args[1]); - v = *args[1]; - } catch (Error & e) { - PathSet context; - e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context)); - throw; - } +static void prim_addErrorContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + try { + state.forceValue(*args[1]); + v = *args[1]; + } catch (Error& e) { + PathSet context; + e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context)); + throw; + } } - /* Try evaluating the argument. Success => {success=true; value=something;}, * else => {success=false; value=false;} */ -static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.mkAttrs(v, 2); - try { - state.forceValue(*args[0]); - v.attrs->push_back(Attr(state.sValue, args[0])); - mkBool(*state.allocAttr(v, state.symbols.create("success")), true); - } catch (AssertionError & e) { - mkBool(*state.allocAttr(v, state.sValue), false); - mkBool(*state.allocAttr(v, state.symbols.create("success")), false); - } - v.attrs->sort(); +static void prim_tryEval(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.mkAttrs(v, 2); + try { + state.forceValue(*args[0]); + v.attrs->push_back(Attr(state.sValue, args[0])); + mkBool(*state.allocAttr(v, state.symbols.create("success")), true); + } catch (AssertionError& e) { + mkBool(*state.allocAttr(v, state.sValue), false); + mkBool(*state.allocAttr(v, state.symbols.create("success")), false); + } + v.attrs->sort(); } - /* Return an environment variable. Use with care. */ -static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - string name = state.forceStringNoCtx(*args[0], pos); - mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name)); +static void prim_getEnv(EvalState& state, const Pos& pos, Value** args, + Value& v) { + string name = state.forceStringNoCtx(*args[0], pos); + mkString(v, evalSettings.restrictEval || evalSettings.pureEval + ? "" + : getEnv(name)); } - /* Evaluate the first argument, then return the second argument. */ -static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - state.forceValue(*args[1]); - v = *args[1]; +static void prim_seq(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0]); + state.forceValue(*args[1]); + v = *args[1]; } - /* Evaluate the first argument deeply (i.e. recursing into lists and attrsets), then return the second argument. */ -static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValueDeep(*args[0]); - state.forceValue(*args[1]); - v = *args[1]; +static void prim_deepSeq(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValueDeep(*args[0]); + state.forceValue(*args[1]); + v = *args[1]; } - /* Evaluate the first expression and print it on standard error. Then return the second expression. Useful for debugging. */ -static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - if (args[0]->type == tString) - printError(format("trace: %1%") % args[0]->string.s); - else - printError(format("trace: %1%") % *args[0]); - state.forceValue(*args[1]); - v = *args[1]; +static void prim_trace(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + if (args[0]->type == tString) + printError(format("trace: %1%") % args[0]->string.s); + else + printError(format("trace: %1%") % *args[0]); + state.forceValue(*args[1]); + v = *args[1]; } - -void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - /* We're not forcing the argument on purpose. */ - mkInt(v, valueSize(*args[0])); +void prim_valueSize(EvalState& state, const Pos& pos, Value** args, Value& v) { + /* We're not forcing the argument on purpose. */ + mkInt(v, valueSize(*args[0])); } - /************************************************************* * Derivations *************************************************************/ - /* Construct (as a unobservable side effect) a Nix derivation expression that performs the derivation described by the argument set. Returns the original set extended with the following @@ -526,255 +528,276 @@ void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & derivation; `drvPath' containing the path of the Nix expression; and `type' set to `derivation' to indicate that this is a derivation. */ -static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceAttrs(*args[0], pos); - - /* Figure out the name first (for stack backtraces). */ - Bindings::iterator attr = args[0]->attrs->find(state.sName); - if (attr == args[0]->attrs->end()) - throw EvalError(format("required attribute 'name' missing, at %1%") % pos); - string drvName; - Pos & posDrvName(*attr->pos); - try { - drvName = state.forceStringNoCtx(*attr->value, pos); - } catch (Error & e) { - e.addPrefix(format("while evaluating the derivation attribute 'name' at %1%:\n") % posDrvName); - throw; - } - - /* Check whether attributes should be passed as a JSON file. */ - std::ostringstream jsonBuf; - std::unique_ptr<JSONObject> jsonObject; - attr = args[0]->attrs->find(state.sStructuredAttrs); - if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) - jsonObject = std::make_unique<JSONObject>(jsonBuf); - - /* Check whether null attributes should be ignored. */ - bool ignoreNulls = false; - attr = args[0]->attrs->find(state.sIgnoreNulls); - if (attr != args[0]->attrs->end()) - ignoreNulls = state.forceBool(*attr->value, pos); - - /* Build the derivation expression by processing the attributes. */ - Derivation drv; - - PathSet context; - - std::optional<std::string> outputHash; - std::string outputHashAlgo; - bool outputHashRecursive = false; - - StringSet outputs; - outputs.insert("out"); - - for (auto & i : args[0]->attrs->lexicographicOrder()) { - if (i->name == state.sIgnoreNulls) continue; - const string & key = i->name; - vomit("processing attribute '%1%'", key); - - auto handleHashMode = [&](const std::string & s) { - if (s == "recursive") outputHashRecursive = true; - else if (s == "flat") outputHashRecursive = false; - else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName); - }; - - auto handleOutputs = [&](const Strings & ss) { - outputs.clear(); - for (auto & j : ss) { - if (outputs.find(j) != outputs.end()) - throw EvalError(format("duplicate derivation output '%1%', at %2%") % j % posDrvName); - /* !!! Check whether j is a valid attribute - name. */ - /* Derivations cannot be named ‘drv’, because - then we'd have an attribute ‘drvPath’ in - the resulting set. */ - if (j == "drv") - throw EvalError(format("invalid derivation output name 'drv', at %1%") % posDrvName); - outputs.insert(j); - } - if (outputs.empty()) - throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName); - }; - - try { - - if (ignoreNulls) { - state.forceValue(*i->value); - if (i->value->type == tNull) continue; - } - - /* The `args' attribute is special: it supplies the - command-line arguments to the builder. */ - if (i->name == state.sArgs) { - state.forceList(*i->value, pos); - for (unsigned int n = 0; n < i->value->listSize(); ++n) { - string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); - drv.args.push_back(s); - } - } - - /* All other attributes are passed to the builder through - the environment. */ - else { - - if (jsonObject) { - - if (i->name == state.sStructuredAttrs) continue; - - auto placeholder(jsonObject->placeholder(key)); - printValueAsJSON(state, true, *i->value, placeholder, context); - - if (i->name == state.sBuilder) - drv.builder = state.forceString(*i->value, context, posDrvName); - else if (i->name == state.sSystem) - drv.platform = state.forceStringNoCtx(*i->value, posDrvName); - else if (i->name == state.sOutputHash) - outputHash = state.forceStringNoCtx(*i->value, posDrvName); - else if (i->name == state.sOutputHashAlgo) - outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); - else if (i->name == state.sOutputHashMode) - handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); - else if (i->name == state.sOutputs) { - /* Require ‘outputs’ to be a list of strings. */ - state.forceList(*i->value, posDrvName); - Strings ss; - for (unsigned int n = 0; n < i->value->listSize(); ++n) - ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName)); - handleOutputs(ss); - } - - } else { - auto s = state.coerceToString(posDrvName, *i->value, context, true); - drv.env.emplace(key, s); - if (i->name == state.sBuilder) drv.builder = s; - else if (i->name == state.sSystem) drv.platform = s; - else if (i->name == state.sOutputHash) outputHash = s; - else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s; - else if (i->name == state.sOutputHashMode) handleHashMode(s); - else if (i->name == state.sOutputs) - handleOutputs(tokenizeString<Strings>(s)); - } - - } - - } catch (Error & e) { - e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n") - % key % drvName % posDrvName); - throw; - } - } +static void prim_derivationStrict(EvalState& state, const Pos& pos, + Value** args, Value& v) { + state.forceAttrs(*args[0], pos); + + /* Figure out the name first (for stack backtraces). */ + Bindings::iterator attr = args[0]->attrs->find(state.sName); + if (attr == args[0]->attrs->end()) + throw EvalError(format("required attribute 'name' missing, at %1%") % pos); + string drvName; + Pos& posDrvName(*attr->pos); + try { + drvName = state.forceStringNoCtx(*attr->value, pos); + } catch (Error& e) { + e.addPrefix( + format("while evaluating the derivation attribute 'name' at %1%:\n") % + posDrvName); + throw; + } + + /* Check whether attributes should be passed as a JSON file. */ + std::ostringstream jsonBuf; + std::unique_ptr<JSONObject> jsonObject; + attr = args[0]->attrs->find(state.sStructuredAttrs); + if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) + jsonObject = std::make_unique<JSONObject>(jsonBuf); + + /* Check whether null attributes should be ignored. */ + bool ignoreNulls = false; + attr = args[0]->attrs->find(state.sIgnoreNulls); + if (attr != args[0]->attrs->end()) + ignoreNulls = state.forceBool(*attr->value, pos); + + /* Build the derivation expression by processing the attributes. */ + Derivation drv; + + PathSet context; + + std::optional<std::string> outputHash; + std::string outputHashAlgo; + bool outputHashRecursive = false; + + StringSet outputs; + outputs.insert("out"); + + for (auto& i : args[0]->attrs->lexicographicOrder()) { + if (i->name == state.sIgnoreNulls) continue; + const string& key = i->name; + vomit("processing attribute '%1%'", key); + + auto handleHashMode = [&](const std::string& s) { + if (s == "recursive") + outputHashRecursive = true; + else if (s == "flat") + outputHashRecursive = false; + else + throw EvalError( + "invalid value '%s' for 'outputHashMode' attribute, at %s", s, + posDrvName); + }; - if (jsonObject) { - jsonObject.reset(); - drv.env.emplace("__json", jsonBuf.str()); - } + auto handleOutputs = [&](const Strings& ss) { + outputs.clear(); + for (auto& j : ss) { + if (outputs.find(j) != outputs.end()) + throw EvalError(format("duplicate derivation output '%1%', at %2%") % + j % posDrvName); + /* !!! Check whether j is a valid attribute + name. */ + /* Derivations cannot be named ‘drv’, because + then we'd have an attribute ‘drvPath’ in + the resulting set. */ + if (j == "drv") + throw EvalError( + format("invalid derivation output name 'drv', at %1%") % + posDrvName); + outputs.insert(j); + } + if (outputs.empty()) + throw EvalError( + format("derivation cannot have an empty set of outputs, at %1%") % + posDrvName); + }; - /* Everything in the context of the strings in the derivation - attributes should be added as dependencies of the resulting - derivation. */ - for (auto & path : context) { - - /* Paths marked with `=' denote that the path of a derivation - is explicitly passed to the builder. Since that allows the - builder to gain access to every path in the dependency - graph of the derivation (including all outputs), all paths - in the graph must be added to this derivation's list of - inputs to ensure that they are available when the builder - runs. */ - if (path.at(0) == '=') { - /* !!! This doesn't work if readOnlyMode is set. */ - PathSet refs; - state.store->computeFSClosure(string(path, 1), refs); - for (auto & j : refs) { - drv.inputSrcs.insert(j); - if (isDerivation(j)) - drv.inputDrvs[j] = state.store->queryDerivationOutputNames(j); - } + try { + if (ignoreNulls) { + state.forceValue(*i->value); + if (i->value->type == tNull) continue; + } + + /* The `args' attribute is special: it supplies the + command-line arguments to the builder. */ + if (i->name == state.sArgs) { + state.forceList(*i->value, pos); + for (unsigned int n = 0; n < i->value->listSize(); ++n) { + string s = state.coerceToString(posDrvName, *i->value->listElems()[n], + context, true); + drv.args.push_back(s); } + } + + /* All other attributes are passed to the builder through + the environment. */ + else { + if (jsonObject) { + if (i->name == state.sStructuredAttrs) continue; + + auto placeholder(jsonObject->placeholder(key)); + printValueAsJSON(state, true, *i->value, placeholder, context); + + if (i->name == state.sBuilder) + drv.builder = state.forceString(*i->value, context, posDrvName); + else if (i->name == state.sSystem) + drv.platform = state.forceStringNoCtx(*i->value, posDrvName); + else if (i->name == state.sOutputHash) + outputHash = state.forceStringNoCtx(*i->value, posDrvName); + else if (i->name == state.sOutputHashAlgo) + outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); + else if (i->name == state.sOutputHashMode) + handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); + else if (i->name == state.sOutputs) { + /* Require ‘outputs’ to be a list of strings. */ + state.forceList(*i->value, posDrvName); + Strings ss; + for (unsigned int n = 0; n < i->value->listSize(); ++n) + ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], + posDrvName)); + handleOutputs(ss); + } - /* Handle derivation outputs of the form ‘!<name>!<path>’. */ - else if (path.at(0) == '!') { - std::pair<string, string> ctx = decodeContext(path); - drv.inputDrvs[ctx.first].insert(ctx.second); + } else { + auto s = state.coerceToString(posDrvName, *i->value, context, true); + drv.env.emplace(key, s); + if (i->name == state.sBuilder) + drv.builder = s; + else if (i->name == state.sSystem) + drv.platform = s; + else if (i->name == state.sOutputHash) + outputHash = s; + else if (i->name == state.sOutputHashAlgo) + outputHashAlgo = s; + else if (i->name == state.sOutputHashMode) + handleHashMode(s); + else if (i->name == state.sOutputs) + handleOutputs(tokenizeString<Strings>(s)); } + } - /* Otherwise it's a source file. */ - else - drv.inputSrcs.insert(path); + } catch (Error& e) { + e.addPrefix(format("while evaluating the attribute '%1%' of the " + "derivation '%2%' at %3%:\n") % + key % drvName % posDrvName); + throw; + } + } + + if (jsonObject) { + jsonObject.reset(); + drv.env.emplace("__json", jsonBuf.str()); + } + + /* Everything in the context of the strings in the derivation + attributes should be added as dependencies of the resulting + derivation. */ + for (auto& path : context) { + /* Paths marked with `=' denote that the path of a derivation + is explicitly passed to the builder. Since that allows the + builder to gain access to every path in the dependency + graph of the derivation (including all outputs), all paths + in the graph must be added to this derivation's list of + inputs to ensure that they are available when the builder + runs. */ + if (path.at(0) == '=') { + /* !!! This doesn't work if readOnlyMode is set. */ + PathSet refs; + state.store->computeFSClosure(string(path, 1), refs); + for (auto& j : refs) { + drv.inputSrcs.insert(j); + if (isDerivation(j)) + drv.inputDrvs[j] = state.store->queryDerivationOutputNames(j); + } } - /* Do we have all required attributes? */ - if (drv.builder == "") - throw EvalError(format("required attribute 'builder' missing, at %1%") % posDrvName); - if (drv.platform == "") - throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName); - - /* Check whether the derivation name is valid. */ - checkStoreName(drvName); - if (isDerivation(drvName)) - throw EvalError(format("derivation names are not allowed to end in '%1%', at %2%") - % drvExtension % posDrvName); - - if (outputHash) { - /* Handle fixed-output derivations. */ - if (outputs.size() != 1 || *(outputs.begin()) != "out") - throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName); - - HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); - Hash h(*outputHash, ht); - - Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); - if (!jsonObject) drv.env["out"] = outPath; - drv.outputs["out"] = DerivationOutput(outPath, - (outputHashRecursive ? "r:" : "") + printHashType(h.type), - h.to_string(Base16, false)); + /* Handle derivation outputs of the form ‘!<name>!<path>’. */ + else if (path.at(0) == '!') { + std::pair<string, string> ctx = decodeContext(path); + drv.inputDrvs[ctx.first].insert(ctx.second); } - else { - /* Construct the "masked" store derivation, which is the final - one except that in the list of outputs, the output paths - are empty, and the corresponding environment variables have - an empty value. This ensures that changes in the set of - output names do get reflected in the hash. */ - for (auto & i : outputs) { - if (!jsonObject) drv.env[i] = ""; - drv.outputs[i] = DerivationOutput("", "", ""); - } + /* Otherwise it's a source file. */ + else + drv.inputSrcs.insert(path); + } + + /* Do we have all required attributes? */ + if (drv.builder == "") + throw EvalError(format("required attribute 'builder' missing, at %1%") % + posDrvName); + if (drv.platform == "") + throw EvalError(format("required attribute 'system' missing, at %1%") % + posDrvName); + + /* Check whether the derivation name is valid. */ + checkStoreName(drvName); + if (isDerivation(drvName)) + throw EvalError( + format("derivation names are not allowed to end in '%1%', at %2%") % + drvExtension % posDrvName); + + if (outputHash) { + /* Handle fixed-output derivations. */ + if (outputs.size() != 1 || *(outputs.begin()) != "out") + throw Error(format("multiple outputs are not supported in fixed-output " + "derivations, at %1%") % + posDrvName); + + HashType ht = + outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); + Hash h(*outputHash, ht); + + Path outPath = + state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); + if (!jsonObject) drv.env["out"] = outPath; + drv.outputs["out"] = DerivationOutput( + outPath, (outputHashRecursive ? "r:" : "") + printHashType(h.type), + h.to_string(Base16, false)); + } + + else { + /* Construct the "masked" store derivation, which is the final + one except that in the list of outputs, the output paths + are empty, and the corresponding environment variables have + an empty value. This ensures that changes in the set of + output names do get reflected in the hash. */ + for (auto& i : outputs) { + if (!jsonObject) drv.env[i] = ""; + drv.outputs[i] = DerivationOutput("", "", ""); + } - /* Use the masked derivation expression to compute the output - path. */ - Hash h = hashDerivationModulo(*state.store, drv); + /* Use the masked derivation expression to compute the output + path. */ + Hash h = hashDerivationModulo(*state.store, drv); - for (auto & i : drv.outputs) - if (i.second.path == "") { - Path outPath = state.store->makeOutputPath(i.first, h, drvName); - if (!jsonObject) drv.env[i.first] = outPath; - i.second.path = outPath; - } - } + for (auto& i : drv.outputs) + if (i.second.path == "") { + Path outPath = state.store->makeOutputPath(i.first, h, drvName); + if (!jsonObject) drv.env[i.first] = outPath; + i.second.path = outPath; + } + } - /* Write the resulting term into the Nix store directory. */ - Path drvPath = writeDerivation(state.store, drv, drvName, state.repair); + /* Write the resulting term into the Nix store directory. */ + Path drvPath = writeDerivation(state.store, drv, drvName, state.repair); - printMsg(lvlChatty, format("instantiated '%1%' -> '%2%'") - % drvName % drvPath); + printMsg(lvlChatty, + format("instantiated '%1%' -> '%2%'") % drvName % drvPath); - /* Optimisation, but required in read-only mode! because in that - case we don't actually write store derivations, so we can't - read them later. */ - drvHashes[drvPath] = hashDerivationModulo(*state.store, drv); + /* Optimisation, but required in read-only mode! because in that + case we don't actually write store derivations, so we can't + read them later. */ + drvHashes[drvPath] = hashDerivationModulo(*state.store, drv); - state.mkAttrs(v, 1 + drv.outputs.size()); - mkString(*state.allocAttr(v, state.sDrvPath), drvPath, {"=" + drvPath}); - for (auto & i : drv.outputs) { - mkString(*state.allocAttr(v, state.symbols.create(i.first)), - i.second.path, {"!" + i.first + "!" + drvPath}); - } - v.attrs->sort(); + state.mkAttrs(v, 1 + drv.outputs.size()); + mkString(*state.allocAttr(v, state.sDrvPath), drvPath, {"=" + drvPath}); + for (auto& i : drv.outputs) { + mkString(*state.allocAttr(v, state.symbols.create(i.first)), i.second.path, + {"!" + i.first + "!" + drvPath}); + } + v.attrs->sort(); } - /* Return a placeholder string for the specified output that will be substituted by the corresponding output path at build time. For example, 'placeholder "out"' returns the string @@ -782,26 +805,23 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * time, any occurence of this string in an derivation attribute will be replaced with the concrete path in the Nix store of the output ‘out’. */ -static void prim_placeholder(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); +static void prim_placeholder(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); } - /************************************************************* * Paths *************************************************************/ - /* Convert the argument to a path. !!! obsolete? */ -static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); - mkString(v, canonPath(path), context); +static void prim_toPath(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + mkString(v, canonPath(path), context); } - /* Allow a valid store path to be used in an expression. This is useful in some generated expressions such as in nix-push, which generates a call to a function with an already existing store path @@ -810,501 +830,512 @@ static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Valu /nix/store/newhash-oldhash-oldname. In the past, `toPath' had special case behaviour for store paths, but that created weird corner cases. */ -static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context)); - /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink - directly in the store. The latter condition is necessary so - e.g. nix-push does the right thing. */ - if (!state.store->isStorePath(path)) path = canonPath(path, true); - if (!state.store->isInStore(path)) - throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos); - Path path2 = state.store->toStorePath(path); - if (!settings.readOnlyMode) - state.store->ensurePath(path2); - context.insert(path2); - mkString(v, path, context); -} - - -static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError(format( - "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); - } - - try { - mkBool(v, pathExists(state.checkSourcePath(path))); - } catch (SysError & e) { - /* Don't give away info from errors while canonicalising - ‘path’ in restricted mode. */ - mkBool(v, false); - } catch (RestrictedPathError & e) { - mkBool(v, false); - } +static void prim_storePath(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context)); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink + directly in the store. The latter condition is necessary so + e.g. nix-push does the right thing. */ + if (!state.store->isStorePath(path)) path = canonPath(path, true); + if (!state.store->isInStore(path)) + throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % + path % pos); + Path path2 = state.store->toStorePath(path); + if (!settings.readOnlyMode) state.store->ensurePath(path2); + context.insert(path2); + mkString(v, path, context); +} + +static void prim_pathExists(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError(format("cannot check the existence of '%1%', since path " + "'%2%' is not valid, at %3%") % + path % e.path % pos); + } + + try { + mkBool(v, pathExists(state.checkSourcePath(path))); + } catch (SysError& e) { + /* Don't give away info from errors while canonicalising + ‘path’ in restricted mode. */ + mkBool(v, false); + } catch (RestrictedPathError& e) { + mkBool(v, false); + } } - /* Return the base name of the given string, i.e., everything following the last slash. */ -static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - mkString(v, baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), context); +static void prim_baseNameOf(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + mkString( + v, baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), + context); } - /* Return the directory of the given path, i.e., everything before the last slash. Return either a path or a string depending on the type of the argument. */ -static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); - if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); +static void prim_dirOf(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); + if (args[0]->type == tPath) + mkPath(v, dir.c_str()); + else + mkString(v, dir, context); } - /* Return the contents of a file as a string. */ -static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); - } - string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); - if (s.find((char) 0) != string::npos) - throw Error(format("the contents of the file '%1%' cannot be represented as a Nix string") % path); - mkString(v, s.c_str()); +static void prim_readFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot read '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); + if (s.find((char)0) != string::npos) + throw Error(format("the contents of the file '%1%' cannot be represented " + "as a Nix string") % + path); + mkString(v, s.c_str()); } - /* Find a file in the Nix search path. Used to implement <x> paths, which are desugared to 'findFile __nixPath "x"'. */ -static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceList(*args[0], pos); - - SearchPath searchPath; +static void prim_findFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); - for (unsigned int n = 0; n < args[0]->listSize(); ++n) { - Value & v2(*args[0]->listElems()[n]); - state.forceAttrs(v2, pos); + SearchPath searchPath; - string prefix; - Bindings::iterator i = v2.attrs->find(state.symbols.create("prefix")); - if (i != v2.attrs->end()) - prefix = state.forceStringNoCtx(*i->value, pos); + for (unsigned int n = 0; n < args[0]->listSize(); ++n) { + Value& v2(*args[0]->listElems()[n]); + state.forceAttrs(v2, pos); - i = v2.attrs->find(state.symbols.create("path")); - if (i == v2.attrs->end()) - throw EvalError(format("attribute 'path' missing, at %1%") % pos); + string prefix; + Bindings::iterator i = v2.attrs->find(state.symbols.create("prefix")); + if (i != v2.attrs->end()) prefix = state.forceStringNoCtx(*i->value, pos); - PathSet context; - string path = state.coerceToString(pos, *i->value, context, false, false); + i = v2.attrs->find(state.symbols.create("path")); + if (i == v2.attrs->end()) + throw EvalError(format("attribute 'path' missing, at %1%") % pos); - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError(format("cannot find '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); - } + PathSet context; + string path = state.coerceToString(pos, *i->value, context, false, false); - searchPath.emplace_back(prefix, path); + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot find '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); } - string path = state.forceStringNoCtx(*args[1], pos); + searchPath.emplace_back(prefix, path); + } + + string path = state.forceStringNoCtx(*args[1], pos); - mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); + mkPath(v, + state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } /* Return the cryptographic hash of a file in base-16. */ -static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - string type = state.forceStringNoCtx(*args[0], pos); - HashType ht = parseHashType(type); - if (ht == htUnknown) - throw Error(format("unknown hash type '%1%', at %2%") % type % pos); +static void prim_hashFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); - PathSet context; // discarded - Path p = state.coerceToPath(pos, *args[1], context); + PathSet context; // discarded + Path p = state.coerceToPath(pos, *args[1], context); - mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context); + mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), + context); } /* Read a directory (without . or ..) */ -static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet ctx; - Path path = state.coerceToPath(pos, *args[0], ctx); - try { - state.realiseContext(ctx); - } catch (InvalidPathError & e) { - throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); - } - - DirEntries entries = readDirectory(state.checkSourcePath(path)); - state.mkAttrs(v, entries.size()); - - for (auto & ent : entries) { - Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name)); - if (ent.type == DT_UNKNOWN) - ent.type = getFileType(path + "/" + ent.name); - mkStringNoCopy(*ent_val, - ent.type == DT_REG ? "regular" : - ent.type == DT_DIR ? "directory" : - ent.type == DT_LNK ? "symlink" : - "unknown"); - } - - v.attrs->sort(); +static void prim_readDir(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet ctx; + Path path = state.coerceToPath(pos, *args[0], ctx); + try { + state.realiseContext(ctx); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot read '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + + DirEntries entries = readDirectory(state.checkSourcePath(path)); + state.mkAttrs(v, entries.size()); + + for (auto& ent : entries) { + Value* ent_val = state.allocAttr(v, state.symbols.create(ent.name)); + if (ent.type == DT_UNKNOWN) ent.type = getFileType(path + "/" + ent.name); + mkStringNoCopy(*ent_val, + ent.type == DT_REG + ? "regular" + : ent.type == DT_DIR + ? "directory" + : ent.type == DT_LNK ? "symlink" : "unknown"); + } + + v.attrs->sort(); } - /************************************************************* * Creating files *************************************************************/ - /* Convert the argument (which can be any Nix expression) to an XML representation returned in a string. Not all Nix expressions can be sensibly or completely represented (e.g., functions). */ -static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - std::ostringstream out; - PathSet context; - printValueAsXML(state, true, false, *args[0], out, context); - mkString(v, out.str(), context); +static void prim_toXML(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::ostringstream out; + PathSet context; + printValueAsXML(state, true, false, *args[0], out, context); + mkString(v, out.str(), context); } - /* Convert the argument (which can be any Nix expression) to a JSON string. Not all Nix expressions can be sensibly or completely represented (e.g., functions). */ -static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - std::ostringstream out; - PathSet context; - printValueAsJSON(state, true, *args[0], out, context); - mkString(v, out.str(), context); +static void prim_toJSON(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::ostringstream out; + PathSet context; + printValueAsJSON(state, true, *args[0], out, context); + mkString(v, out.str(), context); } - /* Parse a JSON string to a value. */ -static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - string s = state.forceStringNoCtx(*args[0], pos); - parseJSON(state, s, v); +static void prim_fromJSON(EvalState& state, const Pos& pos, Value** args, + Value& v) { + string s = state.forceStringNoCtx(*args[0], pos); + parseJSON(state, s, v); } - /* Store a string in the Nix store as a source file that can be used as an input by derivations. */ -static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string name = state.forceStringNoCtx(*args[0], pos); - string contents = state.forceString(*args[1], context, pos); - - PathSet refs; - - for (auto path : context) { - if (path.at(0) != '/') - throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); - refs.insert(path); - } - - Path storePath = settings.readOnlyMode - ? state.store->computeStorePathForText(name, contents, refs) - : state.store->addTextToStore(name, contents, refs, state.repair); - - /* Note: we don't need to add `context' to the context of the - result, since `storePath' itself has references to the paths - used in args[1]. */ - - mkString(v, storePath, {storePath}); -} - - -static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) -{ - const auto path = evalSettings.pureEval && expectedHash ? - path_ : - state.checkSourcePath(path_); - PathFilter filter = filterFun ? ([&](const Path & path) { - auto st = lstat(path); - - /* Call the filter function. The first argument is the path, - the second is a string indicating the type of the file. */ - Value arg1; - mkString(arg1, path); - - Value fun2; - state.callFunction(*filterFun, arg1, fun2, noPos); - - Value arg2; - mkString(arg2, - S_ISREG(st.st_mode) ? "regular" : - S_ISDIR(st.st_mode) ? "directory" : - S_ISLNK(st.st_mode) ? "symlink" : - "unknown" /* not supported, will fail! */); - - Value res; - state.callFunction(fun2, arg2, res, noPos); - - return state.forceBool(res, pos); - }) : defaultPathFilter; - - Path expectedStorePath; - if (expectedHash) { - expectedStorePath = - state.store->makeFixedOutputPath(recursive, expectedHash, name); - } - Path dstPath; - if (!expectedHash || !state.store->isValidPath(expectedStorePath)) { - dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first - : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair); - if (expectedHash && expectedStorePath != dstPath) { - throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path); - } - } else - dstPath = expectedStorePath; - - mkString(v, dstPath, {dstPath}); -} - - -static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); - - state.forceValue(*args[0]); - if (args[0]->type != tLambda) - throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); - - addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v); -} - -static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceAttrs(*args[0], pos); - Path path; - string name; - Value * filterFun = nullptr; - auto recursive = true; - Hash expectedHash; - - for (auto & attr : *args[0]->attrs) { - const string & n(attr.name); - if (n == "path") { - PathSet context; - path = state.coerceToPath(*attr.pos, *attr.value, context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos); - } else if (attr.name == state.sName) - name = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "filter") { - state.forceValue(*attr.value); - filterFun = attr.value; - } else if (n == "recursive") - recursive = state.forceBool(*attr.value, *attr.pos); - else if (n == "sha256") - expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); - else - throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos); +static void prim_toFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + string name = state.forceStringNoCtx(*args[0], pos); + string contents = state.forceString(*args[1], context, pos); + + PathSet refs; + + for (auto path : context) { + if (path.at(0) != '/') + throw EvalError(format("in 'toFile': the file '%1%' cannot refer to " + "derivation outputs, at %2%") % + name % pos); + refs.insert(path); + } + + Path storePath = + settings.readOnlyMode + ? state.store->computeStorePathForText(name, contents, refs) + : state.store->addTextToStore(name, contents, refs, state.repair); + + /* Note: we don't need to add `context' to the context of the + result, since `storePath' itself has references to the paths + used in args[1]. */ + + mkString(v, storePath, {storePath}); +} + +static void addPath(EvalState& state, const Pos& pos, const string& name, + const Path& path_, Value* filterFun, bool recursive, + const Hash& expectedHash, Value& v) { + const auto path = evalSettings.pureEval && expectedHash + ? path_ + : state.checkSourcePath(path_); + PathFilter filter = filterFun ? ([&](const Path& path) { + auto st = lstat(path); + + /* Call the filter function. The first argument is the path, + the second is a string indicating the type of the file. */ + Value arg1; + mkString(arg1, path); + + Value fun2; + state.callFunction(*filterFun, arg1, fun2, noPos); + + Value arg2; + mkString(arg2, S_ISREG(st.st_mode) + ? "regular" + : S_ISDIR(st.st_mode) + ? "directory" + : S_ISLNK(st.st_mode) + ? "symlink" + : "unknown" /* not supported, will fail! */); + + Value res; + state.callFunction(fun2, arg2, res, noPos); + + return state.forceBool(res, pos); + }) + : defaultPathFilter; + + Path expectedStorePath; + if (expectedHash) { + expectedStorePath = + state.store->makeFixedOutputPath(recursive, expectedHash, name); + } + Path dstPath; + if (!expectedHash || !state.store->isValidPath(expectedStorePath)) { + dstPath = settings.readOnlyMode + ? state.store + ->computeStorePathForPath(name, path, recursive, + htSHA256, filter) + .first + : state.store->addToStore(name, path, recursive, htSHA256, + filter, state.repair); + if (expectedHash && expectedStorePath != dstPath) { + throw Error(format("store path mismatch in (possibly filtered) path " + "added from '%1%'") % + path); } - if (path.empty()) - throw EvalError(format("'path' required, at %1%") % pos); - if (name.empty()) - name = baseNameOf(path); + } else + dstPath = expectedStorePath; + + mkString(v, dstPath, {dstPath}); +} + +static void prim_filterSource(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % + path % pos); + + state.forceValue(*args[0]); + if (args[0]->type != tLambda) + throw TypeError(format("first argument in call to 'filterSource' is not a " + "function but %1%, at %2%") % + showType(*args[0]) % pos); + + addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v); +} + +static void prim_path(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + Path path; + string name; + Value* filterFun = nullptr; + auto recursive = true; + Hash expectedHash; + + for (auto& attr : *args[0]->attrs) { + const string& n(attr.name); + if (n == "path") { + PathSet context; + path = state.coerceToPath(*attr.pos, *attr.value, context); + if (!context.empty()) + throw EvalError( + format("string '%1%' cannot refer to other paths, at %2%") % path % + *attr.pos); + } else if (attr.name == state.sName) + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "filter") { + state.forceValue(*attr.value); + filterFun = attr.value; + } else if (n == "recursive") + recursive = state.forceBool(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = + Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else + throw EvalError( + format("unsupported argument '%1%' to 'addPath', at %2%") % + attr.name % *attr.pos); + } + if (path.empty()) throw EvalError(format("'path' required, at %1%") % pos); + if (name.empty()) name = baseNameOf(path); - addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); + addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); } - /************************************************************* * Sets *************************************************************/ - /* Return the names of the attributes in a set as a sorted list of strings. */ -static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceAttrs(*args[0], pos); +static void prim_attrNames(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); - state.mkList(v, args[0]->attrs->size()); + state.mkList(v, args[0]->attrs->size()); - size_t n = 0; - for (auto & i : *args[0]->attrs) - mkString(*(v.listElems()[n++] = state.allocValue()), i.name); + size_t n = 0; + for (auto& i : *args[0]->attrs) + mkString(*(v.listElems()[n++] = state.allocValue()), i.name); - std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); + std::sort(v.listElems(), v.listElems() + n, [](Value* v1, Value* v2) { + return strcmp(v1->string.s, v2->string.s) < 0; + }); } - /* Return the values of the attributes in a set as a list, in the same order as attrNames. */ -static void prim_attrValues(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceAttrs(*args[0], pos); +static void prim_attrValues(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); - state.mkList(v, args[0]->attrs->size()); + state.mkList(v, args[0]->attrs->size()); - unsigned int n = 0; - for (auto & i : *args[0]->attrs) - v.listElems()[n++] = (Value *) &i; + unsigned int n = 0; + for (auto& i : *args[0]->attrs) v.listElems()[n++] = (Value*)&i; - std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return (string) ((Attr *) v1)->name < (string) ((Attr *) v2)->name; }); + std::sort(v.listElems(), v.listElems() + n, [](Value* v1, Value* v2) { + return (string)((Attr*)v1)->name < (string)((Attr*)v2)->name; + }); - for (unsigned int i = 0; i < n; ++i) - v.listElems()[i] = ((Attr *) v.listElems()[i])->value; + for (unsigned int i = 0; i < n; ++i) + v.listElems()[i] = ((Attr*)v.listElems()[i])->value; } - /* Dynamic version of the `.' operator. */ -void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - string attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); - // !!! Should we create a symbol here or just do a lookup? - Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); - if (i == args[1]->attrs->end()) - throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos); - // !!! add to stack trace? - if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; - state.forceValue(*i->value); - v = *i->value; +void prim_getAttr(EvalState& state, const Pos& pos, Value** args, Value& v) { + string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + // !!! Should we create a symbol here or just do a lookup? + Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); + if (i == args[1]->attrs->end()) + throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos); + // !!! add to stack trace? + if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; + state.forceValue(*i->value); + v = *i->value; } - /* Return position information of the specified attribute. */ -void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - string attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); - Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); - if (i == args[1]->attrs->end()) - mkNull(v); - else - state.mkPos(v, i->pos); +void prim_unsafeGetAttrPos(EvalState& state, const Pos& pos, Value** args, + Value& v) { + string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); + if (i == args[1]->attrs->end()) + mkNull(v); + else + state.mkPos(v, i->pos); } - /* Dynamic version of the `?' operator. */ -static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - string attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); - mkBool(v, args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); +static void prim_hasAttr(EvalState& state, const Pos& pos, Value** args, + Value& v) { + string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + mkBool(v, args[1]->attrs->find(state.symbols.create(attr)) != + args[1]->attrs->end()); } - /* Determine whether the argument is a set. */ -static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tAttrs); +static void prim_isAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tAttrs); } +static void prim_removeAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + state.forceList(*args[1], pos); -static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceAttrs(*args[0], pos); - state.forceList(*args[1], pos); - - /* Get the attribute names to be removed. */ - std::set<Symbol> names; - for (unsigned int i = 0; i < args[1]->listSize(); ++i) { - state.forceStringNoCtx(*args[1]->listElems()[i], pos); - names.insert(state.symbols.create(args[1]->listElems()[i]->string.s)); - } + /* Get the attribute names to be removed. */ + std::set<Symbol> names; + for (unsigned int i = 0; i < args[1]->listSize(); ++i) { + state.forceStringNoCtx(*args[1]->listElems()[i], pos); + names.insert(state.symbols.create(args[1]->listElems()[i]->string.s)); + } - /* Copy all attributes not in that set. Note that we don't need - to sort v.attrs because it's a subset of an already sorted - vector. */ - state.mkAttrs(v, args[0]->attrs->size()); - for (auto & i : *args[0]->attrs) { - if (names.find(i.name) == names.end()) - v.attrs->push_back(i); - } + /* Copy all attributes not in that set. Note that we don't need + to sort v.attrs because it's a subset of an already sorted + vector. */ + state.mkAttrs(v, args[0]->attrs->size()); + for (auto& i : *args[0]->attrs) { + if (names.find(i.name) == names.end()) v.attrs->push_back(i); + } } - /* Builds a set from a list specifying (name, value) pairs. To be precise, a list [{name = "name1"; value = value1;} ... {name = "nameN"; value = valueN;}] is transformed to {name1 = value1; ... nameN = valueN;}. In case of duplicate occurences of the same name, the first takes precedence. */ -static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceList(*args[0], pos); - - state.mkAttrs(v, args[0]->listSize()); - - std::set<Symbol> seen; - - for (unsigned int i = 0; i < args[0]->listSize(); ++i) { - Value & v2(*args[0]->listElems()[i]); - state.forceAttrs(v2, pos); - - Bindings::iterator j = v2.attrs->find(state.sName); - if (j == v2.attrs->end()) - throw TypeError(format("'name' attribute missing in a call to 'listToAttrs', at %1%") % pos); - string name = state.forceStringNoCtx(*j->value, pos); - - Symbol sym = state.symbols.create(name); - if (seen.find(sym) == seen.end()) { - Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); - if (j2 == v2.attrs->end()) - throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos); - - v.attrs->push_back(Attr(sym, j2->value, j2->pos)); - seen.insert(sym); - } +static void prim_listToAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + + state.mkAttrs(v, args[0]->listSize()); + + std::set<Symbol> seen; + + for (unsigned int i = 0; i < args[0]->listSize(); ++i) { + Value& v2(*args[0]->listElems()[i]); + state.forceAttrs(v2, pos); + + Bindings::iterator j = v2.attrs->find(state.sName); + if (j == v2.attrs->end()) + throw TypeError( + format( + "'name' attribute missing in a call to 'listToAttrs', at %1%") % + pos); + string name = state.forceStringNoCtx(*j->value, pos); + + Symbol sym = state.symbols.create(name); + if (seen.find(sym) == seen.end()) { + Bindings::iterator j2 = + v2.attrs->find(state.symbols.create(state.sValue)); + if (j2 == v2.attrs->end()) + throw TypeError(format("'value' attribute missing in a call to " + "'listToAttrs', at %1%") % + pos); + + v.attrs->push_back(Attr(sym, j2->value, j2->pos)); + seen.insert(sym); } + } - v.attrs->sort(); + v.attrs->sort(); } - /* Return the right-biased intersection of two sets as1 and as2, i.e. a set that contains every attribute from as2 that is also a member of as1. */ -static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceAttrs(*args[0], pos); - state.forceAttrs(*args[1], pos); +static void prim_intersectAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[1], pos); - state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size())); + state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size())); - for (auto & i : *args[0]->attrs) { - Bindings::iterator j = args[1]->attrs->find(i.name); - if (j != args[1]->attrs->end()) - v.attrs->push_back(*j); - } + for (auto& i : *args[0]->attrs) { + Bindings::iterator j = args[1]->attrs->find(i.name); + if (j != args[1]->attrs->end()) v.attrs->push_back(*j); + } } - /* Collect each attribute named `attr' from a list of attribute sets. Sets that don't contain the named attribute are ignored. @@ -1312,31 +1343,29 @@ static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * ar catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}] => [1 2] */ -static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - Symbol attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos)); - state.forceList(*args[1], pos); - - Value * res[args[1]->listSize()]; - unsigned int found = 0; - - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - Value & v2(*args[1]->listElems()[n]); - state.forceAttrs(v2, pos); - Bindings::iterator i = v2.attrs->find(attrName); - if (i != v2.attrs->end()) - res[found++] = i->value; - } +static void prim_catAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + Symbol attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos)); + state.forceList(*args[1], pos); - state.mkList(v, found); - for (unsigned int n = 0; n < found; ++n) - v.listElems()[n] = res[n]; -} + Value* res[args[1]->listSize()]; + unsigned int found = 0; + + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + Value& v2(*args[1]->listElems()[n]); + state.forceAttrs(v2, pos); + Bindings::iterator i = v2.attrs->find(attrName); + if (i != v2.attrs->end()) res[found++] = i->value; + } + state.mkList(v, found); + for (unsigned int n = 0; n < found; ++n) v.listElems()[n] = res[n]; +} /* Return a set containing the names of the formal arguments expected by the function `f'. The value of each attribute is a Boolean - denoting whether the corresponding argument has a default value. For instance, + denoting whether the corresponding argument has a default value. For + instance, functionArgs ({ x, y ? 123}: ...) => { x = false; y = true; } @@ -1347,983 +1376,936 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va functionArgs (x: ...) => { } */ -static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - if (args[0]->type != tLambda) - throw TypeError(format("'functionArgs' requires a function, at %1%") % pos); +static void prim_functionArgs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + if (args[0]->type != tLambda) + throw TypeError(format("'functionArgs' requires a function, at %1%") % pos); - if (!args[0]->lambda.fun->matchAttrs) { - state.mkAttrs(v, 0); - return; - } + if (!args[0]->lambda.fun->matchAttrs) { + state.mkAttrs(v, 0); + return; + } - state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); - for (auto & i : args[0]->lambda.fun->formals->formals) - // !!! should optimise booleans (allocate only once) - mkBool(*state.allocAttr(v, i.name), i.def); - v.attrs->sort(); + state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); + for (auto& i : args[0]->lambda.fun->formals->formals) + // !!! should optimise booleans (allocate only once) + mkBool(*state.allocAttr(v, i.name), i.def); + v.attrs->sort(); } - /* Apply a function to every element of an attribute set. */ -static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceAttrs(*args[1], pos); - - state.mkAttrs(v, args[1]->attrs->size()); - - for (auto & i : *args[1]->attrs) { - Value * vName = state.allocValue(); - Value * vFun2 = state.allocValue(); - mkString(*vName, i.name); - mkApp(*vFun2, *args[0], *vName); - mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value); - } -} +static void prim_mapAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[1], pos); + state.mkAttrs(v, args[1]->attrs->size()); + for (auto& i : *args[1]->attrs) { + Value* vName = state.allocValue(); + Value* vFun2 = state.allocValue(); + mkString(*vName, i.name); + mkApp(*vFun2, *args[0], *vName); + mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value); + } +} /************************************************************* * Lists *************************************************************/ - /* Determine whether the argument is a list. */ -static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->isList()); +static void prim_isList(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->isList()); } - -static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v) -{ - state.forceList(list, pos); - if (n < 0 || (unsigned int) n >= list.listSize()) - throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); - state.forceValue(*list.listElems()[n]); - v = *list.listElems()[n]; +static void elemAt(EvalState& state, const Pos& pos, Value& list, int n, + Value& v) { + state.forceList(list, pos); + if (n < 0 || (unsigned int)n >= list.listSize()) + throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); + state.forceValue(*list.listElems()[n]); + v = *list.listElems()[n]; } - /* Return the n-1'th element of a list. */ -static void prim_elemAt(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - elemAt(state, pos, *args[0], state.forceInt(*args[1], pos), v); +static void prim_elemAt(EvalState& state, const Pos& pos, Value** args, + Value& v) { + elemAt(state, pos, *args[0], state.forceInt(*args[1], pos), v); } - /* Return the first element of a list. */ -static void prim_head(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - elemAt(state, pos, *args[0], 0, v); +static void prim_head(EvalState& state, const Pos& pos, Value** args, + Value& v) { + elemAt(state, pos, *args[0], 0, v); } - /* Return a list consisting of everything but the first element of a list. Warning: this function takes O(n) time, so you probably don't want to use it! */ -static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceList(*args[0], pos); - if (args[0]->listSize() == 0) - throw Error(format("'tail' called on an empty list, at %1%") % pos); - state.mkList(v, args[0]->listSize() - 1); - for (unsigned int n = 0; n < v.listSize(); ++n) - v.listElems()[n] = args[0]->listElems()[n + 1]; +static void prim_tail(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + if (args[0]->listSize() == 0) + throw Error(format("'tail' called on an empty list, at %1%") % pos); + state.mkList(v, args[0]->listSize() - 1); + for (unsigned int n = 0; n < v.listSize(); ++n) + v.listElems()[n] = args[0]->listElems()[n + 1]; } - /* Apply a function to every element of a list. */ -static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceList(*args[1], pos); +static void prim_map(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceList(*args[1], pos); - state.mkList(v, args[1]->listSize()); + state.mkList(v, args[1]->listSize()); - for (unsigned int n = 0; n < v.listSize(); ++n) - mkApp(*(v.listElems()[n] = state.allocValue()), - *args[0], *args[1]->listElems()[n]); + for (unsigned int n = 0; n < v.listSize(); ++n) + mkApp(*(v.listElems()[n] = state.allocValue()), *args[0], + *args[1]->listElems()[n]); } - /* Filter a list using a predicate; that is, return a list containing every element from the list for which the predicate function returns true. */ -static void prim_filter(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); - - // FIXME: putting this on the stack is risky. - Value * vs[args[1]->listSize()]; - unsigned int k = 0; - - bool same = true; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - Value res; - state.callFunction(*args[0], *args[1]->listElems()[n], res, noPos); - if (state.forceBool(res, pos)) - vs[k++] = args[1]->listElems()[n]; - else - same = false; - } +static void prim_filter(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + // FIXME: putting this on the stack is risky. + Value* vs[args[1]->listSize()]; + unsigned int k = 0; + + bool same = true; + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + Value res; + state.callFunction(*args[0], *args[1]->listElems()[n], res, noPos); + if (state.forceBool(res, pos)) + vs[k++] = args[1]->listElems()[n]; + else + same = false; + } - if (same) - v = *args[1]; - else { - state.mkList(v, k); - for (unsigned int n = 0; n < k; ++n) v.listElems()[n] = vs[n]; - } + if (same) + v = *args[1]; + else { + state.mkList(v, k); + for (unsigned int n = 0; n < k; ++n) v.listElems()[n] = vs[n]; + } } - /* Return true if a list contains a given element. */ -static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - bool res = false; - state.forceList(*args[1], pos); - for (unsigned int n = 0; n < args[1]->listSize(); ++n) - if (state.eqValues(*args[0], *args[1]->listElems()[n])) { - res = true; - break; - } - mkBool(v, res); +static void prim_elem(EvalState& state, const Pos& pos, Value** args, + Value& v) { + bool res = false; + state.forceList(*args[1], pos); + for (unsigned int n = 0; n < args[1]->listSize(); ++n) + if (state.eqValues(*args[0], *args[1]->listElems()[n])) { + res = true; + break; + } + mkBool(v, res); } - /* Concatenate a list of lists. */ -static void prim_concatLists(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceList(*args[0], pos); - state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos); +static void prim_concatLists(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos); } - /* Return the length of a list. This is an O(1) time operation. */ -static void prim_length(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceList(*args[0], pos); - mkInt(v, args[0]->listSize()); +static void prim_length(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + mkInt(v, args[0]->listSize()); } - /* Reduce a list by applying a binary operator, from left to right. The operator is applied strictly. */ -static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceFunction(*args[0], pos); - state.forceList(*args[2], pos); - - if (args[2]->listSize()) { - Value * vCur = args[1]; - - for (unsigned int n = 0; n < args[2]->listSize(); ++n) { - Value vTmp; - state.callFunction(*args[0], *vCur, vTmp, pos); - vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); - state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); - } - state.forceValue(v); - } else { - state.forceValue(*args[1]); - v = *args[1]; +static void prim_foldlStrict(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[2], pos); + + if (args[2]->listSize()) { + Value* vCur = args[1]; + + for (unsigned int n = 0; n < args[2]->listSize(); ++n) { + Value vTmp; + state.callFunction(*args[0], *vCur, vTmp, pos); + vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); + state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); } + state.forceValue(v); + } else { + state.forceValue(*args[1]); + v = *args[1]; + } } +static void anyOrAll(bool any, EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); -static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); - - Value vTmp; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos); - bool res = state.forceBool(vTmp, pos); - if (res == any) { - mkBool(v, any); - return; - } + Value vTmp; + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos); + bool res = state.forceBool(vTmp, pos); + if (res == any) { + mkBool(v, any); + return; } + } - mkBool(v, !any); + mkBool(v, !any); } - -static void prim_any(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - anyOrAll(true, state, pos, args, v); +static void prim_any(EvalState& state, const Pos& pos, Value** args, Value& v) { + anyOrAll(true, state, pos, args, v); } - -static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - anyOrAll(false, state, pos, args, v); +static void prim_all(EvalState& state, const Pos& pos, Value** args, Value& v) { + anyOrAll(false, state, pos, args, v); } +static void prim_genList(EvalState& state, const Pos& pos, Value** args, + Value& v) { + auto len = state.forceInt(*args[1], pos); -static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - auto len = state.forceInt(*args[1], pos); + if (len < 0) + throw EvalError(format("cannot create list of size %1%, at %2%") % len % + pos); - if (len < 0) - throw EvalError(format("cannot create list of size %1%, at %2%") % len % pos); + state.mkList(v, len); - state.mkList(v, len); - - for (unsigned int n = 0; n < (unsigned int) len; ++n) { - Value * arg = state.allocValue(); - mkInt(*arg, n); - mkApp(*(v.listElems()[n] = state.allocValue()), *args[0], *arg); - } + for (unsigned int n = 0; n < (unsigned int)len; ++n) { + Value* arg = state.allocValue(); + mkInt(*arg, n); + mkApp(*(v.listElems()[n] = state.allocValue()), *args[0], *arg); + } } +static void prim_lessThan(EvalState& state, const Pos& pos, Value** args, + Value& v); -static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v); - - -static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); - - auto len = args[1]->listSize(); - state.mkList(v, len); - for (unsigned int n = 0; n < len; ++n) { - state.forceValue(*args[1]->listElems()[n]); - v.listElems()[n] = args[1]->listElems()[n]; - } +static void prim_sort(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + auto len = args[1]->listSize(); + state.mkList(v, len); + for (unsigned int n = 0; n < len; ++n) { + state.forceValue(*args[1]->listElems()[n]); + v.listElems()[n] = args[1]->listElems()[n]; + } - auto comparator = [&](Value * a, Value * b) { - /* Optimization: if the comparator is lessThan, bypass - callFunction. */ - if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan) - return CompareValues()(a, b); + auto comparator = [&](Value* a, Value* b) { + /* Optimization: if the comparator is lessThan, bypass + callFunction. */ + if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan) + return CompareValues()(a, b); - Value vTmp1, vTmp2; - state.callFunction(*args[0], *a, vTmp1, pos); - state.callFunction(vTmp1, *b, vTmp2, pos); - return state.forceBool(vTmp2, pos); - }; + Value vTmp1, vTmp2; + state.callFunction(*args[0], *a, vTmp1, pos); + state.callFunction(vTmp1, *b, vTmp2, pos); + return state.forceBool(vTmp2, pos); + }; - /* FIXME: std::sort can segfault if the comparator is not a strict - weak ordering. What to do? std::stable_sort() seems more - resilient, but no guarantees... */ - std::stable_sort(v.listElems(), v.listElems() + len, comparator); + /* FIXME: std::sort can segfault if the comparator is not a strict + weak ordering. What to do? std::stable_sort() seems more + resilient, but no guarantees... */ + std::stable_sort(v.listElems(), v.listElems() + len, comparator); } +static void prim_partition(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); -static void prim_partition(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + auto len = args[1]->listSize(); - auto len = args[1]->listSize(); + ValueVector right, wrong; - ValueVector right, wrong; - - for (unsigned int n = 0; n < len; ++n) { - auto vElem = args[1]->listElems()[n]; - state.forceValue(*vElem); - Value res; - state.callFunction(*args[0], *vElem, res, pos); - if (state.forceBool(res, pos)) - right.push_back(vElem); - else - wrong.push_back(vElem); - } + for (unsigned int n = 0; n < len; ++n) { + auto vElem = args[1]->listElems()[n]; + state.forceValue(*vElem); + Value res; + state.callFunction(*args[0], *vElem, res, pos); + if (state.forceBool(res, pos)) + right.push_back(vElem); + else + wrong.push_back(vElem); + } - state.mkAttrs(v, 2); + state.mkAttrs(v, 2); - Value * vRight = state.allocAttr(v, state.sRight); - auto rsize = right.size(); - state.mkList(*vRight, rsize); - if (rsize) - memcpy(vRight->listElems(), right.data(), sizeof(Value *) * rsize); + Value* vRight = state.allocAttr(v, state.sRight); + auto rsize = right.size(); + state.mkList(*vRight, rsize); + if (rsize) memcpy(vRight->listElems(), right.data(), sizeof(Value*) * rsize); - Value * vWrong = state.allocAttr(v, state.sWrong); - auto wsize = wrong.size(); - state.mkList(*vWrong, wsize); - if (wsize) - memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wsize); + Value* vWrong = state.allocAttr(v, state.sWrong); + auto wsize = wrong.size(); + state.mkList(*vWrong, wsize); + if (wsize) memcpy(vWrong->listElems(), wrong.data(), sizeof(Value*) * wsize); - v.attrs->sort(); + v.attrs->sort(); } - /* concatMap = f: list: concatLists (map f list); */ /* C++-version is to avoid allocating `mkApp', call `f' eagerly */ -static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); - auto nrLists = args[1]->listSize(); - - Value lists[nrLists]; - size_t len = 0; - - for (unsigned int n = 0; n < nrLists; ++n) { - Value * vElem = args[1]->listElems()[n]; - state.callFunction(*args[0], *vElem, lists[n], pos); - state.forceList(lists[n], pos); - len += lists[n].listSize(); - } - - state.mkList(v, len); - auto out = v.listElems(); - for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { - auto l = lists[n].listSize(); - if (l) - memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *)); - pos += l; - } +static void prim_concatMap(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + auto nrLists = args[1]->listSize(); + + Value lists[nrLists]; + size_t len = 0; + + for (unsigned int n = 0; n < nrLists; ++n) { + Value* vElem = args[1]->listElems()[n]; + state.callFunction(*args[0], *vElem, lists[n], pos); + state.forceList(lists[n], pos); + len += lists[n].listSize(); + } + + state.mkList(v, len); + auto out = v.listElems(); + for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { + auto l = lists[n].listSize(); + if (l) memcpy(out + pos, lists[n].listElems(), l * sizeof(Value*)); + pos += l; + } } - /************************************************************* * Integer arithmetic *************************************************************/ - -static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0], pos); - state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) - mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); - else - mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); +static void prim_add(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) + mkFloat(v, + state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); + else + mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); } - -static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0], pos); - state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) - mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); - else - mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); +static void prim_sub(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) + mkFloat(v, + state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); + else + mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); } - -static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0], pos); - state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) - mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); - else - mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); +static void prim_mul(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) + mkFloat(v, + state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); + else + mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); } +static void prim_div(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); -static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0], pos); - state.forceValue(*args[1], pos); - - NixFloat f2 = state.forceFloat(*args[1], pos); - if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos); + NixFloat f2 = state.forceFloat(*args[1], pos); + if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) { - mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); - } else { - NixInt i1 = state.forceInt(*args[0], pos); - NixInt i2 = state.forceInt(*args[1], pos); - /* Avoid division overflow as it might raise SIGFPE. */ - if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1) - throw EvalError(format("overflow in integer division, at %1%") % pos); - mkInt(v, i1 / i2); - } + if (args[0]->type == tFloat || args[1]->type == tFloat) { + mkFloat(v, + state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); + } else { + NixInt i1 = state.forceInt(*args[0], pos); + NixInt i2 = state.forceInt(*args[1], pos); + /* Avoid division overflow as it might raise SIGFPE. */ + if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1) + throw EvalError(format("overflow in integer division, at %1%") % pos); + mkInt(v, i1 / i2); + } } -static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); +static void prim_bitAnd(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); } -static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); +static void prim_bitOr(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); } -static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); +static void prim_bitXor(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); } -static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - state.forceValue(*args[1]); - CompareValues comp; - mkBool(v, comp(args[0], args[1])); +static void prim_lessThan(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + state.forceValue(*args[1]); + CompareValues comp; + mkBool(v, comp(args[0], args[1])); } - /************************************************************* * String manipulation *************************************************************/ - /* Convert the argument to a string. Paths are *not* copied to the store, so `toString /foo/bar' yields `"/foo/bar"', not `"/nix/store/whatever..."'. */ -static void prim_toString(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context, true, false); - mkString(v, s, context); +static void prim_toString(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + string s = state.coerceToString(pos, *args[0], context, true, false); + mkString(v, s, context); } - /* `substring start len str' returns the substring of `str' starting at character position `min(start, stringLength str)' inclusive and ending at `min(start + len, stringLength str)'. `start' must be non-negative. */ -static void prim_substring(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - int start = state.forceInt(*args[0], pos); - int len = state.forceInt(*args[1], pos); - PathSet context; - string s = state.coerceToString(pos, *args[2], context); +static void prim_substring(EvalState& state, const Pos& pos, Value** args, + Value& v) { + int start = state.forceInt(*args[0], pos); + int len = state.forceInt(*args[1], pos); + PathSet context; + string s = state.coerceToString(pos, *args[2], context); - if (start < 0) throw EvalError(format("negative start position in 'substring', at %1%") % pos); + if (start < 0) + throw EvalError(format("negative start position in 'substring', at %1%") % + pos); - mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); + mkString(v, (unsigned int)start >= s.size() ? "" : string(s, start, len), + context); } - -static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - mkInt(v, s.size()); +static void prim_stringLength(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + mkInt(v, s.size()); } - /* Return the cryptographic hash of a string in base-16. */ -static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - string type = state.forceStringNoCtx(*args[0], pos); - HashType ht = parseHashType(type); - if (ht == htUnknown) - throw Error(format("unknown hash type '%1%', at %2%") % type % pos); +static void prim_hashString(EvalState& state, const Pos& pos, Value** args, + Value& v) { + string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); - PathSet context; // discarded - string s = state.forceString(*args[1], context, pos); + PathSet context; // discarded + string s = state.forceString(*args[1], context, pos); - mkString(v, hashString(ht, s).to_string(Base16, false), context); + mkString(v, hashString(ht, s).to_string(Base16, false), context); } - /* Match a regular expression against a string and return either ‘null’ or a list containing substring matches. */ -static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - auto re = state.forceStringNoCtx(*args[0], pos); - - try { +static void prim_match(EvalState& state, const Pos& pos, Value** args, + Value& v) { + auto re = state.forceStringNoCtx(*args[0], pos); - std::regex regex(re, std::regex::extended); + try { + std::regex regex(re, std::regex::extended); - PathSet context; - const std::string str = state.forceString(*args[1], context, pos); + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); - std::smatch match; - if (!std::regex_match(str, match, regex)) { - mkNull(v); - return; - } + std::smatch match; + if (!std::regex_match(str, match, regex)) { + mkNull(v); + return; + } - // the first match is the whole string - const size_t len = match.size() - 1; - state.mkList(v, len); - for (size_t i = 0; i < len; ++i) { - if (!match[i+1].matched) - mkNull(*(v.listElems()[i] = state.allocValue())); - else - mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str()); - } + // the first match is the whole string + const size_t len = match.size() - 1; + state.mkList(v, len); + for (size_t i = 0; i < len; ++i) { + if (!match[i + 1].matched) + mkNull(*(v.listElems()[i] = state.allocValue())); + else + mkString(*(v.listElems()[i] = state.allocValue()), + match[i + 1].str().c_str()); + } - } catch (std::regex_error &e) { - if (e.code() == std::regex_constants::error_space) { - // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos); - } else { - throw EvalError("invalid regular expression '%s', at %s", re, pos); - } + } catch (std::regex_error& e) { + if (e.code() == std::regex_constants::error_space) { + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError("memory limit exceeded by regular expression '%s', at %s", + re, pos); + } else { + throw EvalError("invalid regular expression '%s', at %s", re, pos); } + } } - /* Split a string with a regular expression, and return a list of the non-matching parts interleaved by the lists of the matching groups. */ -static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - auto re = state.forceStringNoCtx(*args[0], pos); - - try { - - std::regex regex(re, std::regex::extended); - - PathSet context; - const std::string str = state.forceString(*args[1], context, pos); +static void prim_split(EvalState& state, const Pos& pos, Value** args, + Value& v) { + auto re = state.forceStringNoCtx(*args[0], pos); - auto begin = std::sregex_iterator(str.begin(), str.end(), regex); - auto end = std::sregex_iterator(); + try { + std::regex regex(re, std::regex::extended); - // Any matches results are surrounded by non-matching results. - const size_t len = std::distance(begin, end); - state.mkList(v, 2 * len + 1); - size_t idx = 0; - Value * elem; + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); - if (len == 0) { - v.listElems()[idx++] = args[1]; - return; - } + auto begin = std::sregex_iterator(str.begin(), str.end(), regex); + auto end = std::sregex_iterator(); - for (std::sregex_iterator i = begin; i != end; ++i) { - assert(idx <= 2 * len + 1 - 3); - std::smatch match = *i; - - // Add a string for non-matched characters. - elem = v.listElems()[idx++] = state.allocValue(); - mkString(*elem, match.prefix().str().c_str()); - - // Add a list for matched substrings. - const size_t slen = match.size() - 1; - elem = v.listElems()[idx++] = state.allocValue(); - - // Start at 1, beacause the first match is the whole string. - state.mkList(*elem, slen); - for (size_t si = 0; si < slen; ++si) { - if (!match[si + 1].matched) - mkNull(*(elem->listElems()[si] = state.allocValue())); - else - mkString(*(elem->listElems()[si] = state.allocValue()), match[si + 1].str().c_str()); - } - - // Add a string for non-matched suffix characters. - if (idx == 2 * len) { - elem = v.listElems()[idx++] = state.allocValue(); - mkString(*elem, match.suffix().str().c_str()); - } - } - assert(idx == 2 * len + 1); + // Any matches results are surrounded by non-matching results. + const size_t len = std::distance(begin, end); + state.mkList(v, 2 * len + 1); + size_t idx = 0; + Value* elem; - } catch (std::regex_error &e) { - if (e.code() == std::regex_constants::error_space) { - // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos); - } else { - throw EvalError("invalid regular expression '%s', at %s", re, pos); - } + if (len == 0) { + v.listElems()[idx++] = args[1]; + return; } -} + for (std::sregex_iterator i = begin; i != end; ++i) { + assert(idx <= 2 * len + 1 - 3); + std::smatch match = *i; -static void prim_concatStringSep(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - - auto sep = state.forceString(*args[0], context, pos); - state.forceList(*args[1], pos); + // Add a string for non-matched characters. + elem = v.listElems()[idx++] = state.allocValue(); + mkString(*elem, match.prefix().str().c_str()); - string res; - res.reserve((args[1]->listSize() + 32) * sep.size()); - bool first = true; + // Add a list for matched substrings. + const size_t slen = match.size() - 1; + elem = v.listElems()[idx++] = state.allocValue(); - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - if (first) first = false; else res += sep; - res += state.coerceToString(pos, *args[1]->listElems()[n], context); + // Start at 1, beacause the first match is the whole string. + state.mkList(*elem, slen); + for (size_t si = 0; si < slen; ++si) { + if (!match[si + 1].matched) + mkNull(*(elem->listElems()[si] = state.allocValue())); + else + mkString(*(elem->listElems()[si] = state.allocValue()), + match[si + 1].str().c_str()); + } + + // Add a string for non-matched suffix characters. + if (idx == 2 * len) { + elem = v.listElems()[idx++] = state.allocValue(); + mkString(*elem, match.suffix().str().c_str()); + } } + assert(idx == 2 * len + 1); - mkString(v, res, context); + } catch (std::regex_error& e) { + if (e.code() == std::regex_constants::error_space) { + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError("memory limit exceeded by regular expression '%s', at %s", + re, pos); + } else { + throw EvalError("invalid regular expression '%s', at %s", re, pos); + } + } } +static void prim_concatStringSep(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; -static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - state.forceList(*args[0], pos); - state.forceList(*args[1], pos); - if (args[0]->listSize() != args[1]->listSize()) - throw EvalError(format("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%") % pos); - - vector<string> from; - from.reserve(args[0]->listSize()); - for (unsigned int n = 0; n < args[0]->listSize(); ++n) - from.push_back(state.forceString(*args[0]->listElems()[n], pos)); + auto sep = state.forceString(*args[0], context, pos); + state.forceList(*args[1], pos); - vector<std::pair<string, PathSet>> to; - to.reserve(args[1]->listSize()); - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - PathSet ctx; - auto s = state.forceString(*args[1]->listElems()[n], ctx, pos); - to.push_back(std::make_pair(std::move(s), std::move(ctx))); - } + string res; + res.reserve((args[1]->listSize() + 32) * sep.size()); + bool first = true; - PathSet context; - auto s = state.forceString(*args[2], context, pos); - - string res; - // Loops one past last character to handle the case where 'from' contains an empty string. - for (size_t p = 0; p <= s.size(); ) { - bool found = false; - auto i = from.begin(); - auto j = to.begin(); - for (; i != from.end(); ++i, ++j) - if (s.compare(p, i->size(), *i) == 0) { - found = true; - res += j->first; - if (i->empty()) { - if (p < s.size()) - res += s[p]; - p++; - } else { - p += i->size(); - } - for (auto& path : j->second) - context.insert(path); - j->second.clear(); - break; - } - if (!found) { - if (p < s.size()) - res += s[p]; - p++; + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + if (first) + first = false; + else + res += sep; + res += state.coerceToString(pos, *args[1]->listElems()[n], context); + } + + mkString(v, res, context); +} + +static void prim_replaceStrings(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + state.forceList(*args[1], pos); + if (args[0]->listSize() != args[1]->listSize()) + throw EvalError(format("'from' and 'to' arguments to 'replaceStrings' have " + "different lengths, at %1%") % + pos); + + vector<string> from; + from.reserve(args[0]->listSize()); + for (unsigned int n = 0; n < args[0]->listSize(); ++n) + from.push_back(state.forceString(*args[0]->listElems()[n], pos)); + + vector<std::pair<string, PathSet>> to; + to.reserve(args[1]->listSize()); + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + PathSet ctx; + auto s = state.forceString(*args[1]->listElems()[n], ctx, pos); + to.push_back(std::make_pair(std::move(s), std::move(ctx))); + } + + PathSet context; + auto s = state.forceString(*args[2], context, pos); + + string res; + // Loops one past last character to handle the case where 'from' contains an + // empty string. + for (size_t p = 0; p <= s.size();) { + bool found = false; + auto i = from.begin(); + auto j = to.begin(); + for (; i != from.end(); ++i, ++j) + if (s.compare(p, i->size(), *i) == 0) { + found = true; + res += j->first; + if (i->empty()) { + if (p < s.size()) res += s[p]; + p++; + } else { + p += i->size(); } + for (auto& path : j->second) context.insert(path); + j->second.clear(); + break; + } + if (!found) { + if (p < s.size()) res += s[p]; + p++; } + } - mkString(v, res, context); + mkString(v, res, context); } - /************************************************************* * Versions *************************************************************/ - -static void prim_parseDrvName(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - string name = state.forceStringNoCtx(*args[0], pos); - DrvName parsed(name); - state.mkAttrs(v, 2); - mkString(*state.allocAttr(v, state.sName), parsed.name); - mkString(*state.allocAttr(v, state.symbols.create("version")), parsed.version); - v.attrs->sort(); -} - - -static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - string version1 = state.forceStringNoCtx(*args[0], pos); - string version2 = state.forceStringNoCtx(*args[1], pos); - mkInt(v, compareVersions(version1, version2)); +static void prim_parseDrvName(EvalState& state, const Pos& pos, Value** args, + Value& v) { + string name = state.forceStringNoCtx(*args[0], pos); + DrvName parsed(name); + state.mkAttrs(v, 2); + mkString(*state.allocAttr(v, state.sName), parsed.name); + mkString(*state.allocAttr(v, state.symbols.create("version")), + parsed.version); + v.attrs->sort(); +} + +static void prim_compareVersions(EvalState& state, const Pos& pos, Value** args, + Value& v) { + string version1 = state.forceStringNoCtx(*args[0], pos); + string version2 = state.forceStringNoCtx(*args[1], pos); + mkInt(v, compareVersions(version1, version2)); +} + +static void prim_splitVersion(EvalState& state, const Pos& pos, Value** args, + Value& v) { + string version = state.forceStringNoCtx(*args[0], pos); + auto iter = version.cbegin(); + Strings components; + while (iter != version.cend()) { + auto component = nextComponent(iter, version.cend()); + if (component.empty()) break; + components.emplace_back(std::move(component)); + } + state.mkList(v, components.size()); + unsigned int n = 0; + for (auto& component : components) { + auto listElem = v.listElems()[n++] = state.allocValue(); + mkString(*listElem, std::move(component)); + } } - -static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - string version = state.forceStringNoCtx(*args[0], pos); - auto iter = version.cbegin(); - Strings components; - while (iter != version.cend()) { - auto component = nextComponent(iter, version.cend()); - if (component.empty()) - break; - components.emplace_back(std::move(component)); - } - state.mkList(v, components.size()); - unsigned int n = 0; - for (auto & component : components) { - auto listElem = v.listElems()[n++] = state.allocValue(); - mkString(*listElem, std::move(component)); - } -} - - /************************************************************* * Networking *************************************************************/ +void fetch(EvalState& state, const Pos& pos, Value** args, Value& v, + const string& who, bool unpack, const std::string& defaultName) { + CachedDownloadRequest request(""); + request.unpack = unpack; + request.name = defaultName; -void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, - const string & who, bool unpack, const std::string & defaultName) -{ - CachedDownloadRequest request(""); - request.unpack = unpack; - request.name = defaultName; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { + state.forceValue(*args[0]); - state.forceAttrs(*args[0], pos); + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "sha256") - request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); - else if (n == "name") - request.name = state.forceStringNoCtx(*attr.value, *attr.pos); - else - throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); - } + for (auto& attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "sha256") + request.expectedHash = + Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else if (n == "name") + request.name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % + attr.name % who % attr.pos); + } - if (request.uri.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); + if (request.uri.empty()) + throw EvalError(format("'url' argument required, at %1%") % pos); - } else - request.uri = state.forceStringNoCtx(*args[0], pos); + } else + request.uri = state.forceStringNoCtx(*args[0], pos); - state.checkURI(request.uri); + state.checkURI(request.uri); - if (evalSettings.pureEval && !request.expectedHash) - throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + if (evalSettings.pureEval && !request.expectedHash) + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", + who); - auto res = getDownloader()->downloadCached(state.store, request); + auto res = getDownloader()->downloadCached(state.store, request); - if (state.allowedPaths) - state.allowedPaths->insert(res.path); + if (state.allowedPaths) state.allowedPaths->insert(res.path); - mkString(v, res.storePath, PathSet({res.storePath})); + mkString(v, res.storePath, PathSet({res.storePath})); } - -static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchurl", false, ""); +static void prim_fetchurl(EvalState& state, const Pos& pos, Value** args, + Value& v) { + fetch(state, pos, args, v, "fetchurl", false, ""); } - -static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchTarball", true, "source"); +static void prim_fetchTarball(EvalState& state, const Pos& pos, Value** args, + Value& v) { + fetch(state, pos, args, v, "fetchTarball", true, "source"); } - /************************************************************* * Primop registration *************************************************************/ - -RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; - - -RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) -{ - if (!primOps) primOps = new PrimOps; - primOps->emplace_back(name, arity, fun); -} - - -void EvalState::createBaseEnv() -{ - baseEnv.up = 0; - - /* Add global constants such as `true' to the base environment. */ - Value v; - - /* `builtins' must be first! */ - mkAttrs(v, 128); - addConstant("builtins", v); - - mkBool(v, true); - addConstant("true", v); - - mkBool(v, false); - addConstant("false", v); - - mkNull(v); - addConstant("null", v); - - auto vThrow = addPrimOp("throw", 1, prim_throw); - - auto addPurityError = [&](const std::string & name) { - Value * v2 = allocValue(); - mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name)); - mkApp(v, *vThrow, *v2); - addConstant(name, v); - }; - - if (!evalSettings.pureEval) { - mkInt(v, time(0)); - addConstant("__currentTime", v); - } - - if (!evalSettings.pureEval) { - mkString(v, settings.thisSystem); - addConstant("__currentSystem", v); - } - - mkString(v, nixVersion); - addConstant("__nixVersion", v); - - mkString(v, store->storeDir); - addConstant("__storeDir", v); - - /* Language version. This should be increased every time a new - language feature gets added. It's not necessary to increase it - when primops get added, because you can just use `builtins ? - primOp' to check. */ - mkInt(v, 5); - addConstant("__langVersion", v); - - // Miscellaneous - auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport); - Value * v2 = allocValue(); - mkAttrs(*v2, 0); - mkApp(v, *vScopedImport, *v2); - forceValue(v); - addConstant("import", v); - if (evalSettings.enableNativeCode) { - addPrimOp("__importNative", 2, prim_importNative); - addPrimOp("__exec", 1, prim_exec); - } - addPrimOp("__typeOf", 1, prim_typeOf); - addPrimOp("isNull", 1, prim_isNull); - addPrimOp("__isFunction", 1, prim_isFunction); - addPrimOp("__isString", 1, prim_isString); - addPrimOp("__isInt", 1, prim_isInt); - addPrimOp("__isFloat", 1, prim_isFloat); - addPrimOp("__isBool", 1, prim_isBool); - addPrimOp("__isPath", 1, prim_isPath); - addPrimOp("__genericClosure", 1, prim_genericClosure); - addPrimOp("abort", 1, prim_abort); - addPrimOp("__addErrorContext", 2, prim_addErrorContext); - addPrimOp("__tryEval", 1, prim_tryEval); - addPrimOp("__getEnv", 1, prim_getEnv); - - // Strictness - addPrimOp("__seq", 2, prim_seq); - addPrimOp("__deepSeq", 2, prim_deepSeq); - - // Debugging - addPrimOp("__trace", 2, prim_trace); - addPrimOp("__valueSize", 1, prim_valueSize); - - // Paths - addPrimOp("__toPath", 1, prim_toPath); - if (evalSettings.pureEval) - addPurityError("__storePath"); - else - addPrimOp("__storePath", 1, prim_storePath); - addPrimOp("__pathExists", 1, prim_pathExists); - addPrimOp("baseNameOf", 1, prim_baseNameOf); - addPrimOp("dirOf", 1, prim_dirOf); - addPrimOp("__readFile", 1, prim_readFile); - addPrimOp("__readDir", 1, prim_readDir); - addPrimOp("__findFile", 2, prim_findFile); - addPrimOp("__hashFile", 2, prim_hashFile); - - // Creating files - addPrimOp("__toXML", 1, prim_toXML); - addPrimOp("__toJSON", 1, prim_toJSON); - addPrimOp("__fromJSON", 1, prim_fromJSON); - addPrimOp("__toFile", 2, prim_toFile); - addPrimOp("__filterSource", 2, prim_filterSource); - addPrimOp("__path", 1, prim_path); - - // Sets - addPrimOp("__attrNames", 1, prim_attrNames); - addPrimOp("__attrValues", 1, prim_attrValues); - addPrimOp("__getAttr", 2, prim_getAttr); - addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos); - addPrimOp("__hasAttr", 2, prim_hasAttr); - addPrimOp("__isAttrs", 1, prim_isAttrs); - addPrimOp("removeAttrs", 2, prim_removeAttrs); - addPrimOp("__listToAttrs", 1, prim_listToAttrs); - addPrimOp("__intersectAttrs", 2, prim_intersectAttrs); - addPrimOp("__catAttrs", 2, prim_catAttrs); - addPrimOp("__functionArgs", 1, prim_functionArgs); - addPrimOp("__mapAttrs", 2, prim_mapAttrs); - - // Lists - addPrimOp("__isList", 1, prim_isList); - addPrimOp("__elemAt", 2, prim_elemAt); - addPrimOp("__head", 1, prim_head); - addPrimOp("__tail", 1, prim_tail); - addPrimOp("map", 2, prim_map); - addPrimOp("__filter", 2, prim_filter); - addPrimOp("__elem", 2, prim_elem); - addPrimOp("__concatLists", 1, prim_concatLists); - addPrimOp("__length", 1, prim_length); - addPrimOp("__foldl'", 3, prim_foldlStrict); - addPrimOp("__any", 2, prim_any); - addPrimOp("__all", 2, prim_all); - addPrimOp("__genList", 2, prim_genList); - addPrimOp("__sort", 2, prim_sort); - addPrimOp("__partition", 2, prim_partition); - addPrimOp("__concatMap", 2, prim_concatMap); - - // Integer arithmetic - addPrimOp("__add", 2, prim_add); - addPrimOp("__sub", 2, prim_sub); - addPrimOp("__mul", 2, prim_mul); - addPrimOp("__div", 2, prim_div); - addPrimOp("__bitAnd", 2, prim_bitAnd); - addPrimOp("__bitOr", 2, prim_bitOr); - addPrimOp("__bitXor", 2, prim_bitXor); - addPrimOp("__lessThan", 2, prim_lessThan); - - // String manipulation - addPrimOp("toString", 1, prim_toString); - addPrimOp("__substring", 3, prim_substring); - addPrimOp("__stringLength", 1, prim_stringLength); - addPrimOp("__hashString", 2, prim_hashString); - addPrimOp("__match", 2, prim_match); - addPrimOp("__split", 2, prim_split); - addPrimOp("__concatStringsSep", 2, prim_concatStringSep); - addPrimOp("__replaceStrings", 3, prim_replaceStrings); - - // Versions - addPrimOp("__parseDrvName", 1, prim_parseDrvName); - addPrimOp("__compareVersions", 2, prim_compareVersions); - addPrimOp("__splitVersion", 1, prim_splitVersion); - - // Derivations - addPrimOp("derivationStrict", 1, prim_derivationStrict); - addPrimOp("placeholder", 1, prim_placeholder); - - // Networking - addPrimOp("__fetchurl", 1, prim_fetchurl); - addPrimOp("fetchTarball", 1, prim_fetchTarball); - - /* Add a wrapper around the derivation primop that computes the - `drvPath' and `outPath' attributes lazily. */ - string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); - sDerivationNix = symbols.create(path); - evalFile(path, v); - addConstant("derivation", v); - - /* Add a value containing the current Nix expression search path. */ - mkList(v, searchPath.size()); - int n = 0; - for (auto & i : searchPath) { - v2 = v.listElems()[n++] = allocValue(); - mkAttrs(*v2, 2); - mkString(*allocAttr(*v2, symbols.create("path")), i.second); - mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); - v2->attrs->sort(); - } - addConstant("__nixPath", v); - - if (RegisterPrimOp::primOps) - for (auto & primOp : *RegisterPrimOp::primOps) - addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp)); - - /* Now that we've added all primops, sort the `builtins' set, - because attribute lookups expect it to be sorted. */ - baseEnv.values[0]->attrs->sort(); -} - - -} +RegisterPrimOp::PrimOps* RegisterPrimOp::primOps; + +RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) { + if (!primOps) primOps = new PrimOps; + primOps->emplace_back(name, arity, fun); +} + +void EvalState::createBaseEnv() { + baseEnv.up = 0; + + /* Add global constants such as `true' to the base environment. */ + Value v; + + /* `builtins' must be first! */ + mkAttrs(v, 128); + addConstant("builtins", v); + + mkBool(v, true); + addConstant("true", v); + + mkBool(v, false); + addConstant("false", v); + + mkNull(v); + addConstant("null", v); + + auto vThrow = addPrimOp("throw", 1, prim_throw); + + auto addPurityError = [&](const std::string& name) { + Value* v2 = allocValue(); + mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name)); + mkApp(v, *vThrow, *v2); + addConstant(name, v); + }; + + if (!evalSettings.pureEval) { + mkInt(v, time(0)); + addConstant("__currentTime", v); + } + + if (!evalSettings.pureEval) { + mkString(v, settings.thisSystem); + addConstant("__currentSystem", v); + } + + mkString(v, nixVersion); + addConstant("__nixVersion", v); + + mkString(v, store->storeDir); + addConstant("__storeDir", v); + + /* Language version. This should be increased every time a new + language feature gets added. It's not necessary to increase it + when primops get added, because you can just use `builtins ? + primOp' to check. */ + mkInt(v, 5); + addConstant("__langVersion", v); + + // Miscellaneous + auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport); + Value* v2 = allocValue(); + mkAttrs(*v2, 0); + mkApp(v, *vScopedImport, *v2); + forceValue(v); + addConstant("import", v); + if (evalSettings.enableNativeCode) { + addPrimOp("__importNative", 2, prim_importNative); + addPrimOp("__exec", 1, prim_exec); + } + addPrimOp("__typeOf", 1, prim_typeOf); + addPrimOp("isNull", 1, prim_isNull); + addPrimOp("__isFunction", 1, prim_isFunction); + addPrimOp("__isString", 1, prim_isString); + addPrimOp("__isInt", 1, prim_isInt); + addPrimOp("__isFloat", 1, prim_isFloat); + addPrimOp("__isBool", 1, prim_isBool); + addPrimOp("__isPath", 1, prim_isPath); + addPrimOp("__genericClosure", 1, prim_genericClosure); + addPrimOp("abort", 1, prim_abort); + addPrimOp("__addErrorContext", 2, prim_addErrorContext); + addPrimOp("__tryEval", 1, prim_tryEval); + addPrimOp("__getEnv", 1, prim_getEnv); + + // Strictness + addPrimOp("__seq", 2, prim_seq); + addPrimOp("__deepSeq", 2, prim_deepSeq); + + // Debugging + addPrimOp("__trace", 2, prim_trace); + addPrimOp("__valueSize", 1, prim_valueSize); + + // Paths + addPrimOp("__toPath", 1, prim_toPath); + if (evalSettings.pureEval) + addPurityError("__storePath"); + else + addPrimOp("__storePath", 1, prim_storePath); + addPrimOp("__pathExists", 1, prim_pathExists); + addPrimOp("baseNameOf", 1, prim_baseNameOf); + addPrimOp("dirOf", 1, prim_dirOf); + addPrimOp("__readFile", 1, prim_readFile); + addPrimOp("__readDir", 1, prim_readDir); + addPrimOp("__findFile", 2, prim_findFile); + addPrimOp("__hashFile", 2, prim_hashFile); + + // Creating files + addPrimOp("__toXML", 1, prim_toXML); + addPrimOp("__toJSON", 1, prim_toJSON); + addPrimOp("__fromJSON", 1, prim_fromJSON); + addPrimOp("__toFile", 2, prim_toFile); + addPrimOp("__filterSource", 2, prim_filterSource); + addPrimOp("__path", 1, prim_path); + + // Sets + addPrimOp("__attrNames", 1, prim_attrNames); + addPrimOp("__attrValues", 1, prim_attrValues); + addPrimOp("__getAttr", 2, prim_getAttr); + addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos); + addPrimOp("__hasAttr", 2, prim_hasAttr); + addPrimOp("__isAttrs", 1, prim_isAttrs); + addPrimOp("removeAttrs", 2, prim_removeAttrs); + addPrimOp("__listToAttrs", 1, prim_listToAttrs); + addPrimOp("__intersectAttrs", 2, prim_intersectAttrs); + addPrimOp("__catAttrs", 2, prim_catAttrs); + addPrimOp("__functionArgs", 1, prim_functionArgs); + addPrimOp("__mapAttrs", 2, prim_mapAttrs); + + // Lists + addPrimOp("__isList", 1, prim_isList); + addPrimOp("__elemAt", 2, prim_elemAt); + addPrimOp("__head", 1, prim_head); + addPrimOp("__tail", 1, prim_tail); + addPrimOp("map", 2, prim_map); + addPrimOp("__filter", 2, prim_filter); + addPrimOp("__elem", 2, prim_elem); + addPrimOp("__concatLists", 1, prim_concatLists); + addPrimOp("__length", 1, prim_length); + addPrimOp("__foldl'", 3, prim_foldlStrict); + addPrimOp("__any", 2, prim_any); + addPrimOp("__all", 2, prim_all); + addPrimOp("__genList", 2, prim_genList); + addPrimOp("__sort", 2, prim_sort); + addPrimOp("__partition", 2, prim_partition); + addPrimOp("__concatMap", 2, prim_concatMap); + + // Integer arithmetic + addPrimOp("__add", 2, prim_add); + addPrimOp("__sub", 2, prim_sub); + addPrimOp("__mul", 2, prim_mul); + addPrimOp("__div", 2, prim_div); + addPrimOp("__bitAnd", 2, prim_bitAnd); + addPrimOp("__bitOr", 2, prim_bitOr); + addPrimOp("__bitXor", 2, prim_bitXor); + addPrimOp("__lessThan", 2, prim_lessThan); + + // String manipulation + addPrimOp("toString", 1, prim_toString); + addPrimOp("__substring", 3, prim_substring); + addPrimOp("__stringLength", 1, prim_stringLength); + addPrimOp("__hashString", 2, prim_hashString); + addPrimOp("__match", 2, prim_match); + addPrimOp("__split", 2, prim_split); + addPrimOp("__concatStringsSep", 2, prim_concatStringSep); + addPrimOp("__replaceStrings", 3, prim_replaceStrings); + + // Versions + addPrimOp("__parseDrvName", 1, prim_parseDrvName); + addPrimOp("__compareVersions", 2, prim_compareVersions); + addPrimOp("__splitVersion", 1, prim_splitVersion); + + // Derivations + addPrimOp("derivationStrict", 1, prim_derivationStrict); + addPrimOp("placeholder", 1, prim_placeholder); + + // Networking + addPrimOp("__fetchurl", 1, prim_fetchurl); + addPrimOp("fetchTarball", 1, prim_fetchTarball); + + /* Add a wrapper around the derivation primop that computes the + `drvPath' and `outPath' attributes lazily. */ + string path = + canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); + sDerivationNix = symbols.create(path); + evalFile(path, v); + addConstant("derivation", v); + + /* Add a value containing the current Nix expression search path. */ + mkList(v, searchPath.size()); + int n = 0; + for (auto& i : searchPath) { + v2 = v.listElems()[n++] = allocValue(); + mkAttrs(*v2, 2); + mkString(*allocAttr(*v2, symbols.create("path")), i.second); + mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); + v2->attrs->sort(); + } + addConstant("__nixPath", v); + + if (RegisterPrimOp::primOps) + for (auto& primOp : *RegisterPrimOp::primOps) + addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp)); + + /* Now that we've added all primops, sort the `builtins' set, + because attribute lookups expect it to be sorted. */ + baseEnv.values[0]->attrs->sort(); +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops.hh b/third_party/nix/src/libexpr/primops.hh index c790b30f6d0b..60326323b2e0 100644 --- a/third_party/nix/src/libexpr/primops.hh +++ b/third_party/nix/src/libexpr/primops.hh @@ -1,26 +1,25 @@ -#include "eval.hh" - #include <tuple> #include <vector> +#include "eval.hh" namespace nix { -struct RegisterPrimOp -{ - typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps; - static PrimOps * primOps; - /* You can register a constant by passing an arity of 0. fun - will get called during EvalState initialization, so there - may be primops not yet added and builtins is not yet sorted. */ - RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); +struct RegisterPrimOp { + typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps; + static PrimOps* primOps; + /* You can register a constant by passing an arity of 0. fun + will get called during EvalState initialization, so there + may be primops not yet added and builtins is not yet sorted. */ + RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); }; /* These primops are disabled without enableNativeCode, but plugins may wish to use them in limited contexts without globally enabling them. */ /* Load a ValueInitializer from a DSO and return whatever it initializes */ -void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); +void prim_importNative(EvalState& state, const Pos& pos, Value** args, + Value& v); /* Execute a program and parse its output */ -void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v); +void prim_exec(EvalState& state, const Pos& pos, Value** args, Value& v); -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/context.cc b/third_party/nix/src/libexpr/primops/context.cc index 2d79739ea047..13faeef8ce9c 100644 --- a/third_party/nix/src/libexpr/primops/context.cc +++ b/third_party/nix/src/libexpr/primops/context.cc @@ -1,49 +1,47 @@ -#include "primops.hh" -#include "eval-inline.hh" #include "derivations.hh" +#include "eval-inline.hh" +#include "primops.hh" namespace nix { -static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - mkString(v, s, PathSet()); +static void prim_unsafeDiscardStringContext(EvalState& state, const Pos& pos, + Value** args, Value& v) { + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + mkString(v, s, PathSet()); } -static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); - +static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, + prim_unsafeDiscardStringContext); -static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - state.forceString(*args[0], context, pos); - mkBool(v, !context.empty()); +static void prim_hasContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + state.forceString(*args[0], context, pos); + mkBool(v, !context.empty()); } static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); - /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a builder without causing the derivation to be built (for instance, in the derivation that builds NARs in nix-push, when doing source-only deployment). This primop marks the string context so that builtins.derivation adds the path to drv.inputSrcs rather than drv.inputDrvs. */ -static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); +static void prim_unsafeDiscardOutputDependency(EvalState& state, const Pos& pos, + Value** args, Value& v) { + PathSet context; + string s = state.coerceToString(pos, *args[0], context); - PathSet context2; - for (auto & p : context) - context2.insert(p.at(0) == '=' ? string(p, 1) : p); + PathSet context2; + for (auto& p : context) context2.insert(p.at(0) == '=' ? string(p, 1) : p); - mkString(v, s, context2); + mkString(v, s, context2); } -static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); - +static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, + prim_unsafeDiscardOutputDependency); /* Extract the context of a string as a structured Nix value. @@ -64,124 +62,131 @@ static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscar Note that for a given path any combination of the above attributes may be present. */ -static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - struct ContextInfo { - bool path = false; - bool allOutputs = false; - Strings outputs; - }; - PathSet context; - state.forceString(*args[0], context, pos); - auto contextInfos = std::map<Path, ContextInfo>(); - for (const auto & p : context) { - Path drv; - string output; - const Path * path = &p; - if (p.at(0) == '=') { - drv = string(p, 1); - path = &drv; - } else if (p.at(0) == '!') { - std::pair<string, string> ctx = decodeContext(p); - drv = ctx.first; - output = ctx.second; - path = &drv; - } - auto isPath = drv.empty(); - auto isAllOutputs = (!drv.empty()) && output.empty(); - - auto iter = contextInfos.find(*path); - if (iter == contextInfos.end()) { - contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); - } else { - if (isPath) - iter->second.path = true; - else if (isAllOutputs) - iter->second.allOutputs = true; - else - iter->second.outputs.emplace_back(std::move(output)); - } +static void prim_getContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + struct ContextInfo { + bool path = false; + bool allOutputs = false; + Strings outputs; + }; + PathSet context; + state.forceString(*args[0], context, pos); + auto contextInfos = std::map<Path, ContextInfo>(); + for (const auto& p : context) { + Path drv; + string output; + const Path* path = &p; + if (p.at(0) == '=') { + drv = string(p, 1); + path = &drv; + } else if (p.at(0) == '!') { + std::pair<string, string> ctx = decodeContext(p); + drv = ctx.first; + output = ctx.second; + path = &drv; } - - state.mkAttrs(v, contextInfos.size()); - - auto sPath = state.symbols.create("path"); - auto sAllOutputs = state.symbols.create("allOutputs"); - for (const auto & info : contextInfos) { - auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first)); - state.mkAttrs(infoVal, 3); - if (info.second.path) - mkBool(*state.allocAttr(infoVal, sPath), true); - if (info.second.allOutputs) - mkBool(*state.allocAttr(infoVal, sAllOutputs), true); - if (!info.second.outputs.empty()) { - auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); - state.mkList(outputsVal, info.second.outputs.size()); - size_t i = 0; - for (const auto & output : info.second.outputs) { - mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); - } - } - infoVal.attrs->sort(); + auto isPath = drv.empty(); + auto isAllOutputs = (!drv.empty()) && output.empty(); + + auto iter = contextInfos.find(*path); + if (iter == contextInfos.end()) { + contextInfos.emplace( + *path, + ContextInfo{isPath, isAllOutputs, + output.empty() ? Strings{} : Strings{std::move(output)}}); + } else { + if (isPath) + iter->second.path = true; + else if (isAllOutputs) + iter->second.allOutputs = true; + else + iter->second.outputs.emplace_back(std::move(output)); + } + } + + state.mkAttrs(v, contextInfos.size()); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (const auto& info : contextInfos) { + auto& infoVal = *state.allocAttr(v, state.symbols.create(info.first)); + state.mkAttrs(infoVal, 3); + if (info.second.path) mkBool(*state.allocAttr(infoVal, sPath), true); + if (info.second.allOutputs) + mkBool(*state.allocAttr(infoVal, sAllOutputs), true); + if (!info.second.outputs.empty()) { + auto& outputsVal = *state.allocAttr(infoVal, state.sOutputs); + state.mkList(outputsVal, info.second.outputs.size()); + size_t i = 0; + for (const auto& output : info.second.outputs) { + mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); + } } - v.attrs->sort(); + infoVal.attrs->sort(); + } + v.attrs->sort(); } static RegisterPrimOp r4("__getContext", 1, prim_getContext); - /* Append the given context to a given string. See the commentary above unsafeGetContext for details of the context representation. */ -static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - auto orig = state.forceString(*args[0], context, pos); - - state.forceAttrs(*args[1], pos); - - auto sPath = state.symbols.create("path"); - auto sAllOutputs = state.symbols.create("allOutputs"); - for (auto & i : *args[1]->attrs) { - if (!state.store->isStorePath(i.name)) - throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); - if (!settings.readOnlyMode) - state.store->ensurePath(i.name); - state.forceAttrs(*i.value, *i.pos); - auto iter = i.value->attrs->find(sPath); - if (iter != i.value->attrs->end()) { - if (state.forceBool(*iter->value, *iter->pos)) - context.insert(i.name); - } +static void prim_appendContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + auto orig = state.forceString(*args[0], context, pos); + + state.forceAttrs(*args[1], pos); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (auto& i : *args[1]->attrs) { + if (!state.store->isStorePath(i.name)) + throw EvalError("Context key '%s' is not a store path, at %s", i.name, + i.pos); + if (!settings.readOnlyMode) state.store->ensurePath(i.name); + state.forceAttrs(*i.value, *i.pos); + auto iter = i.value->attrs->find(sPath); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) context.insert(i.name); + } - iter = i.value->attrs->find(sAllOutputs); - if (iter != i.value->attrs->end()) { - if (state.forceBool(*iter->value, *iter->pos)) { - if (!isDerivation(i.name)) { - throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); - } - context.insert("=" + string(i.name)); - } + iter = i.value->attrs->find(sAllOutputs); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) { + if (!isDerivation(i.name)) { + throw EvalError( + "Tried to add all-outputs context of %s, which is not a " + "derivation, to a string, at %s", + i.name, i.pos); } + context.insert("=" + string(i.name)); + } + } - iter = i.value->attrs->find(state.sOutputs); - if (iter != i.value->attrs->end()) { - state.forceList(*iter->value, *iter->pos); - if (iter->value->listSize() && !isDerivation(i.name)) { - throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); - } - for (unsigned int n = 0; n < iter->value->listSize(); ++n) { - auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); - context.insert("!" + name + "!" + string(i.name)); - } - } + iter = i.value->attrs->find(state.sOutputs); + if (iter != i.value->attrs->end()) { + state.forceList(*iter->value, *iter->pos); + if (iter->value->listSize() && !isDerivation(i.name)) { + throw EvalError( + "Tried to add derivation output context of %s, which is not a " + "derivation, to a string, at %s", + i.name, i.pos); + } + for (unsigned int n = 0; n < iter->value->listSize(); ++n) { + auto name = + state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); + context.insert("!" + name + "!" + string(i.name)); + } } + } - mkString(v, orig, context); + mkString(v, orig, context); } static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/fetchGit.cc b/third_party/nix/src/libexpr/primops/fetchGit.cc index 90f600284b22..e1edf9a3e58e 100644 --- a/third_party/nix/src/libexpr/primops/fetchGit.cc +++ b/third_party/nix/src/libexpr/primops/fetchGit.cc @@ -1,247 +1,253 @@ -#include "primops.hh" -#include "eval-inline.hh" -#include "download.hh" -#include "store-api.hh" -#include "pathlocks.hh" -#include "hash.hh" - #include <sys/time.h> - -#include <regex> - #include <nlohmann/json.hpp> +#include <regex> +#include "download.hh" +#include "eval-inline.hh" +#include "hash.hh" +#include "pathlocks.hh" +#include "primops.hh" +#include "store-api.hh" using namespace std::string_literals; namespace nix { -struct GitInfo -{ - Path storePath; - std::string rev; - std::string shortRev; - uint64_t revCount = 0; +struct GitInfo { + Path storePath; + std::string rev; + std::string shortRev; + uint64_t revCount = 0; }; std::regex revRegex("^[0-9a-fA-F]{40}$"); -GitInfo exportGit(ref<Store> store, const std::string & uri, - std::optional<std::string> ref, std::string rev, - const std::string & name) -{ - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - - if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { - - bool clean = true; - - try { - runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - clean = false; - } - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked - files. */ - - GitInfo gitInfo; - gitInfo.rev = "0000000000000000000000000000000000000000"; - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - - auto files = tokenizeString<std::set<std::string>>( - runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, uri)); - std::string file(p, uri.size() + 1); - - auto st = lstat(p); +GitInfo exportGit(ref<Store> store, const std::string& uri, + std::optional<std::string> ref, std::string rev, + const std::string& name) { + if (evalSettings.pureEval && rev == "") + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } + if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { + bool clean = true; - return files.count(file); - }; - - gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); - - return gitInfo; - } - - // clean working tree, but no ref or rev specified. Use 'HEAD'. - rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })); - ref = "HEAD"s; + try { + runProgram("git", true, + {"-C", uri, "diff-index", "--quiet", "HEAD", "--"}); + } catch (ExecError& e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + clean = false; } - if (!ref) ref = "HEAD"s; + if (!clean) { + /* This is an unclean working tree. So copy all tracked + files. */ - if (rev != "" && !std::regex_match(rev, revRegex)) - throw Error("invalid Git revision '%s'", rev); + GitInfo gitInfo; + gitInfo.rev = "0000000000000000000000000000000000000000"; + gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - deletePath(getCacheDir() + "/nix/git"); + auto files = tokenizeString<std::set<std::string>>( + runProgram("git", true, {"-C", uri, "ls-files", "-z"}), "\0"s); - Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); + PathFilter filter = [&](const Path& p) -> bool { + assert(hasPrefix(p, uri)); + std::string file(p, uri.size() + 1); - if (!pathExists(cacheDir)) { - createDirs(dirOf(cacheDir)); - runProgram("git", true, { "init", "--bare", cacheDir }); - } + auto st = lstat(p); - Path localRefFile; - if (ref->compare(0, 5, "refs/") == 0) - localRefFile = cacheDir + "/" + *ref; - else - localRefFile = cacheDir + "/refs/heads/" + *ref; - - bool doFetch; - time_t now = time(0); - /* If a rev was specified, we need to fetch if it's not in the - repo. */ - if (rev != "") { - try { - runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); } - } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; - } - if (doFetch) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + return files.count(file); + }; - struct timeval times[2]; - times[0].tv_sec = now; - times[0].tv_usec = 0; - times[1].tv_sec = now; - times[1].tv_usec = 0; + gitInfo.storePath = + store->addToStore("source", uri, true, htSHA256, filter); - utimes(localRefFile.c_str(), times); + return gitInfo; } - // FIXME: check whether rev is an ancestor of ref. - GitInfo gitInfo; - gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + // clean working tree, but no ref or rev specified. Use 'HEAD'. + rev = chomp(runProgram("git", true, {"-C", uri, "rev-parse", "HEAD"})); + ref = "HEAD"s; + } - printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); + if (!ref) ref = "HEAD"s; - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); - Path storeLink = cacheDir + "/" + storeLinkName + ".link"; - PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken + if (rev != "" && !std::regex_match(rev, revRegex)) + throw Error("invalid Git revision '%s'", rev); - try { - auto json = nlohmann::json::parse(readFile(storeLink)); + deletePath(getCacheDir() + "/nix/git"); - assert(json["name"] == name && json["rev"] == gitInfo.rev); + Path cacheDir = getCacheDir() + "/nix/gitv2/" + + hashString(htSHA256, uri).to_string(Base32, false); - gitInfo.storePath = json["storePath"]; + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, {"init", "--bare", cacheDir}); + } - if (store->isValidPath(gitInfo.storePath)) { - gitInfo.revCount = json["revCount"]; - return gitInfo; - } + Path localRefFile; + if (ref->compare(0, 5, "refs/") == 0) + localRefFile = cacheDir + "/" + *ref; + else + localRefFile = cacheDir + "/refs/heads/" + *ref; - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; + bool doFetch; + time_t now = time(0); + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (rev != "") { + try { + runProgram("git", true, {"-C", cacheDir, "cat-file", "-e", rev}); + doFetch = false; + } catch (ExecError& e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now; + } + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, + fmt("fetching Git repository '%s'", uri)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + runProgram("git", true, + {"-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, + fmt("%s:%s", *ref, *ref)}); + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + // FIXME: check whether rev is an ancestor of ref. + GitInfo gitInfo; + gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); + gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + + printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); + + std::string storeLinkName = + hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev) + .to_string(Base32, false); + Path storeLink = cacheDir + "/" + storeLinkName + ".link"; + PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", + storeLink)); // FIXME: broken + + try { + auto json = nlohmann::json::parse(readFile(storeLink)); + + assert(json["name"] == name && json["rev"] == gitInfo.rev); + + gitInfo.storePath = json["storePath"]; + + if (store->isValidPath(gitInfo.storePath)) { + gitInfo.revCount = json["revCount"]; + return gitInfo; + } + + } catch (SysError& e) { + if (e.errNo != ENOENT) throw; + } - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto tar = runProgram("git", true, {"-C", cacheDir, "archive", gitInfo.rev}); - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); - runProgram("tar", true, { "x", "-C", tmpDir }, tar); + runProgram("tar", true, {"x", "-C", tmpDir}, tar); - gitInfo.storePath = store->addToStore(name, tmpDir); + gitInfo.storePath = store->addToStore(name, tmpDir); - gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); + gitInfo.revCount = std::stoull(runProgram( + "git", true, {"-C", cacheDir, "rev-list", "--count", gitInfo.rev})); - nlohmann::json json; - json["storePath"] = gitInfo.storePath; - json["uri"] = uri; - json["name"] = name; - json["rev"] = gitInfo.rev; - json["revCount"] = gitInfo.revCount; + nlohmann::json json; + json["storePath"] = gitInfo.storePath; + json["uri"] = uri; + json["name"] = name; + json["rev"] = gitInfo.rev; + json["revCount"] = gitInfo.revCount; - writeFile(storeLink, json.dump()); + writeFile(storeLink, json.dump()); - return gitInfo; + return gitInfo; } -static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - std::string url; - std::optional<std::string> ref; - std::string rev; - std::string name = "source"; - PathSet context; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - else if (n == "ref") - ref = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); - else - throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); - } +static void prim_fetchGit(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string url; + std::optional<std::string> ref; + std::string rev; + std::string name = "source"; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + for (auto& attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = + state.coerceToString(*attr.pos, *attr.value, context, false, false); + else if (n == "ref") + ref = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "rev") + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", + attr.name, *attr.pos); + } - if (url.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); + if (url.empty()) + throw EvalError(format("'url' argument required, at %1%") % pos); - } else - url = state.coerceToString(pos, *args[0], context, false, false); + } else + url = state.coerceToString(pos, *args[0], context, false, false); - // FIXME: git externals probably can be used to bypass the URI - // whitelist. Ah well. - state.checkURI(url); + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); - auto gitInfo = exportGit(state.store, url, ref, rev, name); + auto gitInfo = exportGit(state.store, url, ref, rev, name); - state.mkAttrs(v, 8); - mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); - v.attrs->sort(); + state.mkAttrs(v, 8); + mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, + PathSet({gitInfo.storePath})); + mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), + gitInfo.shortRev); + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), + gitInfo.revCount); + v.attrs->sort(); - if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); + if (state.allowedPaths) + state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/fetchMercurial.cc b/third_party/nix/src/libexpr/primops/fetchMercurial.cc index a907d0e1cd82..1ee12542ef32 100644 --- a/third_party/nix/src/libexpr/primops/fetchMercurial.cc +++ b/third_party/nix/src/libexpr/primops/fetchMercurial.cc @@ -1,219 +1,229 @@ -#include "primops.hh" -#include "eval-inline.hh" -#include "download.hh" -#include "store-api.hh" -#include "pathlocks.hh" - #include <sys/time.h> - -#include <regex> - #include <nlohmann/json.hpp> +#include <regex> +#include "download.hh" +#include "eval-inline.hh" +#include "pathlocks.hh" +#include "primops.hh" +#include "store-api.hh" using namespace std::string_literals; namespace nix { -struct HgInfo -{ - Path storePath; - std::string branch; - std::string rev; - uint64_t revCount = 0; +struct HgInfo { + Path storePath; + std::string branch; + std::string rev; + uint64_t revCount = 0; }; std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); -HgInfo exportMercurial(ref<Store> store, const std::string & uri, - std::string rev, const std::string & name) -{ - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); - - if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { - - bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; +HgInfo exportMercurial(ref<Store> store, const std::string& uri, + std::string rev, const std::string& name) { + if (evalSettings.pureEval && rev == "") + throw Error( + "in pure evaluation mode, 'fetchMercurial' requires a Mercurial " + "revision"); - if (!clean) { + if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { + bool clean = runProgram("hg", true, + {"status", "-R", uri, "--modified", "--added", + "--removed"}) == ""; - /* This is an unclean working tree. So copy all tracked - files. */ + if (!clean) { + /* This is an unclean working tree. So copy all tracked + files. */ - printTalkative("copying unclean Mercurial working tree '%s'", uri); + printTalkative("copying unclean Mercurial working tree '%s'", uri); - HgInfo hgInfo; - hgInfo.rev = "0000000000000000000000000000000000000000"; - hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri })); + HgInfo hgInfo; + hgInfo.rev = "0000000000000000000000000000000000000000"; + hgInfo.branch = chomp(runProgram("hg", true, {"branch", "-R", uri})); - auto files = tokenizeString<std::set<std::string>>( - runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + auto files = tokenizeString<std::set<std::string>>( + runProgram("hg", true, + {"status", "-R", uri, "--clean", "--modified", "--added", + "--no-status", "--print0"}), + "\0"s); - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, uri)); - std::string file(p, uri.size() + 1); + PathFilter filter = [&](const Path& p) -> bool { + assert(hasPrefix(p, uri)); + std::string file(p, uri.size() + 1); - auto st = lstat(p); + auto st = lstat(p); - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } - return files.count(file); - }; + return files.count(file); + }; - hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); + hgInfo.storePath = + store->addToStore("source", uri, true, htSHA256, filter); - return hgInfo; - } + return hgInfo; } - - if (rev == "") rev = "default"; - - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false)); - - Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false)); - - /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, - do so now. */ - time_t now = time(0); - struct stat st; - if (stat(stampFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) - { - /* Except that if this is a commit hash that we already have, - we don't have to pull again. */ - if (!(std::regex_match(rev, commitHashRegex) - && pathExists(cacheDir) - && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" }) - .killStderr(true)).second == "1")) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri)); - - if (pathExists(cacheDir)) { - try { - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } - catch (ExecError & e) { - string transJournal = cacheDir + "/.hg/store/journal"; - /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) { - runProgram("hg", true, { "recover", "-R", cacheDir }); - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } else { - throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); - } - } - } else { - createDirs(dirOf(cacheDir)); - runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir }); - } + } + + if (rev == "") rev = "default"; + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), + hashString(htSHA256, uri).to_string(Base32, false)); + + Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, + hashString(htSHA512, rev).to_string(Base32, false)); + + /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, + do so now. */ + time_t now = time(0); + struct stat st; + if (stat(stampFile.c_str(), &st) != 0 || + (uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now) { + /* Except that if this is a commit hash that we already have, + we don't have to pull again. */ + if (!(std::regex_match(rev, commitHashRegex) && pathExists(cacheDir) && + runProgram(RunOptions("hg", {"log", "-R", cacheDir, "-r", rev, + "--template", "1"}) + .killStderr(true)) + .second == "1")) { + Activity act(*logger, lvlTalkative, actUnknown, + fmt("fetching Mercurial repository '%s'", uri)); + + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri}); + } catch (ExecError& e) { + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, {"recover", "-R", cacheDir}); + runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri}); + } else { + throw ExecError(e.status, + fmt("'hg pull' %s", statusToString(e.status))); + } } - - writeFile(stampFile, ""); + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, {"clone", "--noupdate", "--", uri, cacheDir}); + } } - auto tokens = tokenizeString<std::vector<std::string>>( - runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" })); - assert(tokens.size() == 3); + writeFile(stampFile, ""); + } - HgInfo hgInfo; - hgInfo.rev = tokens[0]; - hgInfo.revCount = std::stoull(tokens[1]); - hgInfo.branch = tokens[2]; + auto tokens = tokenizeString<std::vector<std::string>>( + runProgram("hg", true, + {"log", "-R", cacheDir, "-r", rev, "--template", + "{node} {rev} {branch}"})); + assert(tokens.size() == 3); - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false); - Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); + HgInfo hgInfo; + hgInfo.rev = tokens[0]; + hgInfo.revCount = std::stoull(tokens[1]); + hgInfo.branch = tokens[2]; - try { - auto json = nlohmann::json::parse(readFile(storeLink)); + std::string storeLinkName = + hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev) + .to_string(Base32, false); + Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); - assert(json["name"] == name && json["rev"] == hgInfo.rev); + try { + auto json = nlohmann::json::parse(readFile(storeLink)); - hgInfo.storePath = json["storePath"]; + assert(json["name"] == name && json["rev"] == hgInfo.rev); - if (store->isValidPath(hgInfo.storePath)) { - printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath); - return hgInfo; - } + hgInfo.storePath = json["storePath"]; - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; + if (store->isValidPath(hgInfo.storePath)) { + printTalkative("using cached Mercurial store path '%s'", + hgInfo.storePath); + return hgInfo; } - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); + } catch (SysError& e) { + if (e.errNo != ENOENT) throw; + } + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir }); + runProgram("hg", true, {"archive", "-R", cacheDir, "-r", rev, tmpDir}); - deletePath(tmpDir + "/.hg_archival.txt"); + deletePath(tmpDir + "/.hg_archival.txt"); - hgInfo.storePath = store->addToStore(name, tmpDir); + hgInfo.storePath = store->addToStore(name, tmpDir); - nlohmann::json json; - json["storePath"] = hgInfo.storePath; - json["uri"] = uri; - json["name"] = name; - json["branch"] = hgInfo.branch; - json["rev"] = hgInfo.rev; - json["revCount"] = hgInfo.revCount; + nlohmann::json json; + json["storePath"] = hgInfo.storePath; + json["uri"] = uri; + json["name"] = name; + json["branch"] = hgInfo.branch; + json["rev"] = hgInfo.rev; + json["revCount"] = hgInfo.revCount; - writeFile(storeLink, json.dump()); + writeFile(storeLink, json.dump()); - return hgInfo; + return hgInfo; } -static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - std::string url; - std::string rev; - std::string name = "source"; - PathSet context; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); - else - throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos); - } +static void prim_fetchMercurial(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string url; + std::string rev; + std::string name = "source"; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + for (auto& attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = + state.coerceToString(*attr.pos, *attr.value, context, false, false); + else if (n == "rev") + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", + attr.name, *attr.pos); + } - if (url.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); + if (url.empty()) + throw EvalError(format("'url' argument required, at %1%") % pos); - } else - url = state.coerceToString(pos, *args[0], context, false, false); + } else + url = state.coerceToString(pos, *args[0], context, false, false); - // FIXME: git externals probably can be used to bypass the URI - // whitelist. Ah well. - state.checkURI(url); + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); - auto hgInfo = exportMercurial(state.store, url, rev, name); + auto hgInfo = exportMercurial(state.store, url, rev, name); - state.mkAttrs(v, 8); - mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch); - mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12)); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount); - v.attrs->sort(); + state.mkAttrs(v, 8); + mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, + PathSet({hgInfo.storePath})); + mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch); + mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), + std::string(hgInfo.rev, 0, 12)); + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount); + v.attrs->sort(); - if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); + if (state.allowedPaths) + state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/fromTOML.cc b/third_party/nix/src/libexpr/primops/fromTOML.cc index a84e569e944d..4b652b379af7 100644 --- a/third_party/nix/src/libexpr/primops/fromTOML.cc +++ b/third_party/nix/src/libexpr/primops/fromTOML.cc @@ -1,90 +1,90 @@ -#include "primops.hh" -#include "eval-inline.hh" - #include "cpptoml/cpptoml.h" +#include "eval-inline.hh" +#include "primops.hh" namespace nix { -static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - using namespace cpptoml; - - auto toml = state.forceStringNoCtx(*args[0], pos); - - std::istringstream tomlStream(toml); +static void prim_fromTOML(EvalState& state, const Pos& pos, Value** args, + Value& v) { + using namespace cpptoml; - std::function<void(Value &, std::shared_ptr<base>)> visit; + auto toml = state.forceStringNoCtx(*args[0], pos); - visit = [&](Value & v, std::shared_ptr<base> t) { + std::istringstream tomlStream(toml); - if (auto t2 = t->as_table()) { + std::function<void(Value&, std::shared_ptr<base>)> visit; - size_t size = 0; - for (auto & i : *t2) { (void) i; size++; } + visit = [&](Value& v, std::shared_ptr<base> t) { + if (auto t2 = t->as_table()) { + size_t size = 0; + for (auto& i : *t2) { + (void)i; + size++; + } - state.mkAttrs(v, size); + state.mkAttrs(v, size); - for (auto & i : *t2) { - auto & v2 = *state.allocAttr(v, state.symbols.create(i.first)); + for (auto& i : *t2) { + auto& v2 = *state.allocAttr(v, state.symbols.create(i.first)); - if (auto i2 = i.second->as_table_array()) { - size_t size2 = i2->get().size(); - state.mkList(v2, size2); - for (size_t j = 0; j < size2; ++j) - visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]); - } - else - visit(v2, i.second); - } + if (auto i2 = i.second->as_table_array()) { + size_t size2 = i2->get().size(); + state.mkList(v2, size2); + for (size_t j = 0; j < size2; ++j) + visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]); + } else + visit(v2, i.second); + } - v.attrs->sort(); - } + v.attrs->sort(); + } - else if (auto t2 = t->as_array()) { - size_t size = t2->get().size(); + else if (auto t2 = t->as_array()) { + size_t size = t2->get().size(); - state.mkList(v, size); + state.mkList(v, size); - for (size_t i = 0; i < size; ++i) - visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); - } + for (size_t i = 0; i < size; ++i) + visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); + } - // Handle cases like 'a = [[{ a = true }]]', which IMHO should be - // parsed as a array containing an array containing a table, - // but instead are parsed as an array containing a table array - // containing a table. - else if (auto t2 = t->as_table_array()) { - size_t size = t2->get().size(); + // Handle cases like 'a = [[{ a = true }]]', which IMHO should be + // parsed as a array containing an array containing a table, + // but instead are parsed as an array containing a table array + // containing a table. + else if (auto t2 = t->as_table_array()) { + size_t size = t2->get().size(); - state.mkList(v, size); + state.mkList(v, size); - for (size_t j = 0; j < size; ++j) - visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); - } + for (size_t j = 0; j < size; ++j) + visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); + } - else if (t->is_value()) { - if (auto val = t->as<int64_t>()) - mkInt(v, val->get()); - else if (auto val = t->as<NixFloat>()) - mkFloat(v, val->get()); - else if (auto val = t->as<bool>()) - mkBool(v, val->get()); - else if (auto val = t->as<std::string>()) - mkString(v, val->get()); - else - throw EvalError("unsupported value type in TOML"); - } + else if (t->is_value()) { + if (auto val = t->as<int64_t>()) + mkInt(v, val->get()); + else if (auto val = t->as<NixFloat>()) + mkFloat(v, val->get()); + else if (auto val = t->as<bool>()) + mkBool(v, val->get()); + else if (auto val = t->as<std::string>()) + mkString(v, val->get()); + else + throw EvalError("unsupported value type in TOML"); + } - else abort(); - }; + else + abort(); + }; - try { - visit(v, parser(tomlStream).parse()); - } catch (std::runtime_error & e) { - throw EvalError("while parsing a TOML string at %s: %s", pos, e.what()); - } + try { + visit(v, parser(tomlStream).parse()); + } catch (std::runtime_error& e) { + throw EvalError("while parsing a TOML string at %s: %s", pos, e.what()); + } } static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/symbol-table.hh b/third_party/nix/src/libexpr/symbol-table.hh index 91faea122ce1..932bff5b46fb 100644 --- a/third_party/nix/src/libexpr/symbol-table.hh +++ b/third_party/nix/src/libexpr/symbol-table.hh @@ -2,7 +2,6 @@ #include <map> #include <unordered_set> - #include "types.hh" namespace nix { @@ -13,75 +12,49 @@ namespace nix { they can be compared efficiently (using a pointer equality test), because the symbol table stores only one copy of each string. */ -class Symbol -{ -private: - const string * s; // pointer into SymbolTable - Symbol(const string * s) : s(s) { }; - friend class SymbolTable; - -public: - Symbol() : s(0) { }; - - bool operator == (const Symbol & s2) const - { - return s == s2.s; - } - - bool operator != (const Symbol & s2) const - { - return s != s2.s; - } - - bool operator < (const Symbol & s2) const - { - return s < s2.s; - } - - operator const string & () const - { - return *s; - } - - bool set() const - { - return s; - } - - bool empty() const - { - return s->empty(); - } - - friend std::ostream & operator << (std::ostream & str, const Symbol & sym); +class Symbol { + private: + const string* s; // pointer into SymbolTable + Symbol(const string* s) : s(s){}; + friend class SymbolTable; + + public: + Symbol() : s(0){}; + + bool operator==(const Symbol& s2) const { return s == s2.s; } + + bool operator!=(const Symbol& s2) const { return s != s2.s; } + + bool operator<(const Symbol& s2) const { return s < s2.s; } + + operator const string&() const { return *s; } + + bool set() const { return s; } + + bool empty() const { return s->empty(); } + + friend std::ostream& operator<<(std::ostream& str, const Symbol& sym); }; -class SymbolTable -{ -private: - typedef std::unordered_set<string> Symbols; - Symbols symbols; - -public: - Symbol create(const string & s) - { - std::pair<Symbols::iterator, bool> res = symbols.insert(s); - return Symbol(&*res.first); - } - - size_t size() const - { - return symbols.size(); - } - - size_t totalSize() const; - - template<typename T> - void dump(T callback) - { - for (auto & s : symbols) - callback(s); - } +class SymbolTable { + private: + typedef std::unordered_set<string> Symbols; + Symbols symbols; + + public: + Symbol create(const string& s) { + std::pair<Symbols::iterator, bool> res = symbols.insert(s); + return Symbol(&*res.first); + } + + size_t size() const { return symbols.size(); } + + size_t totalSize() const; + + template <typename T> + void dump(T callback) { + for (auto& s : symbols) callback(s); + } }; -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-json.cc b/third_party/nix/src/libexpr/value-to-json.cc index 5fe8570adeb4..d96575e734dc 100644 --- a/third_party/nix/src/libexpr/value-to-json.cc +++ b/third_party/nix/src/libexpr/value-to-json.cc @@ -1,100 +1,97 @@ #include "value-to-json.hh" -#include "json.hh" -#include "eval-inline.hh" -#include "util.hh" - #include <cstdlib> #include <iomanip> - +#include "eval-inline.hh" +#include "json.hh" +#include "util.hh" namespace nix { -void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context) -{ - checkInterrupt(); - - if (strict) state.forceValue(v); - - switch (v.type) { - - case tInt: - out.write(v.integer); - break; - - case tBool: - out.write(v.boolean); - break; - - case tString: - copyContext(v, context); - out.write(v.string.s); - break; - - case tPath: - out.write(state.copyPathToStore(context, v.path)); - break; - - case tNull: - out.write(nullptr); - break; - - case tAttrs: { - auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); - if (maybeString) { - out.write(*maybeString); - break; - } - auto i = v.attrs->find(state.sOutPath); - if (i == v.attrs->end()) { - auto obj(out.object()); - StringSet names; - for (auto & j : *v.attrs) - names.insert(j.name); - for (auto & j : names) { - Attr & a(*v.attrs->find(state.symbols.create(j))); - auto placeholder(obj.placeholder(j)); - printValueAsJSON(state, strict, *a.value, placeholder, context); - } - } else - printValueAsJSON(state, strict, *i->value, out, context); - break; +void printValueAsJSON(EvalState& state, bool strict, Value& v, + JSONPlaceholder& out, PathSet& context) { + checkInterrupt(); + + if (strict) state.forceValue(v); + + switch (v.type) { + case tInt: + out.write(v.integer); + break; + + case tBool: + out.write(v.boolean); + break; + + case tString: + copyContext(v, context); + out.write(v.string.s); + break; + + case tPath: + out.write(state.copyPathToStore(context, v.path)); + break; + + case tNull: + out.write(nullptr); + break; + + case tAttrs: { + auto maybeString = + state.tryAttrsToString(noPos, v, context, false, false); + if (maybeString) { + out.write(*maybeString); + break; + } + auto i = v.attrs->find(state.sOutPath); + if (i == v.attrs->end()) { + auto obj(out.object()); + StringSet names; + for (auto& j : *v.attrs) names.insert(j.name); + for (auto& j : names) { + Attr& a(*v.attrs->find(state.symbols.create(j))); + auto placeholder(obj.placeholder(j)); + printValueAsJSON(state, strict, *a.value, placeholder, context); } + } else + printValueAsJSON(state, strict, *i->value, out, context); + break; + } - case tList1: case tList2: case tListN: { - auto list(out.list()); - for (unsigned int n = 0; n < v.listSize(); ++n) { - auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context); - } - break; - } + case tList1: + case tList2: + case tListN: { + auto list(out.list()); + for (unsigned int n = 0; n < v.listSize(); ++n) { + auto placeholder(list.placeholder()); + printValueAsJSON(state, strict, *v.listElems()[n], placeholder, + context); + } + break; + } - case tExternal: - v.external->printValueAsJSON(state, strict, out, context); - break; + case tExternal: + v.external->printValueAsJSON(state, strict, out, context); + break; - case tFloat: - out.write(v.fpoint); - break; + case tFloat: + out.write(v.fpoint); + break; - default: - throw TypeError(format("cannot convert %1% to JSON") % showType(v)); - } + default: + throw TypeError(format("cannot convert %1% to JSON") % showType(v)); + } } -void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context) -{ - JSONPlaceholder out(str); - printValueAsJSON(state, strict, v, out, context); +void printValueAsJSON(EvalState& state, bool strict, Value& v, + std::ostream& str, PathSet& context) { + JSONPlaceholder out(str); + printValueAsJSON(state, strict, v, out, context); } -void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context) const -{ - throw TypeError(format("cannot convert %1% to JSON") % showType()); +void ExternalValueBase::printValueAsJSON(EvalState& state, bool strict, + JSONPlaceholder& out, + PathSet& context) const { + throw TypeError(format("cannot convert %1% to JSON") % showType()); } - -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-json.hh b/third_party/nix/src/libexpr/value-to-json.hh index 67fed6487dd9..62708e4751dd 100644 --- a/third_party/nix/src/libexpr/value-to-json.hh +++ b/third_party/nix/src/libexpr/value-to-json.hh @@ -1,19 +1,18 @@ #pragma once -#include "nixexpr.hh" -#include "eval.hh" - -#include <string> #include <map> +#include <string> +#include "eval.hh" +#include "nixexpr.hh" namespace nix { class JSONPlaceholder; -void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context); +void printValueAsJSON(EvalState& state, bool strict, Value& v, + JSONPlaceholder& out, PathSet& context); -void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context); +void printValueAsJSON(EvalState& state, bool strict, Value& v, + std::ostream& str, PathSet& context); -} +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-xml.cc b/third_party/nix/src/libexpr/value-to-xml.cc index 00b1918a82aa..d9fddaafbbec 100644 --- a/third_party/nix/src/libexpr/value-to-xml.cc +++ b/third_party/nix/src/libexpr/value-to-xml.cc @@ -1,178 +1,173 @@ #include "value-to-xml.hh" -#include "xml-writer.hh" +#include <cstdlib> #include "eval-inline.hh" #include "util.hh" - -#include <cstdlib> - +#include "xml-writer.hh" namespace nix { - -static XMLAttrs singletonAttrs(const string & name, const string & value) -{ - XMLAttrs attrs; - attrs[name] = value; - return attrs; +static XMLAttrs singletonAttrs(const string& name, const string& value) { + XMLAttrs attrs; + attrs[name] = value; + return attrs; } +static void printValueAsXML(EvalState& state, bool strict, bool location, + Value& v, XMLWriter& doc, PathSet& context, + PathSet& drvsSeen); -static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); - - -static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) -{ - xmlAttrs["path"] = pos.file; - xmlAttrs["line"] = (format("%1%") % pos.line).str(); - xmlAttrs["column"] = (format("%1%") % pos.column).str(); +static void posToXML(XMLAttrs& xmlAttrs, const Pos& pos) { + xmlAttrs["path"] = pos.file; + xmlAttrs["line"] = (format("%1%") % pos.line).str(); + xmlAttrs["column"] = (format("%1%") % pos.column).str(); } +static void showAttrs(EvalState& state, bool strict, bool location, + Bindings& attrs, XMLWriter& doc, PathSet& context, + PathSet& drvsSeen) { + StringSet names; -static void showAttrs(EvalState & state, bool strict, bool location, - Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) -{ - StringSet names; + for (auto& i : attrs) names.insert(i.name); - for (auto & i : attrs) - names.insert(i.name); + for (auto& i : names) { + Attr& a(*attrs.find(state.symbols.create(i))); - for (auto & i : names) { - Attr & a(*attrs.find(state.symbols.create(i))); - - XMLAttrs xmlAttrs; - xmlAttrs["name"] = i; - if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos); + XMLAttrs xmlAttrs; + xmlAttrs["name"] = i; + if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos); - XMLOpenElement _(doc, "attr", xmlAttrs); - printValueAsXML(state, strict, location, - *a.value, doc, context, drvsSeen); - } + XMLOpenElement _(doc, "attr", xmlAttrs); + printValueAsXML(state, strict, location, *a.value, doc, context, drvsSeen); + } } +static void printValueAsXML(EvalState& state, bool strict, bool location, + Value& v, XMLWriter& doc, PathSet& context, + PathSet& drvsSeen) { + checkInterrupt(); -static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) -{ - checkInterrupt(); - - if (strict) state.forceValue(v); - - switch (v.type) { + if (strict) state.forceValue(v); - case tInt: - doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); - break; + switch (v.type) { + case tInt: + doc.writeEmptyElement( + "int", singletonAttrs("value", (format("%1%") % v.integer).str())); + break; - case tBool: - doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false")); - break; + case tBool: + doc.writeEmptyElement( + "bool", singletonAttrs("value", v.boolean ? "true" : "false")); + break; - case tString: - /* !!! show the context? */ - copyContext(v, context); - doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); - break; + case tString: + /* !!! show the context? */ + copyContext(v, context); + doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); + break; - case tPath: - doc.writeEmptyElement("path", singletonAttrs("value", v.path)); - break; + case tPath: + doc.writeEmptyElement("path", singletonAttrs("value", v.path)); + break; - case tNull: - doc.writeEmptyElement("null"); - break; + case tNull: + doc.writeEmptyElement("null"); + break; - case tAttrs: - if (state.isDerivation(v)) { - XMLAttrs xmlAttrs; - - Bindings::iterator a = v.attrs->find(state.symbols.create("derivation")); - - Path drvPath; - a = v.attrs->find(state.sDrvPath); - if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); - if (a->value->type == tString) - xmlAttrs["drvPath"] = drvPath = a->value->string.s; - } - - a = v.attrs->find(state.sOutPath); - if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); - if (a->value->type == tString) - xmlAttrs["outPath"] = a->value->string.s; - } - - XMLOpenElement _(doc, "derivation", xmlAttrs); - - if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) { - drvsSeen.insert(drvPath); - showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); - } else - doc.writeEmptyElement("repeated"); - } - - else { - XMLOpenElement _(doc, "attrs"); - showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); - } + case tAttrs: + if (state.isDerivation(v)) { + XMLAttrs xmlAttrs; - break; + Bindings::iterator a = + v.attrs->find(state.symbols.create("derivation")); - case tList1: case tList2: case tListN: { - XMLOpenElement _(doc, "list"); - for (unsigned int n = 0; n < v.listSize(); ++n) - printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); - break; + Path drvPath; + a = v.attrs->find(state.sDrvPath); + if (a != v.attrs->end()) { + if (strict) state.forceValue(*a->value); + if (a->value->type == tString) + xmlAttrs["drvPath"] = drvPath = a->value->string.s; } - case tLambda: { - XMLAttrs xmlAttrs; - if (location) posToXML(xmlAttrs, v.lambda.fun->pos); - XMLOpenElement _(doc, "function", xmlAttrs); - - if (v.lambda.fun->matchAttrs) { - XMLAttrs attrs; - if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg; - if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; - XMLOpenElement _(doc, "attrspat", attrs); - for (auto & i : v.lambda.fun->formals->formals) - doc.writeEmptyElement("attr", singletonAttrs("name", i.name)); - } else - doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg)); - - break; + a = v.attrs->find(state.sOutPath); + if (a != v.attrs->end()) { + if (strict) state.forceValue(*a->value); + if (a->value->type == tString) + xmlAttrs["outPath"] = a->value->string.s; } - case tExternal: - v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); - break; - - case tFloat: - doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); - break; + XMLOpenElement _(doc, "derivation", xmlAttrs); + + if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) { + drvsSeen.insert(drvPath); + showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + } else + doc.writeEmptyElement("repeated"); + } + + else { + XMLOpenElement _(doc, "attrs"); + showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + } + + break; + + case tList1: + case tList2: + case tListN: { + XMLOpenElement _(doc, "list"); + for (unsigned int n = 0; n < v.listSize(); ++n) + printValueAsXML(state, strict, location, *v.listElems()[n], doc, + context, drvsSeen); + break; + } - default: - doc.writeEmptyElement("unevaluated"); + case tLambda: { + XMLAttrs xmlAttrs; + if (location) posToXML(xmlAttrs, v.lambda.fun->pos); + XMLOpenElement _(doc, "function", xmlAttrs); + + if (v.lambda.fun->matchAttrs) { + XMLAttrs attrs; + if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg; + if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; + XMLOpenElement _(doc, "attrspat", attrs); + for (auto& i : v.lambda.fun->formals->formals) + doc.writeEmptyElement("attr", singletonAttrs("name", i.name)); + } else + doc.writeEmptyElement("varpat", + singletonAttrs("name", v.lambda.fun->arg)); + + break; } -} + case tExternal: + v.external->printValueAsXML(state, strict, location, doc, context, + drvsSeen); + break; -void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const -{ - doc.writeEmptyElement("unevaluated"); -} + case tFloat: + doc.writeEmptyElement( + "float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); + break; - -void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context) -{ - XMLWriter doc(true, out); - XMLOpenElement root(doc, "expr"); - PathSet drvsSeen; - printValueAsXML(state, strict, location, v, doc, context, drvsSeen); + default: + doc.writeEmptyElement("unevaluated"); + } } +void ExternalValueBase::printValueAsXML(EvalState& state, bool strict, + bool location, XMLWriter& doc, + PathSet& context, + PathSet& drvsSeen) const { + doc.writeEmptyElement("unevaluated"); +} +void printValueAsXML(EvalState& state, bool strict, bool location, Value& v, + std::ostream& out, PathSet& context) { + XMLWriter doc(true, out); + XMLOpenElement root(doc, "expr"); + PathSet drvsSeen; + printValueAsXML(state, strict, location, v, doc, context, drvsSeen); } + +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-xml.hh b/third_party/nix/src/libexpr/value-to-xml.hh index 97657327edba..e456b4921b24 100644 --- a/third_party/nix/src/libexpr/value-to-xml.hh +++ b/third_party/nix/src/libexpr/value-to-xml.hh @@ -1,14 +1,13 @@ #pragma once -#include "nixexpr.hh" -#include "eval.hh" - -#include <string> #include <map> +#include <string> +#include "eval.hh" +#include "nixexpr.hh" namespace nix { -void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context); - +void printValueAsXML(EvalState& state, bool strict, bool location, Value& v, + std::ostream& out, PathSet& context); + } diff --git a/third_party/nix/src/libexpr/value.hh b/third_party/nix/src/libexpr/value.hh index e1ec87d3b84c..211b1669f8d4 100644 --- a/third_party/nix/src/libexpr/value.hh +++ b/third_party/nix/src/libexpr/value.hh @@ -8,28 +8,26 @@ namespace nix { - typedef enum { - tInt = 1, - tBool, - tString, - tPath, - tNull, - tAttrs, - tList1, - tList2, - tListN, - tThunk, - tApp, - tLambda, - tBlackhole, - tPrimOp, - tPrimOpApp, - tExternal, - tFloat + tInt = 1, + tBool, + tString, + tPath, + tNull, + tAttrs, + tList1, + tList2, + tListN, + tThunk, + tApp, + tLambda, + tBlackhole, + tPrimOp, + tPrimOpApp, + tExternal, + tFloat } ValueType; - class Bindings; struct Env; struct Expr; @@ -42,233 +40,201 @@ class EvalState; class XMLWriter; class JSONPlaceholder; - typedef int64_t NixInt; typedef double NixFloat; /* External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented */ -class ExternalValueBase -{ - friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); - protected: - /* Print out the value */ - virtual std::ostream & print(std::ostream & str) const = 0; - - public: - /* Return a simple string describing the type */ - virtual string showType() const = 0; - - /* Return a string to be used in builtins.typeOf */ - virtual string typeOf() const = 0; - - /* How much space does this value take up */ - virtual size_t valueSize(std::set<const void *> & seen) const = 0; - - /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an - * error - */ - virtual string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; - - /* Compare to another value of the same type. Defaults to uncomparable, - * i.e. always false. - */ - virtual bool operator==(const ExternalValueBase & b) const; - - /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ - virtual void printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context) const; - - /* Print the value as XML. Defaults to unevaluated */ - virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const; - - virtual ~ExternalValueBase() - { - }; -}; - -std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); - - -struct Value -{ - ValueType type; - union - { - NixInt integer; - bool boolean; - - /* Strings in the evaluator carry a so-called `context' which - is a list of strings representing store paths. This is to - allow users to write things like - - "--with-freetype2-library=" + freetype + "/lib" - - where `freetype' is a derivation (or a source to be copied - to the store). If we just concatenated the strings without - keeping track of the referenced store paths, then if the - string is used as a derivation attribute, the derivation - will not have the correct dependencies in its inputDrvs and - inputSrcs. - - The semantics of the context is as follows: when a string - with context C is used as a derivation attribute, then the - derivations in C will be added to the inputDrvs of the - derivation, and the other store paths in C will be added to - the inputSrcs of the derivations. - - For canonicity, the store paths should be in sorted order. */ - struct { - const char * s; - const char * * context; // must be in sorted order - } string; - - const char * path; - Bindings * attrs; - struct { - size_t size; - Value * * elems; - } bigList; - Value * smallList[2]; - struct { - Env * env; - Expr * expr; - } thunk; - struct { - Value * left, * right; - } app; - struct { - Env * env; - ExprLambda * fun; - } lambda; - PrimOp * primOp; - struct { - Value * left, * right; - } primOpApp; - ExternalValueBase * external; - NixFloat fpoint; - }; - - bool isList() const - { - return type == tList1 || type == tList2 || type == tListN; - } - - Value * * listElems() - { - return type == tList1 || type == tList2 ? smallList : bigList.elems; - } - - const Value * const * listElems() const - { - return type == tList1 || type == tList2 ? smallList : bigList.elems; - } - - size_t listSize() const - { - return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; - } -}; +class ExternalValueBase { + friend std::ostream& operator<<(std::ostream& str, + const ExternalValueBase& v); + protected: + /* Print out the value */ + virtual std::ostream& print(std::ostream& str) const = 0; -/* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ -static inline void clearValue(Value & v) -{ - v.app.left = v.app.right = 0; -} + public: + /* Return a simple string describing the type */ + virtual string showType() const = 0; + /* Return a string to be used in builtins.typeOf */ + virtual string typeOf() const = 0; -static inline void mkInt(Value & v, NixInt n) -{ - clearValue(v); - v.type = tInt; - v.integer = n; -} + /* How much space does this value take up */ + virtual size_t valueSize(std::set<const void*>& seen) const = 0; + /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an + * error + */ + virtual string coerceToString(const Pos& pos, PathSet& context, bool copyMore, + bool copyToStore) const; -static inline void mkFloat(Value & v, NixFloat n) -{ - clearValue(v); - v.type = tFloat; - v.fpoint = n; -} + /* Compare to another value of the same type. Defaults to uncomparable, + * i.e. always false. + */ + virtual bool operator==(const ExternalValueBase& b) const; + /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ + virtual void printValueAsJSON(EvalState& state, bool strict, + JSONPlaceholder& out, PathSet& context) const; -static inline void mkBool(Value & v, bool b) -{ - clearValue(v); - v.type = tBool; - v.boolean = b; -} + /* Print the value as XML. Defaults to unevaluated */ + virtual void printValueAsXML(EvalState& state, bool strict, bool location, + XMLWriter& doc, PathSet& context, + PathSet& drvsSeen) const; + virtual ~ExternalValueBase(){}; +}; -static inline void mkNull(Value & v) -{ - clearValue(v); - v.type = tNull; -} +std::ostream& operator<<(std::ostream& str, const ExternalValueBase& v); + +struct Value { + ValueType type; + union { + NixInt integer; + bool boolean; + + /* Strings in the evaluator carry a so-called `context' which + is a list of strings representing store paths. This is to + allow users to write things like + + "--with-freetype2-library=" + freetype + "/lib" + + where `freetype' is a derivation (or a source to be copied + to the store). If we just concatenated the strings without + keeping track of the referenced store paths, then if the + string is used as a derivation attribute, the derivation + will not have the correct dependencies in its inputDrvs and + inputSrcs. + + The semantics of the context is as follows: when a string + with context C is used as a derivation attribute, then the + derivations in C will be added to the inputDrvs of the + derivation, and the other store paths in C will be added to + the inputSrcs of the derivations. + + For canonicity, the store paths should be in sorted order. */ + struct { + const char* s; + const char** context; // must be in sorted order + } string; + + const char* path; + Bindings* attrs; + struct { + size_t size; + Value** elems; + } bigList; + Value* smallList[2]; + struct { + Env* env; + Expr* expr; + } thunk; + struct { + Value *left, *right; + } app; + struct { + Env* env; + ExprLambda* fun; + } lambda; + PrimOp* primOp; + struct { + Value *left, *right; + } primOpApp; + ExternalValueBase* external; + NixFloat fpoint; + }; + + bool isList() const { + return type == tList1 || type == tList2 || type == tListN; + } + + Value** listElems() { + return type == tList1 || type == tList2 ? smallList : bigList.elems; + } + + const Value* const* listElems() const { + return type == tList1 || type == tList2 ? smallList : bigList.elems; + } + + size_t listSize() const { + return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; + } +}; +/* After overwriting an app node, be sure to clear pointers in the + Value to ensure that the target isn't kept alive unnecessarily. */ +static inline void clearValue(Value& v) { v.app.left = v.app.right = 0; } -static inline void mkApp(Value & v, Value & left, Value & right) -{ - v.type = tApp; - v.app.left = &left; - v.app.right = &right; +static inline void mkInt(Value& v, NixInt n) { + clearValue(v); + v.type = tInt; + v.integer = n; } - -static inline void mkPrimOpApp(Value & v, Value & left, Value & right) -{ - v.type = tPrimOpApp; - v.app.left = &left; - v.app.right = &right; +static inline void mkFloat(Value& v, NixFloat n) { + clearValue(v); + v.type = tFloat; + v.fpoint = n; } - -static inline void mkStringNoCopy(Value & v, const char * s) -{ - v.type = tString; - v.string.s = s; - v.string.context = 0; +static inline void mkBool(Value& v, bool b) { + clearValue(v); + v.type = tBool; + v.boolean = b; } - -static inline void mkString(Value & v, const Symbol & s) -{ - mkStringNoCopy(v, ((const string &) s).c_str()); +static inline void mkNull(Value& v) { + clearValue(v); + v.type = tNull; } +static inline void mkApp(Value& v, Value& left, Value& right) { + v.type = tApp; + v.app.left = &left; + v.app.right = &right; +} -void mkString(Value & v, const char * s); +static inline void mkPrimOpApp(Value& v, Value& left, Value& right) { + v.type = tPrimOpApp; + v.app.left = &left; + v.app.right = &right; +} +static inline void mkStringNoCopy(Value& v, const char* s) { + v.type = tString; + v.string.s = s; + v.string.context = 0; +} -static inline void mkPathNoCopy(Value & v, const char * s) -{ - clearValue(v); - v.type = tPath; - v.path = s; +static inline void mkString(Value& v, const Symbol& s) { + mkStringNoCopy(v, ((const string&)s).c_str()); } +void mkString(Value& v, const char* s); -void mkPath(Value & v, const char * s); +static inline void mkPathNoCopy(Value& v, const char* s) { + clearValue(v); + v.type = tPath; + v.path = s; +} +void mkPath(Value& v, const char* s); /* Compute the size in bytes of the given value, including all values and environments reachable from it. Static expressions (Exprs) are not included. */ -size_t valueSize(Value & v); - +size_t valueSize(Value& v); #if HAVE_BOEHMGC -typedef std::vector<Value *, gc_allocator<Value *> > ValueVector; -typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap; +typedef std::vector<Value*, gc_allocator<Value*> > ValueVector; +typedef std::map<Symbol, Value*, std::less<Symbol>, + gc_allocator<std::pair<const Symbol, Value*> > > + ValueMap; #else -typedef std::vector<Value *> ValueVector; -typedef std::map<Symbol, Value *> ValueMap; +typedef std::vector<Value*> ValueVector; +typedef std::map<Symbol, Value*> ValueMap; #endif - -} +} // namespace nix diff --git a/third_party/nix/src/libmain/common-args.cc b/third_party/nix/src/libmain/common-args.cc index 9e1d7cee60e6..e4c7c6daf6a5 100644 --- a/third_party/nix/src/libmain/common-args.cc +++ b/third_party/nix/src/libmain/common-args.cc @@ -3,54 +3,53 @@ namespace nix { -MixCommonArgs::MixCommonArgs(const string & programName) - : programName(programName) -{ - mkFlag() - .longName("verbose") - .shortName('v') - .description("increase verbosity level") - .handler([]() { verbosity = (Verbosity) (verbosity + 1); }); - - mkFlag() - .longName("quiet") - .description("decrease verbosity level") - .handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }); - - mkFlag() - .longName("debug") - .description("enable debug output") - .handler([]() { verbosity = lvlDebug; }); - - mkFlag() - .longName("option") - .labels({"name", "value"}) - .description("set a Nix configuration option (overriding nix.conf)") - .arity(2) - .handler([](std::vector<std::string> ss) { - try { - globalConfig.set(ss[0], ss[1]); - } catch (UsageError & e) { - warn(e.what()); - } - }); - - mkFlag() - .longName("max-jobs") - .shortName('j') - .label("jobs") - .description("maximum number of parallel builds") - .handler([=](std::string s) { - settings.set("max-jobs", s); - }); - - std::string cat = "config"; - globalConfig.convertToArgs(*this, cat); - - // Backward compatibility hack: nix-env already had a --system flag. - if (programName == "nix-env") longFlags.erase("system"); - - hiddenCategories.insert(cat); +MixCommonArgs::MixCommonArgs(const string& programName) + : programName(programName) { + mkFlag() + .longName("verbose") + .shortName('v') + .description("increase verbosity level") + .handler([]() { verbosity = (Verbosity)(verbosity + 1); }); + + mkFlag() + .longName("quiet") + .description("decrease verbosity level") + .handler([]() { + verbosity = + verbosity > lvlError ? (Verbosity)(verbosity - 1) : lvlError; + }); + + mkFlag().longName("debug").description("enable debug output").handler([]() { + verbosity = lvlDebug; + }); + + mkFlag() + .longName("option") + .labels({"name", "value"}) + .description("set a Nix configuration option (overriding nix.conf)") + .arity(2) + .handler([](std::vector<std::string> ss) { + try { + globalConfig.set(ss[0], ss[1]); + } catch (UsageError& e) { + warn(e.what()); + } + }); + + mkFlag() + .longName("max-jobs") + .shortName('j') + .label("jobs") + .description("maximum number of parallel builds") + .handler([=](std::string s) { settings.set("max-jobs", s); }); + + std::string cat = "config"; + globalConfig.convertToArgs(*this, cat); + + // Backward compatibility hack: nix-env already had a --system flag. + if (programName == "nix-env") longFlags.erase("system"); + + hiddenCategories.insert(cat); } -} +} // namespace nix diff --git a/third_party/nix/src/libmain/common-args.hh b/third_party/nix/src/libmain/common-args.hh index a4de3dccf0a5..d8917b836778 100644 --- a/third_party/nix/src/libmain/common-args.hh +++ b/third_party/nix/src/libmain/common-args.hh @@ -4,30 +4,24 @@ namespace nix { -struct MixCommonArgs : virtual Args -{ - string programName; - MixCommonArgs(const string & programName); +struct MixCommonArgs : virtual Args { + string programName; + MixCommonArgs(const string& programName); }; -struct MixDryRun : virtual Args -{ - bool dryRun = false; +struct MixDryRun : virtual Args { + bool dryRun = false; - MixDryRun() - { - mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); - } + MixDryRun() { + mkFlag(0, "dry-run", "show what this command would do without doing it", + &dryRun); + } }; -struct MixJSON : virtual Args -{ - bool json = false; +struct MixJSON : virtual Args { + bool json = false; - MixJSON() - { - mkFlag(0, "json", "produce JSON output", &json); - } + MixJSON() { mkFlag(0, "json", "produce JSON output", &json); } }; -} +} // namespace nix diff --git a/third_party/nix/src/libmain/shared.cc b/third_party/nix/src/libmain/shared.cc index d3dbfbc44592..72b84e28e020 100644 --- a/third_party/nix/src/libmain/shared.cc +++ b/third_party/nix/src/libmain/shared.cc @@ -1,381 +1,352 @@ -#include "globals.hh" #include "shared.hh" -#include "store-api.hh" -#include "util.hh" - +#include <openssl/crypto.h> +#include <signal.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <unistd.h> #include <algorithm> #include <cctype> +#include <cstdlib> #include <exception> #include <iostream> #include <mutex> - -#include <cstdlib> -#include <sys/time.h> -#include <sys/stat.h> -#include <unistd.h> -#include <signal.h> - -#include <openssl/crypto.h> - +#include "globals.hh" +#include "store-api.hh" +#include "util.hh" namespace nix { - static bool gcWarning = true; -void printGCWarning() -{ - if (!gcWarning) return; - static bool haveWarned = false; - warnOnce(haveWarned, - "you did not specify '--add-root'; " - "the result might be removed by the garbage collector"); +void printGCWarning() { + if (!gcWarning) return; + static bool haveWarned = false; + warnOnce(haveWarned, + "you did not specify '--add-root'; " + "the result might be removed by the garbage collector"); } - -void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl) -{ - unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; - store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl); +void printMissing(ref<Store> store, const PathSet& paths, Verbosity lvl) { + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, + narSize); + printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, + lvl); } - -void printMissing(ref<Store> store, const PathSet & willBuild, - const PathSet & willSubstitute, const PathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) -{ - if (!willBuild.empty()) { - printMsg(lvl, "these derivations will be built:"); - Paths sorted = store->topoSortPaths(willBuild); - reverse(sorted.begin(), sorted.end()); - for (auto & i : sorted) - printMsg(lvl, fmt(" %s", i)); - } - - if (!willSubstitute.empty()) { - printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", - downloadSize / (1024.0 * 1024.0), - narSize / (1024.0 * 1024.0))); - for (auto & i : willSubstitute) - printMsg(lvl, fmt(" %s", i)); - } - - if (!unknown.empty()) { - printMsg(lvl, fmt("don't know how to build these paths%s:", - (settings.readOnlyMode ? " (may be caused by read-only store access)" : ""))); - for (auto & i : unknown) - printMsg(lvl, fmt(" %s", i)); - } +void printMissing(ref<Store> store, const PathSet& willBuild, + const PathSet& willSubstitute, const PathSet& unknown, + unsigned long long downloadSize, unsigned long long narSize, + Verbosity lvl) { + if (!willBuild.empty()) { + printMsg(lvl, "these derivations will be built:"); + Paths sorted = store->topoSortPaths(willBuild); + reverse(sorted.begin(), sorted.end()); + for (auto& i : sorted) printMsg(lvl, fmt(" %s", i)); + } + + if (!willSubstitute.empty()) { + printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f " + "MiB unpacked):", + downloadSize / (1024.0 * 1024.0), + narSize / (1024.0 * 1024.0))); + for (auto& i : willSubstitute) printMsg(lvl, fmt(" %s", i)); + } + + if (!unknown.empty()) { + printMsg(lvl, fmt("don't know how to build these paths%s:", + (settings.readOnlyMode + ? " (may be caused by read-only store access)" + : ""))); + for (auto& i : unknown) printMsg(lvl, fmt(" %s", i)); + } } - -string getArg(const string & opt, - Strings::iterator & i, const Strings::iterator & end) -{ - ++i; - if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); - return *i; +string getArg(const string& opt, Strings::iterator& i, + const Strings::iterator& end) { + ++i; + if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); + return *i; } - #if OPENSSL_VERSION_NUMBER < 0x10101000L /* OpenSSL is not thread-safe by default - it will randomly crash unless the user supplies a mutex locking function. So let's do that. */ static std::vector<std::mutex> opensslLocks; -static void opensslLockCallback(int mode, int type, const char * file, int line) -{ - if (mode & CRYPTO_LOCK) - opensslLocks[type].lock(); - else - opensslLocks[type].unlock(); +static void opensslLockCallback(int mode, int type, const char* file, + int line) { + if (mode & CRYPTO_LOCK) + opensslLocks[type].lock(); + else + opensslLocks[type].unlock(); } #endif +static void sigHandler(int signo) {} -static void sigHandler(int signo) { } - - -void initNix() -{ - /* Turn on buffering for cerr. */ +void initNix() { + /* Turn on buffering for cerr. */ #if HAVE_PUBSETBUF - static char buf[1024]; - std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); + static char buf[1024]; + std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif #if OPENSSL_VERSION_NUMBER < 0x10101000L - /* Initialise OpenSSL locking. */ - opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(opensslLockCallback); + /* Initialise OpenSSL locking. */ + opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); + CRYPTO_set_locking_callback(opensslLockCallback); #endif - loadConfFile(); + loadConfFile(); - startSignalHandlerThread(); + startSignalHandlerThread(); - /* Reset SIGCHLD to its default. */ - struct sigaction act; - sigemptyset(&act.sa_mask); - act.sa_handler = SIG_DFL; - act.sa_flags = 0; - if (sigaction(SIGCHLD, &act, 0)) - throw SysError("resetting SIGCHLD"); + /* Reset SIGCHLD to its default. */ + struct sigaction act; + sigemptyset(&act.sa_mask); + act.sa_handler = SIG_DFL; + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD"); - /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ - act.sa_handler = sigHandler; - if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); + /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ + act.sa_handler = sigHandler; + if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); #if __APPLE__ - /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. - * Instead, add a dummy sigaction handler, and signalHandlerThread - * can handle the rest. */ - struct sigaction sa; - sa.sa_handler = sigHandler; - if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); + /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. + * Instead, add a dummy sigaction handler, and signalHandlerThread + * can handle the rest. */ + struct sigaction sa; + sa.sa_handler = sigHandler; + if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); #endif - /* Register a SIGSEGV handler to detect stack overflows. */ - detectStackOverflow(); + /* Register a SIGSEGV handler to detect stack overflows. */ + detectStackOverflow(); - /* There is no privacy in the Nix system ;-) At least not for - now. In particular, store objects should be readable by - everybody. */ - umask(0022); + /* There is no privacy in the Nix system ;-) At least not for + now. In particular, store objects should be readable by + everybody. */ + umask(0022); - /* Initialise the PRNG. */ - struct timeval tv; - gettimeofday(&tv, 0); - srandom(tv.tv_usec); + /* Initialise the PRNG. */ + struct timeval tv; + gettimeofday(&tv, 0); + srandom(tv.tv_usec); - /* On macOS, don't use the per-session TMPDIR (as set e.g. by - sshd). This breaks build users because they don't have access - to the TMPDIR, in particular in ‘nix-store --serve’. */ + /* On macOS, don't use the per-session TMPDIR (as set e.g. by + sshd). This breaks build users because they don't have access + to the TMPDIR, in particular in ‘nix-store --serve’. */ #if __APPLE__ - if (getuid() == 0 && hasPrefix(getEnv("TMPDIR"), "/var/folders/")) - unsetenv("TMPDIR"); + if (getuid() == 0 && hasPrefix(getEnv("TMPDIR"), "/var/folders/")) + unsetenv("TMPDIR"); #endif } - -LegacyArgs::LegacyArgs(const std::string & programName, - std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) - : MixCommonArgs(programName), parseArg(parseArg) -{ - mkFlag() - .longName("no-build-output") - .shortName('Q') - .description("do not show build output") - .set(&settings.verboseBuild, false); - - mkFlag() - .longName("keep-failed") - .shortName('K') - .description("keep temporary directories of failed builds") - .set(&(bool&) settings.keepFailed, true); - - mkFlag() - .longName("keep-going") - .shortName('k') - .description("keep going after a build fails") - .set(&(bool&) settings.keepGoing, true); - - mkFlag() - .longName("fallback") - .description("build from source if substitution fails") - .set(&(bool&) settings.tryFallback, true); - - auto intSettingAlias = [&](char shortName, const std::string & longName, - const std::string & description, const std::string & dest) { - mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { - settings.set(dest, std::to_string(n)); - }); - }; - - intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); - intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); - intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout"); - - mkFlag(0, "readonly-mode", "do not write to the Nix store", - &settings.readOnlyMode); - - mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", - &gcWarning, false); - - mkFlag() - .longName("store") - .label("store-uri") - .description("URI of the Nix store to use") - .dest(&(std::string&) settings.storeUri); +LegacyArgs::LegacyArgs( + const std::string& programName, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg) + : MixCommonArgs(programName), parseArg(parseArg) { + mkFlag() + .longName("no-build-output") + .shortName('Q') + .description("do not show build output") + .set(&settings.verboseBuild, false); + + mkFlag() + .longName("keep-failed") + .shortName('K') + .description("keep temporary directories of failed builds") + .set(&(bool&)settings.keepFailed, true); + + mkFlag() + .longName("keep-going") + .shortName('k') + .description("keep going after a build fails") + .set(&(bool&)settings.keepGoing, true); + + mkFlag() + .longName("fallback") + .description("build from source if substitution fails") + .set(&(bool&)settings.tryFallback, true); + + auto intSettingAlias = [&](char shortName, const std::string& longName, + const std::string& description, + const std::string& dest) { + mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { + settings.set(dest, std::to_string(n)); + }); + }; + + intSettingAlias(0, "cores", + "maximum number of CPU cores to use inside a build", "cores"); + intSettingAlias(0, "max-silent-time", + "number of seconds of silence before a build is killed", + "max-silent-time"); + intSettingAlias(0, "timeout", "number of seconds before a build is killed", + "timeout"); + + mkFlag(0, "readonly-mode", "do not write to the Nix store", + &settings.readOnlyMode); + + mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", + &gcWarning, false); + + mkFlag() + .longName("store") + .label("store-uri") + .description("URI of the Nix store to use") + .dest(&(std::string&)settings.storeUri); } - -bool LegacyArgs::processFlag(Strings::iterator & pos, Strings::iterator end) -{ - if (MixCommonArgs::processFlag(pos, end)) return true; - bool res = parseArg(pos, end); - if (res) ++pos; - return res; +bool LegacyArgs::processFlag(Strings::iterator& pos, Strings::iterator end) { + if (MixCommonArgs::processFlag(pos, end)) return true; + bool res = parseArg(pos, end); + if (res) ++pos; + return res; } - -bool LegacyArgs::processArgs(const Strings & args, bool finish) -{ - if (args.empty()) return true; - assert(args.size() == 1); - Strings ss(args); - auto pos = ss.begin(); - if (!parseArg(pos, ss.end())) - throw UsageError(format("unexpected argument '%1%'") % args.front()); - return true; +bool LegacyArgs::processArgs(const Strings& args, bool finish) { + if (args.empty()) return true; + assert(args.size() == 1); + Strings ss(args); + auto pos = ss.begin(); + if (!parseArg(pos, ss.end())) + throw UsageError(format("unexpected argument '%1%'") % args.front()); + return true; } - -void parseCmdLine(int argc, char * * argv, - std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) -{ - parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg); +void parseCmdLine( + int argc, char** argv, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg) { + parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg); } - -void parseCmdLine(const string & programName, const Strings & args, - std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) -{ - LegacyArgs(programName, parseArg).parseCmdline(args); +void parseCmdLine( + const string& programName, const Strings& args, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg) { + LegacyArgs(programName, parseArg).parseCmdline(args); } - -void printVersion(const string & programName) -{ - std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl; - if (verbosity > lvlInfo) { - Strings cfg; +void printVersion(const string& programName) { + std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl; + if (verbosity > lvlInfo) { + Strings cfg; #if HAVE_BOEHMGC - cfg.push_back("gc"); + cfg.push_back("gc"); #endif #if HAVE_SODIUM - cfg.push_back("signed-caches"); + cfg.push_back("signed-caches"); #endif - std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; - std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; - std::cout << "Store directory: " << settings.nixStore << "\n"; - std::cout << "State directory: " << settings.nixStateDir << "\n"; - } - throw Exit(); + std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; + std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" + << "\n"; + std::cout << "Store directory: " << settings.nixStore << "\n"; + std::cout << "State directory: " << settings.nixStateDir << "\n"; + } + throw Exit(); } - -void showManPage(const string & name) -{ - restoreSignals(); - setenv("MANPATH", settings.nixManDir.c_str(), 1); - execlp("man", "man", name.c_str(), nullptr); - throw SysError(format("command 'man %1%' failed") % name.c_str()); +void showManPage(const string& name) { + restoreSignals(); + setenv("MANPATH", settings.nixManDir.c_str(), 1); + execlp("man", "man", name.c_str(), nullptr); + throw SysError(format("command 'man %1%' failed") % name.c_str()); } +int handleExceptions(const string& programName, std::function<void()> fun) { + ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this -int handleExceptions(const string & programName, std::function<void()> fun) -{ - ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this - - string error = ANSI_RED "error:" ANSI_NORMAL " "; + string error = ANSI_RED "error:" ANSI_NORMAL " "; + try { try { - try { - fun(); - } catch (...) { - /* Subtle: we have to make sure that any `interrupted' - condition is discharged before we reach printMsg() - below, since otherwise it will throw an (uncaught) - exception. */ - setInterruptThrown(); - throw; - } - } catch (Exit & e) { - return e.status; - } catch (UsageError & e) { - printError( - format(error + "%1%\nTry '%2% --help' for more information.") - % e.what() % programName); - return 1; - } catch (BaseError & e) { - printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); - if (e.prefix() != "" && !settings.showTrace) - printError("(use '--show-trace' to show detailed location information)"); - return e.status; - } catch (std::bad_alloc & e) { - printError(error + "out of memory"); - return 1; - } catch (std::exception & e) { - printError(error + e.what()); - return 1; + fun(); + } catch (...) { + /* Subtle: we have to make sure that any `interrupted' + condition is discharged before we reach printMsg() + below, since otherwise it will throw an (uncaught) + exception. */ + setInterruptThrown(); + throw; } - - return 0; + } catch (Exit& e) { + return e.status; + } catch (UsageError& e) { + printError(format(error + "%1%\nTry '%2% --help' for more information.") % + e.what() % programName); + return 1; + } catch (BaseError& e) { + printError(format(error + "%1%%2%") % + (settings.showTrace ? e.prefix() : "") % e.msg()); + if (e.prefix() != "" && !settings.showTrace) + printError("(use '--show-trace' to show detailed location information)"); + return e.status; + } catch (std::bad_alloc& e) { + printError(error + "out of memory"); + return 1; + } catch (std::exception& e) { + printError(error + e.what()); + return 1; + } + + return 0; } +RunPager::RunPager() { + if (!isatty(STDOUT_FILENO)) return; + char* pager = getenv("NIX_PAGER"); + if (!pager) pager = getenv("PAGER"); + if (pager && ((string)pager == "" || (string)pager == "cat")) return; -RunPager::RunPager() -{ - if (!isatty(STDOUT_FILENO)) return; - char * pager = getenv("NIX_PAGER"); - if (!pager) pager = getenv("PAGER"); - if (pager && ((string) pager == "" || (string) pager == "cat")) return; - - Pipe toPager; - toPager.create(); - - pid = startProcess([&]() { - if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (!getenv("LESS")) - setenv("LESS", "FRSXMK", 1); - restoreSignals(); - if (pager) - execl("/bin/sh", "sh", "-c", pager, nullptr); - execlp("pager", "pager", nullptr); - execlp("less", "less", nullptr); - execlp("more", "more", nullptr); - throw SysError(format("executing '%1%'") % pager); - }); + Pipe toPager; + toPager.create(); + + pid = startProcess([&]() { + if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + if (!getenv("LESS")) setenv("LESS", "FRSXMK", 1); + restoreSignals(); + if (pager) execl("/bin/sh", "sh", "-c", pager, nullptr); + execlp("pager", "pager", nullptr); + execlp("less", "less", nullptr); + execlp("more", "more", nullptr); + throw SysError(format("executing '%1%'") % pager); + }); - pid.setKillSignal(SIGINT); + pid.setKillSignal(SIGINT); - if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); + if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); } - -RunPager::~RunPager() -{ - try { - if (pid != -1) { - std::cout.flush(); - close(STDOUT_FILENO); - pid.wait(); - } - } catch (...) { - ignoreException(); +RunPager::~RunPager() { + try { + if (pid != -1) { + std::cout.flush(); + close(STDOUT_FILENO); + pid.wait(); } + } catch (...) { + ignoreException(); + } } - -string showBytes(unsigned long long bytes) -{ - return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); +string showBytes(unsigned long long bytes) { + return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); } - -PrintFreed::~PrintFreed() -{ - if (show) - std::cout << format("%1% store paths deleted, %2% freed\n") - % results.paths.size() - % showBytes(results.bytesFreed); +PrintFreed::~PrintFreed() { + if (show) + std::cout << format("%1% store paths deleted, %2% freed\n") % + results.paths.size() % showBytes(results.bytesFreed); } -Exit::~Exit() { } +Exit::~Exit() {} -} +} // namespace nix diff --git a/third_party/nix/src/libmain/shared.hh b/third_party/nix/src/libmain/shared.hh index 8e4861232db5..8f895109d7c5 100644 --- a/third_party/nix/src/libmain/shared.hh +++ b/third_party/nix/src/libmain/shared.hh @@ -1,126 +1,128 @@ #pragma once -#include "util.hh" -#include "args.hh" -#include "common-args.hh" - #include <signal.h> - #include <locale> - +#include "args.hh" +#include "common-args.hh" +#include "util.hh" namespace nix { -class Exit : public std::exception -{ -public: - int status; - Exit() : status(0) { } - Exit(int status) : status(status) { } - virtual ~Exit(); +class Exit : public std::exception { + public: + int status; + Exit() : status(0) {} + Exit(int status) : status(status) {} + virtual ~Exit(); }; -int handleExceptions(const string & programName, std::function<void()> fun); +int handleExceptions(const string& programName, std::function<void()> fun); /* Don't forget to call initPlugins() after settings are initialized! */ void initNix(); -void parseCmdLine(int argc, char * * argv, - std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg); +void parseCmdLine( + int argc, char** argv, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg); -void parseCmdLine(const string & programName, const Strings & args, - std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg); +void parseCmdLine( + const string& programName, const Strings& args, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg); -void printVersion(const string & programName); +void printVersion(const string& programName); /* Ugh. No better place to put this. */ void printGCWarning(); class Store; -void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl = lvlInfo); - -void printMissing(ref<Store> store, const PathSet & willBuild, - const PathSet & willSubstitute, const PathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo); - -string getArg(const string & opt, - Strings::iterator & i, const Strings::iterator & end); - -template<class N> N getIntArg(const string & opt, - Strings::iterator & i, const Strings::iterator & end, bool allowUnit) -{ - ++i; - if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); - string s = *i; - N multiplier = 1; - if (allowUnit && !s.empty()) { - char u = std::toupper(*s.rbegin()); - if (std::isalpha(u)) { - if (u == 'K') multiplier = 1ULL << 10; - else if (u == 'M') multiplier = 1ULL << 20; - else if (u == 'G') multiplier = 1ULL << 30; - else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError(format("invalid unit specifier '%1%'") % u); - s.resize(s.size() - 1); - } +void printMissing(ref<Store> store, const PathSet& paths, + Verbosity lvl = lvlInfo); + +void printMissing(ref<Store> store, const PathSet& willBuild, + const PathSet& willSubstitute, const PathSet& unknown, + unsigned long long downloadSize, unsigned long long narSize, + Verbosity lvl = lvlInfo); + +string getArg(const string& opt, Strings::iterator& i, + const Strings::iterator& end); + +template <class N> +N getIntArg(const string& opt, Strings::iterator& i, + const Strings::iterator& end, bool allowUnit) { + ++i; + if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); + string s = *i; + N multiplier = 1; + if (allowUnit && !s.empty()) { + char u = std::toupper(*s.rbegin()); + if (std::isalpha(u)) { + if (u == 'K') + multiplier = 1ULL << 10; + else if (u == 'M') + multiplier = 1ULL << 20; + else if (u == 'G') + multiplier = 1ULL << 30; + else if (u == 'T') + multiplier = 1ULL << 40; + else + throw UsageError(format("invalid unit specifier '%1%'") % u); + s.resize(s.size() - 1); } - N n; - if (!string2Int(s, n)) - throw UsageError(format("'%1%' requires an integer argument") % opt); - return n * multiplier; + } + N n; + if (!string2Int(s, n)) + throw UsageError(format("'%1%' requires an integer argument") % opt); + return n * multiplier; } +struct LegacyArgs : public MixCommonArgs { + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg; -struct LegacyArgs : public MixCommonArgs -{ - std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg; - - LegacyArgs(const std::string & programName, - std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg); + LegacyArgs( + const std::string& programName, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg); - bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + bool processFlag(Strings::iterator& pos, Strings::iterator end) override; - bool processArgs(const Strings & args, bool finish) override; + bool processArgs(const Strings& args, bool finish) override; }; - /* Show the manual page for the specified program. */ -void showManPage(const string & name); +void showManPage(const string& name); /* The constructor of this class starts a pager if stdout is a terminal and $PAGER is set. Stdout is redirected to the pager. */ -class RunPager -{ -public: - RunPager(); - ~RunPager(); - -private: - Pid pid; +class RunPager { + public: + RunPager(); + ~RunPager(); + + private: + Pid pid; }; extern volatile ::sig_atomic_t blockInt; - /* GC helpers. */ string showBytes(unsigned long long bytes); struct GCResults; -struct PrintFreed -{ - bool show; - const GCResults & results; - PrintFreed(bool show, const GCResults & results) - : show(show), results(results) { } - ~PrintFreed(); +struct PrintFreed { + bool show; + const GCResults& results; + PrintFreed(bool show, const GCResults& results) + : show(show), results(results) {} + ~PrintFreed(); }; - /* Install a SIGSEGV handler to detect stack overflows. */ void detectStackOverflow(); - -} +} // namespace nix diff --git a/third_party/nix/src/libmain/stack.cc b/third_party/nix/src/libmain/stack.cc index e6224de7d28f..621542d2198a 100644 --- a/third_party/nix/src/libmain/stack.cc +++ b/third_party/nix/src/libmain/stack.cc @@ -1,71 +1,64 @@ -#include "types.hh" - -#include <cstring> +#include <signal.h> +#include <unistd.h> #include <cstddef> #include <cstdlib> - -#include <unistd.h> -#include <signal.h> +#include <cstring> +#include "types.hh" namespace nix { - -static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) -{ - /* Detect stack overflows by comparing the faulting address with - the stack pointer. Unfortunately, getting the stack pointer is - not portable. */ - bool haveSP = true; - char * sp = 0; +static void sigsegvHandler(int signo, siginfo_t* info, void* ctx) { + /* Detect stack overflows by comparing the faulting address with + the stack pointer. Unfortunately, getting the stack pointer is + not portable. */ + bool haveSP = true; + char* sp = 0; #if defined(__x86_64__) && defined(REG_RSP) - sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_RSP]; + sp = (char*)((ucontext_t*)ctx)->uc_mcontext.gregs[REG_RSP]; #elif defined(REG_ESP) - sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_ESP]; + sp = (char*)((ucontext_t*)ctx)->uc_mcontext.gregs[REG_ESP]; #else - haveSP = false; + haveSP = false; #endif - if (haveSP) { - ptrdiff_t diff = (char *) info->si_addr - sp; - if (diff < 0) diff = -diff; - if (diff < 4096) { - char msg[] = "error: stack overflow (possible infinite recursion)\n"; - [[gnu::unused]] auto res = write(2, msg, strlen(msg)); - _exit(1); // maybe abort instead? - } + if (haveSP) { + ptrdiff_t diff = (char*)info->si_addr - sp; + if (diff < 0) diff = -diff; + if (diff < 4096) { + char msg[] = "error: stack overflow (possible infinite recursion)\n"; + [[gnu::unused]] auto res = write(2, msg, strlen(msg)); + _exit(1); // maybe abort instead? } + } - /* Restore default behaviour (i.e. segfault and dump core). */ - struct sigaction act; - sigfillset(&act.sa_mask); - act.sa_handler = SIG_DFL; - act.sa_flags = 0; - if (sigaction(SIGSEGV, &act, 0)) abort(); + /* Restore default behaviour (i.e. segfault and dump core). */ + struct sigaction act; + sigfillset(&act.sa_mask); + act.sa_handler = SIG_DFL; + act.sa_flags = 0; + if (sigaction(SIGSEGV, &act, 0)) abort(); } +void detectStackOverflow() { +#if defined(SA_SIGINFO) && defined(SA_ONSTACK) + /* Install a SIGSEGV handler to detect stack overflows. This + requires an alternative stack, otherwise the signal cannot be + delivered when we're out of stack space. */ + stack_t stack; + stack.ss_size = 4096 * 4 + MINSIGSTKSZ; + static auto stackBuf = std::make_unique<std::vector<char>>(stack.ss_size); + stack.ss_sp = stackBuf->data(); + if (!stack.ss_sp) throw Error("cannot allocate alternative stack"); + stack.ss_flags = 0; + if (sigaltstack(&stack, 0) == -1) + throw SysError("cannot set alternative stack"); -void detectStackOverflow() -{ -#if defined(SA_SIGINFO) && defined (SA_ONSTACK) - /* Install a SIGSEGV handler to detect stack overflows. This - requires an alternative stack, otherwise the signal cannot be - delivered when we're out of stack space. */ - stack_t stack; - stack.ss_size = 4096 * 4 + MINSIGSTKSZ; - static auto stackBuf = std::make_unique<std::vector<char>>(stack.ss_size); - stack.ss_sp = stackBuf->data(); - if (!stack.ss_sp) throw Error("cannot allocate alternative stack"); - stack.ss_flags = 0; - if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack"); - - struct sigaction act; - sigfillset(&act.sa_mask); - act.sa_sigaction = sigsegvHandler; - act.sa_flags = SA_SIGINFO | SA_ONSTACK; - if (sigaction(SIGSEGV, &act, 0)) - throw SysError("resetting SIGSEGV"); + struct sigaction act; + sigfillset(&act.sa_mask); + act.sa_sigaction = sigsegvHandler; + act.sa_flags = SA_SIGINFO | SA_ONSTACK; + if (sigaction(SIGSEGV, &act, 0)) throw SysError("resetting SIGSEGV"); #endif } - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/binary-cache-store.cc b/third_party/nix/src/libstore/binary-cache-store.cc index 10cde8704bb3..a601ef77a369 100644 --- a/third_party/nix/src/libstore/binary-cache-store.cc +++ b/third_party/nix/src/libstore/binary-cache-store.cc @@ -1,361 +1,364 @@ -#include "archive.hh" #include "binary-cache-store.hh" +#include <chrono> +#include <future> +#include "archive.hh" #include "compression.hh" #include "derivations.hh" #include "fs-accessor.hh" #include "globals.hh" +#include "json.hh" +#include "nar-accessor.hh" +#include "nar-info-disk-cache.hh" #include "nar-info.hh" -#include "sync.hh" #include "remote-fs-accessor.hh" -#include "nar-info-disk-cache.hh" -#include "nar-accessor.hh" -#include "json.hh" - -#include <chrono> - -#include <future> +#include "sync.hh" namespace nix { -BinaryCacheStore::BinaryCacheStore(const Params & params) - : Store(params) -{ - if (secretKeyFile != "") - secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); +BinaryCacheStore::BinaryCacheStore(const Params& params) : Store(params) { + if (secretKeyFile != "") + secretKey = + std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); - StringSink sink; - sink << narVersionMagic1; - narMagic = *sink.s; + StringSink sink; + sink << narVersionMagic1; + narMagic = *sink.s; } -void BinaryCacheStore::init() -{ - std::string cacheInfoFile = "nix-cache-info"; - - auto cacheInfo = getFile(cacheInfoFile); - if (!cacheInfo) { - upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); - } else { - for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) { - size_t colon = line.find(':'); - if (colon == std::string::npos) continue; - auto name = line.substr(0, colon); - auto value = trim(line.substr(colon + 1, std::string::npos)); - if (name == "StoreDir") { - if (value != storeDir) - throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'") - % getUri() % value % storeDir); - } else if (name == "WantMassQuery") { - wantMassQuery_ = value == "1"; - } else if (name == "Priority") { - string2Int(value, priority); - } - } +void BinaryCacheStore::init() { + std::string cacheInfoFile = "nix-cache-info"; + + auto cacheInfo = getFile(cacheInfoFile); + if (!cacheInfo) { + upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", + "text/x-nix-cache-info"); + } else { + for (auto& line : tokenizeString<Strings>(*cacheInfo, "\n")) { + size_t colon = line.find(':'); + if (colon == std::string::npos) continue; + auto name = line.substr(0, colon); + auto value = trim(line.substr(colon + 1, std::string::npos)); + if (name == "StoreDir") { + if (value != storeDir) + throw Error(format("binary cache '%s' is for Nix stores with prefix " + "'%s', not '%s'") % + getUri() % value % storeDir); + } else if (name == "WantMassQuery") { + wantMassQuery_ = value == "1"; + } else if (name == "Priority") { + string2Int(value, priority); + } } + } } -void BinaryCacheStore::getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback) noexcept -{ - try { - callback(getFile(path)); - } catch (...) { callback.rethrow(); } +void BinaryCacheStore::getFile( + const std::string& path, + Callback<std::shared_ptr<std::string>> callback) noexcept { + try { + callback(getFile(path)); + } catch (...) { + callback.rethrow(); + } } -void BinaryCacheStore::getFile(const std::string & path, Sink & sink) -{ - std::promise<std::shared_ptr<std::string>> promise; - getFile(path, - {[&](std::future<std::shared_ptr<std::string>> result) { +void BinaryCacheStore::getFile(const std::string& path, Sink& sink) { + std::promise<std::shared_ptr<std::string>> promise; + getFile(path, {[&](std::future<std::shared_ptr<std::string>> result) { try { - promise.set_value(result.get()); + promise.set_value(result.get()); } catch (...) { - promise.set_exception(std::current_exception()); + promise.set_exception(std::current_exception()); } - }}); - auto data = promise.get_future().get(); - sink((unsigned char *) data->data(), data->size()); + }}); + auto data = promise.get_future().get(); + sink((unsigned char*)data->data(), data->size()); } -std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) -{ - StringSink sink; - try { - getFile(path, sink); - } catch (NoSuchBinaryCacheFile &) { - return nullptr; - } - return sink.s; +std::shared_ptr<std::string> BinaryCacheStore::getFile( + const std::string& path) { + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile&) { + return nullptr; + } + return sink.s; } -Path BinaryCacheStore::narInfoFileFor(const Path & storePath) -{ - assertStorePath(storePath); - return storePathToHash(storePath) + ".narinfo"; +Path BinaryCacheStore::narInfoFileFor(const Path& storePath) { + assertStorePath(storePath); + return storePathToHash(storePath) + ".narinfo"; } -void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) -{ - auto narInfoFile = narInfoFileFor(narInfo->path); +void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) { + auto narInfoFile = narInfoFileFor(narInfo->path); - upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo"); + upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo"); - auto hashPart = storePathToHash(narInfo->path); + auto hashPart = storePathToHash(narInfo->path); - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo)); - } + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo)); + } - if (diskCache) - diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); + if (diskCache) + diskCache->upsertNarInfo(getUri(), hashPart, + std::shared_ptr<NarInfo>(narInfo)); } -void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) -{ - if (!repair && isValidPath(info.path)) return; - - /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. */ - for (auto & ref : info.references) - try { - if (ref != info.path) - queryPathInfo(ref); - } catch (InvalidPath &) { - throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid") - % info.path % ref); - } - - assert(nar->compare(0, narMagic.size(), narMagic) == 0); +void BinaryCacheStore::addToStore(const ValidPathInfo& info, + const ref<std::string>& nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + if (!repair && isValidPath(info.path)) return; + + /* Verify that all references are valid. This may do some .narinfo + reads, but typically they'll already be cached. */ + for (auto& ref : info.references) try { + if (ref != info.path) queryPathInfo(ref); + } catch (InvalidPath&) { + throw Error(format("cannot add '%s' to the binary cache because the " + "reference '%s' is not valid") % + info.path % ref); + } - auto narInfo = make_ref<NarInfo>(info); + assert(nar->compare(0, narMagic.size(), narMagic) == 0); - narInfo->narSize = nar->size(); - narInfo->narHash = hashString(htSHA256, *nar); + auto narInfo = make_ref<NarInfo>(info); - if (info.narHash && info.narHash != narInfo->narHash) - throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path); + narInfo->narSize = nar->size(); + narInfo->narHash = hashString(htSHA256, *nar); - auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); + if (info.narHash && info.narHash != narInfo->narHash) + throw Error( + format("refusing to copy corrupted path '%1%' to binary cache") % + info.path); - /* Optionally write a JSON file containing a listing of the - contents of the NAR. */ - if (writeNARListing) { - std::ostringstream jsonOut; + auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); - { - JSONObject jsonRoot(jsonOut); - jsonRoot.attr("version", 1); + /* Optionally write a JSON file containing a listing of the + contents of the NAR. */ + if (writeNARListing) { + std::ostringstream jsonOut; - auto narAccessor = makeNarAccessor(nar); + { + JSONObject jsonRoot(jsonOut); + jsonRoot.attr("version", 1); - if (accessor_) - accessor_->addToCache(info.path, *nar, narAccessor); + auto narAccessor = makeNarAccessor(nar); - { - auto res = jsonRoot.placeholder("root"); - listNar(res, narAccessor, "", true); - } - } - - upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json"); - } + if (accessor_) accessor_->addToCache(info.path, *nar, narAccessor); - else { - if (accessor_) - accessor_->addToCache(info.path, *nar, makeNarAccessor(nar)); + { + auto res = jsonRoot.placeholder("root"); + listNar(res, narAccessor, "", true); + } } - /* Compress the NAR. */ - narInfo->compression = compression; - auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, *nar, parallelCompression); - auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, *narCompressed); - narInfo->fileSize = narCompressed->size(); - - auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); - printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") - % narInfo->path % narInfo->narSize - % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) - % duration); - - /* Atomically write the NAR file. */ - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "br" ? ".br" : - ""); - if (repair || !fileExists(narInfo->url)) { - stats.narWrite++; - upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); - } else - stats.narWriteAverted++; - - stats.narWriteBytes += nar->size(); - stats.narWriteCompressedBytes += narCompressed->size(); - stats.narWriteCompressionTimeMs += duration; - - /* Atomically write the NAR info file.*/ - if (secretKey) narInfo->sign(*secretKey); - - writeNarInfo(narInfo); - - stats.narInfoWrite++; + upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), + "application/json"); + } + + else { + if (accessor_) accessor_->addToCache(info.path, *nar, makeNarAccessor(nar)); + } + + /* Compress the NAR. */ + narInfo->compression = compression; + auto now1 = std::chrono::steady_clock::now(); + auto narCompressed = compress(compression, *nar, parallelCompression); + auto now2 = std::chrono::steady_clock::now(); + narInfo->fileHash = hashString(htSHA256, *narCompressed); + narInfo->fileSize = narCompressed->size(); + + auto duration = + std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) + .count(); + printMsg(lvlTalkative, + format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% " + "ms) to binary cache") % + narInfo->path % narInfo->narSize % + ((1.0 - (double)narCompressed->size() / nar->size()) * 100.0) % + duration); + + /* Atomically write the NAR file. */ + narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" + : compression == "bzip2" + ? ".bz2" + : compression == "br" ? ".br" : ""); + if (repair || !fileExists(narInfo->url)) { + stats.narWrite++; + upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); + } else + stats.narWriteAverted++; + + stats.narWriteBytes += nar->size(); + stats.narWriteCompressedBytes += narCompressed->size(); + stats.narWriteCompressionTimeMs += duration; + + /* Atomically write the NAR info file.*/ + if (secretKey) narInfo->sign(*secretKey); + + writeNarInfo(narInfo); + + stats.narInfoWrite++; } -bool BinaryCacheStore::isValidPathUncached(const Path & storePath) -{ - // FIXME: this only checks whether a .narinfo with a matching hash - // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even - // though they shouldn't. Not easily fixed. - return fileExists(narInfoFileFor(storePath)); +bool BinaryCacheStore::isValidPathUncached(const Path& storePath) { + // FIXME: this only checks whether a .narinfo with a matching hash + // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even + // though they shouldn't. Not easily fixed. + return fileExists(narInfoFileFor(storePath)); } -void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) -{ - auto info = queryPathInfo(storePath).cast<const NarInfo>(); +void BinaryCacheStore::narFromPath(const Path& storePath, Sink& sink) { + auto info = queryPathInfo(storePath).cast<const NarInfo>(); - uint64_t narSize = 0; + uint64_t narSize = 0; - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - narSize += len; - }); + LambdaSink wrapperSink([&](const unsigned char* data, size_t len) { + sink(data, len); + narSize += len; + }); - auto decompressor = makeDecompressionSink(info->compression, wrapperSink); + auto decompressor = makeDecompressionSink(info->compression, wrapperSink); - try { - getFile(info->url, *decompressor); - } catch (NoSuchBinaryCacheFile & e) { - throw SubstituteGone(e.what()); - } + try { + getFile(info->url, *decompressor); + } catch (NoSuchBinaryCacheFile& e) { + throw SubstituteGone(e.what()); + } - decompressor->finish(); + decompressor->finish(); - stats.narRead++; - //stats.narReadCompressedBytes += nar->size(); // FIXME - stats.narReadBytes += narSize; + stats.narRead++; + // stats.narReadCompressedBytes += nar->size(); // FIXME + stats.narReadBytes += narSize; } -void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, - Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept -{ - auto uri = getUri(); - auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo, - fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri}); - PushActivity pact(act->id); +void BinaryCacheStore::queryPathInfoUncached( + const Path& storePath, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { + auto uri = getUri(); + auto act = std::make_shared<Activity>( + *logger, lvlTalkative, actQueryPathInfo, + fmt("querying info about '%s' on '%s'", storePath, uri), + Logger::Fields{storePath, uri}); + PushActivity pact(act->id); - auto narInfoFile = narInfoFileFor(storePath); + auto narInfoFile = narInfoFileFor(storePath); - auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); - getFile(narInfoFile, - {[=](std::future<std::shared_ptr<std::string>> fut) { - try { - auto data = fut.get(); + getFile( + narInfoFile, {[=](std::future<std::shared_ptr<std::string>> fut) { + try { + auto data = fut.get(); - if (!data) return (*callbackPtr)(nullptr); + if (!data) return (*callbackPtr)(nullptr); - stats.narInfoRead++; + stats.narInfoRead++; - (*callbackPtr)((std::shared_ptr<ValidPathInfo>) - std::make_shared<NarInfo>(*this, *data, narInfoFile)); + (*callbackPtr)( + (std::shared_ptr<ValidPathInfo>)std::make_shared<NarInfo>( + *this, *data, narInfoFile)); - (void) act; // force Activity into this lambda to ensure it stays alive - } catch (...) { - callbackPtr->rethrow(); - } - }}); + (void) + act; // force Activity into this lambda to ensure it stays alive + } catch (...) { + callbackPtr->rethrow(); + } + }}); } -Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - // FIXME: some cut&paste from LocalStore::addToStore(). - - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - Hash h; - if (recursive) { - dumpPath(srcPath, sink, filter); - h = hashString(hashAlgo, *sink.s); - } else { - auto s = readFile(srcPath); - dumpString(s, sink); - h = hashString(hashAlgo, s); - } +Path BinaryCacheStore::addToStore(const string& name, const Path& srcPath, + bool recursive, HashType hashAlgo, + PathFilter& filter, RepairFlag repair) { + // FIXME: some cut&paste from LocalStore::addToStore(). + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + Hash h; + if (recursive) { + dumpPath(srcPath, sink, filter); + h = hashString(hashAlgo, *sink.s); + } else { + auto s = readFile(srcPath); + dumpString(s, sink); + h = hashString(hashAlgo, s); + } + + ValidPathInfo info; + info.path = makeFixedOutputPath(recursive, h, name); + + addToStore(info, sink.s, repair, CheckSigs, nullptr); + + return info.path; +} - ValidPathInfo info; - info.path = makeFixedOutputPath(recursive, h, name); +Path BinaryCacheStore::addTextToStore(const string& name, const string& s, + const PathSet& references, + RepairFlag repair) { + ValidPathInfo info; + info.path = computeStorePathForText(name, s, references); + info.references = references; + if (repair || !isValidPath(info.path)) { + StringSink sink; + dumpString(s, sink); addToStore(info, sink.s, repair, CheckSigs, nullptr); + } - return info.path; + return info.path; } -Path BinaryCacheStore::addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) -{ - ValidPathInfo info; - info.path = computeStorePathForText(name, s, references); - info.references = references; - - if (repair || !isValidPath(info.path)) { - StringSink sink; - dumpString(s, sink); - addToStore(info, sink.s, repair, CheckSigs, nullptr); - } - - return info.path; +ref<FSAccessor> BinaryCacheStore::getFSAccessor() { + return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), + localNarCache); } -ref<FSAccessor> BinaryCacheStore::getFSAccessor() -{ - return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache); -} - -void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs) -{ - /* Note: this is inherently racy since there is no locking on - binary caches. In particular, with S3 this unreliable, even - when addSignatures() is called sequentially on a path, because - S3 might return an outdated cached version. */ +void BinaryCacheStore::addSignatures(const Path& storePath, + const StringSet& sigs) { + /* Note: this is inherently racy since there is no locking on + binary caches. In particular, with S3 this unreliable, even + when addSignatures() is called sequentially on a path, because + S3 might return an outdated cached version. */ - auto narInfo = make_ref<NarInfo>((NarInfo &) *queryPathInfo(storePath)); + auto narInfo = make_ref<NarInfo>((NarInfo&)*queryPathInfo(storePath)); - narInfo->sigs.insert(sigs.begin(), sigs.end()); + narInfo->sigs.insert(sigs.begin(), sigs.end()); - auto narInfoFile = narInfoFileFor(narInfo->path); + auto narInfoFile = narInfoFileFor(narInfo->path); - writeNarInfo(narInfo); + writeNarInfo(narInfo); } -std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path) -{ - Path drvPath; +std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path& path) { + Path drvPath; - if (isDerivation(path)) - drvPath = path; - else { - try { - auto info = queryPathInfo(path); - // FIXME: add a "Log" field to .narinfo - if (info->deriver == "") return nullptr; - drvPath = info->deriver; - } catch (InvalidPath &) { - return nullptr; - } + if (isDerivation(path)) + drvPath = path; + else { + try { + auto info = queryPathInfo(path); + // FIXME: add a "Log" field to .narinfo + if (info->deriver == "") return nullptr; + drvPath = info->deriver; + } catch (InvalidPath&) { + return nullptr; } + } - auto logPath = "log/" + baseNameOf(drvPath); + auto logPath = "log/" + baseNameOf(drvPath); - debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); + debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); - return getFile(logPath); + return getFile(logPath); } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/binary-cache-store.hh b/third_party/nix/src/libstore/binary-cache-store.hh index af108880cba8..80121aa22ee8 100644 --- a/third_party/nix/src/libstore/binary-cache-store.hh +++ b/third_party/nix/src/libstore/binary-cache-store.hh @@ -1,115 +1,113 @@ #pragma once +#include <atomic> #include "crypto.hh" -#include "store-api.hh" - #include "pool.hh" - -#include <atomic> +#include "store-api.hh" namespace nix { struct NarInfo; -class BinaryCacheStore : public Store -{ -public: - - const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; - const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; - const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; - const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; - const Setting<bool> parallelCompression{this, false, "parallel-compression", - "enable multi-threading compression, available for xz only currently"}; - -private: - - std::unique_ptr<SecretKey> secretKey; - -protected: - - BinaryCacheStore(const Params & params); - -public: - - virtual bool fileExists(const std::string & path) = 0; - - virtual void upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) = 0; - - /* Note: subclasses must implement at least one of the two - following getFile() methods. */ +class BinaryCacheStore : public Store { + public: + const Setting<std::string> compression{ + this, "xz", "compression", + "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting<bool> writeNARListing{ + this, false, "write-nar-listing", + "whether to write a JSON file listing the files in each NAR"}; + const Setting<Path> secretKeyFile{ + this, "", "secret-key", + "path to secret key used to sign the binary cache"}; + const Setting<Path> localNarCache{this, "", "local-nar-cache", + "path to a local cache of NARs"}; + const Setting<bool> parallelCompression{ + this, false, "parallel-compression", + "enable multi-threading compression, available for xz only currently"}; - /* Dump the contents of the specified file to a sink. */ - virtual void getFile(const std::string & path, Sink & sink); + private: + std::unique_ptr<SecretKey> secretKey; - /* Fetch the specified file and call the specified callback with - the result. A subclass may implement this asynchronously. */ - virtual void getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback) noexcept; + protected: + BinaryCacheStore(const Params& params); - std::shared_ptr<std::string> getFile(const std::string & path); + public: + virtual bool fileExists(const std::string& path) = 0; -protected: + virtual void upsertFile(const std::string& path, const std::string& data, + const std::string& mimeType) = 0; - bool wantMassQuery_ = false; - int priority = 50; + /* Note: subclasses must implement at least one of the two + following getFile() methods. */ -public: + /* Dump the contents of the specified file to a sink. */ + virtual void getFile(const std::string& path, Sink& sink); - virtual void init(); + /* Fetch the specified file and call the specified callback with + the result. A subclass may implement this asynchronously. */ + virtual void getFile( + const std::string& path, + Callback<std::shared_ptr<std::string>> callback) noexcept; -private: + std::shared_ptr<std::string> getFile(const std::string& path); - std::string narMagic; + protected: + bool wantMassQuery_ = false; + int priority = 50; - std::string narInfoFileFor(const Path & storePath); + public: + virtual void init(); - void writeNarInfo(ref<NarInfo> narInfo); + private: + std::string narMagic; -public: + std::string narInfoFileFor(const Path& storePath); - bool isValidPathUncached(const Path & path) override; + void writeNarInfo(ref<NarInfo> narInfo); - void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; + public: + bool isValidPathUncached(const Path& path) override; - Path queryPathFromHashPart(const string & hashPart) override - { unsupported("queryPathFromHashPart"); } + void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; - bool wantMassQuery() override { return wantMassQuery_; } + Path queryPathFromHashPart(const string& hashPart) override { + unsupported("queryPathFromHashPart"); + } - void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; + bool wantMassQuery() override { return wantMassQuery_; } - Path addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override; + void addToStore(const ValidPathInfo& info, const ref<std::string>& nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) override; - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override; + Path addToStore(const string& name, const Path& srcPath, bool recursive, + HashType hashAlgo, PathFilter& filter, + RepairFlag repair) override; - void narFromPath(const Path & path, Sink & sink) override; + Path addTextToStore(const string& name, const string& s, + const PathSet& references, RepairFlag repair) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { unsupported("buildDerivation"); } + void narFromPath(const Path& path, Sink& sink) override; - void ensurePath(const Path & path) override - { unsupported("ensurePath"); } + BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv, + BuildMode buildMode) override { + unsupported("buildDerivation"); + } - ref<FSAccessor> getFSAccessor() override; + void ensurePath(const Path& path) override { unsupported("ensurePath"); } - void addSignatures(const Path & storePath, const StringSet & sigs) override; + ref<FSAccessor> getFSAccessor() override; - std::shared_ptr<std::string> getBuildLog(const Path & path) override; + void addSignatures(const Path& storePath, const StringSet& sigs) override; - int getPriority() override { return priority; } + std::shared_ptr<std::string> getBuildLog(const Path& path) override; + int getPriority() override { return priority; } }; MakeError(NoSuchBinaryCacheFile, Error); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/build.cc b/third_party/nix/src/libstore/build.cc index 539e1ea71d5b..c110bd4e6e44 100644 --- a/third_party/nix/src/libstore/build.cc +++ b/third_party/nix/src/libstore/build.cc @@ -1,64 +1,62 @@ -#include "references.hh" -#include "pathlocks.hh" -#include "globals.hh" -#include "local-store.hh" -#include "util.hh" -#include "archive.hh" +#include <errno.h> +#include <fcntl.h> +#include <grp.h> +#include <limits.h> +#include <netdb.h> +#include <pwd.h> +#include <sys/resource.h> +#include <sys/select.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/utsname.h> +#include <sys/wait.h> +#include <termios.h> +#include <unistd.h> +#include <algorithm> +#include <chrono> +#include <cstring> +#include <future> +#include <iostream> +#include <map> +#include <queue> +#include <regex> +#include <sstream> +#include <thread> #include "affinity.hh" +#include "archive.hh" #include "builtins.hh" +#include "compression.hh" #include "download.hh" #include "finally.hh" -#include "compression.hh" +#include "globals.hh" #include "json.hh" +#include "local-store.hh" +#include "machines.hh" #include "nar-info.hh" #include "parsed-derivations.hh" -#include "machines.hh" - -#include <algorithm> -#include <iostream> -#include <map> -#include <sstream> -#include <thread> -#include <future> -#include <chrono> -#include <regex> -#include <queue> - -#include <limits.h> -#include <sys/time.h> -#include <sys/wait.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/utsname.h> -#include <sys/select.h> -#include <sys/resource.h> -#include <sys/socket.h> -#include <fcntl.h> -#include <netdb.h> -#include <unistd.h> -#include <errno.h> -#include <cstring> -#include <termios.h> - -#include <pwd.h> -#include <grp.h> +#include "pathlocks.hh" +#include "references.hh" +#include "util.hh" /* Includes required for chroot support. */ #if __linux__ -#include <sys/socket.h> -#include <sys/ioctl.h> #include <net/if.h> #include <netinet/ip.h> -#include <sys/personality.h> -#include <sys/mman.h> #include <sched.h> -#include <sys/param.h> +#include <sys/ioctl.h> +#include <sys/mman.h> #include <sys/mount.h> +#include <sys/param.h> +#include <sys/personality.h> +#include <sys/socket.h> #include <sys/syscall.h> #if HAVE_SECCOMP #include <seccomp.h> #endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) +#define pivot_root(new_root, put_old) \ + (syscall(SYS_pivot_root, new_root, put_old)) #endif #if HAVE_STATVFS @@ -67,20 +65,16 @@ #include <nlohmann/json.hpp> - namespace nix { using std::map; - static string pathNullDevice = "/dev/null"; - /* Forward definition. */ class Worker; struct HookInstance; - /* A pointer to a goal. */ class Goal; class DerivationGoal; @@ -88,7 +82,7 @@ typedef std::shared_ptr<Goal> GoalPtr; typedef std::weak_ptr<Goal> WeakGoalPtr; struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; + bool operator()(const GoalPtr& a, const GoalPtr& b) const; }; /* Set of goals. */ @@ -98,4611 +92,4508 @@ typedef list<WeakGoalPtr> WeakGoals; /* A map of paths to goals (and the other way around). */ typedef map<Path, WeakGoalPtr> WeakGoalMap; +class Goal : public std::enable_shared_from_this<Goal> { + public: + typedef enum { + ecBusy, + ecSuccess, + ecFailed, + ecNoSubstituters, + ecIncompleteClosure + } ExitCode; + protected: + /* Backlink to the worker. */ + Worker& worker; -class Goal : public std::enable_shared_from_this<Goal> -{ -public: - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; + /* Goals that this goal is waiting for. */ + Goals waitees; -protected: + /* Goals waiting for this one to finish. Must use weak pointers + here to prevent cycles. */ + WeakGoals waiters; - /* Backlink to the worker. */ - Worker & worker; + /* Number of goals we are/were waiting for that have failed. */ + unsigned int nrFailed; - /* Goals that this goal is waiting for. */ - Goals waitees; + /* Number of substitution goals we are/were waiting for that + failed because there are no substituters. */ + unsigned int nrNoSubstituters; - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; + /* Number of substitution goals we are/were waiting for that + failed because othey had unsubstitutable references. */ + unsigned int nrIncompleteClosure; - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; + /* Name of this goal for debugging purposes. */ + string name; - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; + /* Whether the goal is finished. */ + ExitCode exitCode; - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; + Goal(Worker& worker) : worker(worker) { + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + exitCode = ecBusy; + } - /* Name of this goal for debugging purposes. */ - string name; + virtual ~Goal() { trace("goal destroyed"); } - /* Whether the goal is finished. */ - ExitCode exitCode; + public: + virtual void work() = 0; - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } + void addWaitee(GoalPtr waitee); - virtual ~Goal() - { - trace("goal destroyed"); - } + virtual void waiteeDone(GoalPtr waitee, ExitCode result); -public: - virtual void work() = 0; + virtual void handleChildOutput(int fd, const string& data) { abort(); } - void addWaitee(GoalPtr waitee); + virtual void handleEOF(int fd) { abort(); } - virtual void waiteeDone(GoalPtr waitee, ExitCode result); + void trace(const FormatOrString& fs); - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } + string getName() { return name; } - virtual void handleEOF(int fd) - { - abort(); - } + ExitCode getExitCode() { return exitCode; } - void trace(const FormatOrString & fs); + /* Callback in case of a timeout. It should wake up its waiters, + get rid of any running child processes that are being monitored + by the worker (important!), etc. */ + virtual void timedOut() = 0; - string getName() - { - return name; - } + virtual string key() = 0; - ExitCode getExitCode() - { - return exitCode; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut() = 0; - - virtual string key() = 0; - -protected: - - virtual void amDone(ExitCode result); + protected: + virtual void amDone(ExitCode result); }; - -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - string s1 = a->key(); - string s2 = b->key(); - return s1 < s2; +bool CompareGoalPtrs::operator()(const GoalPtr& a, const GoalPtr& b) const { + string s1 = a->key(); + string s2 = b->key(); + return s1 < s2; } - typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; - /* A mapping used to remember for each child process to what goal it belongs, and file descriptors for receiving log data and output path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set<int> fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; +struct Child { + WeakGoalPtr goal; + Goal* goal2; // ugly hackery + set<int> fds; + bool respectTimeouts; + bool inBuildSlot; + steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + steady_time_point timeStarted; }; - /* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list<Child> children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map<Path, bool> pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; +class Worker { + private: + /* Note: the worker should only have strong pointers to the + top-level goals. */ - LocalStore & store; + /* The top-level goals of the worker. */ + Goals topGoals; - std::unique_ptr<HookInstance> hook; + /* Goals that are ready to do some work. */ + WeakGoals awake; - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; + /* Goals waiting for a build slot. */ + WeakGoals wantingToBuild; - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; + /* Child processes currently running. */ + std::list<Child> children; - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; + /* Number of build slots occupied. This includes local builds and + substitutions but not remote builds via the build hook. */ + unsigned int nrLocalBuilds; - Worker(LocalStore & store); - ~Worker(); + /* Maps used to prevent multiple instantiations of a goal for the + same derivation / path. */ + WeakGoalMap derivationGoals; + WeakGoalMap substitutionGoals; + + /* Goals waiting for busy paths to be unlocked. */ + WeakGoals waitingForAnyGoal; + + /* Goals sleeping for a few seconds (polling a lock). */ + WeakGoals waitingForAWhile; + + /* Last time the goals in `waitingForAWhile' where woken up. */ + steady_time_point lastWokenUp; - /* Make a goal (with caching). */ - GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const Path & drvPath, - const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const Path & storePath, RepairFlag repair = NoRepair); + /* Cache for pathContentsGood(). */ + std::map<Path, bool> pathContentsGoodCache; - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); + public: + const Activity act; + const Activity actDerivations; + const Activity actSubstitutions; - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); + /* Set if at least one derivation had a BuildError (i.e. permanent + failure). */ + bool permanentFailure; - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set<int> & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const Path & path); - - void markContentsGood(const Path & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } + /* Set if at least one derivation had a timeout. */ + bool timedOut; + + /* Set if at least one derivation fails with a hash mismatch. */ + bool hashMismatch; + + /* Set if at least one derivation is not deterministic in check mode. */ + bool checkMismatch; + + LocalStore& store; + + std::unique_ptr<HookInstance> hook; + + uint64_t expectedBuilds = 0; + uint64_t doneBuilds = 0; + uint64_t failedBuilds = 0; + uint64_t runningBuilds = 0; + + uint64_t expectedSubstitutions = 0; + uint64_t doneSubstitutions = 0; + uint64_t failedSubstitutions = 0; + uint64_t runningSubstitutions = 0; + uint64_t expectedDownloadSize = 0; + uint64_t doneDownloadSize = 0; + uint64_t expectedNarSize = 0; + uint64_t doneNarSize = 0; + + /* Whether to ask the build hook if it can build a derivation. If + it answers with "decline-permanently", we don't try again. */ + bool tryBuildHook = true; + + Worker(LocalStore& store); + ~Worker(); + + /* Make a goal (with caching). */ + GoalPtr makeDerivationGoal(const Path& drvPath, + const StringSet& wantedOutputs, + BuildMode buildMode = bmNormal); + std::shared_ptr<DerivationGoal> makeBasicDerivationGoal( + const Path& drvPath, const BasicDerivation& drv, + BuildMode buildMode = bmNormal); + GoalPtr makeSubstitutionGoal(const Path& storePath, + RepairFlag repair = NoRepair); + + /* Remove a dead goal. */ + void removeGoal(GoalPtr goal); + + /* Wake up a goal (i.e., there is something for it to do). */ + void wakeUp(GoalPtr goal); + + /* Return the number of local build and substitution processes + currently running (but not remote builds via the build + hook). */ + unsigned int getNrLocalBuilds(); + + /* Registers a running child process. `inBuildSlot' means that + the process counts towards the jobs limit. */ + void childStarted(GoalPtr goal, const set<int>& fds, bool inBuildSlot, + bool respectTimeouts); + + /* Unregisters a running child process. `wakeSleepers' should be + false if there is no sense in waking up goals that are sleeping + because they can't run yet (e.g., there is no free build slot, + or the hook would still say `postpone'). */ + void childTerminated(Goal* goal, bool wakeSleepers = true); + + /* Put `goal' to sleep until a build slot becomes available (which + might be right away). */ + void waitForBuildSlot(GoalPtr goal); + + /* Wait for any goal to finish. Pretty indiscriminate way to + wait for some resource that some other goal is holding. */ + void waitForAnyGoal(GoalPtr goal); + + /* Wait for a few seconds and then retry this goal. Used when + waiting for a lock held by another process. This kind of + polling is inefficient, but POSIX doesn't really provide a way + to wait for multiple locks in the main select() loop. */ + void waitForAWhile(GoalPtr goal); + + /* Loop until the specified top-level goals have finished. */ + void run(const Goals& topGoals); + + /* Wait for input to become available. */ + void waitForInput(); + + unsigned int exitStatus(); + + /* Check whether the given valid path exists and has the right + contents. */ + bool pathContentsGood(const Path& path); + + void markContentsGood(const Path& path); + + void updateProgress() { + actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, + runningBuilds, failedBuilds); + actSubstitutions.progress(doneSubstitutions, + expectedSubstitutions + doneSubstitutions, + runningSubstitutions, failedSubstitutions); + act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize); + act.setExpected(actCopyPath, expectedNarSize + doneNarSize); + } }; - ////////////////////////////////////////////////////////////////////// - -void addToWeakGoals(WeakGoals & goals, GoalPtr p) -{ - // FIXME: necessary? - // FIXME: O(n) - for (auto & i : goals) - if (i.lock() == p) return; - goals.push_back(p); +void addToWeakGoals(WeakGoals& goals, GoalPtr p) { + // FIXME: necessary? + // FIXME: O(n) + for (auto& i : goals) + if (i.lock() == p) return; + goals.push_back(p); } - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); +void Goal::addWaitee(GoalPtr waitee) { + waitees.insert(waitee); + addToWeakGoals(waitee->waiters, shared_from_this()); } +void Goal::waiteeDone(GoalPtr waitee, ExitCode result) { + assert(waitees.find(waitee) != waitees.end()); + waitees.erase(waitee); -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.find(waitee) != waitees.end()); - waitees.erase(waitee); - - trace(format("waitee '%1%' done; %2% left") % - waitee->name % waitees.size()); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; + trace(format("waitee '%1%' done; %2% left") % waitee->name % waitees.size()); - if (result == ecNoSubstituters) ++nrNoSubstituters; + if (result == ecFailed || result == ecNoSubstituters || + result == ecIncompleteClosure) + ++nrFailed; - if (result == ecIncompleteClosure) ++nrIncompleteClosure; + if (result == ecNoSubstituters) ++nrNoSubstituters; - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { + if (result == ecIncompleteClosure) ++nrIncompleteClosure; - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & goal : waitees) { - WeakGoals waiters2; - for (auto & j : goal->waiters) - if (j.lock() != shared_from_this()) waiters2.push_back(j); - goal->waiters = waiters2; - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); + if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { + /* If we failed and keepGoing is not set, we remove all + remaining waitees. */ + for (auto& goal : waitees) { + WeakGoals waiters2; + for (auto& j : goal->waiters) + if (j.lock() != shared_from_this()) waiters2.push_back(j); + goal->waiters = waiters2; } -} + waitees.clear(); - -void Goal::amDone(ExitCode result) -{ - trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); - exitCode = result; - for (auto & i : waiters) { - GoalPtr goal = i.lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); + worker.wakeUp(shared_from_this()); + } } - -void Goal::trace(const FormatOrString & fs) -{ - debug("%1%: %2%", name, fs.s); +void Goal::amDone(ExitCode result) { + trace("done"); + assert(exitCode == ecBusy); + assert(result == ecSuccess || result == ecFailed || + result == ecNoSubstituters || result == ecIncompleteClosure); + exitCode = result; + for (auto& i : waiters) { + GoalPtr goal = i.lock(); + if (goal) goal->waiteeDone(shared_from_this(), result); + } + waiters.clear(); + worker.removeGoal(shared_from_this()); } - +void Goal::trace(const FormatOrString& fs) { debug("%1%: %2%", name, fs.s); } ////////////////////////////////////////////////////////////////////// - /* Common initialisation performed in child processes. */ -static void commonChildInit(Pipe & logPipe) -{ - restoreSignals(); - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) - throw SysError(format("creating a new session")); - - /* Dup the write side of the logger pipe into stderr. */ - if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) - throw SysError("cannot pipe standard error into log file"); - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError(format("cannot open '%1%'") % pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - close(fdDevNull); +static void commonChildInit(Pipe& logPipe) { + restoreSignals(); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) throw SysError(format("creating a new session")); + + /* Dup the write side of the logger pipe into stderr. */ + if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError(format("cannot open '%1%'") % pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); } -void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) -{ - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { - try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - - auto diffRes = runProgram(diffHookOptions); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first))); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); - } +void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, + Path tmpDir) { + auto diffHook = settings.diffHook; + if (diffHook != "" && settings.runDiffHook) { + try { + RunOptions diffHookOptions(diffHook, {tryA, tryB, drvPath, tmpDir}); + diffHookOptions.searchPath = true; + diffHookOptions.uid = uid; + diffHookOptions.gid = gid; + diffHookOptions.chdir = "/"; + + auto diffRes = runProgram(diffHookOptions); + if (!statusOk(diffRes.first)) + throw ExecError(diffRes.first, + fmt("diff-hook program '%1%' %2%", diffHook, + statusToString(diffRes.first))); + + if (diffRes.second != "") printError(chomp(diffRes.second)); + } catch (Error& error) { + printError("diff hook execution failed: %s", error.what()); } + } } ////////////////////////////////////////////////////////////////////// - -class UserLock -{ -private: - /* POSIX locks suck. If we have a lock on a file, and we open and - close that file again (without closing the original file - descriptor), we lose the lock. So we have to be *very* careful - not to open a lock file on which we are holding a lock. */ - static Sync<PathSet> lockedPaths_; - - Path fnUserLock; - AutoCloseFD fdUserLock; - - string user; - uid_t uid; - gid_t gid; - std::vector<gid_t> supplementaryGIDs; - -public: - UserLock(); - ~UserLock(); - - void kill(); - - string getUser() { return user; } - uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } - std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; } - - bool enabled() { return uid != 0; } - +class UserLock { + private: + /* POSIX locks suck. If we have a lock on a file, and we open and + close that file again (without closing the original file + descriptor), we lose the lock. So we have to be *very* careful + not to open a lock file on which we are holding a lock. */ + static Sync<PathSet> lockedPaths_; + + Path fnUserLock; + AutoCloseFD fdUserLock; + + string user; + uid_t uid; + gid_t gid; + std::vector<gid_t> supplementaryGIDs; + + public: + UserLock(); + ~UserLock(); + + void kill(); + + string getUser() { return user; } + uid_t getUID() { + assert(uid); + return uid; + } + uid_t getGID() { + assert(gid); + return gid; + } + std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; } + + bool enabled() { return uid != 0; } }; - Sync<PathSet> UserLock::lockedPaths_; +UserLock::UserLock() { + assert(settings.buildUsersGroup != ""); -UserLock::UserLock() -{ - assert(settings.buildUsersGroup != ""); - - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error(format("the group '%1%' specified in 'build-users-group' does not exist") - % settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug(format("found build user '%1%'") % *p); - users.push_back(*p); - } + /* Get the members of the build-users-group. */ + struct group* gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (!gr) + throw Error( + format( + "the group '%1%' specified in 'build-users-group' does not exist") % + settings.buildUsersGroup); + gid = gr->gr_gid; - if (users.empty()) - throw Error(format("the build users group '%1%' has no members") - % settings.buildUsersGroup); + /* Copy the result of getgrnam. */ + Strings users; + for (char** p = gr->gr_mem; *p; ++p) { + debug(format("found build user '%1%'") % *p); + users.push_back(*p); + } - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug(format("trying user '%1%'") % i); + if (users.empty()) + throw Error(format("the build users group '%1%' has no members") % + settings.buildUsersGroup); - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error(format("the user '%1%' in the group '%2%' does not exist") - % i % settings.buildUsersGroup); + /* Find a user account that isn't currently in use for another + build. */ + for (auto& i : users) { + debug(format("trying user '%1%'") % i); - createDirs(settings.nixStateDir + "/userpool"); + struct passwd* pw = getpwnam(i.c_str()); + if (!pw) + throw Error(format("the user '%1%' in the group '%2%' does not exist") % + i % settings.buildUsersGroup); - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); + createDirs(settings.nixStateDir + "/userpool"); - { - auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(fnUserLock)) - /* We already have a lock on this one. */ - continue; - lockedPaths->insert(fnUserLock); - } + fnUserLock = + (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - try { + { + auto lockedPaths(lockedPaths_.lock()); + if (lockedPaths->count(fnUserLock)) + /* We already have a lock on this one. */ + continue; + lockedPaths->insert(fnUserLock); + } - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError(format("opening user lock '%1%'") % fnUserLock); + try { + AutoCloseFD fd = + open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) throw SysError(format("opening user lock '%1%'") % fnUserLock); - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; + if (lockFile(fd.get(), ltWrite, false)) { + fdUserLock = std::move(fd); + user = i; + uid = pw->pw_uid; - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error(format("the Nix user should not be a member of '%1%'") - % settings.buildUsersGroup); + /* Sanity check... */ + if (uid == getuid() || uid == geteuid()) + throw Error(format("the Nix user should not be a member of '%1%'") % + settings.buildUsersGroup); #if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name); - - supplementaryGIDs.resize(ngroups); + /* Get the list of supplementary groups of this build user. This + is usually either empty or contains a group such as "kvm". */ + supplementaryGIDs.resize(10); + int ngroups = supplementaryGIDs.size(); + int err = getgrouplist(pw->pw_name, pw->pw_gid, + supplementaryGIDs.data(), &ngroups); + if (err == -1) + throw Error( + format("failed to get list of supplementary groups for '%1%'") % + pw->pw_name); + + supplementaryGIDs.resize(ngroups); #endif - return; - } + return; + } - } catch (...) { - lockedPaths_.lock()->erase(fnUserLock); - } + } catch (...) { + lockedPaths_.lock()->erase(fnUserLock); } + } - throw Error(format("all build users are currently in use; " - "consider creating additional users and adding them to the '%1%' group") - % settings.buildUsersGroup); -} - - -UserLock::~UserLock() -{ - auto lockedPaths(lockedPaths_.lock()); - assert(lockedPaths->count(fnUserLock)); - lockedPaths->erase(fnUserLock); + throw Error(format("all build users are currently in use; " + "consider creating additional users and adding them to " + "the '%1%' group") % + settings.buildUsersGroup); } - -void UserLock::kill() -{ - killUser(uid); +UserLock::~UserLock() { + auto lockedPaths(lockedPaths_.lock()); + assert(lockedPaths->count(fnUserLock)); + lockedPaths->erase(fnUserLock); } +void UserLock::kill() { killUser(uid); } ////////////////////////////////////////////////////////////////////// +struct HookInstance { + /* Pipes for talking to the build hook. */ + Pipe toHook; -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; + /* Pipe for the hook's standard output/error. */ + Pipe fromHook; - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; - /* The process ID of the hook. */ - Pid pid; + /* The process ID of the hook. */ + Pid pid; - FdSink sink; + FdSink sink; - std::map<ActivityId, Activity> activities; + std::map<ActivityId, Activity> activities; - HookInstance(); + HookInstance(); - ~HookInstance(); + ~HookInstance(); }; +HookInstance::HookInstance() { + debug("starting build hook '%s'", settings.buildHook); -HookInstance::HookInstance() -{ - debug("starting build hook '%s'", settings.buildHook); + /* Create a pipe to get the output of the child. */ + fromHook.create(); - /* Create a pipe to get the output of the child. */ - fromHook.create(); + /* Create the communication pipes. */ + toHook.create(); - /* Create the communication pipes. */ - toHook.create(); + /* Create a pipe to get the output of the builder. */ + builderOut.create(); - /* Create a pipe to get the output of the builder. */ - builderOut.create(); + /* Fork the hook. */ + pid = startProcess([&]() { + commonChildInit(fromHook); - /* Fork the hook. */ - pid = startProcess([&]() { + if (chdir("/") == -1) throw SysError("changing into /"); - commonChildInit(fromHook); + /* Dup the communication pipes. */ + if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping to-hook read side"); - if (chdir("/") == -1) throw SysError("changing into /"); + /* Use fd 4 for the builder's stdout/stderr. */ + if (dup2(builderOut.writeSide.get(), 4) == -1) + throw SysError("dupping builder's stdout/stderr"); - /* Dup the communication pipes. */ - if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); + /* Hack: pass the read side of that fd to allow build-remote + to read SSH error messages. */ + if (dup2(builderOut.readSide.get(), 5) == -1) + throw SysError("dupping builder's stdout/stderr"); - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide.get(), 4) == -1) - throw SysError("dupping builder's stdout/stderr"); - - /* Hack: pass the read side of that fd to allow build-remote - to read SSH error messages. */ - if (dup2(builderOut.readSide.get(), 5) == -1) - throw SysError("dupping builder's stdout/stderr"); - - Strings args = { - baseNameOf(settings.buildHook), - std::to_string(verbosity), - }; + Strings args = { + baseNameOf(settings.buildHook), + std::to_string(verbosity), + }; - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); + execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - throw SysError("executing '%s'", settings.buildHook); - }); + throw SysError("executing '%s'", settings.buildHook); + }); - pid.setSeparatePG(true); - fromHook.writeSide = -1; - toHook.readSide = -1; + pid.setSeparatePG(true); + fromHook.writeSide = -1; + toHook.readSide = -1; - sink = FdSink(toHook.writeSide.get()); - std::map<std::string, Config::SettingInfo> settings; - globalConfig.getSettings(settings); - for (auto & setting : settings) - sink << 1 << setting.first << setting.second.value; - sink << 0; + sink = FdSink(toHook.writeSide.get()); + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (auto& setting : settings) + sink << 1 << setting.first << setting.second.value; + sink << 0; } - -HookInstance::~HookInstance() -{ - try { - toHook.writeSide = -1; - if (pid != -1) pid.kill(); - } catch (...) { - ignoreException(); - } +HookInstance::~HookInstance() { + try { + toHook.writeSide = -1; + if (pid != -1) pid.kill(); + } catch (...) { + ignoreException(); + } } - ////////////////////////////////////////////////////////////////////// - typedef map<std::string, std::string> StringRewrites; - -std::string rewriteStrings(std::string s, const StringRewrites & rewrites) -{ - for (auto & i : rewrites) { - size_t j = 0; - while ((j = s.find(i.first, j)) != string::npos) - s.replace(j, i.first.size(), i.second); - } - return s; +std::string rewriteStrings(std::string s, const StringRewrites& rewrites) { + for (auto& i : rewrites) { + size_t j = 0; + while ((j = s.find(i.first, j)) != string::npos) + s.replace(j, i.first.size(), i.second); + } + return s; } - ////////////////////////////////////////////////////////////////////// - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; +typedef enum { rpAccept, rpDecline, rpPostpone } HookReply; class SubstitutionGoal; -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; +class DerivationGoal : public Goal { + private: + /* Whether to use an on-disk .drv file. */ + bool useDerivation; - /* The path of the derivation. */ - Path drvPath; + /* The path of the derivation. */ + Path drvPath; - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; + /* The specific outputs that we need to build. Empty means all of + them. */ + StringSet wantedOutputs; - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; + /* Whether additional wanted outputs have been added. */ + bool needRestart = false; - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; + /* Whether to retry substituting the outputs after building the + inputs. */ + bool retrySubstitution; - /* The derivation stored at drvPath. */ - std::unique_ptr<BasicDerivation> drv; + /* The derivation stored at drvPath. */ + std::unique_ptr<BasicDerivation> drv; - std::unique_ptr<ParsedDerivation> parsedDrv; + std::unique_ptr<ParsedDerivation> parsedDrv; - /* The remainder is state held during the build. */ + /* The remainder is state held during the build. */ - /* Locks on the output paths. */ - PathLocks outputLocks; + /* Locks on the output paths. */ + PathLocks outputLocks; - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - PathSet inputPaths; + /* All input paths (that is, the union of FS closures of the + immediate input paths). */ + PathSet inputPaths; - /* Referenceable paths (i.e., input and output paths). */ - PathSet allPaths; + /* Referenceable paths (i.e., input and output paths). */ + PathSet allPaths; - /* Outputs that are already valid. If we're repairing, these are - the outputs that are valid *and* not corrupt. */ - PathSet validPaths; + /* Outputs that are already valid. If we're repairing, these are + the outputs that are valid *and* not corrupt. */ + PathSet validPaths; - /* Outputs that are corrupt or not valid. */ - PathSet missingPaths; + /* Outputs that are corrupt or not valid. */ + PathSet missingPaths; - /* User selected for running the builder. */ - std::unique_ptr<UserLock> buildUser; + /* User selected for running the builder. */ + std::unique_ptr<UserLock> buildUser; - /* The process ID of the builder. */ - Pid pid; + /* The process ID of the builder. */ + Pid pid; - /* The temporary directory. */ - Path tmpDir; + /* The temporary directory. */ + Path tmpDir; - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; + /* The path of the temporary directory in the sandbox. */ + Path tmpDirInSandbox; - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr<BufferedSink> logFileSink, logSink; + /* File descriptor for the log file. */ + AutoCloseFD fdLogFile; + std::shared_ptr<BufferedSink> logFileSink, logSink; - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; + /* Number of bytes received from the builder's stdout/stderr. */ + unsigned long logSize; - /* The most recent log lines. */ - std::list<std::string> logTail; + /* The most recent log lines. */ + std::list<std::string> logTail; - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return + std::string currentLogLine; + size_t currentLogLinePos = 0; // to handle carriage return - std::string currentHookLine; + std::string currentHookLine; - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; - /* Pipe for synchronising updates to the builder user namespace. */ - Pipe userNamespaceSync; + /* Pipe for synchronising updates to the builder user namespace. */ + Pipe userNamespaceSync; - /* The build hook. */ - std::unique_ptr<HookInstance> hook; + /* The build hook. */ + std::unique_ptr<HookInstance> hook; - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; + /* Whether we're currently doing a chroot build. */ + bool useChroot = false; - Path chrootRootDir; + Path chrootRootDir; - /* RAII object to delete the chroot directory. */ - std::shared_ptr<AutoDelete> autoDelChroot; + /* RAII object to delete the chroot directory. */ + std::shared_ptr<AutoDelete> autoDelChroot; - /* Whether this is a fixed-output derivation. */ - bool fixedOutput; + /* Whether this is a fixed-output derivation. */ + bool fixedOutput; - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; + /* Whether to run the build in a private network namespace. */ + bool privateNetwork = false; - typedef void (DerivationGoal::*GoalState)(); - GoalState state; + typedef void (DerivationGoal::*GoalState)(); + GoalState state; - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; + /* Stuff we need to pass to initChild(). */ + struct ChrootPath { + Path source; + bool optional; + ChrootPath(Path source = "", bool optional = false) + : source(source), optional(optional) {} + }; + typedef map<Path, ChrootPath> + DirsInChroot; // maps target path to source path + DirsInChroot dirsInChroot; - typedef map<string, string> Environment; - Environment env; + typedef map<string, string> Environment; + Environment env; #if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; + typedef string SandboxProfile; + SandboxProfile additionalSandboxProfile; #endif - /* Hash rewriting. */ - StringRewrites inputRewrites, outputRewrites; - typedef map<Path, Path> RedirectedOutputs; - RedirectedOutputs redirectedOutputs; + /* Hash rewriting. */ + StringRewrites inputRewrites, outputRewrites; + typedef map<Path, Path> RedirectedOutputs; + RedirectedOutputs redirectedOutputs; - BuildMode buildMode; + BuildMode buildMode; - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - PathSet redirectedBadOutputs; + /* If we're repairing without a chroot, there may be outputs that + are valid but corrupt. So we redirect these outputs to + temporary paths. */ + PathSet redirectedBadOutputs; - BuildResult result; + BuildResult result; - /* The current round, if we're building multiple times. */ - size_t curRound = 1; + /* The current round, if we're building multiple times. */ + size_t curRound = 1; - size_t nrRounds; + size_t nrRounds; - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map<Path, ValidPathInfo> prevInfos; + /* Path registration info from the previous round, if we're + building multiple times. Since this contains the hash, it + allows us to compare whether two rounds produced the same + result. */ + std::map<Path, ValidPathInfo> prevInfos; - const uid_t sandboxUid = 1000; - const gid_t sandboxGid = 100; + const uid_t sandboxUid = 1000; + const gid_t sandboxGid = 100; - const static Path homeDir; + const static Path homeDir; - std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds; + std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds; - std::unique_ptr<Activity> act; + std::unique_ptr<Activity> act; - std::map<ActivityId, Activity> builderActivities; + std::map<ActivityId, Activity> builderActivities; - /* The remote machine on which we're building. */ - std::string machineName; + /* The remote machine on which we're building. */ + std::string machineName; -public: - DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, - Worker & worker, BuildMode buildMode = bmNormal); - DerivationGoal(const Path & drvPath, const BasicDerivation & drv, - Worker & worker, BuildMode buildMode = bmNormal); - ~DerivationGoal(); + public: + DerivationGoal(const Path& drvPath, const StringSet& wantedOutputs, + Worker& worker, BuildMode buildMode = bmNormal); + DerivationGoal(const Path& drvPath, const BasicDerivation& drv, + Worker& worker, BuildMode buildMode = bmNormal); + ~DerivationGoal(); - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); + /* Whether we need to perform hash rewriting if there are valid output paths. + */ + bool needsHashRewrite(); - void timedOut() override; + void timedOut() override; - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + storePathToName(drvPath) + "$" + drvPath; - } + string key() override { + /* Ensure that derivations get built in order of their name, + i.e. a derivation named "aardvark" always comes before + "baboon". And substitution goals always happen before + derivation goals (due to "b$"). */ + return "b$" + storePathToName(drvPath) + "$" + drvPath; + } - void work() override; + void work() override; - Path getDrvPath() - { - return drvPath; - } + Path getDrvPath() { return drvPath; } - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); + /* Add wanted outputs to an already existing derivation goal. */ + void addWantedOutputs(const StringSet& outputs); - BuildResult getResult() { return result; } + BuildResult getResult() { return result; } -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstituted(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void buildDone(); + private: + /* The states. */ + void getDerivation(); + void loadDerivation(); + void haveDerivation(); + void outputsSubstituted(); + void closureRepaired(); + void inputsRealised(); + void tryToBuild(); + void buildDone(); - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); + /* Is the build hook willing to perform the build? */ + HookReply tryBuildHook(); - /* Start building a derivation. */ - void startBuilder(); + /* Start building a derivation. */ + void startBuilder(); - /* Fill in the environment for the builder. */ - void initEnv(); + /* Fill in the environment for the builder. */ + void initEnv(); - /* Setup tmp dir location. */ - void initTmpDir(); + /* Setup tmp dir location. */ + void initTmpDir(); - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); + /* Write a JSON file containing the derivation attributes. */ + void writeStructuredAttrs(); - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); + /* Make a file owned by the builder. */ + void chownToBuilder(const Path& path); - /* Run the builder's process. */ - void runChild(); + /* Run the builder's process. */ + void runChild(); - friend int childEntry(void *); + friend int childEntry(void*); - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); + /* Check that the derivation outputs all exist and register them + as valid. */ + void registerOutputs(); - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs); + /* Check that an output meets the requirements specified by the + 'outputChecks' attribute (or the legacy + '{allowed,disallowed}{References,Requisites}' attributes). */ + void checkOutputs(const std::map<std::string, ValidPathInfo>& outputs); - /* Open a log file and a pipe to it. */ - Path openLogFile(); + /* Open a log file and a pipe to it. */ + Path openLogFile(); - /* Close the log file. */ - void closeLogFile(); + /* Close the log file. */ + void closeLogFile(); - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); + /* Delete the temporary directory, if we have one. */ + void deleteTmpDir(bool force); - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const string& data) override; + void handleEOF(int fd) override; + void flushLine(); - /* Return the set of (in)valid paths. */ - PathSet checkPathValidity(bool returnValid, bool checkHash); + /* Return the set of (in)valid paths. */ + PathSet checkPathValidity(bool returnValid, bool checkHash); - /* Abort the goal if `path' failed to build. */ - bool pathFailed(const Path & path); + /* Abort the goal if `path' failed to build. */ + bool pathFailed(const Path& path); - /* Forcibly kill the child process, if any. */ - void killChild(); + /* Forcibly kill the child process, if any. */ + void killChild(); - Path addHashRewrite(const Path & path); + Path addHashRewrite(const Path& path); - void repairClosure(); + void repairClosure(); - void amDone(ExitCode result) override - { - Goal::amDone(result); - } + void amDone(ExitCode result) override { Goal::amDone(result); } - void done(BuildResult::Status status, const string & msg = ""); + void done(BuildResult::Status status, const string& msg = ""); - PathSet exportReferences(PathSet storePaths); + PathSet exportReferences(PathSet storePaths); }; - const Path DerivationGoal::homeDir = "/homeless-shelter"; - -DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, - Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(true) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &DerivationGoal::getDerivation; - name = (format("building of '%1%'") % drvPath).str(); - trace("created"); - - mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); - worker.updateProgress(); +DerivationGoal::DerivationGoal(const Path& drvPath, + const StringSet& wantedOutputs, Worker& worker, + BuildMode buildMode) + : Goal(worker), + useDerivation(true), + drvPath(drvPath), + wantedOutputs(wantedOutputs), + buildMode(buildMode) { + state = &DerivationGoal::getDerivation; + name = (format("building of '%1%'") % drvPath).str(); + trace("created"); + + mcExpectedBuilds = + std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); + worker.updateProgress(); } - -DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv, - Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(false) - , drvPath(drvPath) - , buildMode(buildMode) -{ - this->drv = std::unique_ptr<BasicDerivation>(new BasicDerivation(drv)); - state = &DerivationGoal::haveDerivation; - name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); - trace("created"); - - mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); - worker.updateProgress(); - - /* Prevent the .chroot directory from being - garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(drvPath); +DerivationGoal::DerivationGoal(const Path& drvPath, const BasicDerivation& drv, + Worker& worker, BuildMode buildMode) + : Goal(worker), + useDerivation(false), + drvPath(drvPath), + buildMode(buildMode) { + this->drv = std::unique_ptr<BasicDerivation>(new BasicDerivation(drv)); + state = &DerivationGoal::haveDerivation; + name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); + trace("created"); + + mcExpectedBuilds = + std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); + worker.updateProgress(); + + /* Prevent the .chroot directory from being + garbage-collected. (See isActiveTempFile() in gc.cc.) */ + worker.store.addTempRoot(drvPath); } - -DerivationGoal::~DerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { killChild(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { closeLogFile(); } catch (...) { ignoreException(); } +DerivationGoal::~DerivationGoal() { + /* Careful: we should never ever throw an exception from a + destructor. */ + try { + killChild(); + } catch (...) { + ignoreException(); + } + try { + deleteTmpDir(false); + } catch (...) { + ignoreException(); + } + try { + closeLogFile(); + } catch (...) { + ignoreException(); + } } - -inline bool DerivationGoal::needsHashRewrite() -{ +inline bool DerivationGoal::needsHashRewrite() { #if __linux__ - return !useChroot; + return !useChroot; #else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + return true; #endif } +void DerivationGoal::killChild() { + if (pid != -1) { + worker.childTerminated(this); -void DerivationGoal::killChild() -{ - if (pid != -1) { - worker.childTerminated(this); - - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); + if (buildUser) { + /* If we're using a build user, then there is a tricky + race condition: if we kill the build user before the + child has done its setuid() to the build user uid, then + it won't be killed, and we'll potentially lock up in + pid.wait(). So also send a conventional kill to the + child. */ + ::kill(-pid, SIGKILL); /* ignore the result */ + buildUser->kill(); + pid.wait(); + } else + pid.kill(); - assert(pid == -1); - } + assert(pid == -1); + } - hook.reset(); + hook.reset(); } - -void DerivationGoal::timedOut() -{ - killChild(); - done(BuildResult::TimedOut); +void DerivationGoal::timedOut() { + killChild(); + done(BuildResult::TimedOut); } +void DerivationGoal::work() { (this->*state)(); } -void DerivationGoal::work() -{ - (this->*state)(); -} +void DerivationGoal::addWantedOutputs(const StringSet& outputs) { + /* If we already want all outputs, there is nothing to do. */ + if (wantedOutputs.empty()) return; - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); + if (outputs.empty()) { + wantedOutputs.clear(); + needRestart = true; + } else + for (auto& i : outputs) + if (wantedOutputs.find(i) == wantedOutputs.end()) { + wantedOutputs.insert(i); needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.find(i) == wantedOutputs.end()) { - wantedOutputs.insert(i); - needRestart = true; - } + } } +void DerivationGoal::getDerivation() { + trace("init"); -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - loadDerivation(); - return; - } + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { + loadDerivation(); + return; + } - addWaitee(worker.makeSubstitutionGoal(drvPath)); + addWaitee(worker.makeSubstitutionGoal(drvPath)); - state = &DerivationGoal::loadDerivation; + state = &DerivationGoal::loadDerivation; } +void DerivationGoal::loadDerivation() { + trace("loading derivation"); -void DerivationGoal::loadDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - printError(format("cannot build missing derivation '%1%'") % drvPath); - done(BuildResult::MiscFailure); - return; - } + if (nrFailed != 0) { + printError(format("cannot build missing derivation '%1%'") % drvPath); + done(BuildResult::MiscFailure); + return; + } - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); + /* `drvPath' should already be a root, but let's be on the safe + side: if the user forgot to make it a root, we wouldn't want + things being garbage collected while we're busy. */ + worker.store.addTempRoot(drvPath); - assert(worker.store.isValidPath(drvPath)); + assert(worker.store.isValidPath(drvPath)); - /* Get the derivation. */ - drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath))); + /* Get the derivation. */ + drv = std::unique_ptr<BasicDerivation>( + new Derivation(worker.store.derivationFromPath(drvPath))); - haveDerivation(); + haveDerivation(); } +void DerivationGoal::haveDerivation() { + trace("have derivation"); -void DerivationGoal::haveDerivation() -{ - trace("have derivation"); - - retrySubstitution = false; + retrySubstitution = false; - for (auto & i : drv->outputs) - worker.store.addTempRoot(i.second.path); + for (auto& i : drv->outputs) worker.store.addTempRoot(i.second.path); - /* Check what outputs paths are not already valid. */ - PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); + /* Check what outputs paths are not already valid. */ + PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); - /* If they are all valid, then we're done. */ - if (invalidOutputs.size() == 0 && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); - return; - } - - parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & i : invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstituted(); - else - state = &DerivationGoal::outputsSubstituted; + /* If they are all valid, then we're done. */ + if (invalidOutputs.size() == 0 && buildMode == bmNormal) { + done(BuildResult::AlreadyValid); + return; + } + + parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); + + /* We are first going to try to create the invalid output paths + through substitutes. If that doesn't work, we'll build + them. */ + if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) + for (auto& i : invalidOutputs) + addWaitee(worker.makeSubstitutionGoal( + i, buildMode == bmRepair ? Repair : NoRepair)); + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + outputsSubstituted(); + else + state = &DerivationGoal::outputsSubstituted; } - -void DerivationGoal::outputsSubstituted() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, (format("some substitutes for the outputs of derivation '%1%' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ") % drvPath).str()); - return; - } - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - auto nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error(format("some outputs of '%1%' are not valid, so checking is not possible") % drvPath); - - /* Otherwise, at least one of the output paths could not be - produced using a substitute. So we have to build instead. */ - - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs = PathSet(); - - /* The inputs must be built before we can build this goal. */ - if (useDerivation) - for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - - for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) - throw Error(format("dependency '%1%' of '%2%' does not exist, and substitution is disabled") - % i % drvPath); - addWaitee(worker.makeSubstitutionGoal(i)); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; +void DerivationGoal::outputsSubstituted() { + trace("all outputs substituted (maybe)"); + + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && + !settings.tryFallback) { + done(BuildResult::TransientFailure, + (format("some substitutes for the outputs of derivation '%1%' failed " + "(usually happens due to networking issues); try '--fallback' " + "to build derivation from source ") % + drvPath) + .str()); + return; + } + + /* If the substitutes form an incomplete closure, then we should + build the dependencies of this derivation, but after that, we + can still use the substitutes for this derivation itself. */ + if (nrIncompleteClosure > 0) retrySubstitution = true; + + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + + if (needRestart) { + needRestart = false; + haveDerivation(); + return; + } + + auto nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); + if (buildMode == bmNormal && nrInvalid == 0) { + done(BuildResult::Substituted); + return; + } + if (buildMode == bmRepair && nrInvalid == 0) { + repairClosure(); + return; + } + if (buildMode == bmCheck && nrInvalid > 0) + throw Error(format("some outputs of '%1%' are not valid, so checking is " + "not possible") % + drvPath); + + /* Otherwise, at least one of the output paths could not be + produced using a substitute. So we have to build instead. */ + + /* Make sure checkPathValidity() from now on checks all + outputs. */ + wantedOutputs = PathSet(); + + /* The inputs must be built before we can build this goal. */ + if (useDerivation) + for (auto& i : dynamic_cast<Derivation*>(drv.get())->inputDrvs) + addWaitee(worker.makeDerivationGoal( + i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + + for (auto& i : drv->inputSrcs) { + if (worker.store.isValidPath(i)) continue; + if (!settings.useSubstitutes) + throw Error(format("dependency '%1%' of '%2%' does not exist, and " + "substitution is disabled") % + i % drvPath); + addWaitee(worker.makeSubstitutionGoal(i)); + } + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + inputsRealised(); + else + state = &DerivationGoal::inputsRealised; } - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - PathSet outputClosure; - for (auto & i : drv->outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second.path, outputClosure); +void DerivationGoal::repairClosure() { + /* If we're repairing, we now know that our own outputs are valid. + Now check whether the other paths in the outputs closure are + good. If not, then start derivation goals for the derivations + that produced those outputs. */ + + /* Get the output closure. */ + PathSet outputClosure; + for (auto& i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + worker.store.computeFSClosure(i.second.path, outputClosure); + } + + /* Filter out our own outputs (which we have already checked). */ + for (auto& i : drv->outputs) outputClosure.erase(i.second.path); + + /* Get all dependencies of this derivation so that we know which + derivation is responsible for which path in the output + closure. */ + PathSet inputClosure; + if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); + std::map<Path, Path> outputsToDrv; + for (auto& i : inputClosure) + if (isDerivation(i)) { + Derivation drv = worker.store.derivationFromPath(i); + for (auto& j : drv.outputs) outputsToDrv[j.second.path] = i; } - /* Filter out our own outputs (which we have already checked). */ - for (auto & i : drv->outputs) - outputClosure.erase(i.second.path); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - PathSet inputClosure; - if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map<Path, Path> outputsToDrv; - for (auto & i : inputClosure) - if (isDerivation(i)) { - Derivation drv = worker.store.derivationFromPath(i); - for (auto & j : drv.outputs) - outputsToDrv[j.second.path] = i; - } - - /* Check each path (slow!). */ - PathSet broken; - for (auto & i : outputClosure) { - if (worker.pathContentsGood(i)) continue; - printError(format("found corrupted or missing path '%1%' in the output closure of '%2%'") % i % drvPath); - Path drvPath2 = outputsToDrv[i]; - if (drvPath2 == "") - addWaitee(worker.makeSubstitutionGoal(i, Repair)); - else - addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); - } + /* Check each path (slow!). */ + PathSet broken; + for (auto& i : outputClosure) { + if (worker.pathContentsGood(i)) continue; + printError(format("found corrupted or missing path '%1%' in the output " + "closure of '%2%'") % + i % drvPath); + Path drvPath2 = outputsToDrv[i]; + if (drvPath2 == "") + addWaitee(worker.makeSubstitutionGoal(i, Repair)); + else + addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); + } - if (waitees.empty()) { - done(BuildResult::AlreadyValid); - return; - } + if (waitees.empty()) { + done(BuildResult::AlreadyValid); + return; + } - state = &DerivationGoal::closureRepaired; + state = &DerivationGoal::closureRepaired; } - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error(format("some paths in the output closure of derivation '%1%' could not be repaired") % drvPath); - done(BuildResult::AlreadyValid); +void DerivationGoal::closureRepaired() { + trace("closure repaired"); + if (nrFailed > 0) + throw Error(format("some paths in the output closure of derivation '%1%' " + "could not be repaired") % + drvPath); + done(BuildResult::AlreadyValid); } +void DerivationGoal::inputsRealised() { + trace("all inputs realised"); -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - if (!useDerivation) - throw Error(format("some dependencies of '%1%' are missing") % drvPath); - printError( - format("cannot build derivation '%1%': %2% dependencies couldn't be built") - % drvPath % nrFailed); - done(BuildResult::DependencyFailed); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ + if (nrFailed != 0) { + if (!useDerivation) + throw Error(format("some dependencies of '%1%' are missing") % drvPath); + printError(format("cannot build derivation '%1%': %2% dependencies " + "couldn't be built") % + drvPath % nrFailed); + done(BuildResult::DependencyFailed); + return; + } - /* The outputs are referenceable paths. */ - for (auto & i : drv->outputs) { - debug(format("building path '%1%'") % i.second.path); - allPaths.insert(i.second.path); + if (retrySubstitution) { + haveDerivation(); + return; + } + + /* Gather information necessary for computing the closure and/or + running the build hook. */ + + /* The outputs are referenceable paths. */ + for (auto& i : drv->outputs) { + debug(format("building path '%1%'") % i.second.path); + allPaths.insert(i.second.path); + } + + /* Determine the full set of input paths. */ + + /* First, the input derivations. */ + if (useDerivation) + for (auto& i : dynamic_cast<Derivation*>(drv.get())->inputDrvs) { + /* Add the relevant output closures of the input derivation + `i' as input paths. Only add the closures of output paths + that are specified as inputs. */ + assert(worker.store.isValidPath(i.first)); + Derivation inDrv = worker.store.derivationFromPath(i.first); + for (auto& j : i.second) + if (inDrv.outputs.find(j) != inDrv.outputs.end()) + worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths); + else + throw Error(format("derivation '%1%' requires non-existent output " + "'%2%' from input derivation '%3%'") % + drvPath % j % i.first); } - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - if (useDerivation) - for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(i.first)); - Derivation inDrv = worker.store.derivationFromPath(i.first); - for (auto & j : i.second) - if (inDrv.outputs.find(j) != inDrv.outputs.end()) - worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths); - else - throw Error( - format("derivation '%1%' requires non-existent output '%2%' from input derivation '%3%'") - % drvPath % j % i.first); - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); + /* Second, the input sources. */ + worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - debug(format("added input paths %1%") % showPaths(inputPaths)); + debug(format("added input paths %1%") % showPaths(inputPaths)); - allPaths.insert(inputPaths.begin(), inputPaths.end()); + allPaths.insert(inputPaths.begin(), inputPaths.end()); - /* Is this a fixed-output derivation? */ - fixedOutput = drv->isFixedOutput(); + /* Is this a fixed-output derivation? */ + fixedOutput = drv->isFixedOutput(); - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; + /* Don't repeat fixed-output derivations since they're already + verified by their output hash.*/ + nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); + /* Okay, try to build. Note that here we don't wait for a build + slot to become available, since we don't need one if there is a + build hook. */ + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); - result = BuildResult(); + result = BuildResult(); } - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths. The locks are automatically - released when we exit this function or Nix crashes. If we - can't acquire the lock, then continue; hopefully some other - goal can start a build, and if not, the main loop will sleep a - few seconds and then retry this goal. */ - PathSet lockFiles; - for (auto & outPath : drv->outputPaths()) - lockFiles.insert(worker.store.toRealPath(outPath)); - - if (!outputLocks.lockPaths(lockFiles, "", false)) { - worker.waitForAWhile(shared_from_this()); - return; - } - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - validPaths = checkPathValidity(true, buildMode == bmRepair); - if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { - debug(format("skipping build of derivation '%1%', someone beat us to it") % drvPath); - outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); - return; - } - - missingPaths = drv->outputPaths(); - if (buildMode != bmCheck) - for (auto & i : validPaths) missingPaths.erase(i); - - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & i : drv->outputs) { - Path path = i.second.path; - if (worker.store.isValidPath(path)) continue; - debug(format("removing invalid path '%1%'") % path); - deletePath(worker.store.toRealPath(path)); - } - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); - - auto started = [&]() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", drvPath, curRound, nrRounds); - fmt("building '%s'", drvPath); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, - Logger::Fields{drvPath, hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); - worker.updateProgress(); - }; - - /* Is the build hook willing to accept this job? */ - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - result.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); +void DerivationGoal::tryToBuild() { + trace("trying to build"); + + /* Obtain locks on all output paths. The locks are automatically + released when we exit this function or Nix crashes. If we + can't acquire the lock, then continue; hopefully some other + goal can start a build, and if not, the main loop will sleep a + few seconds and then retry this goal. */ + PathSet lockFiles; + for (auto& outPath : drv->outputPaths()) + lockFiles.insert(worker.store.toRealPath(outPath)); + + if (!outputLocks.lockPaths(lockFiles, "", false)) { + worker.waitForAWhile(shared_from_this()); + return; + } + + /* Now check again whether the outputs are valid. This is because + another process may have started building in parallel. After + it has finished and released the locks, we can (and should) + reuse its results. (Strictly speaking the first check can be + omitted, but that would be less efficient.) Note that since we + now hold the locks on the output paths, no other process can + build this derivation, so no further checks are necessary. */ + validPaths = checkPathValidity(true, buildMode == bmRepair); + if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { + debug(format("skipping build of derivation '%1%', someone beat us to it") % + drvPath); + outputLocks.setDeletion(true); + done(BuildResult::AlreadyValid); + return; + } + + missingPaths = drv->outputPaths(); + if (buildMode != bmCheck) + for (auto& i : validPaths) missingPaths.erase(i); + + /* If any of the outputs already exist but are not valid, delete + them. */ + for (auto& i : drv->outputs) { + Path path = i.second.path; + if (worker.store.isValidPath(path)) continue; + debug(format("removing invalid path '%1%'") % path); + deletePath(worker.store.toRealPath(path)); + } + + /* Don't do a remote build if the derivation has the attribute + `preferLocalBuild' set. Also, check and repair modes are only + supported for local builds. */ + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); + + auto started = [&]() { + auto msg = fmt(buildMode == bmRepair + ? "repairing outputs of '%s'" + : buildMode == bmCheck + ? "checking outputs of '%s'" + : nrRounds > 1 ? "building '%s' (round %d/%d)" + : "building '%s'", + drvPath, curRound, nrRounds); + fmt("building '%s'", drvPath); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique<Activity>( + *logger, lvlInfo, actBuild, msg, + Logger::Fields{drvPath, hook ? machineName : "", curRound, nrRounds}); + mcRunningBuilds = + std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); + worker.updateProgress(); + }; + + /* Is the build hook willing to accept this job? */ + if (!buildLocally) { + switch (tryBuildHook()) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + result.startTime = time(0); // inexact + state = &DerivationGoal::buildDone; + started(); return; - } - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - printError(e.msg()); + case rpPostpone: + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ + worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - done(BuildResult::InputRejected, e.msg()); return; + case rpDecline: + /* We should do it ourselves. */ + break; } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - - started(); + } + + /* Make sure that we are allowed to start a build. If this + derivation prefers to be done locally, do it even if + maxBuildJobs is 0. */ + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { + worker.waitForBuildSlot(shared_from_this()); + outputLocks.unlock(); + return; + } + + try { + /* Okay, we have to build. */ + startBuilder(); + + } catch (BuildError& e) { + printError(e.msg()); + outputLocks.unlock(); + buildUser.reset(); + worker.permanentFailure = true; + done(BuildResult::InputRejected, e.msg()); + return; + } + + /* This state will be reached when we get EOF on the child's + log pipe. */ + state = &DerivationGoal::buildDone; + + started(); } - -void replaceValidPath(const Path & storePath, const Path tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); - if (pathExists(storePath)) - rename(storePath.c_str(), oldPath.c_str()); - if (rename(tmpPath.c_str(), storePath.c_str()) == -1) - throw SysError(format("moving '%1%' to '%2%'") % tmpPath % storePath); - deletePath(oldPath); +void replaceValidPath(const Path& storePath, const Path tmpPath) { + /* We can't atomically replace storePath (the original) with + tmpPath (the replacement), so we have to move it out of the + way first. We'd better not be interrupted here, because if + we're repairing (say) Glibc, we end up with a broken system. */ + Path oldPath = + (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); + if (pathExists(storePath)) rename(storePath.c_str(), oldPath.c_str()); + if (rename(tmpPath.c_str(), storePath.c_str()) == -1) + throw SysError(format("moving '%1%' to '%2%'") % tmpPath % storePath); + deletePath(oldPath); } - MakeError(NotDeterministic, BuildError) + void DerivationGoal::buildDone() { + trace("build done"); + + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + Finally releaseBuildUser([&]() { buildUser.reset(); }); + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = hook ? hook->pid.kill() : pid.kill(); + + debug(format("builder process for '%1%' finished") % drvPath); + + result.timesBuilt++; + result.stopTime = time(0); + + /* So the child is gone now. */ + worker.childTerminated(this); + + /* Close the read side of the logger pipe. */ + if (hook) { + hook->builderOut.readSide = -1; + hook->fromHook.readSide = -1; + } else + builderOut.readSide = -1; + + /* Close the log file. */ + closeLogFile(); + + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + if (buildUser) buildUser->kill(); + + bool diskFull = false; + + try { + /* Check the exit status. */ + if (!statusOk(status)) { + /* Heuristically check whether the build failure may have + been caused by a disk full condition. We have no way + of knowing whether the build actually got an ENOSPC. + So instead, check if the disk is (nearly) full now. If + so, we don't mark this build as a permanent failure. */ +#if HAVE_STATVFS + unsigned long long required = + 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && + (unsigned long long)st.f_bavail * st.f_bsize < required) + diskFull = true; + if (statvfs(tmpDir.c_str(), &st) == 0 && + (unsigned long long)st.f_bavail * st.f_bsize < required) + diskFull = true; +#endif -void DerivationGoal::buildDone() -{ - trace("build done"); - - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); + deleteTmpDir(false); - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); + /* Move paths out of the chroot for easier debugging of + build failures. */ + if (useChroot && buildMode == bmNormal) + for (auto& i : missingPaths) + if (pathExists(chrootRootDir + i)) + rename((chrootRootDir + i).c_str(), i.c_str()); - debug(format("builder process for '%1%' finished") % drvPath); + std::string msg = + (format("builder for '%1%' %2%") % drvPath % statusToString(status)) + .str(); - result.timesBuilt++; - result.stopTime = time(0); + if (!settings.verboseBuild && !logTail.empty()) { + msg += (format("; last %d log lines:") % logTail.size()).str(); + for (auto& line : logTail) msg += "\n " + line; + } - /* So the child is gone now. */ - worker.childTerminated(this); + if (diskFull) + msg += + "\nnote: build failure may have been caused by lack of free disk " + "space"; - /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; - - /* Close the log file. */ - closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); + throw BuildError(msg); + } - bool diskFull = false; + /* Compute the FS closure of the outputs and register them as + being valid. */ + registerOutputs(); - try { + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{drvPath}); + PushActivity pact(act.id); + auto outputPaths = drv->outputPaths(); + std::map<std::string, std::string> hookEnvironment = getEnv(); - /* Check the exit status. */ - if (!statusOk(status)) { + hookEnvironment.emplace("DRV_PATH", drvPath); + hookEnvironment.emplace("OUT_PATHS", + chomp(concatStringsSep(" ", outputPaths))); - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) - diskFull = true; -#endif + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; - deleteTmpDir(false); + struct LogSink : Sink { + Activity& act; + std::string currentLine; - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & i : missingPaths) - if (pathExists(chrootRootDir + i)) - rename((chrootRootDir + i).c_str(), i.c_str()); + LogSink(Activity& act) : act(act) {} - std::string msg = (format("builder for '%1%' %2%") - % drvPath % statusToString(status)).str(); + void operator()(const unsigned char* data, size_t len) override { + for (size_t i = 0; i < len; i++) { + auto c = data[i]; - if (!settings.verboseBuild && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; + if (c == '\n') { + flushLine(); + } else { + currentLine += c; } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); + } } - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{drvPath}); - PushActivity pact(act.id); - auto outputPaths = drv->outputPaths(); - std::map<std::string, std::string> hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", drvPath); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - if (settings.verboseBuild) { - printError("post-build-hook: " + currentLine); - } else { - act.result(resPostBuildLogLine, currentLine); - } - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); + void flushLine() { + if (settings.verboseBuild) { + printError("post-build-hook: " + currentLine); + } else { + act.result(resPostBuildLogLine, currentLine); + } + currentLine.clear(); } - if (buildMode == bmCheck) { - done(BuildResult::Built); - return; + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } } + }; + LogSink sink(act); - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(i.second); + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ + if (buildMode == bmCheck) { + done(BuildResult::Built); + return; + } - deleteTmpDir(true); + /* Delete unused redirected outputs (when doing hash rewriting). */ + for (auto& i : redirectedOutputs) deletePath(i.second); - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); + deleteTmpDir(true); - } catch (BuildError & e) { - printError(e.msg()); + /* Repeat the build if necessary. */ + if (curRound++ < nrRounds) { + outputLocks.unlock(); + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + return; + } - outputLocks.unlock(); + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); - BuildResult::Status st = BuildResult::MiscFailure; + } catch (BuildError& e) { + printError(e.msg()); - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; + outputLocks.unlock(); - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } + BuildResult::Status st = BuildResult::MiscFailure; - else { - st = - dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - fixedOutput || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } + if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) + st = BuildResult::TimedOut; - done(st, e.msg()); - return; + else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { } - done(BuildResult::Built); -} - + else { + st = dynamic_cast<NotDeterministic*>(&e) + ? BuildResult::NotDeterministic + : statusOk(status) + ? BuildResult::OutputRejected + : fixedOutput || diskFull ? BuildResult::TransientFailure + : BuildResult::PermanentFailure; + } -HookReply DerivationGoal::tryBuildHook() -{ - if (!worker.tryBuildHook || !useDerivation) return rpDecline; + done(st, e.msg()); + return; + } - if (!worker.hook) - worker.hook = std::make_unique<HookInstance>(); + done(BuildResult::Built); +} - try { +HookReply DerivationGoal::tryBuildHook() { + if (!worker.tryBuildHook || !useDerivation) return rpDecline; - /* Send the request to the hook. */ - worker.hook->sink - << "try" - << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) - << drv->platform - << drvPath - << parsedDrv->getRequiredSystemFeatures(); - worker.hook->sink.flush(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) - ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - else { - s += "\n"; - writeToStderr(s); - } - } + if (!worker.hook) worker.hook = std::make_unique<HookInstance>(); - debug(format("hook reply is '%1%'") % reply); + try { + /* Send the request to the hook. */ + worker.hook->sink << "try" + << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 + : 0) + << drv->platform << drvPath + << parsedDrv->getRequiredSystemFeatures(); + worker.hook->sink.flush(); - if (reply == "decline") - return rpDecline; - else if (reply == "decline-permanently") { - worker.tryBuildHook = false; - worker.hook = 0; - return rpDecline; - } - else if (reply == "postpone") - return rpPostpone; - else if (reply != "accept") - throw Error(format("bad hook reply '%1%'") % reply); - - } catch (SysError & e) { - if (e.errNo == EPIPE) { - printError("build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))); - worker.hook = 0; - return rpDecline; - } else - throw; + /* Read the first line of input, which should be a word indicating + whether the hook wishes to perform the build. */ + string reply; + while (true) { + string s = readLine(worker.hook->fromHook.readSide.get()); + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + ; + else if (string(s, 0, 2) == "# ") { + reply = string(s, 2); + break; + } else { + s += "\n"; + writeToStderr(s); + } } - hook = std::move(worker.hook); - - machineName = readLine(hook->fromHook.readSide.get()); + debug(format("hook reply is '%1%'") % reply); + + if (reply == "decline") + return rpDecline; + else if (reply == "decline-permanently") { + worker.tryBuildHook = false; + worker.hook = 0; + return rpDecline; + } else if (reply == "postpone") + return rpPostpone; + else if (reply != "accept") + throw Error(format("bad hook reply '%1%'") % reply); + + } catch (SysError& e) { + if (e.errNo == EPIPE) { + printError("build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))); + worker.hook = 0; + return rpDecline; + } else + throw; + } - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - hook->sink << inputPaths; + hook = std::move(worker.hook); - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - hook->sink << missingPaths; + machineName = readLine(hook->fromHook.readSide.get()); - hook->sink = FdSink(); - hook->toHook.writeSide = -1; + /* Tell the hook all the inputs that have to be copied to the + remote system. */ + hook->sink << inputPaths; - /* Create the log file and pipe. */ - Path logFile = openLogFile(); + /* Tell the hooks the missing outputs that have to be copied back + from the remote system. */ + hook->sink << missingPaths; - set<int> fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); + hook->sink = FdSink(); + hook->toHook.writeSide = -1; - return rpAccept; -} + /* Create the log file and pipe. */ + Path logFile = openLogFile(); + set<int> fds; + fds.insert(hook->fromHook.readSide.get()); + fds.insert(hook->builderOut.readSide.get()); + worker.childStarted(shared_from_this(), fds, false, false); -void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError(format("setting permissions on '%1%'") % path); + return rpAccept; } - -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->runChild(); - return 1; +void chmod_(const Path& path, mode_t mode) { + if (chmod(path.c_str(), mode) == -1) + throw SysError(format("setting permissions on '%1%'") % path); } +int childEntry(void* arg) { + ((DerivationGoal*)arg)->runChild(); + return 1; +} -PathSet DerivationGoal::exportReferences(PathSet storePaths) -{ - PathSet paths; - - for (auto storePath : storePaths) { - - /* Check that the store path is valid. */ - if (!worker.store.isInStore(storePath)) - throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") - % storePath); - - storePath = worker.store.toStorePath(storePath); - - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", storePath); - - worker.store.computeFSClosure(storePath, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - PathSet paths2(paths); - - for (auto & j : paths2) { - if (isDerivation(j)) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); - } +PathSet DerivationGoal::exportReferences(PathSet storePaths) { + PathSet paths; + + for (auto storePath : storePaths) { + /* Check that the store path is valid. */ + if (!worker.store.isInStore(storePath)) + throw BuildError( + format("'exportReferencesGraph' contains a non-store path '%1%'") % + storePath); + + storePath = worker.store.toStorePath(storePath); + + if (!inputPaths.count(storePath)) + throw BuildError( + "cannot export references of path '%s' because it is not in the " + "input closure of the derivation", + storePath); + + worker.store.computeFSClosure(storePath, paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + PathSet paths2(paths); + + for (auto& j : paths2) { + if (isDerivation(j)) { + Derivation drv = worker.store.derivationFromPath(j); + for (auto& k : drv.outputs) + worker.store.computeFSClosure(k.second.path, paths); } + } - return paths; + return paths; } static std::once_flag dns_resolve_flag; static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a + dynamic library load of one of the glibc NSS libraries in a sandboxed + child, which will fail unless the library's already been loaded in the + parent. So we force a lookup of an invalid domain to force the NSS + machinery to + load its lookup libraries in the parent before any child gets a chance to. + */ + std::call_once(dns_resolve_flag, []() { + struct addrinfo* res = NULL; + + if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", + NULL, &res) != 0) { + if (res) freeaddrinfo(res); + } + }); } -void DerivationGoal::startBuilder() -{ - /* Right platform? */ - if (!parsedDrv->canBuildLocally()) - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - drvPath, - settings.thisSystem, - concatStringsSep(", ", settings.systemFeatures)); +void DerivationGoal::startBuilder() { + /* Right platform? */ + if (!parsedDrv->canBuildLocally()) + throw Error( + "a '%s' with features {%s} is required to build '%s', but I am a '%s' " + "with features {%s}", + drv->platform, + concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), drvPath, + settings.thisSystem, concatStringsSep(", ", settings.systemFeatures)); - if (drv->isBuiltin()) - preloadNSS(); + if (drv->isBuiltin()) preloadNSS(); #if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); + additionalSandboxProfile = + parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error(format("derivation '%1%' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'") % drvPath); + /* Are we doing a chroot build? */ + { + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); + if (settings.sandboxMode == smEnabled) { + if (noChroot) + throw Error(format("derivation '%1%' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'") % + drvPath); #if __APPLE__ - if (additionalSandboxProfile != "") - throw Error(format("derivation '%1%' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'") % drvPath); + if (additionalSandboxProfile != "") + throw Error( + format("derivation '%1%' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'") % + drvPath); #endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = !fixedOutput && !noChroot; - } - - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } + useChroot = true; + } else if (settings.sandboxMode == smDisabled) + useChroot = false; + else if (settings.sandboxMode == smRelaxed) + useChroot = !fixedOutput && !noChroot; + } + + if (worker.store.storeDir != worker.store.realStoreDir) { +#if __linux__ + useChroot = true; +#else + throw Error( + "building using a diverted store is not supported on this platform"); +#endif + } - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "" && getuid() == 0) { #if defined(__linux__) || defined(__APPLE__) - buildUser = std::make_unique<UserLock>(); + buildUser = std::make_unique<UserLock>(); - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); #else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); + /* Don't know how to block the creation of setuid/setgid + binaries on this platform. */ + throw Error( + "build users are not supported on this platform for security reasons"); #endif - } - - /* Create a temporary directory where the build will take - place. */ - auto drvName = storePathToName(drvPath); - tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); - - chownToBuilder(tmpDir); - - /* Substitute output placeholders with the actual output paths. */ - for (auto & output : drv->outputs) - inputRewrites[hashPlaceholder(output.first)] = output.second.path; - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->getStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph"); - Strings ss = tokenizeString<Strings>(s); - if (ss.size() % 2 != 0) - throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - checkStoreName(fileName); /* !!! abuse of this function */ - Path storePath = *i++; - - /* Write closure info to <fileName>. */ - writeFile(tmpDir + "/" + fileName, + } + + /* Create a temporary directory where the build will take + place. */ + auto drvName = storePathToName(drvPath); + tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); + + chownToBuilder(tmpDir); + + /* Substitute output placeholders with the actual output paths. */ + for (auto& output : drv->outputs) + inputRewrites[hashPlaceholder(output.first)] = output.second.path; + + /* Construct the environment passed to the builder. */ + initEnv(); + + writeStructuredAttrs(); + + /* Handle exportReferencesGraph(), if set. */ + if (!parsedDrv->getStructuredAttrs()) { + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + string s = get(drv->env, "exportReferencesGraph"); + Strings ss = tokenizeString<Strings>(s); + if (ss.size() % 2 != 0) + throw BuildError( + format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); + for (Strings::iterator i = ss.begin(); i != ss.end();) { + string fileName = *i++; + checkStoreName(fileName); /* !!! abuse of this function */ + Path storePath = *i++; + + /* Write closure info to <fileName>. */ + writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( exportReferences({storePath}), false, false)); - } } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - - dirsInChroot.clear(); - - for (auto i : dirs) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = {i, optional}; - else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; + } + + if (useChroot) { + /* Allow a user-configurable set of directories from the + host file system. */ + PathSet dirs = settings.sandboxPaths; + PathSet dirs2 = settings.extraSandboxPaths; + dirs.insert(dirs2.begin(), dirs2.end()); + + dirsInChroot.clear(); + + for (auto i : dirs) { + if (i.empty()) continue; + bool optional = false; + if (i[i.size() - 1] == '?') { + optional = true; + i.pop_back(); + } + size_t p = i.find('='); + if (p == string::npos) + dirsInChroot[i] = {i, optional}; + else + dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; + } + dirsInChroot[tmpDirInSandbox] = tmpDir; + + /* Add the closure of store paths to the chroot. */ + PathSet closure; + for (auto& i : dirsInChroot) try { + if (worker.store.isInStore(i.second.source)) + worker.store.computeFSClosure( + worker.store.toStorePath(i.second.source), closure); + } catch (InvalidPath& e) { + } catch (Error& e) { + throw Error(format("while processing 'sandbox-paths': %s") % e.what()); + } + for (auto& i : closure) dirsInChroot[i] = i; + + PathSet allowedPaths = settings.allowedImpureHostPrefixes; + + /* This works like the above, except on a per-derivation level */ + auto impurePaths = + parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); + + for (auto& i : impurePaths) { + bool found = false; + /* Note: we're not resolving symlinks here to prevent + giving a non-root user info about inaccessible + files. */ + Path canonI = canonPath(i); + /* If only we had a trie to do this more efficiently :) luckily, these are + * generally going to be pretty small */ + for (auto& a : allowedPaths) { + Path canonA = canonPath(a); + if (canonI == canonA || isInDir(canonI, canonA)) { + found = true; + break; } - dirsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - PathSet closure; - for (auto & i : dirsInChroot) - try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - throw Error(format("while processing 'sandbox-paths': %s") % e.what()); - } - for (auto & i : closure) - dirsInChroot[i] = i; - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error(format("derivation '%1%' requested impure path '%2%', but it was not in allowed-impure-host-deps") % drvPath % i); + } + if (!found) + throw Error(format("derivation '%1%' requested impure path '%2%', but " + "it was not in allowed-impure-host-deps") % + drvPath % i); - dirsInChroot[i] = i; - } + dirsInChroot[i] = i; + } #if __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir); - - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - - if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError(format("cannot create '%1%'") % chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid, sandboxGid, settings.sandboxBuildDir)); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - (format( - "root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n") % sandboxGid).str()); - - /* Create /etc/hosts with localhost entry. */ - if (!fixedOutput) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % chrootStoreDir); - - for (auto & i : inputPaths) { - Path r = worker.store.toRealPath(i); - struct stat st; - if (lstat(r.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % i); - if (S_ISDIR(st.st_mode)) - dirsInChroot[i] = r; - else { - Path p = chrootRootDir + i; - debug("linking '%1%' to '%2%'", p, r); - if (link(r.c_str(), p.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum - link count on a file (e.g. 32000 of ext3), - which is quite possible after a `nix-store - --optimise'. */ - if (errno != EMLINK) - throw SysError(format("linking '%1%' to '%2%'") % p % i); - StringSink sink; - dumpPath(r, sink); - StringSource source(*sink.s); - restorePath(p, source); - } - } + /* Create a temporary directory in which we set up the chroot + environment using bind-mounts. We put it in the Nix store + to ensure that we can create hard-links to non-directory + inputs in the fake Nix store in the chroot (see below). */ + chrootRootDir = worker.store.toRealPath(drvPath) + ".chroot"; + deletePath(chrootRootDir); + + /* Clean up the chroot directory automatically. */ + autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir); + + printMsg(lvlChatty, + format("setting up chroot environment in '%1%'") % chrootRootDir); + + if (mkdir(chrootRootDir.c_str(), 0750) == -1) + throw SysError(format("cannot create '%1%'") % chrootRootDir); + + if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError(format("cannot change ownership of '%1%'") % + chrootRootDir); + + /* Create a writable /tmp in the chroot. Many builders need + this. (Of course they should really respect $TMPDIR + instead.) */ + Path chrootTmpDir = chrootRootDir + "/tmp"; + createDirs(chrootTmpDir); + chmod_(chrootTmpDir, 01777); + + /* Create a /etc/passwd with entries for the build user and the + nobody account. The latter is kind of a hack to support + Samba-in-QEMU. */ + createDirs(chrootRootDir + "/etc"); + + writeFile(chrootRootDir + "/etc/passwd", + fmt("root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid, sandboxGid, settings.sandboxBuildDir)); + + /* Declare the build user's group so that programs get a consistent + view of the system (e.g., "id -gn"). */ + writeFile(chrootRootDir + "/etc/group", (format("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n") % + sandboxGid) + .str()); + + /* Create /etc/hosts with localhost entry. */ + if (!fixedOutput) + writeFile(chrootRootDir + "/etc/hosts", + "127.0.0.1 localhost\n::1 localhost\n"); + + /* Make the closure of the inputs available in the chroot, + rather than the whole Nix store. This prevents any access + to undeclared dependencies. Directories are bind-mounted, + while other inputs are hard-linked (since only directories + can be bind-mounted). !!! As an extra security + precaution, make the fake Nix store only writable by the + build user. */ + Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + createDirs(chrootStoreDir); + chmod_(chrootStoreDir, 01775); + + if (buildUser && + chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError(format("cannot change ownership of '%1%'") % + chrootStoreDir); + + for (auto& i : inputPaths) { + Path r = worker.store.toRealPath(i); + struct stat st; + if (lstat(r.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % i); + if (S_ISDIR(st.st_mode)) + dirsInChroot[i] = r; + else { + Path p = chrootRootDir + i; + debug("linking '%1%' to '%2%'", p, r); + if (link(r.c_str(), p.c_str()) == -1) { + /* Hard-linking fails if we exceed the maximum + link count on a file (e.g. 32000 of ext3), + which is quite possible after a `nix-store + --optimise'. */ + if (errno != EMLINK) + throw SysError(format("linking '%1%' to '%2%'") % p % i); + StringSink sink; + dumpPath(r, sink); + StringSource source(*sink.s); + restorePath(p, source); } + } + } - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputs) - dirsInChroot.erase(i.second.path); + /* If we're repairing, checking or rebuilding part of a + multiple-outputs derivation, it's possible that we're + rebuilding a path that is in settings.dirsInChroot + (typically the dependencies of /bin/sh). Throw them + out. */ + for (auto& i : drv->outputs) dirsInChroot.erase(i.second.path); #elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ + /* We don't really have any parent prep work to do (yet?) + All work happens in the child, instead. */ #else - throw Error("sandboxing builds is not supported on this platform"); + throw Error("sandboxing builds is not supported on this platform"); #endif - } - - if (needsHashRewrite()) { - - if (pathExists(homeDir)) - throw Error(format("directory '%1%' exists; please remove it") % homeDir); - - /* We're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - if (validPaths.size() > 0) - for (auto & i : validPaths) - addHashRewrite(i); - - /* If we're repairing, then we don't want to delete the - corrupt outputs in advance. So rewrite them as well. */ - if (buildMode == bmRepair) - for (auto & i : missingPaths) - if (worker.store.isValidPath(i) && pathExists(i)) { - addHashRewrite(i); - redirectedBadOutputs.insert(i); - } - } + } + + if (needsHashRewrite()) { + if (pathExists(homeDir)) + throw Error(format("directory '%1%' exists; please remove it") % homeDir); + + /* We're not doing a chroot build, but we have some valid + output paths. Since we can't just overwrite or delete + them, we have to do hash rewriting: i.e. in the + environment/arguments passed to the build, we replace the + hashes of the valid outputs with unique dummy strings; + after the build, we discard the redirected outputs + corresponding to the valid outputs, and rewrite the + contents of the new outputs to replace the dummy strings + with the actual hashes. */ + if (validPaths.size() > 0) + for (auto& i : validPaths) addHashRewrite(i); + + /* If we're repairing, then we don't want to delete the + corrupt outputs in advance. So rewrite them as well. */ + if (buildMode == bmRepair) + for (auto& i : missingPaths) + if (worker.store.isValidPath(i) && pathExists(i)) { + addHashRewrite(i); + redirectedBadOutputs.insert(i); + } + } - if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); - auto args = useChroot ? Strings({drvPath, chrootRootDir}) : - Strings({ drvPath }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error(format("unknown pre-build hook command '%1%'") - % line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == string::npos) - dirsInChroot[line] = line; - else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); - } - } + if (useChroot && settings.preBuildHook != "" && + dynamic_cast<Derivation*>(drv.get())) { + printMsg(lvlChatty, + format("executing pre-build hook '%1%'") % settings.preBuildHook); + auto args = + useChroot ? Strings({drvPath, chrootRootDir}) : Strings({drvPath}); + enum BuildHookState { stBegin, stExtraChrootDirs }; + auto state = stBegin; + auto lines = runProgram(settings.preBuildHook, false, args); + auto lastPos = std::string::size_type{0}; + for (auto nlPos = lines.find('\n'); nlPos != string::npos; + nlPos = lines.find('\n', lastPos)) { + auto line = std::string{lines, lastPos, nlPos - lastPos}; + lastPos = nlPos + 1; + if (state == stBegin) { + if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { + state = stExtraChrootDirs; + } else { + throw Error(format("unknown pre-build hook command '%1%'") % line); + } + } else if (state == stExtraChrootDirs) { + if (line == "") { + state = stBegin; + } else { + auto p = line.find('='); + if (p == string::npos) + dirsInChroot[line] = line; + else + dirsInChroot[string(line, 0, p)] = string(line, p + 1); } + } } + } - /* Run the builder. */ - printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); + /* Run the builder. */ + printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); - /* Create the log file. */ - Path logFile = openLogFile(); + /* Create the log file. */ + Path logFile = openLogFile(); - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); + /* Create a pipe to get the output of the builder. */ + // builderOut.create(); - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) - throw SysError("opening pseudoterminal master"); + builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut.readSide) throw SysError("opening pseudoterminal master"); - std::string slaveName(ptsname(builderOut.readSide.get())); + std::string slaveName(ptsname(builderOut.readSide.get())); - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } else { - if (grantpt(builderOut.readSide.get())) - throw SysError("granting access to pseudoterminal slave"); - } + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } else { + if (grantpt(builderOut.readSide.get())) + throw SysError("granting access to pseudoterminal slave"); + } - #if 0 +#if 0 // Mount the pt in the sandbox so that the "tty" command works. // FIXME: this doesn't work with the new devpts in the sandbox. if (useChroot) dirsInChroot[slaveName] = {slaveName, false}; - #endif +#endif - if (unlockpt(builderOut.readSide.get())) - throw SysError("unlocking pseudoterminal"); + if (unlockpt(builderOut.readSide.get())) + throw SysError("unlocking pseudoterminal"); - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); + builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut.writeSide) throw SysError("opening pseudoterminal slave"); - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term)) + throw SysError("getting pseudoterminal attributes"); - cfmakeraw(&term); + cfmakeraw(&term); - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); - result.startTime = time(0); + result.startTime = time(0); - /* Fork a child to build the package. */ - ProcessOptions options; + /* Fork a child to build the package. */ + ProcessOptions options; #if __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - if (!fixedOutput) - privateNetwork = true; - - userNamespaceSync.create(); - - options.allowVfork = false; - - Pid helper = startProcess([&]() { - - /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - flags |= CLONE_NEWNET; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unpriveleged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - - writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); - _exit(0); - }, options); - - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) - throw Error("unable to start build process"); - - userNamespaceSync.readSide = -1; - - pid_t tmp; - if (!string2Int<pid_t>(readLine(builderOut.readSide.get()), tmp)) abort(); - pid = tmp; - - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - (format("%d %d 1") % sandboxUid % hostUid).str()); - - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - (format("%d %d 1") % sandboxGid % hostGid).str()); - - /* Signal the builder that we've updated its user - namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - userNamespaceSync.writeSide = -1; - - } else + if (useChroot) { + /* Set up private namespaces for the build: + + - The PID namespace causes the build to start as PID 1. + Processes outside of the chroot are not visible to those + on the inside, but processes inside the chroot are + visible from the outside (though with different PIDs). + + - The private mount namespace ensures that all the bind + mounts we do will only show up in this process and its + children, and will disappear automatically when we're + done. + + - The private network namespace ensures that the builder + cannot talk to the outside world (or vice versa). It + only has a private loopback interface. (Fixed-output + derivations are not run in a private network namespace + to allow functions like fetchurl to work.) + + - The IPC namespace prevents the builder from communicating + with outside processes using SysV IPC mechanisms (shared + memory, message queues, semaphores). It also ensures + that all IPC objects are destroyed when the builder + exits. + + - The UTS namespace ensures that builders see a hostname of + localhost rather than the actual hostname. + + We use a helper process to do the clone() to work around + clone() being broken in multi-threaded programs due to + at-fork handlers not being run. Note that we use + CLONE_PARENT to ensure that the real builder is parented to + us. + */ + + if (!fixedOutput) privateNetwork = true; + + userNamespaceSync.create(); + + options.allowVfork = false; + + Pid helper = startProcess( + [&]() { + /* Drop additional groups here because we can't do it + after we've created the new user namespace. FIXME: + this means that if we're not root in the parent + namespace, we can't drop additional groups; they will + be mapped to nogroup in the child namespace. There does + not seem to be a workaround for this. (But who can tell + from reading user_namespaces(7)?) + See also https://lwn.net/Articles/621612/. */ + if (getuid() == 0 && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + + size_t stackSize = 1 * 1024 * 1024; + char* stack = + (char*)mmap(0, stackSize, PROT_WRITE | PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + int flags = CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | + CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + if (privateNetwork) flags |= CLONE_NEWNET; + + pid_t child = clone(childEntry, stack + stackSize, flags, this); + if (child == -1 && errno == EINVAL) { + /* Fallback for Linux < 2.13 where CLONE_NEWPID and + CLONE_PARENT are not allowed together. */ + flags &= ~CLONE_NEWPID; + child = clone(childEntry, stack + stackSize, flags, this); + } + if (child == -1 && (errno == EPERM || errno == EINVAL)) { + /* Some distros patch Linux to not allow unpriveleged + * user namespaces. If we get EPERM or EINVAL, try + * without CLONE_NEWUSER and see if that works. + */ + flags &= ~CLONE_NEWUSER; + child = clone(childEntry, stack + stackSize, flags, this); + } + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (child == -1 && (errno == EPERM || errno == EINVAL) && + settings.sandboxFallback) + _exit(1); + if (child == -1) throw SysError("cloning builder process"); + + writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); + _exit(0); + }, + options); + + int res = helper.wait(); + if (res != 0 && settings.sandboxFallback) { + useChroot = false; + initTmpDir(); + goto fallback; + } else if (res != 0) + throw Error("unable to start build process"); + + userNamespaceSync.readSide = -1; + + pid_t tmp; + if (!string2Int<pid_t>(readLine(builderOut.readSide.get()), tmp)) abort(); + pid = tmp; + + /* Set the UID/GID mapping of the builder's user namespace + such that the sandbox user maps to the build user, or to + the calling user (if build users are disabled). */ + uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); + uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + + writeFile("/proc/" + std::to_string(pid) + "/uid_map", + (format("%d %d 1") % sandboxUid % hostUid).str()); + + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + + writeFile("/proc/" + std::to_string(pid) + "/gid_map", + (format("%d %d 1") % sandboxGid % hostGid).str()); + + /* Signal the builder that we've updated its user + namespace. */ + writeFull(userNamespaceSync.writeSide.get(), "1"); + userNamespaceSync.writeSide = -1; + + } else #endif - { - fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); - } - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); - - /* Check if setting up the build environment failed. */ - while (true) { - string msg = readLine(builderOut.readSide.get()); - if (string(msg, 0, 1) == "\1") { - if (msg.size() == 1) break; - throw Error(string(msg, 1)); - } - debug(msg); + { + fallback: + options.allowVfork = !buildUser && !drv->isBuiltin(); + pid = startProcess([&]() { runChild(); }, options); + } + + /* parent */ + pid.setSeparatePG(true); + builderOut.writeSide = -1; + worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, + true); + + /* Check if setting up the build environment failed. */ + while (true) { + string msg = readLine(builderOut.readSide.get()); + if (string(msg, 0, 1) == "\1") { + if (msg.size() == 1) break; + throw Error(string(msg, 1)); } + debug(msg); + } } - void DerivationGoal::initTmpDir() { - /* In a sandbox, for determinism, always use the same temporary - directory. */ + /* In a sandbox, for determinism, always use the same temporary + directory. */ #if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; #else - tmpDirInSandbox = tmpDir; + tmpDirInSandbox = tmpDir; #endif - /* In non-structured mode, add all bindings specified in the - derivation via the environment, except those listed in the - passAsFile attribute. Those are passed as file names pointing - to temporary files containing the contents. Note that - passAsFile is ignored in structure mode because it's not - needed (attributes are not passed through the environment, so - there is no size constraint). */ - if (!parsedDrv->getStructuredAttrs()) { - - StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile")); - int fileNr = 0; - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - string fn = ".attr-" + std::to_string(fileNr++); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - + /* In non-structured mode, add all bindings specified in the + derivation via the environment, except those listed in the + passAsFile attribute. Those are passed as file names pointing + to temporary files containing the contents. Note that + passAsFile is ignored in structure mode because it's not + needed (attributes are not passed through the environment, so + there is no size constraint). */ + if (!parsedDrv->getStructuredAttrs()) { + StringSet passAsFile = + tokenizeString<StringSet>(get(drv->env, "passAsFile")); + int fileNr = 0; + for (auto& i : drv->env) { + if (passAsFile.find(i.first) == passAsFile.end()) { + env[i.first] = i.second; + } else { + string fn = ".attr-" + std::to_string(fileNr++); + Path p = tmpDir + "/" + fn; + writeFile(p, rewriteStrings(i.second, inputRewrites)); + chownToBuilder(p); + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; + } } + } - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDirInSandbox; - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; + /* Explicitly set PWD to prevent problems with chroot builds. In + particular, dietlibc cannot figure out the cwd because the + inode of the current directory doesn't appear in .. (because + getdents returns the inode of the mount point). */ + env["PWD"] = tmpDirInSandbox; } -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (fixedOutput) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i); - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; +void DerivationGoal::initEnv() { + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = worker.store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + initTmpDir(); + + /* Compatibility hack with Nix <= 0.7: if this is a fixed-output + derivation, tell the builder, so that for instance `fetchurl' + can skip checking the output. On older Nixes, this environment + variable won't be set, so `fetchurl' will do the check. */ + if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; + + /* *Only* if this is a fixed-output derivation, propagate the + values of the environment variables specified in the + `impureEnvVars' attribute to the builder. This allows for + instance environment variables for proxy configuration such as + `http_proxy' to be easily passed to downloaders like + `fetchurl'. Passing such environment variables from the caller + to the builder is generally impure, but the output of + fixed-output derivations is by definition pure (since we + already know the cryptographic hash of the output). */ + if (fixedOutput) { + for (auto& i : + parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) + env[i] = getEnv(i); + } + + /* Currently structured log messages piggyback on stderr, but we + may change that in the future. So tell the builder which file + descriptor to use for that. */ + env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; } - static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - -void DerivationGoal::writeStructuredAttrs() -{ - auto & structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - PathSet storePaths; - for (auto & p : *i) - storePaths.insert(p.get<std::string>()); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } +void DerivationGoal::writeStructuredAttrs() { + auto& structuredAttrs = parsedDrv->getStructuredAttrs(); + if (!structuredAttrs) return; + + auto json = *structuredAttrs; + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto& i : drv->outputs) + outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + PathSet storePaths; + for (auto& p : *i) storePaths.insert(p.get<std::string>()); + worker.store.pathInfoToJSON(jsonRoot, exportReferences(storePaths), + false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh } + } - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ + writeFile(tmpDir + "/.attrs.json", + rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get<float>(); - if (std::ceil(f) == f) - return std::to_string(value.get<int>()); - } + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ - if (value.is_null()) - return std::string("''"); + auto handleSimpleType = + [](const nlohmann::json& value) -> std::optional<std::string> { + if (value.is_string()) return shellEscape(value); - if (value.is_boolean()) - return value.get<bool>() ? std::string("1") : std::string(""); + if (value.is_number()) { + auto f = value.get<float>(); + if (std::ceil(f) == f) return std::to_string(value.get<int>()); + } - return {}; - }; + if (value.is_null()) return std::string("''"); - std::string jsonSh; + if (value.is_boolean()) + return value.get<bool>() ? std::string("1") : std::string(""); - for (auto i = json.begin(); i != json.end(); ++i) { + return {}; + }; - if (!std::regex_match(i.key(), shVarName)) continue; + std::string jsonSh; - auto & value = i.value(); + for (auto i = json.begin(); i != json.end(); ++i) { + if (!std::regex_match(i.key(), shVarName)) continue; - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); + auto& value = i.value(); - else if (value.is_array()) { - std::string s2; - bool good = true; + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } + else if (value.is_array()) { + std::string s2; + bool good = true; - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { + good = false; + break; } + s2 += *s3; + s2 += ' '; + } - else if (value.is_object()) { - std::string s2; - bool good = true; + if (good) jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } + else if (value.is_object()) { + std::string s2; + bool good = true; - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { + good = false; + break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); } + } - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); } - -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % path); +void DerivationGoal::chownToBuilder(const Path& path) { + if (!buildUser) return; + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError(format("cannot change ownership of '%1%'") % path); } - -void setupSeccomp() -{ +void setupSeccomp() { #if __linux__ - if (!settings.filterSyscalls) return; + if (!settings.filterSyscalls) return; #if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from creating EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); + scmp_filter_ctx ctx; + + if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) + throw SysError("unable to initialize seccomp mode 2"); + + Finally cleanup([&]() { seccomp_release(ctx); }); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) + throw SysError("unable to add 32-bit seccomp architecture"); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) + throw SysError("unable to add X32 seccomp architecture"); + + if (nativeSystem == "aarch64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) + printError( + "unable to add ARM seccomp architecture; this may result in spurious " + "build failures if running 32-bit ARM processes"); + + /* Prevent builders from creating setuid/setgid binaries. */ + for (int perm : {S_ISUID, S_ISGID}) { + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t)perm, + (scmp_datum_t)perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t)perm, + (scmp_datum_t)perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t)perm, + (scmp_datum_t)perm)) != 0) + throw SysError("unable to add seccomp rule"); + } + + /* Prevent builders from creating EAs or ACLs. Not all filesystems + support these, and they're not allowed in the Nix store because + they're not representable in the NAR serialisation. */ + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != + 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != + 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != + 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, + settings.allowNewPrivileges ? 0 : 1) != 0) + throw SysError("unable to set 'no new privileges' seccomp attribute"); + + if (seccomp_load(ctx) != 0) + throw SysError("unable to load seccomp BPF program"); #else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); + throw Error( + "seccomp is not supported on this platform; " + "you can bypass this error by setting the option 'filter-syscalls' to " + "false, but note that untrusted builds can then create setuid binaries!"); #endif #endif } +void DerivationGoal::runChild() { + /* Warning: in the child we should absolutely not make any SQLite + calls! */ -void DerivationGoal::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ + try { /* child */ - try { /* child */ + commonChildInit(builderOut); - commonChildInit(builderOut); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } + try { + setupSeccomp(); + } catch (...) { + if (buildUser) throw; + } - bool setUser = true; + bool setUser = true; - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } + /* Make the contents of netrc available to builtin:fetchurl + (which may run under a different uid and/or in a sandbox). */ + std::string netrcData; + try { + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") + netrcData = readFile(settings.netrcFile); + } catch (SysError&) { + } #if __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; + if (useChroot) { + userNamespaceSync.writeSide = -1; + + if (drainFD(userNamespaceSync.readSide.get()) != "1") + throw Error("user namespace initialisation failed"); + + userNamespaceSync.readSide = -1; + + if (privateNetwork) { + /* Initialise the loopback interface. */ + AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (!fd) throw SysError("cannot open IP socket"); + + struct ifreq ifr; + strcpy(ifr.ifr_name, "lo"); + ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; + if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) + throw SysError("cannot set loopback interface flags"); + } + + /* Set the hostname etc. to fixed values. */ + char hostname[] = "localhost"; + if (sethostname(hostname, sizeof(hostname)) == -1) + throw SysError("cannot set host name"); + char domainname[] = "(none)"; // kernel default + if (setdomainname(domainname, sizeof(domainname)) == -1) + throw SysError("cannot set domain name"); + + /* Make all filesystems private. This is necessary + because subtrees may have been mounted as "shared" + (MS_SHARED). (Systemd does this, for instance.) Even + though we have a private mount namespace, mounting + filesystems on top of a shared subtree still propagates + outside of the namespace. Making a subtree private is + local to the namespace, though, so setting MS_PRIVATE + does not affect the outside world. */ + if (mount(0, "/", 0, MS_REC | MS_PRIVATE, 0) == -1) { + throw SysError("unable to make '/' private mount"); + } + + /* Bind-mount chroot directory to itself, to treat it as a + different filesystem from /, as needed for pivot_root. */ + if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == + -1) + throw SysError(format("unable to bind mount '%1%'") % chrootRootDir); + + /* Set up a nearly empty /dev, unless the user asked to + bind-mount the host /dev. */ + Strings ss; + if (dirsInChroot.find("/dev") == dirsInChroot.end()) { + createDirs(chrootRootDir + "/dev/shm"); + createDirs(chrootRootDir + "/dev/pts"); + ss.push_back("/dev/full"); + if (settings.systemFeatures.get().count("kvm") && + pathExists("/dev/kvm")) + ss.push_back("/dev/kvm"); + ss.push_back("/dev/null"); + ss.push_back("/dev/random"); + ss.push_back("/dev/tty"); + ss.push_back("/dev/urandom"); + ss.push_back("/dev/zero"); + createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); + createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); + createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); + createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (fixedOutput) { + ss.push_back("/etc/resolv.conf"); + + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", + "hosts: files dns\nservices: files\n"); + + ss.push_back("/etc/services"); + ss.push_back("/etc/hosts"); + if (pathExists("/var/run/nscd/socket")) + ss.push_back("/var/run/nscd/socket"); + } + + for (auto& i : ss) dirsInChroot.emplace(i, i); + + /* Bind-mount all the directories from the "host" + filesystem that we want in the chroot + environment. */ + auto doBind = [&](const Path& source, const Path& target, + bool optional = false) { + debug(format("bind mounting '%1%' to '%2%'") % source % target); + struct stat st; + if (stat(source.c_str(), &st) == -1) { + if (optional && errno == ENOENT) + return; + else + throw SysError("getting attributes of path '%1%'", source); + } + if (S_ISDIR(st.st_mode)) + createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == + -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, + target); + }; + + for (auto& i : dirsInChroot) { + if (i.second.source == "/proc") continue; // backwards compatibility + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + } + + /* Bind a new instance of procfs on /proc. */ + createDirs(chrootRootDir + "/proc"); + if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) + throw SysError("mounting /proc"); + + /* Mount a new tmpfs on /dev/shm to ensure that whatever + the builder puts in /dev/shm is cleaned up automatically. */ + if (pathExists("/dev/shm") && + mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, + fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) + throw SysError("mounting /dev/shm"); + + /* Mount a new devpts on /dev/pts. Note that this + requires the kernel to be compiled with + CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case + if /dev/ptx/ptmx exists). */ + if (pathExists("/dev/pts/ptmx") && + !pathExists(chrootRootDir + "/dev/ptmx") && + !dirsInChroot.count("/dev/pts")) { + if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, + "newinstance,mode=0620") == 0) { + createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + } else { + if (errno != EINVAL) throw SysError("mounting /dev/pts"); + doBind("/dev/pts", chrootRootDir + "/dev/pts"); + doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); + } + } - if (privateNetwork) { + /* Do the chroot(). */ + if (chdir(chrootRootDir.c_str()) == -1) + throw SysError(format("cannot change directory to '%1%'") % + chrootRootDir); - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); + if (mkdir("real-root", 0) == -1) + throw SysError("cannot create real-root directory"); - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } + if (pivot_root(".", "real-root") == -1) + throw SysError(format("cannot pivot old root directory onto '%1%'") % + (chrootRootDir + "/real-root")); - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) { - throw SysError("unable to make '/' private mount"); - } + if (chroot(".") == -1) + throw SysError(format("cannot change root directory to '%1%'") % + chrootRootDir); - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError(format("unable to bind mount '%1%'") % chrootRootDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } + if (umount2("real-root", MNT_DETACH) == -1) + throw SysError("cannot unmount real root filesystem"); - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (fixedOutput) { - ss.push_back("/etc/resolv.conf"); - - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); - } + if (rmdir("real-root") == -1) + throw SysError("cannot remove real-root directory"); - for (auto & i : ss) dirsInChroot.emplace(i, i); - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug(format("bind mounting '%1%' to '%2%'") % source % target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } + /* Switch to the sandbox uid/gid in the user namespace, + which corresponds to the build user or calling user in + the parent namespace. */ + if (setgid(sandboxGid) == -1) throw SysError("setgid failed"); + if (setuid(sandboxUid) == -1) throw SysError("setuid failed"); - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError(format("cannot change directory to '%1%'") % chrootRootDir); - - if (mkdir("real-root", 0) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError(format("cannot pivot old root directory onto '%1%'") % (chrootRootDir + "/real-root")); + setUser = false; + } +#endif - if (chroot(".") == -1) - throw SysError(format("cannot change root directory to '%1%'") % chrootRootDir); + if (chdir(tmpDirInSandbox.c_str()) == -1) + throw SysError(format("changing into '%1%'") % tmpDir); - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); + /* Close all other file descriptors. */ + closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); +#if __linux__ + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if (drv->platform == "i686-linux" && + (settings.thisSystem == "x86_64-linux" || + (!strcmp(utsbuf.sysname, "Linux") && + !strcmp(utsbuf.machine, "x86_64")))) { + if (personality(PER_LINUX32) == -1) + throw SysError("cannot set i686-linux personality"); + } - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid) == -1) - throw SysError("setuid failed"); + /* Impersonate a Linux 2.6 machine to get some determinism in + builds that depend on the kernel version. */ + if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && + settings.impersonateLinux26) { + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); + } - setUser = false; - } + /* Disable address space randomization for improved + determinism. */ + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); #endif - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError(format("changing into '%1%'") % tmpDir); + /* Disable core dumps by default. */ + struct rlimit limit = {0, RLIM_INFINITY}; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + + /* Fill in the environment. */ + Strings envStrs; + for (auto& i : env) + envStrs.push_back( + rewriteStrings(i.first + "=" + i.second, inputRewrites)); + + /* If we are running in `build-users' mode, then switch to the + user we allocated above. Make sure that we drop all root + privileges. Note that above we have closed all file + descriptors except std*, so that's safe. Also note that + setuid() when run as root sets the real, effective and + saved UIDs. */ + if (setUser && buildUser) { + /* Preserve supplementary groups of the build user, to allow + admins to specify groups such as "kvm". */ + if (!buildUser->getSupplementaryGIDs().empty() && + setgroups(buildUser->getSupplementaryGIDs().size(), + buildUser->getSupplementaryGIDs().data()) == -1) + throw SysError("cannot set supplementary groups of build user"); + + if (setgid(buildUser->getGID()) == -1 || + getgid() != buildUser->getGID() || getegid() != buildUser->getGID()) + throw SysError("setgid failed"); + + if (setuid(buildUser->getUID()) == -1 || + getuid() != buildUser->getUID() || geteuid() != buildUser->getUID()) + throw SysError("setuid failed"); + } + + /* Fill in the arguments. */ + Strings args; - /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); + const char* builder = "invalid"; -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); + if (drv->isBuiltin()) { + ; + } +#if __APPLE__ + else if (getEnv("_NIX_TEST_NO_SANDBOX") == "") { + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; + + if (useChroot) { + /* Lots and lots and lots of file functions freak out if they can't stat + * their full ancestry */ + PathSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store + because we know they'll all have the same parents (the store), and + there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice + */ + for (auto& i : dirsInChroot) { + Path cur = i.first; + while (cur.compare("/") != 0) { + cur = dirOf(cur); + ancestry.insert(cur); + } } - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); + /* And we want the store in there regardless of how empty dirsInChroot. + We include the innermost path component this time, since it's + typically /nix/store and we care about that. */ + Path cur = worker.store.storeDir; + while (cur.compare("/") != 0) { + ancestry.insert(cur); + cur = dirOf(cur); } - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif + /* Add all our input paths to the chroot */ + for (auto& i : inputPaths) dirsInChroot[i] = i; - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); + /* Violations will go to the syslog if you set this. Unfortunately the + * destination does not appear to be configurable */ + if (settings.darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; } - /* Fill in the arguments. */ - Strings args; + sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - const char *builder = "invalid"; + if (fixedOutput) sandboxProfile += "(import \"sandbox-network.sb\")\n"; - if (drv->isBuiltin()) { - ; + /* Our rwx outputs */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto& i : missingPaths) { + sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); } -#if __APPLE__ - else if (getEnv("_NIX_TEST_NO_SANDBOX") == "") { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) - dirsInChroot[i] = i; - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - - if (fixedOutput) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; - - /* Our rwx outputs */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : missingPaths) { - sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); - } - /* Also add redirected outputs to the chroot */ - for (auto & i : redirectedOutputs) { - sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str(); - } - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error(format( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") - % i.first % i.second.source); - - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError(format("getting attributes of path '%1%'") % path); - } - if (S_ISDIR(st.st_mode)) - sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str(); - else - sandboxProfile += (format("\t(literal \"%1%\")\n") % path).str(); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); + /* Also add redirected outputs to the chroot */ + for (auto& i : redirectedOutputs) { + sandboxProfile += + (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str(); } -#endif - else { - builder = drv->builder.c_str(); - string builderBasename = baseNameOf(drv->builder); - args.push_back(builderBasename); + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed + above) + + without file-write* allowed, access() incorrectly returns EPERM + */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto& i : dirsInChroot) { + if (i.first != i.second.source) + throw Error(format("can't map '%1%' to '%2%': mismatched impure " + "paths not supported on Darwin") % + i.first % i.second.source); + + string path = i.first; + struct stat st; + if (lstat(path.c_str(), &st)) { + if (i.second.optional && errno == ENOENT) continue; + throw SysError(format("getting attributes of path '%1%'") % path); + } + if (S_ISDIR(st.st_mode)) + sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str(); + else + sandboxProfile += (format("\t(literal \"%1%\")\n") % path).str(); } + sandboxProfile += ")\n"; - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\1\n")); - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(*logger); - - BasicDerivation drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); - else - throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); - _exit(1); - } + /* Allow file-read* on full directory hierarchy to self. Allows + * realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto& i : ancestry) { + sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); } - - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - - throw SysError(format("executing '%1%'") % drv->builder); - - } catch (std::exception & e) { - writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); - _exit(1); + sandboxProfile += ")\n"; + + sandboxProfile += additionalSandboxProfile; + } else + sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + Path sandboxFile = tmpDir + "/.sandbox.sb"; + + writeFile(sandboxFile, sandboxProfile); + + bool allowLocalNetworking = + parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); + + /* The tmpDir in scope points at the temporary build directory for our + derivation. Some packages try different mechanisms to find temporary + directories, so we want to open up a broader place for them to dump + their files, if needed. */ + Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true); + + /* They don't like trailing slashes on subpath directives */ + if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); + + builder = "/usr/bin/sandbox-exec"; + args.push_back("sandbox-exec"); + args.push_back("-f"); + args.push_back(sandboxFile); + args.push_back("-D"); + args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); + args.push_back("-D"); + args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); + if (allowLocalNetworking) { + args.push_back("-D"); + args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); + } + args.push_back(drv->builder); } -} - - -/* Parse a list of reference specifiers. Each element must either be - a store path, or the symbolic name of the output of the derivation - (such as `out'). */ -PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) -{ - PathSet result; - for (auto & i : paths) { - if (store.isStorePath(i)) - result.insert(i); - else if (drv.outputs.find(i) != drv.outputs.end()) - result.insert(drv.outputs.find(i)->second.path); - else throw BuildError( - format("derivation contains an illegal reference specifier '%1%'") % i); +#endif + else { + builder = drv->builder.c_str(); + string builderBasename = baseNameOf(drv->builder); + args.push_back(builderBasename); } - return result; -} + for (auto& i : drv->args) args.push_back(rewriteStrings(i, inputRewrites)); -void DerivationGoal::registerOutputs() -{ - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. */ - if (hook) { - bool allValid = true; - for (auto & i : drv->outputs) - if (!worker.store.isValidPath(i.second.path)) allValid = false; - if (allValid) return; - } + /* Indicate that we managed to set up the build environment. */ + writeFull(STDERR_FILENO, string("\1\n")); - std::map<std::string, ValidPathInfo> infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - - std::exception_ptr delayedException; - - /* Check whether the output paths were created, and grep each - output path to determine what other paths it references. Also make all - output paths read-only. */ - for (auto & i : drv->outputs) { - Path path = i.second.path; - if (missingPaths.find(path) == missingPaths.end()) continue; - - ValidPathInfo info; - - Path actualPath = path; - if (useChroot) { - actualPath = chrootRootDir + path; - if (pathExists(actualPath)) { - /* Move output paths from the chroot to the Nix store. */ - if (buildMode == bmRepair) - replaceValidPath(path, actualPath); - else - if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1) - throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); - } - if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); - } + /* Execute the program. This should not return. */ + if (drv->isBuiltin()) { + try { + logger = makeJSONLogger(*logger); - if (needsHashRewrite()) { - Path redirected = redirectedOutputs[path]; - if (buildMode == bmRepair - && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() - && pathExists(redirected)) - replaceValidPath(path, redirected); - if (buildMode == bmCheck && redirected != "") - actualPath = redirected; - } + BasicDerivation drv2(*drv); + for (auto& e : drv2.env) + e.second = rewriteStrings(e.second, inputRewrites); - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - format("builder for '%1%' failed to produce output path '%2%'") - % drvPath % path); - throw SysError(format("getting attributes of path '%1%'") % actualPath); - } + if (drv->builder == "builtin:fetchurl") + builtinFetchurl(drv2, netrcData); + else if (drv->builder == "builtin:buildenv") + builtinBuildenv(drv2); + else + throw Error(format("unsupported builtin function '%1%'") % + string(drv->builder, 8)); + _exit(0); + } catch (std::exception& e) { + writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); + _exit(1); + } + } -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError(format("suspicious ownership or permission on '%1%'; rejecting this build output") % path); -#endif + execve(builder, stringsToCharPtrs(args).data(), + stringsToCharPtrs(envStrs).data()); - /* Apply hash rewriting if necessary. */ - bool rewritten = false; - if (!outputRewrites.empty()) { - printError(format("warning: rewriting hashes in '%1%'; cross fingers") % path); - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - rewritten = true; - } + throw SysError(format("executing '%1%'") % drv->builder); - /* Check that fixed-output derivations produced the right - outputs (i.e., the content hash should match the specified - hash). */ - if (fixedOutput) { + } catch (std::exception& e) { + writeFull(STDERR_FILENO, "\1while setting up the build environment: " + + string(e.what()) + "\n"); + _exit(1); + } +} - bool recursive; Hash h; - i.second.parseHashInfo(recursive, h); +/* Parse a list of reference specifiers. Each element must either be + a store path, or the symbolic name of the output of the derivation + (such as `out'). */ +PathSet parseReferenceSpecifiers(Store& store, const BasicDerivation& drv, + const Strings& paths) { + PathSet result; + for (auto& i : paths) { + if (store.isStorePath(i)) + result.insert(i); + else if (drv.outputs.find(i) != drv.outputs.end()) + result.insert(drv.outputs.find(i)->second.path); + else + throw BuildError( + format("derivation contains an illegal reference specifier '%1%'") % + i); + } + return result; +} - if (!recursive) { - /* The output path should be a regular file without - execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - format("output path '%1%' should be a non-executable regular file") % path); - } +void DerivationGoal::registerOutputs() { + /* When using a build hook, the build hook can register the output + as valid (by doing `nix-store --import'). If so we don't have + to do anything here. */ + if (hook) { + bool allValid = true; + for (auto& i : drv->outputs) + if (!worker.store.isValidPath(i.second.path)) allValid = false; + if (allValid) return; + } - /* Check the hash. In hash mode, move the path produced by - the derivation to its content-addressed location. */ - Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); + std::map<std::string, ValidPathInfo> infos; - Path dest = worker.store.makeFixedOutputPath(recursive, h2, storePathToName(path)); + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; - if (h != h2) { + Path checkSuffix = ".check"; + bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - dest, h.to_string(), h2.to_string())); + std::exception_ptr delayedException; - Path actualDest = worker.store.toRealPath(dest); + /* Check whether the output paths were created, and grep each + output path to determine what other paths it references. Also make all + output paths read-only. */ + for (auto& i : drv->outputs) { + Path path = i.second.path; + if (missingPaths.find(path) == missingPaths.end()) continue; - if (worker.store.isValidPath(dest)) - std::rethrow_exception(delayedException); + ValidPathInfo info; - if (actualPath != actualDest) { - PathLocks outputLocks({actualDest}); - deletePath(actualDest); - if (rename(actualPath.c_str(), actualDest.c_str()) == -1) - throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest); - } + Path actualPath = path; + if (useChroot) { + actualPath = chrootRootDir + path; + if (pathExists(actualPath)) { + /* Move output paths from the chroot to the Nix store. */ + if (buildMode == bmRepair) + replaceValidPath(path, actualPath); + else if (buildMode != bmCheck && + rename(actualPath.c_str(), + worker.store.toRealPath(path).c_str()) == -1) + throw SysError(format("moving build output '%1%' from the sandbox to " + "the Nix store") % + path); + } + if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); + } - path = dest; - actualPath = actualDest; - } - else - assert(path == dest); + if (needsHashRewrite()) { + Path redirected = redirectedOutputs[path]; + if (buildMode == bmRepair && + redirectedBadOutputs.find(path) != redirectedBadOutputs.end() && + pathExists(redirected)) + replaceValidPath(path, redirected); + if (buildMode == bmCheck && redirected != "") actualPath = redirected; + } - info.ca = makeFixedOutputCA(recursive, h2); - } + struct stat st; + if (lstat(actualPath.c_str(), &st) == -1) { + if (errno == ENOENT) + throw BuildError( + format("builder for '%1%' failed to produce output path '%2%'") % + drvPath % path); + throw SysError(format("getting attributes of path '%1%'") % actualPath); + } - /* Get rid of all weird permissions. This also checks that - all files are owned by the build user, if applicable. */ - canonicalisePathMetaData(actualPath, - buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen); - - /* For this output path, find the references to other paths - contained in it. Compute the SHA-256 NAR hash at the same - time. The hash is stored in the database so that we can - verify later on whether nobody has messed with the store. */ - debug("scanning for references inside '%1%'", path); - HashResult hash; - PathSet references = scanForReferences(actualPath, allPaths, hash); - - if (buildMode == bmCheck) { - if (!worker.store.isValidPath(path)) continue; - auto info = *worker.store.queryPathInfo(path); - if (hash.first != info.narHash) { - worker.checkMismatch = true; - if (settings.runDiffHook || settings.keepFailed) { - Path dst = worker.store.toRealPath(path + checkSuffix); - deletePath(dst); - if (rename(actualPath.c_str(), dst.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - path, dst, drvPath, tmpDir); - - throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") - % drvPath % path % dst); - } else - throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs") - % drvPath % path); - } +#ifndef __CYGWIN__ + /* Check that the output is not group or world writable, as + that means that someone else can have interfered with the + build. Also, the output should be owned by the build + user. */ + if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || + (buildUser && st.st_uid != buildUser->getUID())) + throw BuildError(format("suspicious ownership or permission on '%1%'; " + "rejecting this build output") % + path); +#endif - /* Since we verified the build, it's now ultimately - trusted. */ - if (!info.ultimate) { - info.ultimate = true; - worker.store.signPathInfo(info); - worker.store.registerValidPaths({info}); - } + /* Apply hash rewriting if necessary. */ + bool rewritten = false; + if (!outputRewrites.empty()) { + printError(format("warning: rewriting hashes in '%1%'; cross fingers") % + path); + + /* Canonicalise first. This ensures that the path we're + rewriting doesn't contain a hard link to /etc/shadow or + something like that. */ + canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, + inodesSeen); + + /* FIXME: this is in-memory. */ + StringSink sink; + dumpPath(actualPath, sink); + deletePath(actualPath); + sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites)); + StringSource source(*sink.s); + restorePath(actualPath, source); + + rewritten = true; + } - continue; + /* Check that fixed-output derivations produced the right + outputs (i.e., the content hash should match the specified + hash). */ + if (fixedOutput) { + bool recursive; + Hash h; + i.second.parseHashInfo(recursive, h); + + if (!recursive) { + /* The output path should be a regular file without + execute permission. */ + if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) + throw BuildError( + format( + "output path '%1%' should be a non-executable regular file") % + path); + } + + /* Check the hash. In hash mode, move the path produced by + the derivation to its content-addressed location. */ + Hash h2 = recursive ? hashPath(h.type, actualPath).first + : hashFile(h.type, actualPath); + + Path dest = worker.store.makeFixedOutputPath(recursive, h2, + storePathToName(path)); + + if (h != h2) { + /* Throw an error after registering the path as + valid. */ + worker.hashMismatch = true; + delayedException = std::make_exception_ptr( + BuildError("hash mismatch in fixed-output derivation '%s':\n " + "wanted: %s\n got: %s", + dest, h.to_string(), h2.to_string())); + + Path actualDest = worker.store.toRealPath(dest); + + if (worker.store.isValidPath(dest)) + std::rethrow_exception(delayedException); + + if (actualPath != actualDest) { + PathLocks outputLocks({actualDest}); + deletePath(actualDest); + if (rename(actualPath.c_str(), actualDest.c_str()) == -1) + throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest); } - /* For debugging, print out the referenced and unreferenced - paths. */ - for (auto & i : inputPaths) { - PathSet::iterator j = references.find(i); - if (j == references.end()) - debug(format("unreferenced input: '%1%'") % i); - else - debug(format("referenced input: '%1%'") % i); - } + path = dest; + actualPath = actualDest; + } else + assert(path == dest); - if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(path); - } + info.ca = makeFixedOutputCA(recursive, h2); + } - info.path = path; - info.narHash = hash.first; - info.narSize = hash.second; - info.references = references; - info.deriver = drvPath; + /* Get rid of all weird permissions. This also checks that + all files are owned by the build user, if applicable. */ + canonicalisePathMetaData(actualPath, + buildUser && !rewritten ? buildUser->getUID() : -1, + inodesSeen); + + /* For this output path, find the references to other paths + contained in it. Compute the SHA-256 NAR hash at the same + time. The hash is stored in the database so that we can + verify later on whether nobody has messed with the store. */ + debug("scanning for references inside '%1%'", path); + HashResult hash; + PathSet references = scanForReferences(actualPath, allPaths, hash); + + if (buildMode == bmCheck) { + if (!worker.store.isValidPath(path)) continue; + auto info = *worker.store.queryPathInfo(path); + if (hash.first != info.narHash) { + worker.checkMismatch = true; + if (settings.runDiffHook || settings.keepFailed) { + Path dst = worker.store.toRealPath(path + checkSuffix); + deletePath(dst); + if (rename(actualPath.c_str(), dst.c_str())) + throw SysError(format("renaming '%1%' to '%2%'") % actualPath % + dst); + + handleDiffHook(buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), path, dst, + drvPath, tmpDir); + + throw NotDeterministic( + format("derivation '%1%' may not be deterministic: output '%2%' " + "differs from '%3%'") % + drvPath % path % dst); + } else + throw NotDeterministic(format("derivation '%1%' may not be " + "deterministic: output '%2%' differs") % + drvPath % path); + } + + /* Since we verified the build, it's now ultimately + trusted. */ + if (!info.ultimate) { info.ultimate = true; worker.store.signPathInfo(info); + worker.store.registerValidPaths({info}); + } - if (!info.references.empty()) info.ca.clear(); - - infos[i.first] = info; - } - - if (buildMode == bmCheck) return; - - /* Apply output checks. */ - checkOutputs(infos); - - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - result.isNonDeterministic = true; - Path prev = i->second.path + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - auto msg = prevExists - ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) - : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, i->second.path, drvPath, tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(msg); - - printError(msg); - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out - of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & i : drv->outputs) { - Path prev = i.second.path + checkSuffix; - deletePath(prev); - Path dst = i.second.path + checkSuffix; - if (rename(i.second.path.c_str(), dst.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % i.second.path % dst); - } + continue; } - if (curRound < nrRounds) { - prevInfos = infos; - return; + /* For debugging, print out the referenced and unreferenced + paths. */ + for (auto& i : inputPaths) { + PathSet::iterator j = references.find(i); + if (j == references.end()) + debug(format("unreferenced input: '%1%'") % i); + else + debug(format("referenced input: '%1%'") % i); } - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ if (curRound == nrRounds) { - for (auto & i : drv->outputs) { - Path prev = i.second.path + checkSuffix; - deletePath(prev); - } + worker.store.optimisePath( + actualPath); // FIXME: combine with scanForReferences() + worker.markContentsGood(path); } - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - ValidPathInfos infos2; - for (auto & i : infos) infos2.push_back(i.second); - worker.store.registerValidPaths(infos2); + info.path = path; + info.narHash = hash.first; + info.narSize = hash.second; + info.references = references; + info.deriver = drvPath; + info.ultimate = true; + worker.store.signPathInfo(info); + + if (!info.references.empty()) info.ca.clear(); + + infos[i.first] = info; + } + + if (buildMode == bmCheck) return; + + /* Apply output checks. */ + checkOutputs(infos); + + /* Compare the result with the previous round, and report which + path is different, if any.*/ + if (curRound > 1 && prevInfos != infos) { + assert(prevInfos.size() == infos.size()); + for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); + ++i, ++j) + if (!(*i == *j)) { + result.isNonDeterministic = true; + Path prev = i->second.path + checkSuffix; + bool prevExists = keepPreviousRound && pathExists(prev); + auto msg = + prevExists + ? fmt("output '%1%' of '%2%' differs from '%3%' from previous " + "round", + i->second.path, drvPath, prev) + : fmt("output '%1%' of '%2%' differs from previous round", + i->second.path, drvPath); + + handleDiffHook(buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), prev, + i->second.path, drvPath, tmpDir); + + if (settings.enforceDeterminism) throw NotDeterministic(msg); + + printError(msg); + curRound = nrRounds; // we know enough, bail out early + } + } + + /* If this is the first round of several, then move the output out + of the way. */ + if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && + keepPreviousRound) { + for (auto& i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + deletePath(prev); + Path dst = i.second.path + checkSuffix; + if (rename(i.second.path.c_str(), dst.c_str())) + throw SysError(format("renaming '%1%' to '%2%'") % i.second.path % dst); } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); + } + + if (curRound < nrRounds) { + prevInfos = infos; + return; + } + + /* Remove the .check directories if we're done. FIXME: keep them + if the result was not determistic? */ + if (curRound == nrRounds) { + for (auto& i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + deletePath(prev); + } + } + + /* Register each output path as valid, and register the sets of + paths referenced by each of them. If there are cycles in the + outputs, this will fail. */ + { + ValidPathInfos infos2; + for (auto& i : infos) infos2.push_back(i.second); + worker.store.registerValidPaths(infos2); + } + + /* In case of a fixed-output derivation hash mismatch, throw an + exception now that we have registered the output as valid. */ + if (delayedException) std::rethrow_exception(delayedException); } +void DerivationGoal::checkOutputs( + const std::map<Path, ValidPathInfo>& outputs) { + std::map<Path, const ValidPathInfo&> outputsByPath; + for (auto& output : outputs) + outputsByPath.emplace(output.second.path, output.second); + + for (auto& output : outputs) { + auto& outputName = output.first; + auto& info = output.second; + + struct Checks { + bool ignoreSelfRefs = false; + std::optional<uint64_t> maxSize, maxClosureSize; + std::optional<Strings> allowedReferences, allowedRequisites, + disallowedReferences, disallowedRequisites; + }; -void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) -{ - std::map<Path, const ValidPathInfo &> outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(output.second.path, output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - struct Checks - { - bool ignoreSelfRefs = false; - std::optional<uint64_t> maxSize, maxClosureSize; - std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; - }; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const Path & path) - { - uint64_t closureSize = 0; - PathSet pathsDone; - std::queue<Path> pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(path); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = worker.store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const Path& path) { + uint64_t closureSize = 0; + PathSet pathsDone; + std::queue<Path> pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) continue; + + auto i = outputsByPath.find(path); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto& ref : i->second.references) pathsLeft.push(ref); + } else { + auto info = worker.store.queryPathInfo(path); + closureSize += info->narSize; + for (auto& ref : info->references) pathsLeft.push(ref); + } + } - return std::make_pair(pathsDone, closureSize); - }; + return std::make_pair(pathsDone, closureSize); + }; - auto applyChecks = [&](const Checks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - info.path, info.narSize, *checks.maxSize); + auto applyChecks = [&](const Checks& checks) { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError( + "path '%s' is too large at %d bytes; limit is %d bytes", info.path, + info.narSize, *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError( + "closure of path '%s' is too large at %d bytes; limit is %d " + "bytes", + info.path, closureSize, *checks.maxClosureSize); + } + + auto checkRefs = [&](const std::optional<Strings>& value, bool allowed, + bool recursive) { + if (!value) return; + + PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value); + + PathSet used = + recursive ? getClosure(info.path).first : info.references; + + if (recursive && checks.ignoreSelfRefs) used.erase(info.path); + + PathSet badPaths; + + for (auto& i : used) + if (allowed) { + if (!spec.count(i)) badPaths.insert(i); + } else { + if (spec.count(i)) badPaths.insert(i); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto& i : badPaths) { + badPathsStr += "\n "; + badPathsStr += i; + } + throw BuildError( + "output '%s' is not allowed to refer to the following paths:%s", + info.path, badPathsStr); + } + }; - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - info.path, closureSize, *checks.maxClosureSize); - } + checkRefs(checks.allowedReferences, true, false); + checkRefs(checks.allowedRequisites, true, true); + checkRefs(checks.disallowedReferences, false, false); + checkRefs(checks.disallowedRequisites, false, true); + }; - auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive) - { - if (!value) return; - - PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value); - - PathSet used = recursive ? getClosure(info.path).first : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - PathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += i; - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr); - } - }; - - checkRefs(checks.allowedReferences, true, false); - checkRefs(checks.allowedRequisites, true, true); - checkRefs(checks.disallowedReferences, false, false); - checkRefs(checks.disallowedRequisites, false, true); - }; - - if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { - Checks checks; - - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) - checks.maxSize = maxSize->get<uint64_t>(); - - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) - checks.maxClosureSize = maxClosureSize->get<uint64_t>(); - - auto get = [&](const std::string & name) -> std::optional<Strings> { - auto i = output->find(name); - if (i != output->end()) { - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); - res.push_back(j->get<std::string>()); - } - checks.disallowedRequisites = res; - return res; - } - return {}; - }; - - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); - - applyChecks(checks); - } + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + auto outputChecks = structuredAttrs->find("outputChecks"); + if (outputChecks != structuredAttrs->end()) { + auto output = outputChecks->find(outputName); + + if (output != outputChecks->end()) { + Checks checks; + + auto maxSize = output->find("maxSize"); + if (maxSize != output->end()) + checks.maxSize = maxSize->get<uint64_t>(); + + auto maxClosureSize = output->find("maxClosureSize"); + if (maxClosureSize != output->end()) + checks.maxClosureSize = maxClosureSize->get<uint64_t>(); + + auto get = [&](const std::string& name) -> std::optional<Strings> { + auto i = output->find(name); + if (i != output->end()) { + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error( + "attribute '%s' of derivation '%s' must be a list of " + "strings", + name, drvPath); + res.push_back(j->get<std::string>()); + } + checks.disallowedRequisites = res; + return res; } - } else { - // legacy non-structured-attributes case - Checks checks; - checks.ignoreSelfRefs = true; - checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); - checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); - checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); - checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); - applyChecks(checks); + return {}; + }; + + checks.allowedReferences = get("allowedReferences"); + checks.allowedRequisites = get("allowedRequisites"); + checks.disallowedReferences = get("disallowedReferences"); + checks.disallowedRequisites = get("disallowedRequisites"); + + applyChecks(checks); } + } + } else { + // legacy non-structured-attributes case + Checks checks; + checks.ignoreSelfRefs = true; + checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); + checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); + checks.disallowedReferences = + parsedDrv->getStringsAttr("disallowedReferences"); + checks.disallowedRequisites = + parsedDrv->getStringsAttr("disallowedRequisites"); + applyChecks(checks); } + } } +Path DerivationGoal::openLogFile() { + logSize = 0; -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; + if (!settings.keepLog) return ""; - string baseName = baseNameOf(drvPath); + string baseName = baseNameOf(drvPath); - /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); - createDirs(dir); + /* Create a log file. */ + Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, + string(baseName, 0, 2)); + createDirs(dir); - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), - settings.compressLog ? ".bz2" : ""); + Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), + settings.compressLog ? ".bz2" : ""); - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError(format("creating log file '%1%'") % logFileName); + fdLogFile = + open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); + if (!fdLogFile) + throw SysError(format("creating log file '%1%'") % logFileName); - logFileSink = std::make_shared<FdSink>(fdLogFile.get()); + logFileSink = std::make_shared<FdSink>(fdLogFile.get()); - if (settings.compressLog) - logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink)); - else - logSink = logFileSink; + if (settings.compressLog) + logSink = std::shared_ptr<CompressionSink>( + makeCompressionSink("bzip2", *logFileSink)); + else + logSink = logFileSink; - return logFileName; + return logFileName; } - -void DerivationGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink); - if (logSink2) logSink2->finish(); - if (logFileSink) logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile = -1; +void DerivationGoal::closeLogFile() { + auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink); + if (logSink2) logSink2->finish(); + if (logFileSink) logFileSink->flush(); + logSink = logFileSink = 0; + fdLogFile = -1; } - -void DerivationGoal::deleteTmpDir(bool force) -{ - if (tmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError( - format("note: keeping build directory '%2%'") - % drvPath % tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } +void DerivationGoal::deleteTmpDir(bool force) { + if (tmpDir != "") { + /* Don't keep temporary directories for builtins because they + might have privileged stuff (like a copy of netrc). */ + if (settings.keepFailed && !force && !drv->isBuiltin()) { + printError(format("note: keeping build directory '%2%'") % drvPath % + tmpDir); + chmod(tmpDir.c_str(), 0755); + } else + deletePath(tmpDir); + tmpDir = ""; + } } - -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - printError( - format("%1% killed after writing more than %2% bytes of log output") - % getName() % settings.maxLogSize); - killChild(); - done(BuildResult::LogLimitExceeded); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) (*logSink)(data); +void DerivationGoal::handleChildOutput(int fd, const string& data) { + if ((hook && fd == hook->builderOut.readSide.get()) || + (!hook && fd == builderOut.readSide.get())) { + logSize += data.size(); + if (settings.maxLogSize && logSize > settings.maxLogSize) { + printError( + format("%1% killed after writing more than %2% bytes of log output") % + getName() % settings.maxLogSize); + killChild(); + done(BuildResult::LogLimitExceeded); + return; } - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); - currentHookLine.clear(); - } else - currentHookLine += c; - } + for (auto c : data) + if (c == '\r') + currentLogLinePos = 0; + else if (c == '\n') + flushLine(); + else { + if (currentLogLinePos >= currentLogLine.size()) + currentLogLine.resize(currentLogLinePos + 1); + currentLogLine[currentLogLinePos++] = c; + } + + if (logSink) (*logSink)(data); + } + + if (hook && fd == hook->fromHook.readSide.get()) { + for (auto c : data) + if (c == '\n') { + handleJSONLogMessage(currentHookLine, worker.act, hook->activities, + true); + currentHookLine.clear(); + } else + currentHookLine += c; + } } - -void DerivationGoal::handleEOF(int fd) -{ - if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); +void DerivationGoal::handleEOF(int fd) { + if (!currentLogLine.empty()) flushLine(); + worker.wakeUp(shared_from_this()); } +void DerivationGoal::flushLine() { + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + ; -void DerivationGoal::flushLine() -{ - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) - ; - + else { + if (settings.verboseBuild && + (settings.printRepeatedBuilds || curRound == 1)) + printError(currentLogLine); else { - if (settings.verboseBuild && - (settings.printRepeatedBuilds || curRound == 1)) - printError(currentLogLine); - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - } - - act->result(resBuildLogLine, currentLogLine); + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); } - currentLogLine = ""; - currentLogLinePos = 0; -} - + act->result(resBuildLogLine, currentLogLine); + } -PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) -{ - PathSet result; - for (auto & i : drv->outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - bool good = - worker.store.isValidPath(i.second.path) && - (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path); - } - return result; + currentLogLine = ""; + currentLogLinePos = 0; } - -Path DerivationGoal::addHashRewrite(const Path & path) -{ - string h1 = string(path, worker.store.storeDir.size() + 1, 32); - string h2 = string(hashString(htSHA256, "rewrite:" + drvPath + ":" + path).to_string(Base32, false), 0, 32); - Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33); - deletePath(p); - assert(path.size() == p.size()); - inputRewrites[h1] = h2; - outputRewrites[h2] = h1; - redirectedOutputs[path] = p; - return p; +PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { + PathSet result; + for (auto& i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + bool good = worker.store.isValidPath(i.second.path) && + (!checkHash || worker.pathContentsGood(i.second.path)); + if (good == returnValid) result.insert(i.second.path); + } + return result; } +Path DerivationGoal::addHashRewrite(const Path& path) { + string h1 = string(path, worker.store.storeDir.size() + 1, 32); + string h2 = string(hashString(htSHA256, "rewrite:" + drvPath + ":" + path) + .to_string(Base32, false), + 0, 32); + Path p = worker.store.storeDir + "/" + h2 + + string(path, worker.store.storeDir.size() + 33); + deletePath(p); + assert(path.size() == p.size()); + inputRewrites[h1] = h2; + outputRewrites[h2] = h1; + redirectedOutputs[path] = p; + return p; +} -void DerivationGoal::done(BuildResult::Status status, const string & msg) -{ - result.status = status; - result.errorMsg = msg; - amDone(result.success() ? ecSuccess : ecFailed); - if (result.status == BuildResult::TimedOut) - worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; +void DerivationGoal::done(BuildResult::Status status, const string& msg) { + result.status = status; + result.errorMsg = msg; + amDone(result.success() ? ecSuccess : ecFailed); + if (result.status == BuildResult::TimedOut) worker.timedOut = true; + if (result.status == BuildResult::PermanentFailure) + worker.permanentFailure = true; - mcExpectedBuilds.reset(); - mcRunningBuilds.reset(); + mcExpectedBuilds.reset(); + mcRunningBuilds.reset(); - if (result.success()) { - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } + if (result.success()) { + if (status == BuildResult::Built) worker.doneBuilds++; + } else { + if (status != BuildResult::DependencyFailed) worker.failedBuilds++; + } - worker.updateProgress(); + worker.updateProgress(); } - ////////////////////////////////////////////////////////////////////// +class SubstitutionGoal : public Goal { + friend class Worker; -class SubstitutionGoal : public Goal -{ - friend class Worker; + private: + /* The store path that should be realised through a substitute. */ + Path storePath; -private: - /* The store path that should be realised through a substitute. */ - Path storePath; + /* The remaining substituters. */ + std::list<ref<Store>> subs; - /* The remaining substituters. */ - std::list<ref<Store>> subs; + /* The current substituter. */ + std::shared_ptr<Store> sub; - /* The current substituter. */ - std::shared_ptr<Store> sub; + /* Whether a substituter failed. */ + bool substituterFailed = false; - /* Whether a substituter failed. */ - bool substituterFailed = false; + /* Path info returned by the substituter's query info operation. */ + std::shared_ptr<const ValidPathInfo> info; - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr<const ValidPathInfo> info; + /* Pipe for the substituter's standard output. */ + Pipe outPipe; - /* Pipe for the substituter's standard output. */ - Pipe outPipe; + /* The substituter thread. */ + std::thread thr; - /* The substituter thread. */ - std::thread thr; + std::promise<void> promise; - std::promise<void> promise; + /* Whether to try to repair a valid path. */ + RepairFlag repair; - /* Whether to try to repair a valid path. */ - RepairFlag repair; + /* Location where we're downloading the substitute. Differs from + storePath when doing a repair. */ + Path destPath; - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; + std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions, + maintainRunningSubstitutions, maintainExpectedNar, + maintainExpectedDownload; - std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; + typedef void (SubstitutionGoal::*GoalState)(); + GoalState state; - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; + public: + SubstitutionGoal(const Path& storePath, Worker& worker, + RepairFlag repair = NoRepair); + ~SubstitutionGoal(); -public: - SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair = NoRepair); - ~SubstitutionGoal(); + void timedOut() override { abort(); }; - void timedOut() override { abort(); }; + string key() override { + /* "a$" ensures substitution goals happen before derivation + goals. */ + return "a$" + storePathToName(storePath) + "$" + storePath; + } - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + storePathToName(storePath) + "$" + storePath; - } - - void work() override; + void work() override; - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); + /* The states. */ + void init(); + void tryNext(); + void gotInfo(); + void referencesValid(); + void tryToRun(); + void finished(); - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const string& data) override; + void handleEOF(int fd) override; - Path getStorePath() { return storePath; } + Path getStorePath() { return storePath; } - void amDone(ExitCode result) override - { - Goal::amDone(result); - } + void amDone(ExitCode result) override { Goal::amDone(result); } }; - -SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair) - : Goal(worker) - , repair(repair) -{ - this->storePath = storePath; - state = &SubstitutionGoal::init; - name = (format("substitution of '%1%'") % storePath).str(); - trace("created"); - maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions); +SubstitutionGoal::SubstitutionGoal(const Path& storePath, Worker& worker, + RepairFlag repair) + : Goal(worker), repair(repair) { + this->storePath = storePath; + state = &SubstitutionGoal::init; + name = (format("substitution of '%1%'") % storePath).str(); + trace("created"); + maintainExpectedSubstitutions = + std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions); } - -SubstitutionGoal::~SubstitutionGoal() -{ - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); +SubstitutionGoal::~SubstitutionGoal() { + try { + if (thr.joinable()) { + // FIXME: signal worker thread to quit. + thr.join(); + worker.childTerminated(this); } + } catch (...) { + ignoreException(); + } } +void SubstitutionGoal::work() { (this->*state)(); } -void SubstitutionGoal::work() -{ - (this->*state)(); -} - - -void SubstitutionGoal::init() -{ - trace("init"); +void SubstitutionGoal::init() { + trace("init"); - worker.store.addTempRoot(storePath); + worker.store.addTempRoot(storePath); - /* If the path already exists we're done. */ - if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); - return; - } + /* If the path already exists we're done. */ + if (!repair && worker.store.isValidPath(storePath)) { + amDone(ecSuccess); + return; + } - if (settings.readOnlyMode) - throw Error(format("cannot substitute path '%1%' - no write access to the Nix store") % storePath); + if (settings.readOnlyMode) + throw Error( + format( + "cannot substitute path '%1%' - no write access to the Nix store") % + storePath); - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>(); + subs = settings.useSubstitutes ? getDefaultSubstituters() + : std::list<ref<Store>>(); - tryNext(); + tryNext(); } +void SubstitutionGoal::tryNext() { + trace("trying next substituter"); -void SubstitutionGoal::tryNext() -{ - trace("trying next substituter"); + if (subs.size() == 0) { + /* None left. Terminate this goal and let someone else deal + with it. */ + debug(format("path '%1%' is required, but there is no substituter that can " + "build it") % + storePath); - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug(format("path '%1%' is required, but there is no substituter that can build it") % storePath); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + amDone(substituterFailed ? ecFailed : ecNoSubstituters); - try { - // FIXME: make async - info = sub->queryPathInfo(storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - printError(e.what()); - tryNext(); - return; - } - throw; + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); } - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info); - - maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize); + return; + } - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; + sub = subs.front(); + subs.pop_front(); - worker.updateProgress(); + if (sub->storeDir != worker.store.storeDir) { + tryNext(); + return; + } - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) - { - printError("warning: substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), storePath); - tryNext(); - return; + try { + // FIXME: make async + info = sub->queryPathInfo(storePath); + } catch (InvalidPath&) { + tryNext(); + return; + } catch (SubstituterDisabled&) { + if (settings.tryFallback) { + tryNext(); + return; } - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &SubstitutionGoal::referencesValid; + throw; + } catch (Error& e) { + if (settings.tryFallback) { + printError(e.what()); + tryNext(); + return; + } + throw; + } + + /* Update the total expected download size. */ + auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info); + + maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>( + worker.expectedNarSize, info->narSize); + + maintainExpectedDownload = + narInfo && narInfo->fileSize + ? std::make_unique<MaintainCount<uint64_t>>( + worker.expectedDownloadSize, narInfo->fileSize) + : nullptr; + + worker.updateProgress(); + + /* Bail out early if this substituter lacks a valid + signature. LocalStore::addToStore() also checks for this, but + only after we've downloaded the path. */ + if (worker.store.requireSigs && !sub->isTrusted && + !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { + printError( + "warning: substituter '%s' does not have a valid signature for path " + "'%s'", + sub->getUri(), storePath); + tryNext(); + return; + } + + /* To maintain the closure invariant, we first have to realise the + paths referenced by this one. */ + for (auto& i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makeSubstitutionGoal(i)); + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + referencesValid(); + else + state = &SubstitutionGoal::referencesValid; } +void SubstitutionGoal::referencesValid() { + trace("all references realised"); -void SubstitutionGoal::referencesValid() -{ - trace("all references realised"); - - if (nrFailed > 0) { - debug(format("some references of path '%1%' could not be realised") % storePath); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; - } + if (nrFailed > 0) { + debug(format("some references of path '%1%' could not be realised") % + storePath); + amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure + : ecFailed); + return; + } - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + for (auto& i : info->references) + if (i != storePath) /* ignore self-references */ + assert(worker.store.isValidPath(i)); - state = &SubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); + state = &SubstitutionGoal::tryToRun; + worker.wakeUp(shared_from_this()); } +void SubstitutionGoal::tryToRun() { + trace("trying to run"); -void SubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a build. Note that even - if maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return; - } + /* Make sure that we are allowed to start a build. Note that even + if maxBuildJobs == 0 (no local builds allowed), we still allow + a substituter to run. This is because substitutions cannot be + distributed to another machine via the build hook. */ + if (worker.getNrLocalBuilds() >= + std::max(1U, (unsigned int)settings.maxBuildJobs)) { + worker.waitForBuildSlot(shared_from_this()); + return; + } - maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); - worker.updateProgress(); + maintainRunningSubstitutions = + std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); + worker.updateProgress(); - outPipe.create(); + outPipe.create(); - promise = std::promise<void>(); + promise = std::promise<void>(); - thr = std::thread([this]() { - try { - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); + thr = std::thread([this]() { + try { + /* Wake up the worker loop when we're done. */ + Finally updateStats([this]() { outPipe.writeSide = -1; }); - Activity act(*logger, actSubstitute, Logger::Fields{storePath, sub->getUri()}); - PushActivity pact(act.id); + Activity act(*logger, actSubstitute, + Logger::Fields{storePath, sub->getUri()}); + PushActivity pact(act.id); - copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), - storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + copyStorePath(ref<Store>(sub), + ref<Store>(worker.store.shared_from_this()), storePath, + repair, sub->isTrusted ? NoCheckSigs : CheckSigs); - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }); + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }); - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, + false); - state = &SubstitutionGoal::finished; + state = &SubstitutionGoal::finished; } +void SubstitutionGoal::finished() { + trace("substitute finished"); -void SubstitutionGoal::finished() -{ - trace("substitute finished"); + thr.join(); + worker.childTerminated(this); - thr.join(); - worker.childTerminated(this); + try { + promise.get_future().get(); + } catch (std::exception& e) { + printError(e.what()); + /* Cause the parent build to fail unless --fallback is given, + or the substitute has disappeared. The latter case behaves + the same as the substitute never having existed in the + first place. */ try { - promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ - try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; - } - - /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; + throw; + } catch (SubstituteGone&) { + } catch (...) { + substituterFailed = true; } - worker.markContentsGood(storePath); - - printMsg(lvlChatty, - format("substitution of path '%1%' succeeded") % storePath); + /* Try the next substitute. */ + state = &SubstitutionGoal::tryNext; + worker.wakeUp(shared_from_this()); + return; + } - maintainRunningSubstitutions.reset(); + worker.markContentsGood(storePath); - maintainExpectedSubstitutions.reset(); - worker.doneSubstitutions++; + printMsg(lvlChatty, + format("substitution of path '%1%' succeeded") % storePath); - if (maintainExpectedDownload) { - auto fileSize = maintainExpectedDownload->delta; - maintainExpectedDownload.reset(); - worker.doneDownloadSize += fileSize; - } + maintainRunningSubstitutions.reset(); - worker.doneNarSize += maintainExpectedNar->delta; - maintainExpectedNar.reset(); + maintainExpectedSubstitutions.reset(); + worker.doneSubstitutions++; - worker.updateProgress(); + if (maintainExpectedDownload) { + auto fileSize = maintainExpectedDownload->delta; + maintainExpectedDownload.reset(); + worker.doneDownloadSize += fileSize; + } - amDone(ecSuccess); -} + worker.doneNarSize += maintainExpectedNar->delta; + maintainExpectedNar.reset(); + worker.updateProgress(); -void SubstitutionGoal::handleChildOutput(int fd, const string & data) -{ + amDone(ecSuccess); } +void SubstitutionGoal::handleChildOutput(int fd, const string& data) {} -void SubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); +void SubstitutionGoal::handleEOF(int fd) { + if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } - ////////////////////////////////////////////////////////////////////// - static bool working = false; - -Worker::Worker(LocalStore & store) - : act(*logger, actRealise) - , actDerivations(*logger, actBuilds) - , actSubstitutions(*logger, actCopyPaths) - , store(store) -{ - /* Debugging: prevent recursive workers. */ - if (working) abort(); - working = true; - nrLocalBuilds = 0; - lastWokenUp = steady_time_point::min(); - permanentFailure = false; - timedOut = false; - hashMismatch = false; - checkMismatch = false; +Worker::Worker(LocalStore& store) + : act(*logger, actRealise), + actDerivations(*logger, actBuilds), + actSubstitutions(*logger, actCopyPaths), + store(store) { + /* Debugging: prevent recursive workers. */ + if (working) abort(); + working = true; + nrLocalBuilds = 0; + lastWokenUp = steady_time_point::min(); + permanentFailure = false; + timedOut = false; + hashMismatch = false; + checkMismatch = false; } +Worker::~Worker() { + working = false; -Worker::~Worker() -{ - working = false; - - /* Explicitly get rid of all strong pointers now. After this all - goals that refer to this worker should be gone. (Otherwise we - are in trouble, since goals may call childTerminated() etc. in - their destructors). */ - topGoals.clear(); - - assert(expectedSubstitutions == 0); - assert(expectedDownloadSize == 0); - assert(expectedNarSize == 0); -} - + /* Explicitly get rid of all strong pointers now. After this all + goals that refer to this worker should be gone. (Otherwise we + are in trouble, since goals may call childTerminated() etc. in + their destructors). */ + topGoals.clear(); -GoalPtr Worker::makeDerivationGoal(const Path & path, - const StringSet & wantedOutputs, BuildMode buildMode) -{ - GoalPtr goal = derivationGoals[path].lock(); - if (!goal) { - goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode); - derivationGoals[path] = goal; - wakeUp(goal); - } else - (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs); - return goal; + assert(expectedSubstitutions == 0); + assert(expectedDownloadSize == 0); + assert(expectedNarSize == 0); } - -std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const Path & drvPath, - const BasicDerivation & drv, BuildMode buildMode) -{ - auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode); +GoalPtr Worker::makeDerivationGoal(const Path& path, + const StringSet& wantedOutputs, + BuildMode buildMode) { + GoalPtr goal = derivationGoals[path].lock(); + if (!goal) { + goal = + std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode); + derivationGoals[path] = goal; wakeUp(goal); - return goal; + } else + (dynamic_cast<DerivationGoal*>(goal.get())) + ->addWantedOutputs(wantedOutputs); + return goal; } - -GoalPtr Worker::makeSubstitutionGoal(const Path & path, RepairFlag repair) -{ - GoalPtr goal = substitutionGoals[path].lock(); - if (!goal) { - goal = std::make_shared<SubstitutionGoal>(path, *this, repair); - substitutionGoals[path] = goal; - wakeUp(goal); - } - return goal; +std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal( + const Path& drvPath, const BasicDerivation& drv, BuildMode buildMode) { + auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode); + wakeUp(goal); + return goal; } - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) -{ - /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; +GoalPtr Worker::makeSubstitutionGoal(const Path& path, RepairFlag repair) { + GoalPtr goal = substitutionGoals[path].lock(); + if (!goal) { + goal = std::make_shared<SubstitutionGoal>(path, *this, repair); + substitutionGoals[path] = goal; + wakeUp(goal); + } + return goal; } - -void Worker::removeGoal(GoalPtr goal) -{ - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - - waitingForAnyGoal.clear(); +static void removeGoal(GoalPtr goal, WeakGoalMap& goalMap) { + /* !!! inefficient */ + for (WeakGoalMap::iterator i = goalMap.begin(); i != goalMap.end();) + if (i->second.lock() == goal) { + WeakGoalMap::iterator j = i; + ++j; + goalMap.erase(i); + i = j; + } else + ++i; } - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - addToWeakGoals(awake, goal); +void Worker::removeGoal(GoalPtr goal) { + nix::removeGoal(goal, derivationGoals); + nix::removeGoal(goal, substitutionGoals); + if (topGoals.find(goal) != topGoals.end()) { + topGoals.erase(goal); + /* If a top-level goal failed, then kill all other goals + (unless keepGoing was set). */ + if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) + topGoals.clear(); + } + + /* Wake up goals waiting for any goal to finish. */ + for (auto& i : waitingForAnyGoal) { + GoalPtr goal = i.lock(); + if (goal) wakeUp(goal); + } + + waitingForAnyGoal.clear(); } - -unsigned Worker::getNrLocalBuilds() -{ - return nrLocalBuilds; +void Worker::wakeUp(GoalPtr goal) { + goal->trace("woken up"); + addToWeakGoals(awake, goal); } - -void Worker::childStarted(GoalPtr goal, const set<int> & fds, - bool inBuildSlot, bool respectTimeouts) -{ - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - child.respectTimeouts = respectTimeouts; - children.emplace_back(child); - if (inBuildSlot) nrLocalBuilds++; +unsigned Worker::getNrLocalBuilds() { return nrLocalBuilds; } + +void Worker::childStarted(GoalPtr goal, const set<int>& fds, bool inBuildSlot, + bool respectTimeouts) { + Child child; + child.goal = goal; + child.goal2 = goal.get(); + child.fds = fds; + child.timeStarted = child.lastOutput = steady_time_point::clock::now(); + child.inBuildSlot = inBuildSlot; + child.respectTimeouts = respectTimeouts; + children.emplace_back(child); + if (inBuildSlot) nrLocalBuilds++; } +void Worker::childTerminated(Goal* goal, bool wakeSleepers) { + auto i = + std::find_if(children.begin(), children.end(), + [&](const Child& child) { return child.goal2 == goal; }); + if (i == children.end()) return; -void Worker::childTerminated(Goal * goal, bool wakeSleepers) -{ - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; - - if (i->inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - } - - children.erase(i); - - if (wakeSleepers) { + if (i->inBuildSlot) { + assert(nrLocalBuilds > 0); + nrLocalBuilds--; + } - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } + children.erase(i); - wantingToBuild.clear(); + if (wakeSleepers) { + /* Wake up goals waiting for a build slot. */ + for (auto& j : wantingToBuild) { + GoalPtr goal = j.lock(); + if (goal) wakeUp(goal); } -} - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) - wakeUp(goal); /* we can do it right away */ - else - addToWeakGoals(wantingToBuild, goal); + wantingToBuild.clear(); + } } - -void Worker::waitForAnyGoal(GoalPtr goal) -{ - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); +void Worker::waitForBuildSlot(GoalPtr goal) { + debug("wait for build slot"); + if (getNrLocalBuilds() < settings.maxBuildJobs) + wakeUp(goal); /* we can do it right away */ + else + addToWeakGoals(wantingToBuild, goal); } +void Worker::waitForAnyGoal(GoalPtr goal) { + debug("wait for any goal"); + addToWeakGoals(waitingForAnyGoal, goal); +} -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); +void Worker::waitForAWhile(GoalPtr goal) { + debug("wait for a while"); + addToWeakGoals(waitingForAWhile, goal); } +void Worker::run(const Goals& _topGoals) { + for (auto& i : _topGoals) topGoals.insert(i); -void Worker::run(const Goals & _topGoals) -{ - for (auto & i : _topGoals) topGoals.insert(i); + debug("entered goal loop"); - debug("entered goal loop"); + while (1) { + checkInterrupt(); - while (1) { + store.autoGC(false); + /* Call every wake goal (in the ordering established by + CompareGoalPtrs). */ + while (!awake.empty() && !topGoals.empty()) { + Goals awake2; + for (auto& i : awake) { + GoalPtr goal = i.lock(); + if (goal) awake2.insert(goal); + } + awake.clear(); + for (auto& goal : awake2) { checkInterrupt(); + goal->work(); + if (topGoals.empty()) break; // stuff may have been cancelled + } + } - store.autoGC(false); - - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); - for (auto & goal : awake2) { - checkInterrupt(); - goal->work(); - if (topGoals.empty()) break; // stuff may have been cancelled - } - } - - if (topGoals.empty()) break; + if (topGoals.empty()) break; - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( - "unable to start any build; either increase '--max-jobs' " - "or enable remote builds"); - assert(!awake.empty()); - } + /* Wait for input. */ + if (!children.empty() || !waitingForAWhile.empty()) + waitForInput(); + else { + if (awake.empty() && 0 == settings.maxBuildJobs) + throw Error( + "unable to start any build; either increase '--max-jobs' " + "or enable remote builds"); + assert(!awake.empty()); } - - /* If --keep-going is not set, it's possible that the main goal - exited while some of its subgoals were still active. But if - --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); + } + + /* If --keep-going is not set, it's possible that the main goal + exited while some of its subgoals were still active. But if + --keep-going *is* set, then they must all be finished now. */ + assert(!settings.keepGoing || awake.empty()); + assert(!settings.keepGoing || wantingToBuild.empty()); + assert(!settings.keepGoing || children.empty()); } +void Worker::waitForInput() { + printMsg(lvlVomit, "waiting for children"); + + /* Process output from the file descriptors attached to the + children, namely log output and output path creation commands. + We also use this to detect child termination: if we get EOF on + the logger pipe of a build, we assume that the builder has + terminated. */ + + bool useTimeout = false; + struct timeval timeout; + timeout.tv_usec = 0; + auto before = steady_time_point::clock::now(); + + /* If we're monitoring for silence on stdout/stderr, or if there + is a build timeout, then wait for input until the first + deadline for any child. */ + auto nearest = steady_time_point::max(); // nearest deadline + if (settings.minFree.get() != 0) + // Periodicallty wake up to see if we need to run the garbage collector. + nearest = before + std::chrono::seconds(10); + for (auto& i : children) { + if (!i.respectTimeouts) continue; + if (0 != settings.maxSilentTime) + nearest = std::min( + nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); + if (0 != settings.buildTimeout) + nearest = std::min( + nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); + } + if (nearest != steady_time_point::max()) { + timeout.tv_sec = std::max( + 1L, + (long)std::chrono::duration_cast<std::chrono::seconds>(nearest - before) + .count()); + useTimeout = true; + } + + /* If we are polling goals that are waiting for a lock, then wake + up after a few seconds at most. */ + if (!waitingForAWhile.empty()) { + useTimeout = true; + if (lastWokenUp == steady_time_point::min()) + printError("waiting for locks or build slots..."); + if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) + lastWokenUp = before; + timeout.tv_sec = std::max( + 1L, + (long)std::chrono::duration_cast<std::chrono::seconds>( + lastWokenUp + std::chrono::seconds(settings.pollInterval) - before) + .count()); + } else + lastWokenUp = steady_time_point::min(); -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - struct timeval timeout; - timeout.tv_usec = 0; - auto before = steady_time_point::clock::now(); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (!i.respectTimeouts) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); + if (useTimeout) vomit("sleeping %d seconds", timeout.tv_sec); + + /* Use select() to wait for the input side of any logger pipe to + become `available'. Note that `available' (i.e., non-blocking) + includes EOF. */ + fd_set fds; + FD_ZERO(&fds); + int fdMax = 0; + for (auto& i : children) { + for (auto& j : i.fds) { + if (j >= FD_SETSIZE) throw Error("reached FD_SETSIZE limit"); + FD_SET(j, &fds); + if (j >= fdMax) fdMax = j + 1; } - if (nearest != steady_time_point::max()) { - timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count()); - useTimeout = true; - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == steady_time_point::min()) - printError("waiting for locks or build slots..."); - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout.tv_sec = std::max(1L, - (long) std::chrono::duration_cast<std::chrono::seconds>( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - - if (useTimeout) - vomit("sleeping %d seconds", timeout.tv_sec); - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - fd_set fds; - FD_ZERO(&fds); - int fdMax = 0; - for (auto & i : children) { - for (auto & j : i.fds) { - if (j >= FD_SETSIZE) - throw Error("reached FD_SETSIZE limit"); - FD_SET(j, &fds); - if (j >= fdMax) fdMax = j + 1; + } + + if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) { + if (errno == EINTR) return; + throw SysError("waiting for input"); + } + + auto after = steady_time_point::clock::now(); + + /* Process all available file descriptors. FIXME: this is + O(children * fds). */ + decltype(children)::iterator i; + for (auto j = children.begin(); j != children.end(); j = i) { + i = std::next(j); + + checkInterrupt(); + + GoalPtr goal = j->goal.lock(); + assert(goal); + + set<int> fds2(j->fds); + std::vector<unsigned char> buffer(4096); + for (auto& k : fds2) { + if (FD_ISSET(k, &fds)) { + ssize_t rd = read(k, buffer.data(), buffer.size()); + // FIXME: is there a cleaner way to handle pt close + // than EIO? Is this even standard? + if (rd == 0 || (rd == -1 && errno == EIO)) { + debug(format("%1%: got EOF") % goal->getName()); + goal->handleEOF(k); + j->fds.erase(k); + } else if (rd == -1) { + if (errno != EINTR) + throw SysError("%s: read failed", goal->getName()); + } else { + printMsg(lvlVomit, + format("%1%: read %2% bytes") % goal->getName() % rd); + string data((char*)buffer.data(), rd); + j->lastOutput = after; + goal->handleChildOutput(k, data); } + } } - if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); + if (goal->getExitCode() == Goal::ecBusy && 0 != settings.maxSilentTime && + j->respectTimeouts && + after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { + printError(format("%1% timed out after %2% seconds of silence") % + goal->getName() % settings.maxSilentTime); + goal->timedOut(); } - auto after = steady_time_point::clock::now(); - - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - set<int> fds2(j->fds); - std::vector<unsigned char> buffer(4096); - for (auto & k : fds2) { - if (FD_ISSET(k, &fds)) { - ssize_t rd = read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug(format("%1%: got EOF") % goal->getName()); - goal->handleEOF(k); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, format("%1%: read %2% bytes") - % goal->getName() % rd); - string data((char *) buffer.data(), rd); - j->lastOutput = after; - goal->handleChildOutput(k, data); - } - } - } - - if (goal->getExitCode() == Goal::ecBusy && - 0 != settings.maxSilentTime && - j->respectTimeouts && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - printError( - format("%1% timed out after %2% seconds of silence") - % goal->getName() % settings.maxSilentTime); - goal->timedOut(); - } - - else if (goal->getExitCode() == Goal::ecBusy && - 0 != settings.buildTimeout && - j->respectTimeouts && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - printError( - format("%1% timed out after %2% seconds") - % goal->getName() % settings.buildTimeout); - goal->timedOut(); - } + else if (goal->getExitCode() == Goal::ecBusy && + 0 != settings.buildTimeout && j->respectTimeouts && + after - j->timeStarted >= + std::chrono::seconds(settings.buildTimeout)) { + printError(format("%1% timed out after %2% seconds") % goal->getName() % + settings.buildTimeout); + goal->timedOut(); } - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); + } + + if (!waitingForAWhile.empty() && + lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { + lastWokenUp = after; + for (auto& i : waitingForAWhile) { + GoalPtr goal = i.lock(); + if (goal) wakeUp(goal); } + waitingForAWhile.clear(); + } } - -unsigned int Worker::exitStatus() -{ - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ - unsigned int mask = 0; - bool buildFailure = permanentFailure || timedOut || hashMismatch; - if (buildFailure) - mask |= 0x04; // 100 - if (timedOut) - mask |= 0x01; // 101 - if (hashMismatch) - mask |= 0x02; // 102 - if (checkMismatch) { - mask |= 0x08; // 104 - } - - if (mask) - mask |= 0x60; - return mask ? mask : 1; +unsigned int Worker::exitStatus() { + /* + * 1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + */ + unsigned int mask = 0; + bool buildFailure = permanentFailure || timedOut || hashMismatch; + if (buildFailure) mask |= 0x04; // 100 + if (timedOut) mask |= 0x01; // 101 + if (hashMismatch) mask |= 0x02; // 102 + if (checkMismatch) { + mask |= 0x08; // 104 + } + + if (mask) mask |= 0x60; + return mask ? mask : 1; } - -bool Worker::pathContentsGood(const Path & path) -{ - std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printInfo(format("checking path '%1%'...") % path); - auto info = store.queryPathInfo(path); - bool res; - if (!pathExists(path)) - res = false; - else { - HashResult current = hashPath(info->narHash.type, path); - Hash nullHash(htSHA256); - res = info->narHash == nullHash || info->narHash == current.first; - } - pathContentsGoodCache[path] = res; - if (!res) printError(format("path '%1%' is corrupted or missing!") % path); - return res; +bool Worker::pathContentsGood(const Path& path) { + std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); + if (i != pathContentsGoodCache.end()) return i->second; + printInfo(format("checking path '%1%'...") % path); + auto info = store.queryPathInfo(path); + bool res; + if (!pathExists(path)) + res = false; + else { + HashResult current = hashPath(info->narHash.type, path); + Hash nullHash(htSHA256); + res = info->narHash == nullHash || info->narHash == current.first; + } + pathContentsGoodCache[path] = res; + if (!res) printError(format("path '%1%' is corrupted or missing!") % path); + return res; } - -void Worker::markContentsGood(const Path & path) -{ - pathContentsGoodCache[path] = true; +void Worker::markContentsGood(const Path& path) { + pathContentsGoodCache[path] = true; } - ////////////////////////////////////////////////////////////////////// +static void primeCache(Store& store, const PathSet& paths) { + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, + narSize); -static void primeCache(Store & store, const PathSet & paths) -{ - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); + if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) + throw Error( + "%d derivations need to be built, but neither local builds " + "('--max-jobs') " + "nor remote builds ('--builders') are enabled", + willBuild.size()); } +void LocalStore::buildPaths(const PathSet& drvPaths, BuildMode buildMode) { + Worker worker(*this); -void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) -{ - Worker worker(*this); - - primeCache(*this, drvPaths); + primeCache(*this, drvPaths); - Goals goals; - for (auto & i : drvPaths) { - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); - if (isDerivation(i2.first)) - goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); - } - - worker.run(goals); - - PathSet failed; - for (auto & i : goals) { - if (i->getExitCode() != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath()); - } + Goals goals; + for (auto& i : drvPaths) { + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); + if (isDerivation(i2.first)) + goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode)); + else + goals.insert(worker.makeSubstitutionGoal( + i, buildMode == bmRepair ? Repair : NoRepair)); + } + + worker.run(goals); + + PathSet failed; + for (auto& i : goals) { + if (i->getExitCode() != Goal::ecSuccess) { + DerivationGoal* i2 = dynamic_cast<DerivationGoal*>(i.get()); + if (i2) + failed.insert(i2->getDrvPath()); + else + failed.insert(dynamic_cast<SubstitutionGoal*>(i.get())->getStorePath()); } + } - if (!failed.empty()) - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); + if (!failed.empty()) + throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); } +BuildResult LocalStore::buildDerivation(const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) { + Worker worker(*this); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); -BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); + BuildResult result; - BuildResult result; + try { + worker.run(Goals{goal}); + result = goal->getResult(); + } catch (Error& e) { + result.status = BuildResult::MiscFailure; + result.errorMsg = e.msg(); + } - try { - worker.run(Goals{goal}); - result = goal->getResult(); - } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - - return result; + return result; } +void LocalStore::ensurePath(const Path& path) { + /* If the path is already valid, we're done. */ + if (isValidPath(path)) return; -void LocalStore::ensurePath(const Path & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; + primeCache(*this, {path}); - primeCache(*this, {path}); + Worker worker(*this); + GoalPtr goal = worker.makeSubstitutionGoal(path); + Goals goals = {goal}; - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = {goal}; + worker.run(goals); - worker.run(goals); - - if (goal->getExitCode() != Goal::ecSuccess) - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", path); + if (goal->getExitCode() != Goal::ecSuccess) + throw Error(worker.exitStatus(), + "path '%s' does not exist and cannot be created", path); } - -void LocalStore::repairPath(const Path & path) -{ - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->getExitCode() != Goal::ecSuccess) { - /* Since substituting the path didn't work, if we have a valid - deriver, then rebuild the deriver. */ - auto deriver = queryPathInfo(path)->deriver; - if (deriver != "" && isValidPath(deriver)) { - goals.clear(); - goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); - worker.run(goals); - } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", path); - } +void LocalStore::repairPath(const Path& path) { + Worker worker(*this); + GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); + Goals goals = {goal}; + + worker.run(goals); + + if (goal->getExitCode() != Goal::ecSuccess) { + /* Since substituting the path didn't work, if we have a valid + deriver, then rebuild the deriver. */ + auto deriver = queryPathInfo(path)->deriver; + if (deriver != "" && isValidPath(deriver)) { + goals.clear(); + goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); + worker.run(goals); + } else + throw Error(worker.exitStatus(), "cannot repair path '%s'", path); + } } - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/builtins.hh b/third_party/nix/src/libstore/builtins.hh index 0d2da873ece4..07601be0f50c 100644 --- a/third_party/nix/src/libstore/builtins.hh +++ b/third_party/nix/src/libstore/builtins.hh @@ -5,7 +5,7 @@ namespace nix { // TODO: make pluggable. -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); -void builtinBuildenv(const BasicDerivation & drv); +void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData); +void builtinBuildenv(const BasicDerivation& drv); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/builtins/buildenv.cc b/third_party/nix/src/libstore/builtins/buildenv.cc index 74e706664694..ce054a430309 100644 --- a/third_party/nix/src/libstore/builtins/buildenv.cc +++ b/third_party/nix/src/libstore/builtins/buildenv.cc @@ -1,13 +1,12 @@ -#include "builtins.hh" - +#include <fcntl.h> #include <sys/stat.h> #include <sys/types.h> -#include <fcntl.h> #include <algorithm> +#include "builtins.hh" namespace nix { -typedef std::map<Path,int> Priorities; +typedef std::map<Path, int> Priorities; // FIXME: change into local variables. @@ -16,102 +15,104 @@ static Priorities priorities; static unsigned long symlinks; /* For each activated package, create symlinks */ -static void createLinks(const Path & srcDir, const Path & dstDir, int priority) -{ - DirEntries srcFiles; - - try { - srcFiles = readDirectory(srcDir); - } catch (SysError & e) { - if (e.errNo == ENOTDIR) { - printError("warning: not including '%s' in the user environment because it's not a directory", srcDir); - return; - } - throw; +static void createLinks(const Path& srcDir, const Path& dstDir, int priority) { + DirEntries srcFiles; + + try { + srcFiles = readDirectory(srcDir); + } catch (SysError& e) { + if (e.errNo == ENOTDIR) { + printError( + "warning: not including '%s' in the user environment because it's " + "not a directory", + srcDir); + return; } + throw; + } - for (const auto & ent : srcFiles) { - if (ent.name[0] == '.') - /* not matched by glob */ - continue; - auto srcFile = srcDir + "/" + ent.name; - auto dstFile = dstDir + "/" + ent.name; - - struct stat srcSt; - try { - if (stat(srcFile.c_str(), &srcSt) == -1) - throw SysError("getting status of '%1%'", srcFile); - } catch (SysError & e) { - if (e.errNo == ENOENT || e.errNo == ENOTDIR) { - printError("warning: skipping dangling symlink '%s'", dstFile); - continue; - } - throw; - } + for (const auto& ent : srcFiles) { + if (ent.name[0] == '.') /* not matched by glob */ + continue; + auto srcFile = srcDir + "/" + ent.name; + auto dstFile = dstDir + "/" + ent.name; - /* The files below are special-cased to that they don't show up - * in user profiles, either because they are useless, or - * because they would cauase pointless collisions (e.g., each - * Python package brings its own - * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.) - */ - if (hasSuffix(srcFile, "/propagated-build-inputs") || - hasSuffix(srcFile, "/nix-support") || - hasSuffix(srcFile, "/perllocal.pod") || - hasSuffix(srcFile, "/info/dir") || - hasSuffix(srcFile, "/log")) - continue; - - else if (S_ISDIR(srcSt.st_mode)) { - struct stat dstSt; - auto res = lstat(dstFile.c_str(), &dstSt); - if (res == 0) { - if (S_ISDIR(dstSt.st_mode)) { - createLinks(srcFile, dstFile, priority); - continue; - } else if (S_ISLNK(dstSt.st_mode)) { - auto target = canonPath(dstFile, true); - if (!S_ISDIR(lstat(target).st_mode)) - throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); - if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); - if (mkdir(dstFile.c_str(), 0755) == -1) - throw SysError(format("creating directory '%1%'")); - createLinks(target, dstFile, priorities[dstFile]); - createLinks(srcFile, dstFile, priority); - continue; - } - } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); - } + struct stat srcSt; + try { + if (stat(srcFile.c_str(), &srcSt) == -1) + throw SysError("getting status of '%1%'", srcFile); + } catch (SysError& e) { + if (e.errNo == ENOENT || e.errNo == ENOTDIR) { + printError("warning: skipping dangling symlink '%s'", dstFile); + continue; + } + throw; + } - else { - struct stat dstSt; - auto res = lstat(dstFile.c_str(), &dstSt); - if (res == 0) { - if (S_ISLNK(dstSt.st_mode)) { - auto prevPriority = priorities[dstFile]; - if (prevPriority == priority) - throw Error( - "packages '%1%' and '%2%' have the same priority %3%; " - "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "to change the priority of one of the conflicting packages" - " (0 being the highest priority)", - srcFile, readLink(dstFile), priority); - if (prevPriority < priority) - continue; - if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); - } else if (S_ISDIR(dstSt.st_mode)) - throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); - } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); + /* The files below are special-cased to that they don't show up + * in user profiles, either because they are useless, or + * because they would cauase pointless collisions (e.g., each + * Python package brings its own + * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.) + */ + if (hasSuffix(srcFile, "/propagated-build-inputs") || + hasSuffix(srcFile, "/nix-support") || + hasSuffix(srcFile, "/perllocal.pod") || + hasSuffix(srcFile, "/info/dir") || hasSuffix(srcFile, "/log")) + continue; + + else if (S_ISDIR(srcSt.st_mode)) { + struct stat dstSt; + auto res = lstat(dstFile.c_str(), &dstSt); + if (res == 0) { + if (S_ISDIR(dstSt.st_mode)) { + createLinks(srcFile, dstFile, priority); + continue; + } else if (S_ISLNK(dstSt.st_mode)) { + auto target = canonPath(dstFile, true); + if (!S_ISDIR(lstat(target).st_mode)) + throw Error("collision between '%1%' and non-directory '%2%'", + srcFile, target); + if (unlink(dstFile.c_str()) == -1) + throw SysError(format("unlinking '%1%'") % dstFile); + if (mkdir(dstFile.c_str(), 0755) == -1) + throw SysError(format("creating directory '%1%'")); + createLinks(target, dstFile, priorities[dstFile]); + createLinks(srcFile, dstFile, priority); + continue; } + } else if (errno != ENOENT) + throw SysError(format("getting status of '%1%'") % dstFile); + } - createSymlink(srcFile, dstFile); - priorities[dstFile] = priority; - symlinks++; + else { + struct stat dstSt; + auto res = lstat(dstFile.c_str(), &dstSt); + if (res == 0) { + if (S_ISLNK(dstSt.st_mode)) { + auto prevPriority = priorities[dstFile]; + if (prevPriority == priority) + throw Error( + "packages '%1%' and '%2%' have the same priority %3%; " + "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " + "to change the priority of one of the conflicting packages" + " (0 being the highest priority)", + srcFile, readLink(dstFile), priority); + if (prevPriority < priority) continue; + if (unlink(dstFile.c_str()) == -1) + throw SysError(format("unlinking '%1%'") % dstFile); + } else if (S_ISDIR(dstSt.st_mode)) + throw Error( + "collision between non-directory '%1%' and directory '%2%'", + srcFile, dstFile); + } else if (errno != ENOENT) + throw SysError(format("getting status of '%1%'") % dstFile); } + + createSymlink(srcFile, dstFile); + priorities[dstFile] = priority; + symlinks++; + } } typedef std::set<Path> FileProp; @@ -121,84 +122,87 @@ static FileProp postponed = FileProp{}; static Path out; -static void addPkg(const Path & pkgDir, int priority) -{ - if (done.count(pkgDir)) return; - done.insert(pkgDir); - createLinks(pkgDir, out, priority); - - try { - for (const auto & p : tokenizeString<std::vector<string>>( - readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) - if (!done.count(p)) - postponed.insert(p); - } catch (SysError & e) { - if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; - } +static void addPkg(const Path& pkgDir, int priority) { + if (done.count(pkgDir)) return; + done.insert(pkgDir); + createLinks(pkgDir, out, priority); + + try { + for (const auto& p : tokenizeString<std::vector<string>>( + readFile(pkgDir + "/nix-support/propagated-user-env-packages"), + " \n")) + if (!done.count(p)) postponed.insert(p); + } catch (SysError& e) { + if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; + } } struct Package { - Path path; - bool active; - int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} + Path path; + bool active; + int priority; + Package(Path path, bool active, int priority) + : path{path}, active{active}, priority{priority} {} }; typedef std::vector<Package> Packages; -void builtinBuildenv(const BasicDerivation & drv) -{ - auto getAttr = [&](const string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); - return i->second; - }; - - out = getAttr("out"); - createDirs(out); - - /* Convert the stuff we get from the environment back into a - * coherent data type. */ - Packages pkgs; - auto derivations = tokenizeString<Strings>(getAttr("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); - } - } - - /* Symlink to the packages that have been installed explicitly by the - * user. Process in priority order to reduce unnecessary - * symlink/unlink steps. - */ - std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) { - return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); - }); - for (const auto & pkg : pkgs) - if (pkg.active) - addPkg(pkg.path, pkg.priority); - - /* Symlink to the packages that have been "propagated" by packages - * installed by the user (i.e., package X declares that it wants Y - * installed as well). We do these later because they have a lower - * priority in case of collisions. +void builtinBuildenv(const BasicDerivation& drv) { + auto getAttr = [&](const string& name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + out = getAttr("out"); + createDirs(out); + + /* Convert the stuff we get from the environment back into a + * coherent data type. */ + Packages pkgs; + auto derivations = tokenizeString<Strings>(getAttr("derivations")); + while (!derivations.empty()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ - auto priorityCounter = 1000; - while (!postponed.empty()) { - auto pkgDirs = postponed; - postponed = FileProp{}; - for (const auto & pkgDir : pkgDirs) - addPkg(pkgDir, priorityCounter++); + auto active = derivations.front(); + derivations.pop_front(); + auto priority = stoi(derivations.front()); + derivations.pop_front(); + auto outputs = stoi(derivations.front()); + derivations.pop_front(); + for (auto n = 0; n < outputs; n++) { + auto path = derivations.front(); + derivations.pop_front(); + pkgs.emplace_back(path, active != "false", priority); } - - printError("created %d symlinks in user environment", symlinks); - - createSymlink(getAttr("manifest"), out + "/manifest.nix"); + } + + /* Symlink to the packages that have been installed explicitly by the + * user. Process in priority order to reduce unnecessary + * symlink/unlink steps. + */ + std::sort(pkgs.begin(), pkgs.end(), [](const Package& a, const Package& b) { + return a.priority < b.priority || + (a.priority == b.priority && a.path < b.path); + }); + for (const auto& pkg : pkgs) + if (pkg.active) addPkg(pkg.path, pkg.priority); + + /* Symlink to the packages that have been "propagated" by packages + * installed by the user (i.e., package X declares that it wants Y + * installed as well). We do these later because they have a lower + * priority in case of collisions. + */ + auto priorityCounter = 1000; + while (!postponed.empty()) { + auto pkgDirs = postponed; + postponed = FileProp{}; + for (const auto& pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++); + } + + printError("created %d symlinks in user environment", symlinks); + + createSymlink(getAttr("manifest"), out + "/manifest.nix"); } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/builtins/fetchurl.cc b/third_party/nix/src/libstore/builtins/fetchurl.cc index b1af3b4fc316..ae4e9a71c802 100644 --- a/third_party/nix/src/libstore/builtins/fetchurl.cc +++ b/third_party/nix/src/libstore/builtins/fetchurl.cc @@ -1,78 +1,76 @@ +#include "archive.hh" #include "builtins.hh" +#include "compression.hh" #include "download.hh" #include "store-api.hh" -#include "archive.hh" -#include "compression.hh" namespace nix { -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) -{ - /* Make the host's netrc data available. Too bad curl requires - this to be stored in a file. It would be nice if we could just - pass a pointer to the data. */ - if (netrcData != "") { - settings.netrcFile = "netrc"; - writeFile(settings.netrcFile, netrcData, 0600); - } +void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData) { + /* Make the host's netrc data available. Too bad curl requires + this to be stored in a file. It would be nice if we could just + pass a pointer to the data. */ + if (netrcData != "") { + settings.netrcFile = "netrc"; + writeFile(settings.netrcFile, netrcData, 0600); + } - auto getAttr = [&](const string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name); - return i->second; - }; + auto getAttr = [&](const string& name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) + throw Error(format("attribute '%s' missing") % name); + return i->second; + }; - Path storePath = getAttr("out"); - auto mainUrl = getAttr("url"); - bool unpack = get(drv.env, "unpack", "") == "1"; + Path storePath = getAttr("out"); + auto mainUrl = getAttr("url"); + bool unpack = get(drv.env, "unpack", "") == "1"; - /* Note: have to use a fresh downloader here because we're in - a forked process. */ - auto downloader = makeDownloader(); + /* Note: have to use a fresh downloader here because we're in + a forked process. */ + auto downloader = makeDownloader(); - auto fetch = [&](const std::string & url) { + auto fetch = [&](const std::string& url) { + auto source = sinkToSource([&](Sink& sink) { + /* No need to do TLS verification, because we check the hash of + the result anyway. */ + DownloadRequest request(url); + request.verifyTLS = false; + request.decompress = false; - auto source = sinkToSource([&](Sink & sink) { + auto decompressor = makeDecompressionSink( + unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); + downloader->download(std::move(request), *decompressor); + decompressor->finish(); + }); - /* No need to do TLS verification, because we check the hash of - the result anyway. */ - DownloadRequest request(url); - request.verifyTLS = false; - request.decompress = false; + if (unpack) + restorePath(storePath, *source); + else + writeFile(storePath, *source); - auto decompressor = makeDecompressionSink( - unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); - downloader->download(std::move(request), *decompressor); - decompressor->finish(); - }); - - if (unpack) - restorePath(storePath, *source); - else - writeFile(storePath, *source); - - auto executable = drv.env.find("executable"); - if (executable != drv.env.end() && executable->second == "1") { - if (chmod(storePath.c_str(), 0755) == -1) - throw SysError(format("making '%1%' executable") % storePath); - } - }; + auto executable = drv.env.find("executable"); + if (executable != drv.env.end() && executable->second == "1") { + if (chmod(storePath.c_str(), 0755) == -1) + throw SysError(format("making '%1%' executable") % storePath); + } + }; - /* Try the hashed mirrors first. */ - if (getAttr("outputHashMode") == "flat") - for (auto hashedMirror : settings.hashedMirrors.get()) - try { - if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto ht = parseHashType(getAttr("outputHashAlgo")); - auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); - return; - } catch (Error & e) { - debug(e.what()); - } + /* Try the hashed mirrors first. */ + if (getAttr("outputHashMode") == "flat") + for (auto hashedMirror : settings.hashedMirrors.get()) try { + if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; + auto ht = parseHashType(getAttr("outputHashAlgo")); + auto h = Hash(getAttr("outputHash"), ht); + fetch(hashedMirror + printHashType(h.type) + "/" + + h.to_string(Base16, false)); + return; + } catch (Error& e) { + debug(e.what()); + } - /* Otherwise try the specified URL. */ - fetch(mainUrl); + /* Otherwise try the specified URL. */ + fetch(mainUrl); } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/crypto.cc b/third_party/nix/src/libstore/crypto.cc index 9ec8abd228e9..7d1fab8161af 100644 --- a/third_party/nix/src/libstore/crypto.cc +++ b/third_party/nix/src/libstore/crypto.cc @@ -1,6 +1,6 @@ #include "crypto.hh" -#include "util.hh" #include "globals.hh" +#include "util.hh" #if HAVE_SODIUM #include <sodium.h> @@ -8,119 +8,107 @@ namespace nix { -static std::pair<std::string, std::string> split(const string & s) -{ - size_t colon = s.find(':'); - if (colon == std::string::npos || colon == 0) - return {"", ""}; - return {std::string(s, 0, colon), std::string(s, colon + 1)}; +static std::pair<std::string, std::string> split(const string& s) { + size_t colon = s.find(':'); + if (colon == std::string::npos || colon == 0) return {"", ""}; + return {std::string(s, 0, colon), std::string(s, colon + 1)}; } -Key::Key(const string & s) -{ - auto ss = split(s); +Key::Key(const string& s) { + auto ss = split(s); - name = ss.first; - key = ss.second; + name = ss.first; + key = ss.second; - if (name == "" || key == "") - throw Error("secret key is corrupt"); + if (name == "" || key == "") throw Error("secret key is corrupt"); - key = base64Decode(key); + key = base64Decode(key); } -SecretKey::SecretKey(const string & s) - : Key(s) -{ +SecretKey::SecretKey(const string& s) : Key(s) { #if HAVE_SODIUM - if (key.size() != crypto_sign_SECRETKEYBYTES) - throw Error("secret key is not valid"); + if (key.size() != crypto_sign_SECRETKEYBYTES) + throw Error("secret key is not valid"); #endif } #if !HAVE_SODIUM -[[noreturn]] static void noSodium() -{ - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); +[[noreturn]] static void noSodium() { + throw Error( + "Nix was not compiled with libsodium, required for signed binary cache " + "support"); } #endif -std::string SecretKey::signDetached(const std::string & data) const -{ +std::string SecretKey::signDetached(const std::string& data) const { #if HAVE_SODIUM - unsigned char sig[crypto_sign_BYTES]; - unsigned long long sigLen; - crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), - (unsigned char *) key.data()); - return name + ":" + base64Encode(std::string((char *) sig, sigLen)); + unsigned char sig[crypto_sign_BYTES]; + unsigned long long sigLen; + crypto_sign_detached(sig, &sigLen, (unsigned char*)data.data(), data.size(), + (unsigned char*)key.data()); + return name + ":" + base64Encode(std::string((char*)sig, sigLen)); #else - noSodium(); + noSodium(); #endif } -PublicKey SecretKey::toPublicKey() const -{ +PublicKey SecretKey::toPublicKey() const { #if HAVE_SODIUM - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; - crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); - return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + crypto_sign_ed25519_sk_to_pk(pk, (unsigned char*)key.data()); + return PublicKey(name, std::string((char*)pk, crypto_sign_PUBLICKEYBYTES)); #else - noSodium(); + noSodium(); #endif } -PublicKey::PublicKey(const string & s) - : Key(s) -{ +PublicKey::PublicKey(const string& s) : Key(s) { #if HAVE_SODIUM - if (key.size() != crypto_sign_PUBLICKEYBYTES) - throw Error("public key is not valid"); + if (key.size() != crypto_sign_PUBLICKEYBYTES) + throw Error("public key is not valid"); #endif } -bool verifyDetached(const std::string & data, const std::string & sig, - const PublicKeys & publicKeys) -{ +bool verifyDetached(const std::string& data, const std::string& sig, + const PublicKeys& publicKeys) { #if HAVE_SODIUM - auto ss = split(sig); + auto ss = split(sig); - auto key = publicKeys.find(ss.first); - if (key == publicKeys.end()) return false; + auto key = publicKeys.find(ss.first); + if (key == publicKeys.end()) return false; - auto sig2 = base64Decode(ss.second); - if (sig2.size() != crypto_sign_BYTES) - throw Error("signature is not valid"); + auto sig2 = base64Decode(ss.second); + if (sig2.size() != crypto_sign_BYTES) throw Error("signature is not valid"); - return crypto_sign_verify_detached((unsigned char *) sig2.data(), - (unsigned char *) data.data(), data.size(), - (unsigned char *) key->second.key.data()) == 0; + return crypto_sign_verify_detached( + (unsigned char*)sig2.data(), (unsigned char*)data.data(), + data.size(), (unsigned char*)key->second.key.data()) == 0; #else - noSodium(); + noSodium(); #endif } -PublicKeys getDefaultPublicKeys() -{ - PublicKeys publicKeys; +PublicKeys getDefaultPublicKeys() { + PublicKeys publicKeys; - // FIXME: filter duplicates + // FIXME: filter duplicates - for (auto s : settings.trustedPublicKeys.get()) { - PublicKey key(s); - publicKeys.emplace(key.name, key); - } + for (auto s : settings.trustedPublicKeys.get()) { + PublicKey key(s); + publicKeys.emplace(key.name, key); + } - for (auto secretKeyFile : settings.secretKeyFiles.get()) { - try { - SecretKey secretKey(readFile(secretKeyFile)); - publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); - } catch (SysError & e) { - /* Ignore unreadable key files. That's normal in a - multi-user installation. */ - } + for (auto secretKeyFile : settings.secretKeyFiles.get()) { + try { + SecretKey secretKey(readFile(secretKeyFile)); + publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); + } catch (SysError& e) { + /* Ignore unreadable key files. That's normal in a + multi-user installation. */ } + } - return publicKeys; + return publicKeys; } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/crypto.hh b/third_party/nix/src/libstore/crypto.hh index 9110af3aa9e5..ffafe6560d15 100644 --- a/third_party/nix/src/libstore/crypto.hh +++ b/third_party/nix/src/libstore/crypto.hh @@ -1,54 +1,48 @@ #pragma once -#include "types.hh" - #include <map> +#include "types.hh" namespace nix { -struct Key -{ - std::string name; - std::string key; +struct Key { + std::string name; + std::string key; - /* Construct Key from a string in the format - ‘<name>:<key-in-base64>’. */ - Key(const std::string & s); + /* Construct Key from a string in the format + ‘<name>:<key-in-base64>’. */ + Key(const std::string& s); -protected: - Key(const std::string & name, const std::string & key) - : name(name), key(key) { } + protected: + Key(const std::string& name, const std::string& key) : name(name), key(key) {} }; struct PublicKey; -struct SecretKey : Key -{ - SecretKey(const std::string & s); +struct SecretKey : Key { + SecretKey(const std::string& s); - /* Return a detached signature of the given string. */ - std::string signDetached(const std::string & s) const; + /* Return a detached signature of the given string. */ + std::string signDetached(const std::string& s) const; - PublicKey toPublicKey() const; + PublicKey toPublicKey() const; }; -struct PublicKey : Key -{ - PublicKey(const std::string & data); +struct PublicKey : Key { + PublicKey(const std::string& data); -private: - PublicKey(const std::string & name, const std::string & key) - : Key(name, key) { } - friend struct SecretKey; + private: + PublicKey(const std::string& name, const std::string& key) : Key(name, key) {} + friend struct SecretKey; }; typedef std::map<std::string, PublicKey> PublicKeys; /* Return true iff ‘sig’ is a correct signature over ‘data’ using one of the given public keys. */ -bool verifyDetached(const std::string & data, const std::string & sig, - const PublicKeys & publicKeys); +bool verifyDetached(const std::string& data, const std::string& sig, + const PublicKeys& publicKeys); PublicKeys getDefaultPublicKeys(); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/derivations.cc b/third_party/nix/src/libstore/derivations.cc index 23fcfb281a72..e75c0d77c3ac 100644 --- a/third_party/nix/src/libstore/derivations.cc +++ b/third_party/nix/src/libstore/derivations.cc @@ -1,289 +1,294 @@ #include "derivations.hh" -#include "store-api.hh" +#include "fs-accessor.hh" #include "globals.hh" +#include "istringstream_nocopy.hh" +#include "store-api.hh" #include "util.hh" #include "worker-protocol.hh" -#include "fs-accessor.hh" -#include "istringstream_nocopy.hh" namespace nix { +void DerivationOutput::parseHashInfo(bool& recursive, Hash& hash) const { + recursive = false; + string algo = hashAlgo; -void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const -{ - recursive = false; - string algo = hashAlgo; - - if (string(algo, 0, 2) == "r:") { - recursive = true; - algo = string(algo, 2); - } + if (string(algo, 0, 2) == "r:") { + recursive = true; + algo = string(algo, 2); + } - HashType hashType = parseHashType(algo); - if (hashType == htUnknown) - throw Error(format("unknown hash algorithm '%1%'") % algo); + HashType hashType = parseHashType(algo); + if (hashType == htUnknown) + throw Error(format("unknown hash algorithm '%1%'") % algo); - hash = Hash(this->hash, hashType); + hash = Hash(this->hash, hashType); } - -Path BasicDerivation::findOutput(const string & id) const -{ - auto i = outputs.find(id); - if (i == outputs.end()) - throw Error(format("derivation has no output '%1%'") % id); - return i->second.path; +Path BasicDerivation::findOutput(const string& id) const { + auto i = outputs.find(id); + if (i == outputs.end()) + throw Error(format("derivation has no output '%1%'") % id); + return i->second.path; } - -bool BasicDerivation::isBuiltin() const -{ - return string(builder, 0, 8) == "builtin:"; +bool BasicDerivation::isBuiltin() const { + return string(builder, 0, 8) == "builtin:"; } - -Path writeDerivation(ref<Store> store, - const Derivation & drv, const string & name, RepairFlag repair) -{ - PathSet references; - references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); - for (auto & i : drv.inputDrvs) - references.insert(i.first); - /* Note that the outputs of a derivation are *not* references - (that can be missing (of course) and should not necessarily be - held during a garbage collection). */ - string suffix = name + drvExtension; - string contents = drv.unparse(); - return settings.readOnlyMode - ? store->computeStorePathForText(suffix, contents, references) - : store->addTextToStore(suffix, contents, references, repair); +Path writeDerivation(ref<Store> store, const Derivation& drv, + const string& name, RepairFlag repair) { + PathSet references; + references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); + for (auto& i : drv.inputDrvs) references.insert(i.first); + /* Note that the outputs of a derivation are *not* references + (that can be missing (of course) and should not necessarily be + held during a garbage collection). */ + string suffix = name + drvExtension; + string contents = drv.unparse(); + return settings.readOnlyMode + ? store->computeStorePathForText(suffix, contents, references) + : store->addTextToStore(suffix, contents, references, repair); } - /* Read string `s' from stream `str'. */ -static void expect(std::istream & str, const string & s) -{ - char s2[s.size()]; - str.read(s2, s.size()); - if (string(s2, s.size()) != s) - throw FormatError(format("expected string '%1%'") % s); +static void expect(std::istream& str, const string& s) { + char s2[s.size()]; + str.read(s2, s.size()); + if (string(s2, s.size()) != s) + throw FormatError(format("expected string '%1%'") % s); } - /* Read a C-style string from stream `str'. */ -static string parseString(std::istream & str) -{ - string res; - expect(str, "\""); - int c; - while ((c = str.get()) != '"') - if (c == '\\') { - c = str.get(); - if (c == 'n') res += '\n'; - else if (c == 'r') res += '\r'; - else if (c == 't') res += '\t'; - else res += c; - } - else res += c; - return res; +static string parseString(std::istream& str) { + string res; + expect(str, "\""); + int c; + while ((c = str.get()) != '"') + if (c == '\\') { + c = str.get(); + if (c == 'n') + res += '\n'; + else if (c == 'r') + res += '\r'; + else if (c == 't') + res += '\t'; + else + res += c; + } else + res += c; + return res; } - -static Path parsePath(std::istream & str) -{ - string s = parseString(str); - if (s.size() == 0 || s[0] != '/') - throw FormatError(format("bad path '%1%' in derivation") % s); - return s; +static Path parsePath(std::istream& str) { + string s = parseString(str); + if (s.size() == 0 || s[0] != '/') + throw FormatError(format("bad path '%1%' in derivation") % s); + return s; } - -static bool endOfList(std::istream & str) -{ - if (str.peek() == ',') { - str.get(); - return false; - } - if (str.peek() == ']') { - str.get(); - return true; - } +static bool endOfList(std::istream& str) { + if (str.peek() == ',') { + str.get(); return false; + } + if (str.peek() == ']') { + str.get(); + return true; + } + return false; } - -static StringSet parseStrings(std::istream & str, bool arePaths) -{ - StringSet res; - while (!endOfList(str)) - res.insert(arePaths ? parsePath(str) : parseString(str)); - return res; +static StringSet parseStrings(std::istream& str, bool arePaths) { + StringSet res; + while (!endOfList(str)) + res.insert(arePaths ? parsePath(str) : parseString(str)); + return res; } - -static Derivation parseDerivation(const string & s) -{ - Derivation drv; - istringstream_nocopy str(s); - expect(str, "Derive(["); - - /* Parse the list of outputs. */ - while (!endOfList(str)) { - DerivationOutput out; - expect(str, "("); string id = parseString(str); - expect(str, ","); out.path = parsePath(str); - expect(str, ","); out.hashAlgo = parseString(str); - expect(str, ","); out.hash = parseString(str); - expect(str, ")"); - drv.outputs[id] = out; - } - - /* Parse the list of input derivations. */ - expect(str, ",["); - while (!endOfList(str)) { - expect(str, "("); - Path drvPath = parsePath(str); - expect(str, ",["); - drv.inputDrvs[drvPath] = parseStrings(str, false); - expect(str, ")"); - } - - expect(str, ",["); drv.inputSrcs = parseStrings(str, true); - expect(str, ","); drv.platform = parseString(str); - expect(str, ","); drv.builder = parseString(str); - - /* Parse the builder arguments. */ - expect(str, ",["); - while (!endOfList(str)) - drv.args.push_back(parseString(str)); - - /* Parse the environment variables. */ +static Derivation parseDerivation(const string& s) { + Derivation drv; + istringstream_nocopy str(s); + expect(str, "Derive(["); + + /* Parse the list of outputs. */ + while (!endOfList(str)) { + DerivationOutput out; + expect(str, "("); + string id = parseString(str); + expect(str, ","); + out.path = parsePath(str); + expect(str, ","); + out.hashAlgo = parseString(str); + expect(str, ","); + out.hash = parseString(str); + expect(str, ")"); + drv.outputs[id] = out; + } + + /* Parse the list of input derivations. */ + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + Path drvPath = parsePath(str); expect(str, ",["); - while (!endOfList(str)) { - expect(str, "("); string name = parseString(str); - expect(str, ","); string value = parseString(str); - expect(str, ")"); - drv.env[name] = value; - } - + drv.inputDrvs[drvPath] = parseStrings(str, false); expect(str, ")"); - return drv; -} - + } + + expect(str, ",["); + drv.inputSrcs = parseStrings(str, true); + expect(str, ","); + drv.platform = parseString(str); + expect(str, ","); + drv.builder = parseString(str); + + /* Parse the builder arguments. */ + expect(str, ",["); + while (!endOfList(str)) drv.args.push_back(parseString(str)); + + /* Parse the environment variables. */ + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + string name = parseString(str); + expect(str, ","); + string value = parseString(str); + expect(str, ")"); + drv.env[name] = value; + } -Derivation readDerivation(const Path & drvPath) -{ - try { - return parseDerivation(readFile(drvPath)); - } catch (FormatError & e) { - throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); - } + expect(str, ")"); + return drv; } - -Derivation Store::derivationFromPath(const Path & drvPath) -{ - assertStorePath(drvPath); - ensurePath(drvPath); - auto accessor = getFSAccessor(); - try { - return parseDerivation(accessor->readFile(drvPath)); - } catch (FormatError & e) { - throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); - } +Derivation readDerivation(const Path& drvPath) { + try { + return parseDerivation(readFile(drvPath)); + } catch (FormatError& e) { + throw Error(format("error parsing derivation '%1%': %2%") % drvPath % + e.msg()); + } } - -static void printString(string & res, const string & s) -{ - res += '"'; - for (const char * i = s.c_str(); *i; i++) - if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; } - else if (*i == '\n') res += "\\n"; - else if (*i == '\r') res += "\\r"; - else if (*i == '\t') res += "\\t"; - else res += *i; - res += '"'; +Derivation Store::derivationFromPath(const Path& drvPath) { + assertStorePath(drvPath); + ensurePath(drvPath); + auto accessor = getFSAccessor(); + try { + return parseDerivation(accessor->readFile(drvPath)); + } catch (FormatError& e) { + throw Error(format("error parsing derivation '%1%': %2%") % drvPath % + e.msg()); + } } - -template<class ForwardIterator> -static void printStrings(string & res, ForwardIterator i, ForwardIterator j) -{ - res += '['; - bool first = true; - for ( ; i != j; ++i) { - if (first) first = false; else res += ','; - printString(res, *i); - } - res += ']'; +static void printString(string& res, const string& s) { + res += '"'; + for (const char* i = s.c_str(); *i; i++) + if (*i == '\"' || *i == '\\') { + res += "\\"; + res += *i; + } else if (*i == '\n') + res += "\\n"; + else if (*i == '\r') + res += "\\r"; + else if (*i == '\t') + res += "\\t"; + else + res += *i; + res += '"'; } - -string Derivation::unparse() const -{ - string s; - s.reserve(65536); - s += "Derive(["; - - bool first = true; - for (auto & i : outputs) { - if (first) first = false; else s += ','; - s += '('; printString(s, i.first); - s += ','; printString(s, i.second.path); - s += ','; printString(s, i.second.hashAlgo); - s += ','; printString(s, i.second.hash); - s += ')'; - } - - s += "],["; - first = true; - for (auto & i : inputDrvs) { - if (first) first = false; else s += ','; - s += '('; printString(s, i.first); - s += ','; printStrings(s, i.second.begin(), i.second.end()); - s += ')'; - } - - s += "],"; - printStrings(s, inputSrcs.begin(), inputSrcs.end()); - - s += ','; printString(s, platform); - s += ','; printString(s, builder); - s += ','; printStrings(s, args.begin(), args.end()); - - s += ",["; - first = true; - for (auto & i : env) { - if (first) first = false; else s += ','; - s += '('; printString(s, i.first); - s += ','; printString(s, i.second); - s += ')'; - } - - s += "])"; - - return s; +template <class ForwardIterator> +static void printStrings(string& res, ForwardIterator i, ForwardIterator j) { + res += '['; + bool first = true; + for (; i != j; ++i) { + if (first) + first = false; + else + res += ','; + printString(res, *i); + } + res += ']'; } - -bool isDerivation(const string & fileName) -{ - return hasSuffix(fileName, drvExtension); +string Derivation::unparse() const { + string s; + s.reserve(65536); + s += "Derive(["; + + bool first = true; + for (auto& i : outputs) { + if (first) + first = false; + else + s += ','; + s += '('; + printString(s, i.first); + s += ','; + printString(s, i.second.path); + s += ','; + printString(s, i.second.hashAlgo); + s += ','; + printString(s, i.second.hash); + s += ')'; + } + + s += "],["; + first = true; + for (auto& i : inputDrvs) { + if (first) + first = false; + else + s += ','; + s += '('; + printString(s, i.first); + s += ','; + printStrings(s, i.second.begin(), i.second.end()); + s += ')'; + } + + s += "],"; + printStrings(s, inputSrcs.begin(), inputSrcs.end()); + + s += ','; + printString(s, platform); + s += ','; + printString(s, builder); + s += ','; + printStrings(s, args.begin(), args.end()); + + s += ",["; + first = true; + for (auto& i : env) { + if (first) + first = false; + else + s += ','; + s += '('; + printString(s, i.first); + s += ','; + printString(s, i.second); + s += ')'; + } + + s += "])"; + + return s; } - -bool BasicDerivation::isFixedOutput() const -{ - return outputs.size() == 1 && - outputs.begin()->first == "out" && - outputs.begin()->second.hash != ""; +bool isDerivation(const string& fileName) { + return hasSuffix(fileName, drvExtension); } +bool BasicDerivation::isFixedOutput() const { + return outputs.size() == 1 && outputs.begin()->first == "out" && + outputs.begin()->second.hash != ""; +} DrvHashes drvHashes; - /* Returns the hash of a derivation modulo fixed-output subderivations. A fixed-output derivation is a derivation with one output (`out') for which an expected hash and hash algorithm are @@ -304,113 +309,95 @@ DrvHashes drvHashes; paths have been replaced by the result of a recursive call to this function, and that for fixed-output derivations we return a hash of its output path. */ -Hash hashDerivationModulo(Store & store, Derivation drv) -{ - /* Return a fixed hash for fixed-output derivations. */ - if (drv.isFixedOutput()) { - DerivationOutputs::const_iterator i = drv.outputs.begin(); - return hashString(htSHA256, "fixed:out:" - + i->second.hashAlgo + ":" - + i->second.hash + ":" - + i->second.path); - } - - /* For other derivations, replace the inputs paths with recursive - calls to this function.*/ - DerivationInputs inputs2; - for (auto & i : drv.inputDrvs) { - Hash h = drvHashes[i.first]; - if (!h) { - assert(store.isValidPath(i.first)); - Derivation drv2 = readDerivation(store.toRealPath(i.first)); - h = hashDerivationModulo(store, drv2); - drvHashes[i.first] = h; - } - inputs2[h.to_string(Base16, false)] = i.second; +Hash hashDerivationModulo(Store& store, Derivation drv) { + /* Return a fixed hash for fixed-output derivations. */ + if (drv.isFixedOutput()) { + DerivationOutputs::const_iterator i = drv.outputs.begin(); + return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" + + i->second.hash + ":" + i->second.path); + } + + /* For other derivations, replace the inputs paths with recursive + calls to this function.*/ + DerivationInputs inputs2; + for (auto& i : drv.inputDrvs) { + Hash h = drvHashes[i.first]; + if (!h) { + assert(store.isValidPath(i.first)); + Derivation drv2 = readDerivation(store.toRealPath(i.first)); + h = hashDerivationModulo(store, drv2); + drvHashes[i.first] = h; } - drv.inputDrvs = inputs2; + inputs2[h.to_string(Base16, false)] = i.second; + } + drv.inputDrvs = inputs2; - return hashString(htSHA256, drv.unparse()); + return hashString(htSHA256, drv.unparse()); } - -DrvPathWithOutputs parseDrvPathWithOutputs(const string & s) -{ - size_t n = s.find("!"); - return n == s.npos - ? DrvPathWithOutputs(s, std::set<string>()) - : DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ",")); +DrvPathWithOutputs parseDrvPathWithOutputs(const string& s) { + size_t n = s.find("!"); + return n == s.npos ? DrvPathWithOutputs(s, std::set<string>()) + : DrvPathWithOutputs(string(s, 0, n), + tokenizeString<std::set<string> >( + string(s, n + 1), ",")); } - -Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs) -{ - return outputs.empty() - ? drvPath - : drvPath + "!" + concatStringsSep(",", outputs); +Path makeDrvPathWithOutputs(const Path& drvPath, + const std::set<string>& outputs) { + return outputs.empty() ? drvPath + : drvPath + "!" + concatStringsSep(",", outputs); } - -bool wantOutput(const string & output, const std::set<string> & wanted) -{ - return wanted.empty() || wanted.find(output) != wanted.end(); +bool wantOutput(const string& output, const std::set<string>& wanted) { + return wanted.empty() || wanted.find(output) != wanted.end(); } - -PathSet BasicDerivation::outputPaths() const -{ - PathSet paths; - for (auto & i : outputs) - paths.insert(i.second.path); - return paths; +PathSet BasicDerivation::outputPaths() const { + PathSet paths; + for (auto& i : outputs) paths.insert(i.second.path); + return paths; } - -Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) -{ - drv.outputs.clear(); - auto nr = readNum<size_t>(in); - for (size_t n = 0; n < nr; n++) { - auto name = readString(in); - DerivationOutput o; - in >> o.path >> o.hashAlgo >> o.hash; - store.assertStorePath(o.path); - drv.outputs[name] = o; - } - - drv.inputSrcs = readStorePaths<PathSet>(store, in); - in >> drv.platform >> drv.builder; - drv.args = readStrings<Strings>(in); - - nr = readNum<size_t>(in); - for (size_t n = 0; n < nr; n++) { - auto key = readString(in); - auto value = readString(in); - drv.env[key] = value; - } - - return in; +Source& readDerivation(Source& in, Store& store, BasicDerivation& drv) { + drv.outputs.clear(); + auto nr = readNum<size_t>(in); + for (size_t n = 0; n < nr; n++) { + auto name = readString(in); + DerivationOutput o; + in >> o.path >> o.hashAlgo >> o.hash; + store.assertStorePath(o.path); + drv.outputs[name] = o; + } + + drv.inputSrcs = readStorePaths<PathSet>(store, in); + in >> drv.platform >> drv.builder; + drv.args = readStrings<Strings>(in); + + nr = readNum<size_t>(in); + for (size_t n = 0; n < nr; n++) { + auto key = readString(in); + auto value = readString(in); + drv.env[key] = value; + } + + return in; } - -Sink & operator << (Sink & out, const BasicDerivation & drv) -{ - out << drv.outputs.size(); - for (auto & i : drv.outputs) - out << i.first << i.second.path << i.second.hashAlgo << i.second.hash; - out << drv.inputSrcs << drv.platform << drv.builder << drv.args; - out << drv.env.size(); - for (auto & i : drv.env) - out << i.first << i.second; - return out; +Sink& operator<<(Sink& out, const BasicDerivation& drv) { + out << drv.outputs.size(); + for (auto& i : drv.outputs) + out << i.first << i.second.path << i.second.hashAlgo << i.second.hash; + out << drv.inputSrcs << drv.platform << drv.builder << drv.args; + out << drv.env.size(); + for (auto& i : drv.env) out << i.first << i.second; + return out; } - -std::string hashPlaceholder(const std::string & outputName) -{ - // FIXME: memoize? - return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); +std::string hashPlaceholder(const std::string& outputName) { + // FIXME: memoize? + return "/" + hashString(htSHA256, "nix-output:" + outputName) + .to_string(Base32, false); } - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/derivations.hh b/third_party/nix/src/libstore/derivations.hh index 8e02c9bc57c1..51a2dc44b695 100644 --- a/third_party/nix/src/libstore/derivations.hh +++ b/third_party/nix/src/libstore/derivations.hh @@ -1,36 +1,28 @@ #pragma once -#include "types.hh" +#include <map> #include "hash.hh" #include "store-api.hh" - -#include <map> - +#include "types.hh" namespace nix { - /* Extension of derivations in the Nix store. */ const string drvExtension = ".drv"; - /* Abstract syntax of derivations. */ -struct DerivationOutput -{ - Path path; - string hashAlgo; /* hash used for expected hash computation */ - string hash; /* expected hash, may be null */ - DerivationOutput() - { - } - DerivationOutput(Path path, string hashAlgo, string hash) - { - this->path = path; - this->hashAlgo = hashAlgo; - this->hash = hash; - } - void parseHashInfo(bool & recursive, Hash & hash) const; +struct DerivationOutput { + Path path; + string hashAlgo; /* hash used for expected hash computation */ + string hash; /* expected hash, may be null */ + DerivationOutput() {} + DerivationOutput(Path path, string hashAlgo, string hash) { + this->path = path; + this->hashAlgo = hashAlgo; + this->hash = hash; + } + void parseHashInfo(bool& recursive, Hash& hash) const; }; typedef std::map<string, DerivationOutput> DerivationOutputs; @@ -41,77 +33,73 @@ typedef std::map<Path, StringSet> DerivationInputs; typedef std::map<string, string> StringPairs; -struct BasicDerivation -{ - DerivationOutputs outputs; /* keyed on symbolic IDs */ - PathSet inputSrcs; /* inputs that are sources */ - string platform; - Path builder; - Strings args; - StringPairs env; - - virtual ~BasicDerivation() { }; +struct BasicDerivation { + DerivationOutputs outputs; /* keyed on symbolic IDs */ + PathSet inputSrcs; /* inputs that are sources */ + string platform; + Path builder; + Strings args; + StringPairs env; - /* Return the path corresponding to the output identifier `id' in - the given derivation. */ - Path findOutput(const string & id) const; + virtual ~BasicDerivation(){}; - bool isBuiltin() const; + /* Return the path corresponding to the output identifier `id' in + the given derivation. */ + Path findOutput(const string& id) const; - /* Return true iff this is a fixed-output derivation. */ - bool isFixedOutput() const; + bool isBuiltin() const; - /* Return the output paths of a derivation. */ - PathSet outputPaths() const; + /* Return true iff this is a fixed-output derivation. */ + bool isFixedOutput() const; + /* Return the output paths of a derivation. */ + PathSet outputPaths() const; }; -struct Derivation : BasicDerivation -{ - DerivationInputs inputDrvs; /* inputs that are sub-derivations */ +struct Derivation : BasicDerivation { + DerivationInputs inputDrvs; /* inputs that are sub-derivations */ - /* Print a derivation. */ - std::string unparse() const; + /* Print a derivation. */ + std::string unparse() const; }; - class Store; - /* Write a derivation to the Nix store, and return its path. */ -Path writeDerivation(ref<Store> store, - const Derivation & drv, const string & name, RepairFlag repair = NoRepair); +Path writeDerivation(ref<Store> store, const Derivation& drv, + const string& name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ -Derivation readDerivation(const Path & drvPath); +Derivation readDerivation(const Path& drvPath); /* Check whether a file name ends with the extension for derivations. */ -bool isDerivation(const string & fileName); +bool isDerivation(const string& fileName); -Hash hashDerivationModulo(Store & store, Derivation drv); +Hash hashDerivationModulo(Store& store, Derivation drv); /* Memoisation of hashDerivationModulo(). */ typedef std::map<Path, Hash> DrvHashes; -extern DrvHashes drvHashes; // FIXME: global, not thread-safe +extern DrvHashes drvHashes; // FIXME: global, not thread-safe /* Split a string specifying a derivation and a set of outputs (/nix/store/hash-foo!out1,out2,...) into the derivation path and the outputs. */ typedef std::pair<string, std::set<string> > DrvPathWithOutputs; -DrvPathWithOutputs parseDrvPathWithOutputs(const string & s); +DrvPathWithOutputs parseDrvPathWithOutputs(const string& s); -Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs); +Path makeDrvPathWithOutputs(const Path& drvPath, + const std::set<string>& outputs); -bool wantOutput(const string & output, const std::set<string> & wanted); +bool wantOutput(const string& output, const std::set<string>& wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, Store & store, BasicDerivation & drv); -Sink & operator << (Sink & out, const BasicDerivation & drv); +Source& readDerivation(Source& in, Store& store, BasicDerivation& drv); +Sink& operator<<(Sink& out, const BasicDerivation& drv); -std::string hashPlaceholder(const std::string & outputName); +std::string hashPlaceholder(const std::string& outputName); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/download.cc b/third_party/nix/src/libstore/download.cc index 80674a9e7a43..66d6cdd6cc48 100644 --- a/third_party/nix/src/libstore/download.cc +++ b/third_party/nix/src/libstore/download.cc @@ -1,23 +1,21 @@ #include "download.hh" -#include "util.hh" -#include "globals.hh" -#include "hash.hh" -#include "store-api.hh" #include "archive.hh" -#include "s3.hh" #include "compression.hh" -#include "pathlocks.hh" #include "finally.hh" +#include "globals.hh" +#include "hash.hh" +#include "pathlocks.hh" +#include "s3.hh" +#include "store-api.hh" +#include "util.hh" #ifdef ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> #endif -#include <unistd.h> -#include <fcntl.h> - #include <curl/curl.h> - +#include <fcntl.h> +#include <unistd.h> #include <algorithm> #include <cmath> #include <cstring> @@ -34,913 +32,939 @@ DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); -std::string resolveUri(const std::string & uri) -{ - if (uri.compare(0, 8, "channel:") == 0) - return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz"; - else - return uri; +std::string resolveUri(const std::string& uri) { + if (uri.compare(0, 8, "channel:") == 0) + return "https://nixos.org/channels/" + std::string(uri, 8) + + "/nixexprs.tar.xz"; + else + return uri; } -struct CurlDownloader : public Downloader -{ - CURLM * curlm = 0; - - std::random_device rd; - std::mt19937 mt19937; - - struct DownloadItem : public std::enable_shared_from_this<DownloadItem> - { - CurlDownloader & downloader; - DownloadRequest request; - DownloadResult result; - Activity act; - bool done = false; // whether either the success or failure function has been called - Callback<DownloadResult> callback; - CURL * req = 0; - bool active = false; // whether the handle has been added to the multi object - std::string status; - - unsigned int attempt = 0; - - /* Don't start this download until the specified time point - has been reached. */ - std::chrono::steady_clock::time_point embargo; - - struct curl_slist * requestHeaders = 0; - - std::string encoding; - - bool acceptRanges = false; - - curl_off_t writtenToSink = 0; - - DownloadItem(CurlDownloader & downloader, - const DownloadRequest & request, - Callback<DownloadResult> && callback) - : downloader(downloader) - , request(request) - , act(*logger, lvlTalkative, actDownload, - fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), - {request.uri}, request.parentAct) - , callback(std::move(callback)) - , finalSink([this](const unsigned char * data, size_t len) { - if (this->request.dataCallback) { - long httpStatus = 0; - curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - - /* Only write data to the sink if this is a - successful response. */ - if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { - writtenToSink += len; - this->request.dataCallback((char *) data, len); - } - } else - this->result.data->append((char *) data, len); - }) - { - if (!request.expectedETag.empty()) - requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); - if (!request.mimeType.empty()) - requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); - } - - ~DownloadItem() - { - if (req) { - if (active) - curl_multi_remove_handle(downloader.curlm, req); - curl_easy_cleanup(req); - } - if (requestHeaders) curl_slist_free_all(requestHeaders); - try { - if (!done) - fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)); - } catch (...) { - ignoreException(); - } - } - - void failEx(std::exception_ptr ex) - { - assert(!done); - done = true; - callback.rethrow(ex); - } +struct CurlDownloader : public Downloader { + CURLM* curlm = 0; + + std::random_device rd; + std::mt19937 mt19937; + + struct DownloadItem : public std::enable_shared_from_this<DownloadItem> { + CurlDownloader& downloader; + DownloadRequest request; + DownloadResult result; + Activity act; + bool done = false; // whether either the success or failure function has + // been called + Callback<DownloadResult> callback; + CURL* req = 0; + bool active = + false; // whether the handle has been added to the multi object + std::string status; + + unsigned int attempt = 0; + + /* Don't start this download until the specified time point + has been reached. */ + std::chrono::steady_clock::time_point embargo; + + struct curl_slist* requestHeaders = 0; + + std::string encoding; + + bool acceptRanges = false; + + curl_off_t writtenToSink = 0; + + DownloadItem(CurlDownloader& downloader, const DownloadRequest& request, + Callback<DownloadResult>&& callback) + : downloader(downloader), + request(request), + act(*logger, lvlTalkative, actDownload, + fmt(request.data ? "uploading '%s'" : "downloading '%s'", + request.uri), + {request.uri}, request.parentAct), + callback(std::move(callback)), + finalSink([this](const unsigned char* data, size_t len) { + if (this->request.dataCallback) { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + + /* Only write data to the sink if this is a + successful response. */ + if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || + httpStatus == 206) { + writtenToSink += len; + this->request.dataCallback((char*)data, len); + } + } else + this->result.data->append((char*)data, len); + }) { + if (!request.expectedETag.empty()) + requestHeaders = curl_slist_append( + requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); + if (!request.mimeType.empty()) + requestHeaders = curl_slist_append( + requestHeaders, ("Content-Type: " + request.mimeType).c_str()); + } - template<class T> - void fail(const T & e) - { - failEx(std::make_exception_ptr(e)); - } + ~DownloadItem() { + if (req) { + if (active) curl_multi_remove_handle(downloader.curlm, req); + curl_easy_cleanup(req); + } + if (requestHeaders) curl_slist_free_all(requestHeaders); + try { + if (!done) + fail(DownloadError( + Interrupted, + format("download of '%s' was interrupted") % request.uri)); + } catch (...) { + ignoreException(); + } + } - LambdaSink finalSink; - std::shared_ptr<CompressionSink> decompressionSink; + void failEx(std::exception_ptr ex) { + assert(!done); + done = true; + callback.rethrow(ex); + } - std::exception_ptr writeException; + template <class T> + void fail(const T& e) { + failEx(std::make_exception_ptr(e)); + } - size_t writeCallback(void * contents, size_t size, size_t nmemb) - { - try { - size_t realSize = size * nmemb; - result.bodySize += realSize; + LambdaSink finalSink; + std::shared_ptr<CompressionSink> decompressionSink; - if (!decompressionSink) - decompressionSink = makeDecompressionSink(encoding, finalSink); + std::exception_ptr writeException; - (*decompressionSink)((unsigned char *) contents, realSize); + size_t writeCallback(void* contents, size_t size, size_t nmemb) { + try { + size_t realSize = size * nmemb; + result.bodySize += realSize; - return realSize; - } catch (...) { - writeException = std::current_exception(); - return 0; - } - } + if (!decompressionSink) + decompressionSink = makeDecompressionSink(encoding, finalSink); - static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) - { - return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb); - } + (*decompressionSink)((unsigned char*)contents, realSize); - size_t headerCallback(void * contents, size_t size, size_t nmemb) - { - size_t realSize = size * nmemb; - std::string line((char *) contents, realSize); - printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); - if (line.compare(0, 5, "HTTP/") == 0) { // new response starts - result.etag = ""; - auto ss = tokenizeString<vector<string>>(line, " "); - status = ss.size() >= 2 ? ss[1] : ""; - result.data = std::make_shared<std::string>(); - result.bodySize = 0; - acceptRanges = false; - encoding = ""; - } else { - auto i = line.find(':'); - if (i != string::npos) { - string name = toLower(trim(string(line, 0, i))); - if (name == "etag") { - result.etag = trim(string(line, i + 1)); - /* Hack to work around a GitHub bug: it sends - ETags, but ignores If-None-Match. So if we get - the expected ETag on a 200 response, then shut - down the connection because we already have the - data. */ - if (result.etag == request.expectedETag && status == "200") { - debug(format("shutting down on 200 HTTP response with expected ETag")); - return 0; - } - } else if (name == "content-encoding") - encoding = trim(string(line, i + 1)); - else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes") - acceptRanges = true; - } - } - return realSize; - } + return realSize; + } catch (...) { + writeException = std::current_exception(); + return 0; + } + } - static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) - { - return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb); - } + static size_t writeCallbackWrapper(void* contents, size_t size, + size_t nmemb, void* userp) { + return ((DownloadItem*)userp)->writeCallback(contents, size, nmemb); + } - int progressCallback(double dltotal, double dlnow) - { - try { - act.progress(dlnow, dltotal); - } catch (nix::Interrupted &) { - assert(_isInterrupted); + size_t headerCallback(void* contents, size_t size, size_t nmemb) { + size_t realSize = size * nmemb; + std::string line((char*)contents, realSize); + printMsg(lvlVomit, + format("got header for '%s': %s") % request.uri % trim(line)); + if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + result.etag = ""; + auto ss = tokenizeString<vector<string>>(line, " "); + status = ss.size() >= 2 ? ss[1] : ""; + result.data = std::make_shared<std::string>(); + result.bodySize = 0; + acceptRanges = false; + encoding = ""; + } else { + auto i = line.find(':'); + if (i != string::npos) { + string name = toLower(trim(string(line, 0, i))); + if (name == "etag") { + result.etag = trim(string(line, i + 1)); + /* Hack to work around a GitHub bug: it sends + ETags, but ignores If-None-Match. So if we get + the expected ETag on a 200 response, then shut + down the connection because we already have the + data. */ + if (result.etag == request.expectedETag && status == "200") { + debug(format( + "shutting down on 200 HTTP response with expected ETag")); + return 0; } - return _isInterrupted; - } - - static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) - { - return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow); - } - - static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) - { - if (type == CURLINFO_TEXT) - vomit("curl: %s", chomp(std::string(data, size))); - return 0; - } - - size_t readOffset = 0; - size_t readCallback(char *buffer, size_t size, size_t nitems) - { - if (readOffset == request.data->length()) - return 0; - auto count = std::min(size * nitems, request.data->length() - readOffset); - assert(count); - memcpy(buffer, request.data->data() + readOffset, count); - readOffset += count; - return count; + } else if (name == "content-encoding") + encoding = trim(string(line, i + 1)); + else if (name == "accept-ranges" && + toLower(trim(std::string(line, i + 1))) == "bytes") + acceptRanges = true; } + } + return realSize; + } - static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) - { - return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); - } + static size_t headerCallbackWrapper(void* contents, size_t size, + size_t nmemb, void* userp) { + return ((DownloadItem*)userp)->headerCallback(contents, size, nmemb); + } - void init() - { - if (!req) req = curl_easy_init(); + int progressCallback(double dltotal, double dlnow) { + try { + act.progress(dlnow, dltotal); + } catch (nix::Interrupted&) { + assert(_isInterrupted); + } + return _isInterrupted; + } - curl_easy_reset(req); + static int progressCallbackWrapper(void* userp, double dltotal, + double dlnow, double ultotal, + double ulnow) { + return ((DownloadItem*)userp)->progressCallback(dltotal, dlnow); + } - if (verbosity >= lvlVomit) { - curl_easy_setopt(req, CURLOPT_VERBOSE, 1); - curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); - } + static int debugCallback(CURL* handle, curl_infotype type, char* data, + size_t size, void* userptr) { + if (type == CURLINFO_TEXT) + vomit("curl: %s", chomp(std::string(data, size))); + return 0; + } - curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); - curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10); - curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(req, CURLOPT_USERAGENT, - ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + - (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str()); - #if LIBCURL_VERSION_NUM >= 0x072b00 - curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); - #endif - #if LIBCURL_VERSION_NUM >= 0x072f00 - if (downloadSettings.enableHttp2) - curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); - else - curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); - #endif - curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); - curl_easy_setopt(req, CURLOPT_WRITEDATA, this); - curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper); - curl_easy_setopt(req, CURLOPT_HEADERDATA, this); - - curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); - curl_easy_setopt(req, CURLOPT_PROGRESSDATA, this); - curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0); - - curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); - - if (request.head) - curl_easy_setopt(req, CURLOPT_NOBODY, 1); - - if (request.data) { - curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); - curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); - curl_easy_setopt(req, CURLOPT_READDATA, this); - curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length()); - } + size_t readOffset = 0; + size_t readCallback(char* buffer, size_t size, size_t nitems) { + if (readOffset == request.data->length()) return 0; + auto count = std::min(size * nitems, request.data->length() - readOffset); + assert(count); + memcpy(buffer, request.data->data() + readOffset, count); + readOffset += count; + return count; + } - if (request.verifyTLS) { - if (settings.caFile != "") - curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); - } else { - curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); - curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); - } + static size_t readCallbackWrapper(char* buffer, size_t size, size_t nitems, + void* userp) { + return ((DownloadItem*)userp)->readCallback(buffer, size, nitems); + } - curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); + void init() { + if (!req) req = curl_easy_init(); + + curl_easy_reset(req); + + if (verbosity >= lvlVomit) { + curl_easy_setopt(req, CURLOPT_VERBOSE, 1); + curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, + DownloadItem::debugCallback); + } + + curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); + curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10); + curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); + curl_easy_setopt(req, CURLOPT_USERAGENT, + ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + + (downloadSettings.userAgentSuffix != "" + ? " " + downloadSettings.userAgentSuffix.get() + : "")) + .c_str()); +#if LIBCURL_VERSION_NUM >= 0x072b00 + curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); +#endif +#if LIBCURL_VERSION_NUM >= 0x072f00 + if (downloadSettings.enableHttp2) + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); + else + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); +#endif + curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, + DownloadItem::writeCallbackWrapper); + curl_easy_setopt(req, CURLOPT_WRITEDATA, this); + curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, + DownloadItem::headerCallbackWrapper); + curl_easy_setopt(req, CURLOPT_HEADERDATA, this); + + curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); + curl_easy_setopt(req, CURLOPT_PROGRESSDATA, this); + curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0); + + curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + + if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1); + + if (request.data) { + curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); + curl_easy_setopt(req, CURLOPT_READDATA, this); + curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, + (curl_off_t)request.data->length()); + } + + if (request.verifyTLS) { + if (settings.caFile != "") + curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); + } else { + curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); + } + + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, + downloadSettings.connectTimeout.get()); + + curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, + downloadSettings.stalledDownloadTimeout.get()); + + /* If no file exist in the specified path, curl continues to work + anyway as if netrc support was disabled. */ + curl_easy_setopt(req, CURLOPT_NETRC_FILE, + settings.netrcFile.get().c_str()); + curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + + if (writtenToSink) + curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + + result.data = std::make_shared<std::string>(); + result.bodySize = 0; + } - curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); + void finish(CURLcode code) { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - /* If no file exist in the specified path, curl continues to work - anyway as if netrc support was disabled. */ - curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); - curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + char* effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr) result.effectiveUri = effectiveUriCStr; - if (writtenToSink) - curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + debug( + "finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d " + "bytes", + request.verb(), request.uri, code, httpStatus, result.bodySize); - result.data = std::make_shared<std::string>(); - result.bodySize = 0; + if (decompressionSink) { + try { + decompressionSink->finish(); + } catch (...) { + writeException = std::current_exception(); } - - void finish(CURLcode code) - { - long httpStatus = 0; - curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - - char * effectiveUriCStr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); - if (effectiveUriCStr) - result.effectiveUri = effectiveUriCStr; - - debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", - request.verb(), request.uri, code, httpStatus, result.bodySize); - - if (decompressionSink) { - try { - decompressionSink->finish(); - } catch (...) { - writeException = std::current_exception(); - } - } - - if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { - code = CURLE_OK; - httpStatus = 304; - } - - if (writeException) - failEx(writeException); - - else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) - { - result.cached = httpStatus == 304; - act.progress(result.bodySize, result.bodySize); - done = true; - callback(std::move(result)); - } - - else { - // We treat most errors as transient, but won't retry when hopeless - Error err = Transient; - - if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) { - // The file is definitely not there - err = NotFound; - } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) { - // Don't retry on authentication/authorization failures - err = Forbidden; - } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) { - // Most 4xx errors are client errors and are probably not worth retrying: - // * 408 means the server timed out waiting for us, so we try again - // * 429 means too many requests, so we retry (with a delay) - err = Misc; - } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) { - // Let's treat most 5xx (server) errors as transient, except for a handful: - // * 501 not implemented - // * 505 http version not supported - // * 511 we're behind a captive portal - err = Misc; - } else { - // Don't bother retrying on certain cURL errors either - switch (code) { - case CURLE_FAILED_INIT: - case CURLE_URL_MALFORMAT: - case CURLE_NOT_BUILT_IN: - case CURLE_REMOTE_ACCESS_DENIED: - case CURLE_FILE_COULDNT_READ_FILE: - case CURLE_FUNCTION_NOT_FOUND: - case CURLE_ABORTED_BY_CALLBACK: - case CURLE_BAD_FUNCTION_ARGUMENT: - case CURLE_INTERFACE_FAILED: - case CURLE_UNKNOWN_OPTION: - case CURLE_SSL_CACERT_BADFILE: - case CURLE_TOO_MANY_REDIRECTS: - case CURLE_WRITE_ERROR: - case CURLE_UNSUPPORTED_PROTOCOL: - err = Misc; - break; - default: // Shut up warnings - break; - } - } - - attempt++; - - auto exc = - code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) - : httpStatus != 0 - ? DownloadError(err, - fmt("unable to %s '%s': HTTP error %d", - request.verb(), request.uri, httpStatus) - + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) - ) - : DownloadError(err, - fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code)); - - /* If this is a transient error, then maybe retry the - download after a while. If we're writing to a - sink, we can only retry if the server supports - ranged requests. */ - if (err == Transient - && attempt < request.tries - && (!this->request.dataCallback - || writtenToSink == 0 - || (acceptRanges && encoding.empty()))) - { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - if (writtenToSink) - warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); - else - warn("%s; retrying in %d ms", exc.what(), ms); - embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); - } - else - fail(exc); - } + } + + if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { + code = CURLE_OK; + httpStatus = 304; + } + + if (writeException) + failEx(writeException); + + else if (code == CURLE_OK && + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || + httpStatus == 206 || httpStatus == 304 || + httpStatus == 226 /* FTP */ || + httpStatus == 0 /* other protocol */)) { + result.cached = httpStatus == 304; + act.progress(result.bodySize, result.bodySize); + done = true; + callback(std::move(result)); + } + + else { + // We treat most errors as transient, but won't retry when hopeless + Error err = Transient; + + if (httpStatus == 404 || httpStatus == 410 || + code == CURLE_FILE_COULDNT_READ_FILE) { + // The file is definitely not there + err = NotFound; + } else if (httpStatus == 401 || httpStatus == 403 || + httpStatus == 407) { + // Don't retry on authentication/authorization failures + err = Forbidden; + } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && + httpStatus != 429) { + // Most 4xx errors are client errors and are probably not worth + // retrying: + // * 408 means the server timed out waiting for us, so we try again + // * 429 means too many requests, so we retry (with a delay) + err = Misc; + } else if (httpStatus == 501 || httpStatus == 505 || + httpStatus == 511) { + // Let's treat most 5xx (server) errors as transient, except for a + // handful: + // * 501 not implemented + // * 505 http version not supported + // * 511 we're behind a captive portal + err = Misc; + } else { + // Don't bother retrying on certain cURL errors either + switch (code) { + case CURLE_FAILED_INIT: + case CURLE_URL_MALFORMAT: + case CURLE_NOT_BUILT_IN: + case CURLE_REMOTE_ACCESS_DENIED: + case CURLE_FILE_COULDNT_READ_FILE: + case CURLE_FUNCTION_NOT_FOUND: + case CURLE_ABORTED_BY_CALLBACK: + case CURLE_BAD_FUNCTION_ARGUMENT: + case CURLE_INTERFACE_FAILED: + case CURLE_UNKNOWN_OPTION: + case CURLE_SSL_CACERT_BADFILE: + case CURLE_TOO_MANY_REDIRECTS: + case CURLE_WRITE_ERROR: + case CURLE_UNSUPPORTED_PROTOCOL: + err = Misc; + break; + default: // Shut up warnings + break; + } } - }; - struct State - { - struct EmbargoComparator { - bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) { - return i1->embargo > i2->embargo; - } - }; - bool quit = false; - std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming; + attempt++; + + auto exc = + code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted + ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", + request.verb(), request.uri)) + : httpStatus != 0 + ? DownloadError( + err, fmt("unable to %s '%s': HTTP error %d", + request.verb(), request.uri, httpStatus) + + (code == CURLE_OK + ? "" + : fmt(" (curl error: %s)", + curl_easy_strerror(code)))) + : DownloadError(err, fmt("unable to %s '%s': %s (%d)", + request.verb(), request.uri, + curl_easy_strerror(code), code)); + + /* If this is a transient error, then maybe retry the + download after a while. If we're writing to a + sink, we can only retry if the server supports + ranged requests. */ + if (err == Transient && attempt < request.tries && + (!this->request.dataCallback || writtenToSink == 0 || + (acceptRanges && encoding.empty()))) { + int ms = request.baseRetryTimeMs * + std::pow(2.0f, attempt - 1 + + std::uniform_real_distribution<>( + 0.0, 0.5)(downloader.mt19937)); + if (writtenToSink) + warn("%s; retrying from offset %d in %d ms", exc.what(), + writtenToSink, ms); + else + warn("%s; retrying in %d ms", exc.what(), ms); + embargo = + std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); + downloader.enqueueItem(shared_from_this()); + } else + fail(exc); + } + } + }; + + struct State { + struct EmbargoComparator { + bool operator()(const std::shared_ptr<DownloadItem>& i1, + const std::shared_ptr<DownloadItem>& i2) { + return i1->embargo > i2->embargo; + } }; + bool quit = false; + std::priority_queue<std::shared_ptr<DownloadItem>, + std::vector<std::shared_ptr<DownloadItem>>, + EmbargoComparator> + incoming; + }; - Sync<State> state_; + Sync<State> state_; - /* We can't use a std::condition_variable to wake up the curl - thread, because it only monitors file descriptors. So use a - pipe instead. */ - Pipe wakeupPipe; + /* We can't use a std::condition_variable to wake up the curl + thread, because it only monitors file descriptors. So use a + pipe instead. */ + Pipe wakeupPipe; - std::thread workerThread; + std::thread workerThread; - CurlDownloader() - : mt19937(rd()) - { - static std::once_flag globalInit; - std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); + CurlDownloader() : mt19937(rd()) { + static std::once_flag globalInit; + std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); - curlm = curl_multi_init(); + curlm = curl_multi_init(); - #if LIBCURL_VERSION_NUM >= 0x072b00 // Multiplex requires >= 7.43.0 - curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); - #endif - #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 - curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - downloadSettings.httpConnections.get()); - #endif +#if LIBCURL_VERSION_NUM >= 0x072b00 // Multiplex requires >= 7.43.0 + curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); +#endif +#if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 + curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, + downloadSettings.httpConnections.get()); +#endif - wakeupPipe.create(); - fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); + wakeupPipe.create(); + fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); - workerThread = std::thread([&]() { workerThreadEntry(); }); - } + workerThread = std::thread([&]() { workerThreadEntry(); }); + } - ~CurlDownloader() - { - stopWorkerThread(); + ~CurlDownloader() { + stopWorkerThread(); - workerThread.join(); + workerThread.join(); - if (curlm) curl_multi_cleanup(curlm); - } + if (curlm) curl_multi_cleanup(curlm); + } - void stopWorkerThread() + void stopWorkerThread() { + /* Signal the worker thread to exit. */ { - /* Signal the worker thread to exit. */ - { - auto state(state_.lock()); - state->quit = true; + auto state(state_.lock()); + state->quit = true; + } + writeFull(wakeupPipe.writeSide.get(), " ", false); + } + + void workerThreadMain() { + /* Cause this thread to be notified on SIGINT. */ + auto callback = createInterruptCallback([&]() { stopWorkerThread(); }); + + std::map<CURL*, std::shared_ptr<DownloadItem>> items; + + bool quit = false; + + std::chrono::steady_clock::time_point nextWakeup; + + while (!quit) { + checkInterrupt(); + + /* Let curl do its thing. */ + int running; + CURLMcode mc = curl_multi_perform(curlm, &running); + if (mc != CURLM_OK) + throw nix::Error( + format("unexpected error from curl_multi_perform(): %s") % + curl_multi_strerror(mc)); + + /* Set the promises of any finished requests. */ + CURLMsg* msg; + int left; + while ((msg = curl_multi_info_read(curlm, &left))) { + if (msg->msg == CURLMSG_DONE) { + auto i = items.find(msg->easy_handle); + assert(i != items.end()); + i->second->finish(msg->data.result); + curl_multi_remove_handle(curlm, i->second->req); + i->second->active = false; + items.erase(i); + } + } + + /* Wait for activity, including wakeup events. */ + int numfds = 0; + struct curl_waitfd extraFDs[1]; + extraFDs[0].fd = wakeupPipe.readSide.get(); + extraFDs[0].events = CURL_WAIT_POLLIN; + extraFDs[0].revents = 0; + long maxSleepTimeMs = items.empty() ? 10000 : 100; + auto sleepTimeMs = + nextWakeup != std::chrono::steady_clock::time_point() + ? std::max( + 0, + (int)std::chrono::duration_cast<std::chrono::milliseconds>( + nextWakeup - std::chrono::steady_clock::now()) + .count()) + : maxSleepTimeMs; + vomit("download thread waiting for %d ms", sleepTimeMs); + mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); + if (mc != CURLM_OK) + throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % + curl_multi_strerror(mc)); + + nextWakeup = std::chrono::steady_clock::time_point(); + + /* Add new curl requests from the incoming requests queue, + except for requests that are embargoed (waiting for a + retry timeout to expire). */ + if (extraFDs[0].revents & CURL_WAIT_POLLIN) { + char buf[1024]; + auto res = read(extraFDs[0].fd, buf, sizeof(buf)); + if (res == -1 && errno != EINTR) + throw SysError("reading curl wakeup socket"); + } + + std::vector<std::shared_ptr<DownloadItem>> incoming; + auto now = std::chrono::steady_clock::now(); + + { + auto state(state_.lock()); + while (!state->incoming.empty()) { + auto item = state->incoming.top(); + if (item->embargo <= now) { + incoming.push_back(item); + state->incoming.pop(); + } else { + if (nextWakeup == std::chrono::steady_clock::time_point() || + item->embargo < nextWakeup) + nextWakeup = item->embargo; + break; + } } - writeFull(wakeupPipe.writeSide.get(), " ", false); + quit = state->quit; + } + + for (auto& item : incoming) { + debug("starting %s of %s", item->request.verb(), item->request.uri); + item->init(); + curl_multi_add_handle(curlm, item->req); + item->active = true; + items[item->req] = item; + } } - void workerThreadMain() - { - /* Cause this thread to be notified on SIGINT. */ - auto callback = createInterruptCallback([&]() { - stopWorkerThread(); - }); - - std::map<CURL *, std::shared_ptr<DownloadItem>> items; - - bool quit = false; - - std::chrono::steady_clock::time_point nextWakeup; - - while (!quit) { - checkInterrupt(); - - /* Let curl do its thing. */ - int running; - CURLMcode mc = curl_multi_perform(curlm, &running); - if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc)); - - /* Set the promises of any finished requests. */ - CURLMsg * msg; - int left; - while ((msg = curl_multi_info_read(curlm, &left))) { - if (msg->msg == CURLMSG_DONE) { - auto i = items.find(msg->easy_handle); - assert(i != items.end()); - i->second->finish(msg->data.result); - curl_multi_remove_handle(curlm, i->second->req); - i->second->active = false; - items.erase(i); - } - } - - /* Wait for activity, including wakeup events. */ - int numfds = 0; - struct curl_waitfd extraFDs[1]; - extraFDs[0].fd = wakeupPipe.readSide.get(); - extraFDs[0].events = CURL_WAIT_POLLIN; - extraFDs[0].revents = 0; - long maxSleepTimeMs = items.empty() ? 10000 : 100; - auto sleepTimeMs = - nextWakeup != std::chrono::steady_clock::time_point() - ? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count()) - : maxSleepTimeMs; - vomit("download thread waiting for %d ms", sleepTimeMs); - mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); - if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc)); - - nextWakeup = std::chrono::steady_clock::time_point(); - - /* Add new curl requests from the incoming requests queue, - except for requests that are embargoed (waiting for a - retry timeout to expire). */ - if (extraFDs[0].revents & CURL_WAIT_POLLIN) { - char buf[1024]; - auto res = read(extraFDs[0].fd, buf, sizeof(buf)); - if (res == -1 && errno != EINTR) - throw SysError("reading curl wakeup socket"); - } - - std::vector<std::shared_ptr<DownloadItem>> incoming; - auto now = std::chrono::steady_clock::now(); - - { - auto state(state_.lock()); - while (!state->incoming.empty()) { - auto item = state->incoming.top(); - if (item->embargo <= now) { - incoming.push_back(item); - state->incoming.pop(); - } else { - if (nextWakeup == std::chrono::steady_clock::time_point() - || item->embargo < nextWakeup) - nextWakeup = item->embargo; - break; - } - } - quit = state->quit; - } - - for (auto & item : incoming) { - debug("starting %s of %s", item->request.verb(), item->request.uri); - item->init(); - curl_multi_add_handle(curlm, item->req); - item->active = true; - items[item->req] = item; - } - } + debug("download thread shutting down"); + } - debug("download thread shutting down"); + void workerThreadEntry() { + try { + workerThreadMain(); + } catch (nix::Interrupted& e) { + } catch (std::exception& e) { + printError("unexpected error in download thread: %s", e.what()); } - void workerThreadEntry() { - try { - workerThreadMain(); - } catch (nix::Interrupted & e) { - } catch (std::exception & e) { - printError("unexpected error in download thread: %s", e.what()); - } - - { - auto state(state_.lock()); - while (!state->incoming.empty()) state->incoming.pop(); - state->quit = true; - } + auto state(state_.lock()); + while (!state->incoming.empty()) state->incoming.pop(); + state->quit = true; } + } + + void enqueueItem(std::shared_ptr<DownloadItem> item) { + if (item->request.data && !hasPrefix(item->request.uri, "http://") && + !hasPrefix(item->request.uri, "https://")) + throw nix::Error("uploading to '%s' is not supported", item->request.uri); - void enqueueItem(std::shared_ptr<DownloadItem> item) { - if (item->request.data - && !hasPrefix(item->request.uri, "http://") - && !hasPrefix(item->request.uri, "https://")) - throw nix::Error("uploading to '%s' is not supported", item->request.uri); - - { - auto state(state_.lock()); - if (state->quit) - throw nix::Error("cannot enqueue download request because the download thread is shutting down"); - state->incoming.push(item); - } - writeFull(wakeupPipe.writeSide.get(), " "); + auto state(state_.lock()); + if (state->quit) + throw nix::Error( + "cannot enqueue download request because the download thread is " + "shutting down"); + state->incoming.push(item); } + writeFull(wakeupPipe.writeSide.get(), " "); + } #ifdef ENABLE_S3 - std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri) - { - auto [path, params] = splitUriAndParams(uri); + std::tuple<std::string, std::string, Store::Params> parseS3Uri( + std::string uri) { + auto [path, params] = splitUriAndParams(uri); - auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix - if (slash == std::string::npos) - throw nix::Error("bad S3 URI '%s'", path); + auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix + if (slash == std::string::npos) throw nix::Error("bad S3 URI '%s'", path); - std::string bucketName(path, 5, slash - 5); - std::string key(path, slash + 1); + std::string bucketName(path, 5, slash - 5); + std::string key(path, slash + 1); - return {bucketName, key, params}; - } + return {bucketName, key, params}; + } #endif - void enqueueDownload(const DownloadRequest & request, - Callback<DownloadResult> callback) override - { - /* Ugly hack to support s3:// URIs. */ - if (hasPrefix(request.uri, "s3://")) { - // FIXME: do this on a worker thread - try { + void enqueueDownload(const DownloadRequest& request, + Callback<DownloadResult> callback) override { + /* Ugly hack to support s3:// URIs. */ + if (hasPrefix(request.uri, "s3://")) { + // FIXME: do this on a worker thread + try { #ifdef ENABLE_S3 - auto [bucketName, key, params] = parseS3Uri(request.uri); - - std::string profile = get(params, "profile", ""); - std::string region = get(params, "region", Aws::Region::US_EAST_1); - std::string scheme = get(params, "scheme", ""); - std::string endpoint = get(params, "endpoint", ""); - - S3Helper s3Helper(profile, region, scheme, endpoint); - - // FIXME: implement ETag - auto s3Res = s3Helper.getObject(bucketName, key); - DownloadResult res; - if (!s3Res.data) - throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); - res.data = s3Res.data; - callback(std::move(res)); + auto [bucketName, key, params] = parseS3Uri(request.uri); + + std::string profile = get(params, "profile", ""); + std::string region = get(params, "region", Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme", ""); + std::string endpoint = get(params, "endpoint", ""); + + S3Helper s3Helper(profile, region, scheme, endpoint); + + // FIXME: implement ETag + auto s3Res = s3Helper.getObject(bucketName, key); + DownloadResult res; + if (!s3Res.data) + throw DownloadError( + NotFound, fmt("S3 object '%s' does not exist", request.uri)); + res.data = s3Res.data; + callback(std::move(res)); #else - throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri); + throw nix::Error( + "cannot download '%s' because Nix is not built with S3 support", + request.uri); #endif - } catch (...) { callback.rethrow(); } - return; - } - - enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback))); + } catch (...) { + callback.rethrow(); + } + return; } + + enqueueItem( + std::make_shared<DownloadItem>(*this, request, std::move(callback))); + } }; -ref<Downloader> getDownloader() -{ - static ref<Downloader> downloader = makeDownloader(); - return downloader; +ref<Downloader> getDownloader() { + static ref<Downloader> downloader = makeDownloader(); + return downloader; } -ref<Downloader> makeDownloader() -{ - return make_ref<CurlDownloader>(); -} +ref<Downloader> makeDownloader() { return make_ref<CurlDownloader>(); } -std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request) -{ - auto promise = std::make_shared<std::promise<DownloadResult>>(); - enqueueDownload(request, - {[promise](std::future<DownloadResult> fut) { - try { - promise->set_value(fut.get()); - } catch (...) { - promise->set_exception(std::current_exception()); - } - }}); - return promise->get_future(); +std::future<DownloadResult> Downloader::enqueueDownload( + const DownloadRequest& request) { + auto promise = std::make_shared<std::promise<DownloadResult>>(); + enqueueDownload(request, {[promise](std::future<DownloadResult> fut) { + try { + promise->set_value(fut.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + }}); + return promise->get_future(); } -DownloadResult Downloader::download(const DownloadRequest & request) -{ - return enqueueDownload(request).get(); +DownloadResult Downloader::download(const DownloadRequest& request) { + return enqueueDownload(request).get(); } -void Downloader::download(DownloadRequest && request, Sink & sink) -{ - /* Note: we can't call 'sink' via request.dataCallback, because - that would cause the sink to execute on the downloader - thread. If 'sink' is a coroutine, this will fail. Also, if the - sink is expensive (e.g. one that does decompression and writing - to the Nix store), it would stall the download thread too much. - Therefore we use a buffer to communicate data between the - download thread and the calling thread. */ - - struct State { - bool quit = false; - std::exception_ptr exc; - std::string data; - std::condition_variable avail, request; - }; - - auto _state = std::make_shared<Sync<State>>(); +void Downloader::download(DownloadRequest&& request, Sink& sink) { + /* Note: we can't call 'sink' via request.dataCallback, because + that would cause the sink to execute on the downloader + thread. If 'sink' is a coroutine, this will fail. Also, if the + sink is expensive (e.g. one that does decompression and writing + to the Nix store), it would stall the download thread too much. + Therefore we use a buffer to communicate data between the + download thread and the calling thread. */ + + struct State { + bool quit = false; + std::exception_ptr exc; + std::string data; + std::condition_variable avail, request; + }; + + auto _state = std::make_shared<Sync<State>>(); + + /* In case of an exception, wake up the download thread. FIXME: + abort the download request. */ + Finally finally([&]() { + auto state(_state->lock()); + state->quit = true; + state->request.notify_one(); + }); + + request.dataCallback = [_state](char* buf, size_t len) { + auto state(_state->lock()); + + if (state->quit) return; + + /* If the buffer is full, then go to sleep until the calling + thread wakes us up (i.e. when it has removed data from the + buffer). We don't wait forever to prevent stalling the + download thread. (Hopefully sleeping will throttle the + sender.) */ + if (state->data.size() > 1024 * 1024) { + debug("download buffer is full; going to sleep"); + state.wait_for(state->request, std::chrono::seconds(10)); + } - /* In case of an exception, wake up the download thread. FIXME: - abort the download request. */ - Finally finally([&]() { - auto state(_state->lock()); - state->quit = true; - state->request.notify_one(); - }); + /* Append data to the buffer and wake up the calling + thread. */ + state->data.append(buf, len); + state->avail.notify_one(); + }; + + enqueueDownload(request, {[_state](std::future<DownloadResult> fut) { + auto state(_state->lock()); + state->quit = true; + try { + fut.get(); + } catch (...) { + state->exc = std::current_exception(); + } + state->avail.notify_one(); + state->request.notify_one(); + }}); - request.dataCallback = [_state](char * buf, size_t len) { + while (true) { + checkInterrupt(); - auto state(_state->lock()); + std::string chunk; - if (state->quit) return; + /* Grab data if available, otherwise wait for the download + thread to wake us up. */ + { + auto state(_state->lock()); - /* If the buffer is full, then go to sleep until the calling - thread wakes us up (i.e. when it has removed data from the - buffer). We don't wait forever to prevent stalling the - download thread. (Hopefully sleeping will throttle the - sender.) */ - if (state->data.size() > 1024 * 1024) { - debug("download buffer is full; going to sleep"); - state.wait_for(state->request, std::chrono::seconds(10)); + while (state->data.empty()) { + if (state->quit) { + if (state->exc) std::rethrow_exception(state->exc); + return; } - /* Append data to the buffer and wake up the calling - thread. */ - state->data.append(buf, len); - state->avail.notify_one(); - }; - - enqueueDownload(request, - {[_state](std::future<DownloadResult> fut) { - auto state(_state->lock()); - state->quit = true; - try { - fut.get(); - } catch (...) { - state->exc = std::current_exception(); - } - state->avail.notify_one(); - state->request.notify_one(); - }}); - - while (true) { - checkInterrupt(); - - std::string chunk; - - /* Grab data if available, otherwise wait for the download - thread to wake us up. */ - { - auto state(_state->lock()); + state.wait(state->avail); + } - while (state->data.empty()) { + chunk = std::move(state->data); - if (state->quit) { - if (state->exc) std::rethrow_exception(state->exc); - return; - } - - state.wait(state->avail); - } - - chunk = std::move(state->data); - - state->request.notify_one(); - } - - /* Flush the data to the sink and wake up the download thread - if it's blocked on a full buffer. We don't hold the state - lock while doing this to prevent blocking the download - thread if sink() takes a long time. */ - sink((unsigned char *) chunk.data(), chunk.size()); + state->request.notify_one(); } + + /* Flush the data to the sink and wake up the download thread + if it's blocked on a full buffer. We don't hold the state + lock while doing this to prevent blocking the download + thread if sink() takes a long time. */ + sink((unsigned char*)chunk.data(), chunk.size()); + } } CachedDownloadResult Downloader::downloadCached( - ref<Store> store, const CachedDownloadRequest & request) -{ - auto url = resolveUri(request.uri); - - auto name = request.name; - if (name == "") { - auto p = url.rfind('/'); - if (p != string::npos) name = string(url, p + 1); + ref<Store> store, const CachedDownloadRequest& request) { + auto url = resolveUri(request.uri); + + auto name = request.name; + if (name == "") { + auto p = url.rfind('/'); + if (p != string::npos) name = string(url, p + 1); + } + + Path expectedStorePath; + if (request.expectedHash) { + expectedStorePath = + store->makeFixedOutputPath(request.unpack, request.expectedHash, name); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.storePath = expectedStorePath; + result.path = store->toRealPath(expectedStorePath); + return result; } + } - Path expectedStorePath; - if (request.expectedHash) { - expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); - if (store->isValidPath(expectedStorePath)) { - CachedDownloadResult result; - result.storePath = expectedStorePath; - result.path = store->toRealPath(expectedStorePath); - return result; - } - } + Path cacheDir = getCacheDir() + "/nix/tarballs"; + createDirs(cacheDir); - Path cacheDir = getCacheDir() + "/nix/tarballs"; - createDirs(cacheDir); + string urlHash = hashString(htSHA256, name + std::string("\0"s) + url) + .to_string(Base32, false); - string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false); + Path dataFile = cacheDir + "/" + urlHash + ".info"; + Path fileLink = cacheDir + "/" + urlHash + "-file"; - Path dataFile = cacheDir + "/" + urlHash + ".info"; - Path fileLink = cacheDir + "/" + urlHash + "-file"; + PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); - PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); + Path storePath; - Path storePath; + string expectedETag; - string expectedETag; + bool skip = false; - bool skip = false; + CachedDownloadResult result; - CachedDownloadResult result; - - if (pathExists(fileLink) && pathExists(dataFile)) { - storePath = readLink(fileLink); - store->addTempRoot(storePath); - if (store->isValidPath(storePath)) { - auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); - if (ss.size() >= 3 && ss[0] == url) { - time_t lastChecked; - if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { - skip = true; - result.effectiveUri = request.uri; - result.etag = ss[1]; - } else if (!ss[1].empty()) { - debug(format("verifying previous ETag '%1%'") % ss[1]); - expectedETag = ss[1]; - } - } - } else - storePath = ""; - } - - if (!skip) { - - try { - DownloadRequest request2(url); - request2.expectedETag = expectedETag; - auto res = download(request2); - result.effectiveUri = res.effectiveUri; - result.etag = res.etag; - - if (!res.cached) { - ValidPathInfo info; - StringSink sink; - dumpString(*res.data, sink); - Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); - info.path = store->makeFixedOutputPath(false, hash, name); - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); - store->addToStore(info, sink.s, NoRepair, NoCheckSigs); - storePath = info.path; - } - - assert(!storePath.empty()); - replaceSymlink(storePath, fileLink); - - writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); - } catch (DownloadError & e) { - if (storePath.empty()) throw; - warn("warning: %s; using cached result", e.msg()); - result.etag = expectedETag; + if (pathExists(fileLink) && pathExists(dataFile)) { + storePath = readLink(fileLink); + store->addTempRoot(storePath); + if (store->isValidPath(storePath)) { + auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); + if (ss.size() >= 3 && ss[0] == url) { + time_t lastChecked; + if (string2Int(ss[2], lastChecked) && + (uint64_t)lastChecked + request.ttl >= (uint64_t)time(0)) { + skip = true; + result.effectiveUri = request.uri; + result.etag = ss[1]; + } else if (!ss[1].empty()) { + debug(format("verifying previous ETag '%1%'") % ss[1]); + expectedETag = ss[1]; } + } + } else + storePath = ""; + } + + if (!skip) { + try { + DownloadRequest request2(url); + request2.expectedETag = expectedETag; + auto res = download(request2); + result.effectiveUri = res.effectiveUri; + result.etag = res.etag; + + if (!res.cached) { + ValidPathInfo info; + StringSink sink; + dumpString(*res.data, sink); + Hash hash = hashString( + request.expectedHash ? request.expectedHash.type : htSHA256, + *res.data); + info.path = store->makeFixedOutputPath(false, hash, name); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(false, hash); + store->addToStore(info, sink.s, NoRepair, NoCheckSigs); + storePath = info.path; + } + + assert(!storePath.empty()); + replaceSymlink(storePath, fileLink); + + writeFile(dataFile, + url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); + } catch (DownloadError& e) { + if (storePath.empty()) throw; + warn("warning: %s; using cached result", e.msg()); + result.etag = expectedETag; } - - if (request.unpack) { - Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; - PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); - Path unpackedStorePath; - if (pathExists(unpackedLink)) { - unpackedStorePath = readLink(unpackedLink); - store->addTempRoot(unpackedStorePath); - if (!store->isValidPath(unpackedStorePath)) - unpackedStorePath = ""; - } - if (unpackedStorePath.empty()) { - printInfo(format("unpacking '%1%'...") % url); - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); - } - replaceSymlink(unpackedStorePath, unpackedLink); - storePath = unpackedStorePath; + } + + if (request.unpack) { + Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; + PathLocks lock2({unpackedLink}, + fmt("waiting for lock on '%1%'...", unpackedLink)); + Path unpackedStorePath; + if (pathExists(unpackedLink)) { + unpackedStorePath = readLink(unpackedLink); + store->addTempRoot(unpackedStorePath); + if (!store->isValidPath(unpackedStorePath)) unpackedStorePath = ""; } - - if (expectedStorePath != "" && storePath != expectedStorePath) { - unsigned int statusCode = 102; - Hash gotHash = request.unpack - ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first - : hashFile(request.expectedHash.type, store->toRealPath(storePath)); - throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, request.expectedHash.to_string(), gotHash.to_string()); + if (unpackedStorePath.empty()) { + printInfo(format("unpacking '%1%'...") % url); + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + // FIXME: this requires GNU tar for decompression. + runProgram("tar", true, + {"xf", store->toRealPath(storePath), "-C", tmpDir, + "--strip-components", "1"}); + unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, + defaultPathFilter, NoRepair); } - - result.storePath = storePath; - result.path = store->toRealPath(storePath); - return result; + replaceSymlink(unpackedStorePath, unpackedLink); + storePath = unpackedStorePath; + } + + if (expectedStorePath != "" && storePath != expectedStorePath) { + unsigned int statusCode = 102; + Hash gotHash = + request.unpack + ? hashPath(request.expectedHash.type, store->toRealPath(storePath)) + .first + : hashFile(request.expectedHash.type, store->toRealPath(storePath)); + throw nix::Error(statusCode, + "hash mismatch in file downloaded from '%s':\n wanted: " + "%s\n got: %s", + url, request.expectedHash.to_string(), + gotHash.to_string()); + } + + result.storePath = storePath; + result.path = store->toRealPath(storePath); + return result; } - -bool isUri(const string & s) -{ - if (s.compare(0, 8, "channel:") == 0) return true; - size_t pos = s.find("://"); - if (pos == string::npos) return false; - string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; +bool isUri(const string& s) { + if (s.compare(0, 8, "channel:") == 0) return true; + size_t pos = s.find("://"); + if (pos == string::npos) return false; + string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file" || + scheme == "channel" || scheme == "git" || scheme == "s3" || + scheme == "ssh"; } - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/download.hh b/third_party/nix/src/libstore/download.hh index 68565bf462a7..8e07403bcccd 100644 --- a/third_party/nix/src/libstore/download.hh +++ b/third_party/nix/src/libstore/download.hh @@ -1,120 +1,118 @@ #pragma once -#include "types.hh" -#include "hash.hh" -#include "globals.hh" - -#include <string> #include <future> +#include <string> +#include "globals.hh" +#include "hash.hh" +#include "types.hh" namespace nix { -struct DownloadSettings : Config -{ - Setting<bool> enableHttp2{this, true, "http2", - "Whether to enable HTTP/2 support."}; - - Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", - "String appended to the user agent in HTTP requests."}; - - Setting<size_t> httpConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", - {"binary-caches-parallel-connections"}}; - - Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; - - Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout", - "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; - - Setting<unsigned int> tries{this, 5, "download-attempts", - "How often Nix will attempt to download a file before giving up."}; +struct DownloadSettings : Config { + Setting<bool> enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting<std::string> userAgentSuffix{ + this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting<size_t> httpConnections{this, + 25, + "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting<unsigned long> connectTimeout{ + this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's " + "builtin default."}; + + Setting<unsigned long> stalledDownloadTimeout{ + this, 300, "stalled-download-timeout", + "Timeout (in seconds) for receiving data from servers during download. " + "Nix cancels idle downloads after this timeout's duration."}; + + Setting<unsigned int> tries{ + this, 5, "download-attempts", + "How often Nix will attempt to download a file before giving up."}; }; extern DownloadSettings downloadSettings; -struct DownloadRequest -{ - std::string uri; - std::string expectedETag; - bool verifyTLS = true; - bool head = false; - size_t tries = downloadSettings.tries; - unsigned int baseRetryTimeMs = 250; - ActivityId parentAct; - bool decompress = true; - std::shared_ptr<std::string> data; - std::string mimeType; - std::function<void(char *, size_t)> dataCallback; - - DownloadRequest(const std::string & uri) - : uri(uri), parentAct(getCurActivity()) { } - - std::string verb() - { - return data ? "upload" : "download"; - } +struct DownloadRequest { + std::string uri; + std::string expectedETag; + bool verifyTLS = true; + bool head = false; + size_t tries = downloadSettings.tries; + unsigned int baseRetryTimeMs = 250; + ActivityId parentAct; + bool decompress = true; + std::shared_ptr<std::string> data; + std::string mimeType; + std::function<void(char*, size_t)> dataCallback; + + DownloadRequest(const std::string& uri) + : uri(uri), parentAct(getCurActivity()) {} + + std::string verb() { return data ? "upload" : "download"; } }; -struct DownloadResult -{ - bool cached = false; - std::string etag; - std::string effectiveUri; - std::shared_ptr<std::string> data; - uint64_t bodySize = 0; +struct DownloadResult { + bool cached = false; + std::string etag; + std::string effectiveUri; + std::shared_ptr<std::string> data; + uint64_t bodySize = 0; }; -struct CachedDownloadRequest -{ - std::string uri; - bool unpack = false; - std::string name; - Hash expectedHash; - unsigned int ttl = settings.tarballTtl; +struct CachedDownloadRequest { + std::string uri; + bool unpack = false; + std::string name; + Hash expectedHash; + unsigned int ttl = settings.tarballTtl; - CachedDownloadRequest(const std::string & uri) - : uri(uri) { } + CachedDownloadRequest(const std::string& uri) : uri(uri) {} }; -struct CachedDownloadResult -{ - // Note: 'storePath' may be different from 'path' when using a - // chroot store. - Path storePath; - Path path; - std::optional<std::string> etag; - std::string effectiveUri; +struct CachedDownloadResult { + // Note: 'storePath' may be different from 'path' when using a + // chroot store. + Path storePath; + Path path; + std::optional<std::string> etag; + std::string effectiveUri; }; class Store; -struct Downloader -{ - virtual ~Downloader() { } +struct Downloader { + virtual ~Downloader() {} - /* Enqueue a download request, returning a future to the result of - the download. The future may throw a DownloadError - exception. */ - virtual void enqueueDownload(const DownloadRequest & request, - Callback<DownloadResult> callback) = 0; + /* Enqueue a download request, returning a future to the result of + the download. The future may throw a DownloadError + exception. */ + virtual void enqueueDownload(const DownloadRequest& request, + Callback<DownloadResult> callback) = 0; - std::future<DownloadResult> enqueueDownload(const DownloadRequest & request); + std::future<DownloadResult> enqueueDownload(const DownloadRequest& request); - /* Synchronously download a file. */ - DownloadResult download(const DownloadRequest & request); + /* Synchronously download a file. */ + DownloadResult download(const DownloadRequest& request); - /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ - void download(DownloadRequest && request, Sink & sink); + /* Download a file, writing its data to a sink. The sink will be + invoked on the thread of the caller. */ + void download(DownloadRequest&& request, Sink& sink); - /* Check if the specified file is already in ~/.cache/nix/tarballs - and is more recent than ‘tarball-ttl’ seconds. Otherwise, - use the recorded ETag to verify if the server has a more - recent version, and if so, download it to the Nix store. */ - CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request); + /* Check if the specified file is already in ~/.cache/nix/tarballs + and is more recent than ‘tarball-ttl’ seconds. Otherwise, + use the recorded ETag to verify if the server has a more + recent version, and if so, download it to the Nix store. */ + CachedDownloadResult downloadCached(ref<Store> store, + const CachedDownloadRequest& request); - enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; + enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; /* Return a shared Downloader object. Using this object is preferred @@ -124,15 +122,13 @@ ref<Downloader> getDownloader(); /* Return a new Downloader object. */ ref<Downloader> makeDownloader(); -class DownloadError : public Error -{ -public: - Downloader::Error error; - DownloadError(Downloader::Error error, const FormatOrString & fs) - : Error(fs), error(error) - { } +class DownloadError : public Error { + public: + Downloader::Error error; + DownloadError(Downloader::Error error, const FormatOrString& fs) + : Error(fs), error(error) {} }; -bool isUri(const string & s); +bool isUri(const string& s); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/export-import.cc b/third_party/nix/src/libstore/export-import.cc index ef3fea7c838b..b38b15c6f3bc 100644 --- a/third_party/nix/src/libstore/export-import.cc +++ b/third_party/nix/src/libstore/export-import.cc @@ -1,106 +1,100 @@ -#include "store-api.hh" +#include <algorithm> #include "archive.hh" +#include "store-api.hh" #include "worker-protocol.hh" -#include <algorithm> - namespace nix { -struct HashAndWriteSink : Sink -{ - Sink & writeSink; - HashSink hashSink; - HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) - { - } - virtual void operator () (const unsigned char * data, size_t len) - { - writeSink(data, len); - hashSink(data, len); - } - Hash currentHash() - { - return hashSink.currentHash().first; - } +struct HashAndWriteSink : Sink { + Sink& writeSink; + HashSink hashSink; + HashAndWriteSink(Sink& writeSink) + : writeSink(writeSink), hashSink(htSHA256) {} + virtual void operator()(const unsigned char* data, size_t len) { + writeSink(data, len); + hashSink(data, len); + } + Hash currentHash() { return hashSink.currentHash().first; } }; -void Store::exportPaths(const Paths & paths, Sink & sink) -{ - Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end())); - std::reverse(sorted.begin(), sorted.end()); +void Store::exportPaths(const Paths& paths, Sink& sink) { + Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end())); + std::reverse(sorted.begin(), sorted.end()); - std::string doneLabel("paths exported"); - //logger->incExpected(doneLabel, sorted.size()); + std::string doneLabel("paths exported"); + // logger->incExpected(doneLabel, sorted.size()); - for (auto & path : sorted) { - //Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); - sink << 1; - exportPath(path, sink); - //logger->incProgress(doneLabel); - } + for (auto& path : sorted) { + // Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); + sink << 1; + exportPath(path, sink); + // logger->incProgress(doneLabel); + } - sink << 0; + sink << 0; } -void Store::exportPath(const Path & path, Sink & sink) -{ - auto info = queryPathInfo(path); +void Store::exportPath(const Path& path, Sink& sink) { + auto info = queryPathInfo(path); - HashAndWriteSink hashAndWriteSink(sink); + HashAndWriteSink hashAndWriteSink(sink); - narFromPath(path, hashAndWriteSink); + narFromPath(path, hashAndWriteSink); - /* Refuse to export paths that have changed. This prevents - filesystem corruption from spreading to other machines. - Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashAndWriteSink.currentHash(); - if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) - throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % path - % info->narHash.to_string() % hash.to_string()); + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ + Hash hash = hashAndWriteSink.currentHash(); + if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) + throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % + path % info->narHash.to_string() % hash.to_string()); - hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0; + hashAndWriteSink << exportMagic << path << info->references << info->deriver + << 0; } -Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) -{ - Paths res; - while (true) { - auto n = readNum<uint64_t>(source); - if (n == 0) break; - if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); +Paths Store::importPaths(Source& source, std::shared_ptr<FSAccessor> accessor, + CheckSigsFlag checkSigs) { + Paths res; + while (true) { + auto n = readNum<uint64_t>(source); + if (n == 0) break; + if (n != 1) + throw Error( + "input doesn't look like something created by 'nix-store --export'"); - /* Extract the NAR from the source. */ - TeeSink tee(source); - parseDump(tee, tee.source); + /* Extract the NAR from the source. */ + TeeSink tee(source); + parseDump(tee, tee.source); - uint32_t magic = readInt(source); - if (magic != exportMagic) - throw Error("Nix archive cannot be imported; wrong format"); + uint32_t magic = readInt(source); + if (magic != exportMagic) + throw Error("Nix archive cannot be imported; wrong format"); - ValidPathInfo info; + ValidPathInfo info; - info.path = readStorePath(*this, source); + info.path = readStorePath(*this, source); - //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); + // Activity act(*logger, lvlInfo, format("importing path '%s'") % + // info.path); - info.references = readStorePaths<PathSet>(*this, source); + info.references = readStorePaths<PathSet>(*this, source); - info.deriver = readString(source); - if (info.deriver != "") assertStorePath(info.deriver); + info.deriver = readString(source); + if (info.deriver != "") assertStorePath(info.deriver); - info.narHash = hashString(htSHA256, *tee.source.data); - info.narSize = tee.source.data->size(); + info.narHash = hashString(htSHA256, *tee.source.data); + info.narSize = tee.source.data->size(); - // Ignore optional legacy signature. - if (readInt(source) == 1) - readString(source); + // Ignore optional legacy signature. + if (readInt(source) == 1) readString(source); - addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); + addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); - res.push_back(info.path); - } + res.push_back(info.path); + } - return res; + return res; } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/fs-accessor.hh b/third_party/nix/src/libstore/fs-accessor.hh index 64780a6daf40..ad0d7f0ed9fd 100644 --- a/third_party/nix/src/libstore/fs-accessor.hh +++ b/third_party/nix/src/libstore/fs-accessor.hh @@ -6,28 +6,26 @@ namespace nix { /* An abstract class for accessing a filesystem-like structure, such as a (possibly remote) Nix store or the contents of a NAR file. */ -class FSAccessor -{ -public: - enum Type { tMissing, tRegular, tSymlink, tDirectory }; +class FSAccessor { + public: + enum Type { tMissing, tRegular, tSymlink, tDirectory }; - struct Stat - { - Type type = tMissing; - uint64_t fileSize = 0; // regular files only - bool isExecutable = false; // regular files only - uint64_t narOffset = 0; // regular files only - }; + struct Stat { + Type type = tMissing; + uint64_t fileSize = 0; // regular files only + bool isExecutable = false; // regular files only + uint64_t narOffset = 0; // regular files only + }; - virtual ~FSAccessor() { } + virtual ~FSAccessor() {} - virtual Stat stat(const Path & path) = 0; + virtual Stat stat(const Path& path) = 0; - virtual StringSet readDirectory(const Path & path) = 0; + virtual StringSet readDirectory(const Path& path) = 0; - virtual std::string readFile(const Path & path) = 0; + virtual std::string readFile(const Path& path) = 0; - virtual std::string readLink(const Path & path) = 0; + virtual std::string readLink(const Path& path) = 0; }; -} +} // namespace nix diff --git a/third_party/nix/src/libstore/gc.cc b/third_party/nix/src/libstore/gc.cc index a166f4ee2483..94cf1b7d4187 100644 --- a/third_party/nix/src/libstore/gc.cc +++ b/third_party/nix/src/libstore/gc.cc @@ -1,948 +1,894 @@ -#include "derivations.hh" -#include "globals.hh" -#include "local-store.hh" -#include "finally.hh" - -#include <functional> -#include <queue> -#include <algorithm> -#include <regex> -#include <random> - -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/statvfs.h> #include <errno.h> #include <fcntl.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/types.h> #include <unistd.h> +#include <algorithm> #include <climits> +#include <functional> +#include <queue> +#include <random> +#include <regex> +#include "derivations.hh" +#include "finally.hh" +#include "globals.hh" +#include "local-store.hh" namespace nix { - static string gcLockName = "gc.lock"; static string gcRootsDir = "gcroots"; - /* Acquire the global GC lock. This is used to prevent new Nix processes from starting after the temporary root files have been read. To be precise: when they try to create a new temporary root file, they will block until the garbage collector has finished / yielded the GC lock. */ -AutoCloseFD LocalStore::openGCLock(LockType lockType) -{ - Path fnGCLock = (format("%1%/%2%") - % stateDir % gcLockName).str(); +AutoCloseFD LocalStore::openGCLock(LockType lockType) { + Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str(); - debug(format("acquiring global GC lock '%1%'") % fnGCLock); + debug(format("acquiring global GC lock '%1%'") % fnGCLock); - AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fdGCLock) - throw SysError(format("opening global GC lock '%1%'") % fnGCLock); + AutoCloseFD fdGCLock = + open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fdGCLock) + throw SysError(format("opening global GC lock '%1%'") % fnGCLock); - if (!lockFile(fdGCLock.get(), lockType, false)) { - printError(format("waiting for the big garbage collector lock...")); - lockFile(fdGCLock.get(), lockType, true); - } + if (!lockFile(fdGCLock.get(), lockType, false)) { + printError(format("waiting for the big garbage collector lock...")); + lockFile(fdGCLock.get(), lockType, true); + } - /* !!! Restrict read permission on the GC root. Otherwise any - process that can open the file for reading can DoS the - collector. */ + /* !!! Restrict read permission on the GC root. Otherwise any + process that can open the file for reading can DoS the + collector. */ - return fdGCLock; + return fdGCLock; } +static void makeSymlink(const Path& link, const Path& target) { + /* Create directories up to `gcRoot'. */ + createDirs(dirOf(link)); -static void makeSymlink(const Path & link, const Path & target) -{ - /* Create directories up to `gcRoot'. */ - createDirs(dirOf(link)); - - /* Create the new symlink. */ - Path tempLink = (format("%1%.tmp-%2%-%3%") - % link % getpid() % random()).str(); - createSymlink(target, tempLink); - - /* Atomically replace the old one. */ - if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError(format("cannot rename '%1%' to '%2%'") - % tempLink % link); -} - + /* Create the new symlink. */ + Path tempLink = + (format("%1%.tmp-%2%-%3%") % link % getpid() % random()).str(); + createSymlink(target, tempLink); -void LocalStore::syncWithGC() -{ - AutoCloseFD fdGCLock = openGCLock(ltRead); + /* Atomically replace the old one. */ + if (rename(tempLink.c_str(), link.c_str()) == -1) + throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % link); } +void LocalStore::syncWithGC() { AutoCloseFD fdGCLock = openGCLock(ltRead); } -void LocalStore::addIndirectRoot(const Path & path) -{ - string hash = hashString(htSHA1, path).to_string(Base32, false); - Path realRoot = canonPath((format("%1%/%2%/auto/%3%") - % stateDir % gcRootsDir % hash).str()); - makeSymlink(realRoot, path); +void LocalStore::addIndirectRoot(const Path& path) { + string hash = hashString(htSHA1, path).to_string(Base32, false); + Path realRoot = canonPath( + (format("%1%/%2%/auto/%3%") % stateDir % gcRootsDir % hash).str()); + makeSymlink(realRoot, path); } - -Path LocalFSStore::addPermRoot(const Path & _storePath, - const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) -{ - Path storePath(canonPath(_storePath)); - Path gcRoot(canonPath(_gcRoot)); - assertStorePath(storePath); - - if (isInStore(gcRoot)) - throw Error(format( - "creating a garbage collector root (%1%) in the Nix store is forbidden " - "(are you running nix-build inside the store?)") % gcRoot); - - if (indirect) { - /* Don't clobber the link if it already exists and doesn't - point to the Nix store. */ - if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) - throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot); - makeSymlink(gcRoot, storePath); - addIndirectRoot(gcRoot); +Path LocalFSStore::addPermRoot(const Path& _storePath, const Path& _gcRoot, + bool indirect, bool allowOutsideRootsDir) { + Path storePath(canonPath(_storePath)); + Path gcRoot(canonPath(_gcRoot)); + assertStorePath(storePath); + + if (isInStore(gcRoot)) + throw Error(format("creating a garbage collector root (%1%) in the Nix " + "store is forbidden " + "(are you running nix-build inside the store?)") % + gcRoot); + + if (indirect) { + /* Don't clobber the link if it already exists and doesn't + point to the Nix store. */ + if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) + throw Error(format("cannot create symlink '%1%'; already exists") % + gcRoot); + makeSymlink(gcRoot, storePath); + addIndirectRoot(gcRoot); + } + + else { + if (!allowOutsideRootsDir) { + Path rootsDir = + canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str()); + + if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") + throw Error(format("path '%1%' is not a valid garbage collector root; " + "it's not in the directory '%2%'") % + gcRoot % rootsDir); } - else { - if (!allowOutsideRootsDir) { - Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str()); - - if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") - throw Error(format( - "path '%1%' is not a valid garbage collector root; " - "it's not in the directory '%2%'") - % gcRoot % rootsDir); - } - - if (baseNameOf(gcRoot) == baseNameOf(storePath)) - writeFile(gcRoot, ""); - else - makeSymlink(gcRoot, storePath); - } - - /* Check that the root can be found by the garbage collector. - !!! This can be very slow on machines that have many roots. - Instead of reading all the roots, it would be more efficient to - check if the root is in a directory in or linked from the - gcroots directory. */ - if (settings.checkRootReachability) { - Roots roots = findRoots(false); - if (roots[storePath].count(gcRoot) == 0) - printError( - format( - "warning: '%1%' is not in a directory where the garbage collector looks for roots; " - "therefore, '%2%' might be removed by the garbage collector") - % gcRoot % storePath); - } - - /* Grab the global GC root, causing us to block while a GC is in - progress. This prevents the set of permanent roots from - increasing while a GC is in progress. */ - syncWithGC(); - - return gcRoot; + if (baseNameOf(gcRoot) == baseNameOf(storePath)) + writeFile(gcRoot, ""); + else + makeSymlink(gcRoot, storePath); + } + + /* Check that the root can be found by the garbage collector. + !!! This can be very slow on machines that have many roots. + Instead of reading all the roots, it would be more efficient to + check if the root is in a directory in or linked from the + gcroots directory. */ + if (settings.checkRootReachability) { + Roots roots = findRoots(false); + if (roots[storePath].count(gcRoot) == 0) + printError( + format("warning: '%1%' is not in a directory where the garbage " + "collector looks for roots; " + "therefore, '%2%' might be removed by the garbage collector") % + gcRoot % storePath); + } + + /* Grab the global GC root, causing us to block while a GC is in + progress. This prevents the set of permanent roots from + increasing while a GC is in progress. */ + syncWithGC(); + + return gcRoot; } +void LocalStore::addTempRoot(const Path& path) { + auto state(_state.lock()); -void LocalStore::addTempRoot(const Path & path) -{ - auto state(_state.lock()); + /* Create the temporary roots file for this process. */ + if (!state->fdTempRoots) { + while (1) { + AutoCloseFD fdGCLock = openGCLock(ltRead); - /* Create the temporary roots file for this process. */ - if (!state->fdTempRoots) { + if (pathExists(fnTempRoots)) + /* It *must* be stale, since there can be no two + processes with the same pid. */ + unlink(fnTempRoots.c_str()); - while (1) { - AutoCloseFD fdGCLock = openGCLock(ltRead); + state->fdTempRoots = openLockFile(fnTempRoots, true); - if (pathExists(fnTempRoots)) - /* It *must* be stale, since there can be no two - processes with the same pid. */ - unlink(fnTempRoots.c_str()); + fdGCLock = -1; - state->fdTempRoots = openLockFile(fnTempRoots, true); + debug(format("acquiring read lock on '%1%'") % fnTempRoots); + lockFile(state->fdTempRoots.get(), ltRead, true); - fdGCLock = -1; - - debug(format("acquiring read lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltRead, true); - - /* Check whether the garbage collector didn't get in our - way. */ - struct stat st; - if (fstat(state->fdTempRoots.get(), &st) == -1) - throw SysError(format("statting '%1%'") % fnTempRoots); - if (st.st_size == 0) break; - - /* The garbage collector deleted this file before we could - get a lock. (It won't delete the file after we get a - lock.) Try again. */ - } + /* Check whether the garbage collector didn't get in our + way. */ + struct stat st; + if (fstat(state->fdTempRoots.get(), &st) == -1) + throw SysError(format("statting '%1%'") % fnTempRoots); + if (st.st_size == 0) break; + /* The garbage collector deleted this file before we could + get a lock. (It won't delete the file after we get a + lock.) Try again. */ } + } - /* Upgrade the lock to a write lock. This will cause us to block - if the garbage collector is holding our lock. */ - debug(format("acquiring write lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltWrite, true); + /* Upgrade the lock to a write lock. This will cause us to block + if the garbage collector is holding our lock. */ + debug(format("acquiring write lock on '%1%'") % fnTempRoots); + lockFile(state->fdTempRoots.get(), ltWrite, true); - string s = path + '\0'; - writeFull(state->fdTempRoots.get(), s); + string s = path + '\0'; + writeFull(state->fdTempRoots.get(), s); - /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltRead, true); + /* Downgrade to a read lock. */ + debug(format("downgrading to read lock on '%1%'") % fnTempRoots); + lockFile(state->fdTempRoots.get(), ltRead, true); } - static std::string censored = "{censored}"; +void LocalStore::findTempRoots(FDs& fds, Roots& tempRoots, bool censor) { + /* Read the `temproots' directory for per-process temporary root + files. */ + for (auto& i : readDirectory(tempRootsDir)) { + Path path = tempRootsDir + "/" + i.name; -void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) -{ - /* Read the `temproots' directory for per-process temporary root - files. */ - for (auto & i : readDirectory(tempRootsDir)) { - Path path = tempRootsDir + "/" + i.name; - - pid_t pid = std::stoi(i.name); - - debug(format("reading temporary root file '%1%'") % path); - FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666))); - if (!*fd) { - /* It's okay if the file has disappeared. */ - if (errno == ENOENT) continue; - throw SysError(format("opening temporary roots file '%1%'") % path); - } - - /* This should work, but doesn't, for some reason. */ - //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); - //if (*fd == -1) continue; - - /* Try to acquire a write lock without blocking. This can - only succeed if the owning process has died. In that case - we don't care about its temporary roots. */ - if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file '%1%'") % path); - unlink(path.c_str()); - writeFull(fd->get(), "d"); - continue; - } - - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on '%1%'") % path); - lockFile(fd->get(), ltRead, true); - - /* Read the entire file. */ - string contents = readFile(fd->get()); + pid_t pid = std::stoi(i.name); - /* Extract the roots. */ - string::size_type pos = 0, end; - - while ((end = contents.find((char) 0, pos)) != string::npos) { - Path root(contents, pos, end - pos); - debug("got temporary root '%s'", root); - assertStorePath(root); - tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid)); - pos = end + 1; - } + debug(format("reading temporary root file '%1%'") % path); + FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666))); + if (!*fd) { + /* It's okay if the file has disappeared. */ + if (errno == ENOENT) continue; + throw SysError(format("opening temporary roots file '%1%'") % path); + } - fds.push_back(fd); /* keep open */ + /* This should work, but doesn't, for some reason. */ + // FDPtr fd(new AutoCloseFD(openLockFile(path, false))); + // if (*fd == -1) continue; + + /* Try to acquire a write lock without blocking. This can + only succeed if the owning process has died. In that case + we don't care about its temporary roots. */ + if (lockFile(fd->get(), ltWrite, false)) { + printError(format("removing stale temporary roots file '%1%'") % path); + unlink(path.c_str()); + writeFull(fd->get(), "d"); + continue; } -} + /* Acquire a read lock. This will prevent the owning process + from upgrading to a write lock, therefore it will block in + addTempRoot(). */ + debug(format("waiting for read lock on '%1%'") % path); + lockFile(fd->get(), ltRead, true); -void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) -{ - auto foundRoot = [&](const Path & path, const Path & target) { - Path storePath = toStorePath(target); - if (isStorePath(storePath) && isValidPath(storePath)) - roots[storePath].emplace(path); - else - printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath); - }; + /* Read the entire file. */ + string contents = readFile(fd->get()); - try { + /* Extract the roots. */ + string::size_type pos = 0, end; - if (type == DT_UNKNOWN) - type = getFileType(path); + while ((end = contents.find((char)0, pos)) != string::npos) { + Path root(contents, pos, end - pos); + debug("got temporary root '%s'", root); + assertStorePath(root); + tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid)); + pos = end + 1; + } - if (type == DT_DIR) { - for (auto & i : readDirectory(path)) - findRoots(path + "/" + i.name, i.type, roots); - } + fds.push_back(fd); /* keep open */ + } +} - else if (type == DT_LNK) { - Path target = readLink(path); - if (isInStore(target)) - foundRoot(path, target); - - /* Handle indirect roots. */ - else { - target = absPath(target, dirOf(path)); - if (!pathExists(target)) { - if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { - printInfo(format("removing stale link from '%1%' to '%2%'") % path % target); - unlink(path.c_str()); - } - } else { - struct stat st2 = lstat(target); - if (!S_ISLNK(st2.st_mode)) return; - Path target2 = readLink(target); - if (isInStore(target2)) foundRoot(target, target2); - } - } - } +void LocalStore::findRoots(const Path& path, unsigned char type, Roots& roots) { + auto foundRoot = [&](const Path& path, const Path& target) { + Path storePath = toStorePath(target); + if (isStorePath(storePath) && isValidPath(storePath)) + roots[storePath].emplace(path); + else + printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % + storePath); + }; + + try { + if (type == DT_UNKNOWN) type = getFileType(path); + + if (type == DT_DIR) { + for (auto& i : readDirectory(path)) + findRoots(path + "/" + i.name, i.type, roots); + } - else if (type == DT_REG) { - Path storePath = storeDir + "/" + baseNameOf(path); - if (isStorePath(storePath) && isValidPath(storePath)) - roots[storePath].emplace(path); + else if (type == DT_LNK) { + Path target = readLink(path); + if (isInStore(target)) foundRoot(path, target); + + /* Handle indirect roots. */ + else { + target = absPath(target, dirOf(path)); + if (!pathExists(target)) { + if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { + printInfo(format("removing stale link from '%1%' to '%2%'") % path % + target); + unlink(path.c_str()); + } + } else { + struct stat st2 = lstat(target); + if (!S_ISLNK(st2.st_mode)) return; + Path target2 = readLink(target); + if (isInStore(target2)) foundRoot(target, target2); } - + } } - catch (SysError & e) { - /* We only ignore permanent failures. */ - if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) - printInfo(format("cannot read potential root '%1%'") % path); - else - throw; + else if (type == DT_REG) { + Path storePath = storeDir + "/" + baseNameOf(path); + if (isStorePath(storePath) && isValidPath(storePath)) + roots[storePath].emplace(path); } -} - -void LocalStore::findRootsNoTemp(Roots & roots, bool censor) -{ - /* Process direct roots in {gcroots,profiles}. */ - findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); - findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); + } - /* Add additional roots returned by different platforms-specific - heuristics. This is typically used to add running programs to - the set of roots (to prevent them from being garbage collected). */ - findRuntimeRoots(roots, censor); + catch (SysError& e) { + /* We only ignore permanent failures. */ + if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) + printInfo(format("cannot read potential root '%1%'") % path); + else + throw; + } } +void LocalStore::findRootsNoTemp(Roots& roots, bool censor) { + /* Process direct roots in {gcroots,profiles}. */ + findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); + findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); -Roots LocalStore::findRoots(bool censor) -{ - Roots roots; - findRootsNoTemp(roots, censor); + /* Add additional roots returned by different platforms-specific + heuristics. This is typically used to add running programs to + the set of roots (to prevent them from being garbage collected). */ + findRuntimeRoots(roots, censor); +} - FDs fds; - findTempRoots(fds, roots, censor); +Roots LocalStore::findRoots(bool censor) { + Roots roots; + findRootsNoTemp(roots, censor); - return roots; + FDs fds; + findTempRoots(fds, roots, censor); + + return roots; } -static void readProcLink(const string & file, Roots & roots) -{ - /* 64 is the starting buffer size gnu readlink uses... */ - auto bufsiz = ssize_t{64}; +static void readProcLink(const string& file, Roots& roots) { + /* 64 is the starting buffer size gnu readlink uses... */ + auto bufsiz = ssize_t{64}; try_again: - char buf[bufsiz]; - auto res = readlink(file.c_str(), buf, bufsiz); - if (res == -1) { - if (errno == ENOENT || errno == EACCES || errno == ESRCH) - return; - throw SysError("reading symlink"); - } - if (res == bufsiz) { - if (SSIZE_MAX / 2 < bufsiz) - throw Error("stupidly long symlink"); - bufsiz *= 2; - goto try_again; - } - if (res > 0 && buf[0] == '/') - roots[std::string(static_cast<char *>(buf), res)] - .emplace(file); + char buf[bufsiz]; + auto res = readlink(file.c_str(), buf, bufsiz); + if (res == -1) { + if (errno == ENOENT || errno == EACCES || errno == ESRCH) return; + throw SysError("reading symlink"); + } + if (res == bufsiz) { + if (SSIZE_MAX / 2 < bufsiz) throw Error("stupidly long symlink"); + bufsiz *= 2; + goto try_again; + } + if (res > 0 && buf[0] == '/') + roots[std::string(static_cast<char*>(buf), res)].emplace(file); } -static string quoteRegexChars(const string & raw) -{ - static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])"); - return std::regex_replace(raw, specialRegex, R"(\$&)"); +static string quoteRegexChars(const string& raw) { + static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])"); + return std::regex_replace(raw, specialRegex, R"(\$&)"); } -static void readFileRoots(const char * path, Roots & roots) -{ - try { - roots[readFile(path)].emplace(path); - } catch (SysError & e) { - if (e.errNo != ENOENT && e.errNo != EACCES) - throw; - } +static void readFileRoots(const char* path, Roots& roots) { + try { + roots[readFile(path)].emplace(path); + } catch (SysError& e) { + if (e.errNo != ENOENT && e.errNo != EACCES) throw; + } } -void LocalStore::findRuntimeRoots(Roots & roots, bool censor) -{ - Roots unchecked; - - auto procDir = AutoCloseDir{opendir("/proc")}; - if (procDir) { - struct dirent * ent; - auto digitsRegex = std::regex(R"(^\d+$)"); - auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); - auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); - while (errno = 0, ent = readdir(procDir.get())) { - checkInterrupt(); - if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); - readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - - auto fdStr = fmt("/proc/%s/fd", ent->d_name); - auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); - if (!fdDir) { - if (errno == ENOENT || errno == EACCES) - continue; - throw SysError(format("opening %1%") % fdStr); - } - struct dirent * fd_ent; - while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') - readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); - } - if (errno) { - if (errno == ESRCH) - continue; - throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); - } - fdDir.reset(); - - try { - auto mapFile = fmt("/proc/%s/maps", ent->d_name); - auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n"); - for (const auto & line : mapLines) { - auto match = std::smatch{}; - if (std::regex_match(line, match, mapRegex)) - unchecked[match[1]].emplace(mapFile); - } - - auto envFile = fmt("/proc/%s/environ", ent->d_name); - auto envString = readFile(envFile, true); - auto env_end = std::sregex_iterator{}; - for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - unchecked[i->str()].emplace(envFile); - } catch (SysError & e) { - if (errno == ENOENT || errno == EACCES || errno == ESRCH) - continue; - throw; - } - } +void LocalStore::findRuntimeRoots(Roots& roots, bool censor) { + Roots unchecked; + + auto procDir = AutoCloseDir{opendir("/proc")}; + if (procDir) { + struct dirent* ent; + auto digitsRegex = std::regex(R"(^\d+$)"); + auto mapRegex = + std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); + auto storePathRegex = std::regex(quoteRegexChars(storeDir) + + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); + while (errno = 0, ent = readdir(procDir.get())) { + checkInterrupt(); + if (std::regex_match(ent->d_name, digitsRegex)) { + readProcLink(fmt("/proc/%s/exe", ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); + + auto fdStr = fmt("/proc/%s/fd", ent->d_name); + auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); + if (!fdDir) { + if (errno == ENOENT || errno == EACCES) continue; + throw SysError(format("opening %1%") % fdStr); } - if (errno) - throw SysError("iterating /proc"); - } + struct dirent* fd_ent; + while (errno = 0, fd_ent = readdir(fdDir.get())) { + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); + } + if (errno) { + if (errno == ESRCH) continue; + throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); + } + fdDir.reset(); -#if !defined(__linux__) - // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. - // See: https://github.com/NixOS/nix/issues/3011 - // Because of this we disable lsof when running the tests. - if (getEnv("_NIX_TEST_NO_LSOF") == "") { try { - std::regex lsofRegex(R"(^n(/.*)$)"); - auto lsofLines = - tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); - for (const auto & line : lsofLines) { - std::smatch match; - if (std::regex_match(line, match, lsofRegex)) - unchecked[match[1]].emplace("{lsof}"); - } - } catch (ExecError & e) { - /* lsof not installed, lsof failed */ + auto mapFile = fmt("/proc/%s/maps", ent->d_name); + auto mapLines = tokenizeString<std::vector<string>>( + readFile(mapFile, true), "\n"); + for (const auto& line : mapLines) { + auto match = std::smatch{}; + if (std::regex_match(line, match, mapRegex)) + unchecked[match[1]].emplace(mapFile); + } + + auto envFile = fmt("/proc/%s/environ", ent->d_name); + auto envString = readFile(envFile, true); + auto env_end = std::sregex_iterator{}; + for (auto i = std::sregex_iterator{envString.begin(), envString.end(), + storePathRegex}; + i != env_end; ++i) + unchecked[i->str()].emplace(envFile); + } catch (SysError& e) { + if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; + throw; } + } + } + if (errno) throw SysError("iterating /proc"); + } + +#if !defined(__linux__) + // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test + // to fail. See: https://github.com/NixOS/nix/issues/3011 Because of this we + // disable lsof when running the tests. + if (getEnv("_NIX_TEST_NO_LSOF") == "") { + try { + std::regex lsofRegex(R"(^n(/.*)$)"); + auto lsofLines = tokenizeString<std::vector<string>>( + runProgram(LSOF, true, {"-n", "-w", "-F", "n"}), "\n"); + for (const auto& line : lsofLines) { + std::smatch match; + if (std::regex_match(line, match, lsofRegex)) + unchecked[match[1]].emplace("{lsof}"); + } + } catch (ExecError& e) { + /* lsof not installed, lsof failed */ } + } #endif #if defined(__linux__) - readFileRoots("/proc/sys/kernel/modprobe", unchecked); - readFileRoots("/proc/sys/kernel/fbsplash", unchecked); - readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); #endif - for (auto & [target, links] : unchecked) { - if (isInStore(target)) { - Path path = toStorePath(target); - if (isStorePath(path) && isValidPath(path)) { - debug(format("got additional root '%1%'") % path); - if (censor) - roots[path].insert(censored); - else - roots[path].insert(links.begin(), links.end()); - } - } + for (auto& [target, links] : unchecked) { + if (isInStore(target)) { + Path path = toStorePath(target); + if (isStorePath(path) && isValidPath(path)) { + debug(format("got additional root '%1%'") % path); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); + } } + } } - -struct GCLimitReached { }; - - -struct LocalStore::GCState -{ - GCOptions options; - GCResults & results; - PathSet roots; - PathSet tempRoots; - PathSet dead; - PathSet alive; - bool gcKeepOutputs; - bool gcKeepDerivations; - unsigned long long bytesInvalidated; - bool moveToTrash = true; - bool shouldDelete; - GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { } +struct GCLimitReached {}; + +struct LocalStore::GCState { + GCOptions options; + GCResults& results; + PathSet roots; + PathSet tempRoots; + PathSet dead; + PathSet alive; + bool gcKeepOutputs; + bool gcKeepDerivations; + unsigned long long bytesInvalidated; + bool moveToTrash = true; + bool shouldDelete; + GCState(GCResults& results_) : results(results_), bytesInvalidated(0) {} }; - -bool LocalStore::isActiveTempFile(const GCState & state, - const Path & path, const string & suffix) -{ - return hasSuffix(path, suffix) - && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end(); +bool LocalStore::isActiveTempFile(const GCState& state, const Path& path, + const string& suffix) { + return hasSuffix(path, suffix) && + state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != + state.tempRoots.end(); } - -void LocalStore::deleteGarbage(GCState & state, const Path & path) -{ - unsigned long long bytesFreed; - deletePath(path, bytesFreed); - state.results.bytesFreed += bytesFreed; +void LocalStore::deleteGarbage(GCState& state, const Path& path) { + unsigned long long bytesFreed; + deletePath(path, bytesFreed); + state.results.bytesFreed += bytesFreed; } - -void LocalStore::deletePathRecursive(GCState & state, const Path & path) -{ - checkInterrupt(); - - unsigned long long size = 0; - - if (isStorePath(path) && isValidPath(path)) { - PathSet referrers; - queryReferrers(path, referrers); - for (auto & i : referrers) - if (i != path) deletePathRecursive(state, i); - size = queryPathInfo(path)->narSize; - invalidatePathChecked(path); - } - - Path realPath = realStoreDir + "/" + baseNameOf(path); - - struct stat st; - if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT) return; - throw SysError(format("getting status of %1%") % realPath); - } - - printInfo(format("deleting '%1%'") % path); - - state.results.paths.insert(path); - - /* If the path is not a regular file or symlink, move it to the - trash directory. The move is to ensure that later (when we're - not holding the global GC lock) we can delete the path without - being afraid that the path has become alive again. Otherwise - delete it right away. */ - if (state.moveToTrash && S_ISDIR(st.st_mode)) { - // Estimate the amount freed using the narSize field. FIXME: - // if the path was not valid, need to determine the actual - // size. - try { - if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("making '%1%' writable") % realPath); - Path tmp = trashDir + "/" + baseNameOf(path); - if (rename(realPath.c_str(), tmp.c_str())) - throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp); - state.bytesInvalidated += size; - } catch (SysError & e) { - if (e.errNo == ENOSPC) { - printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg()); - deleteGarbage(state, realPath); - } - } - } else +void LocalStore::deletePathRecursive(GCState& state, const Path& path) { + checkInterrupt(); + + unsigned long long size = 0; + + if (isStorePath(path) && isValidPath(path)) { + PathSet referrers; + queryReferrers(path, referrers); + for (auto& i : referrers) + if (i != path) deletePathRecursive(state, i); + size = queryPathInfo(path)->narSize; + invalidatePathChecked(path); + } + + Path realPath = realStoreDir + "/" + baseNameOf(path); + + struct stat st; + if (lstat(realPath.c_str(), &st)) { + if (errno == ENOENT) return; + throw SysError(format("getting status of %1%") % realPath); + } + + printInfo(format("deleting '%1%'") % path); + + state.results.paths.insert(path); + + /* If the path is not a regular file or symlink, move it to the + trash directory. The move is to ensure that later (when we're + not holding the global GC lock) we can delete the path without + being afraid that the path has become alive again. Otherwise + delete it right away. */ + if (state.moveToTrash && S_ISDIR(st.st_mode)) { + // Estimate the amount freed using the narSize field. FIXME: + // if the path was not valid, need to determine the actual + // size. + try { + if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("making '%1%' writable") % realPath); + Path tmp = trashDir + "/" + baseNameOf(path); + if (rename(realPath.c_str(), tmp.c_str())) + throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % + tmp); + state.bytesInvalidated += size; + } catch (SysError& e) { + if (e.errNo == ENOSPC) { + printInfo(format("note: can't create move '%1%': %2%") % realPath % + e.msg()); deleteGarbage(state, realPath); - - if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) { - printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed); - throw GCLimitReached(); + } } + } else + deleteGarbage(state, realPath); + + if (state.results.bytesFreed + state.bytesInvalidated > + state.options.maxFreed) { + printInfo(format("deleted or invalidated more than %1% bytes; stopping") % + state.options.maxFreed); + throw GCLimitReached(); + } } +bool LocalStore::canReachRoot(GCState& state, PathSet& visited, + const Path& path) { + if (visited.count(path)) return false; -bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path) -{ - if (visited.count(path)) return false; + if (state.alive.count(path)) return true; - if (state.alive.count(path)) return true; + if (state.dead.count(path)) return false; - if (state.dead.count(path)) return false; + if (state.roots.count(path)) { + debug(format("cannot delete '%1%' because it's a root") % path); + state.alive.insert(path); + return true; + } - if (state.roots.count(path)) { - debug(format("cannot delete '%1%' because it's a root") % path); - state.alive.insert(path); - return true; - } + visited.insert(path); - visited.insert(path); + if (!isStorePath(path) || !isValidPath(path)) return false; - if (!isStorePath(path) || !isValidPath(path)) return false; + PathSet incoming; - PathSet incoming; + /* Don't delete this path if any of its referrers are alive. */ + queryReferrers(path, incoming); - /* Don't delete this path if any of its referrers are alive. */ - queryReferrers(path, incoming); + /* If keep-derivations is set and this is a derivation, then + don't delete the derivation if any of the outputs are alive. */ + if (state.gcKeepDerivations && isDerivation(path)) { + PathSet outputs = queryDerivationOutputs(path); + for (auto& i : outputs) + if (isValidPath(i) && queryPathInfo(i)->deriver == path) + incoming.insert(i); + } - /* If keep-derivations is set and this is a derivation, then - don't delete the derivation if any of the outputs are alive. */ - if (state.gcKeepDerivations && isDerivation(path)) { - PathSet outputs = queryDerivationOutputs(path); - for (auto & i : outputs) - if (isValidPath(i) && queryPathInfo(i)->deriver == path) - incoming.insert(i); - } + /* If keep-outputs is set, then don't delete this path if there + are derivers of this path that are not garbage. */ + if (state.gcKeepOutputs) { + PathSet derivers = queryValidDerivers(path); + for (auto& i : derivers) incoming.insert(i); + } - /* If keep-outputs is set, then don't delete this path if there - are derivers of this path that are not garbage. */ - if (state.gcKeepOutputs) { - PathSet derivers = queryValidDerivers(path); - for (auto & i : derivers) - incoming.insert(i); - } - - for (auto & i : incoming) - if (i != path) - if (canReachRoot(state, visited, i)) { - state.alive.insert(path); - return true; - } + for (auto& i : incoming) + if (i != path) + if (canReachRoot(state, visited, i)) { + state.alive.insert(path); + return true; + } - return false; + return false; } - -void LocalStore::tryToDelete(GCState & state, const Path & path) -{ - checkInterrupt(); - - auto realPath = realStoreDir + "/" + baseNameOf(path); - if (realPath == linksDir || realPath == trashDir) return; - - //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); - - if (!isStorePath(path) || !isValidPath(path)) { - /* A lock file belonging to a path that we're building right - now isn't garbage. */ - if (isActiveTempFile(state, path, ".lock")) return; - - /* Don't delete .chroot directories for derivations that are - currently being built. */ - if (isActiveTempFile(state, path, ".chroot")) return; - - /* Don't delete .check directories for derivations that are - currently being built, because we may need to run - diff-hook. */ - if (isActiveTempFile(state, path, ".check")) return; - } - - PathSet visited; - - if (canReachRoot(state, visited, path)) { - debug(format("cannot delete '%1%' because it's still reachable") % path); - } else { - /* No path we visited was a root, so everything is garbage. - But we only delete ‘path’ and its referrers here so that - ‘nix-store --delete’ doesn't have the unexpected effect of - recursing into derivations and outputs. */ - state.dead.insert(visited.begin(), visited.end()); - if (state.shouldDelete) - deletePathRecursive(state, path); - } +void LocalStore::tryToDelete(GCState& state, const Path& path) { + checkInterrupt(); + + auto realPath = realStoreDir + "/" + baseNameOf(path); + if (realPath == linksDir || realPath == trashDir) return; + + // Activity act(*logger, lvlDebug, format("considering whether to delete + // '%1%'") % path); + + if (!isStorePath(path) || !isValidPath(path)) { + /* A lock file belonging to a path that we're building right + now isn't garbage. */ + if (isActiveTempFile(state, path, ".lock")) return; + + /* Don't delete .chroot directories for derivations that are + currently being built. */ + if (isActiveTempFile(state, path, ".chroot")) return; + + /* Don't delete .check directories for derivations that are + currently being built, because we may need to run + diff-hook. */ + if (isActiveTempFile(state, path, ".check")) return; + } + + PathSet visited; + + if (canReachRoot(state, visited, path)) { + debug(format("cannot delete '%1%' because it's still reachable") % path); + } else { + /* No path we visited was a root, so everything is garbage. + But we only delete ‘path’ and its referrers here so that + ‘nix-store --delete’ doesn't have the unexpected effect of + recursing into derivations and outputs. */ + state.dead.insert(visited.begin(), visited.end()); + if (state.shouldDelete) deletePathRecursive(state, path); + } } - /* Unlink all files in /nix/store/.links that have a link count of 1, which indicates that there are no other links and so they can be safely deleted. FIXME: race condition with optimisePath(): we might see a link count of 1 just before optimisePath() increases the link count. */ -void LocalStore::removeUnusedLinks(const GCState & state) -{ - AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); +void LocalStore::removeUnusedLinks(const GCState& state) { + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); - long long actualSize = 0, unsharedSize = 0; + long long actualSize = 0, unsharedSize = 0; - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { - checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = linksDir + "/" + name; + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; - struct stat st; - if (lstat(path.c_str(), &st) == -1) - throw SysError(format("statting '%1%'") % path); + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("statting '%1%'") % path); - if (st.st_nlink != 1) { - actualSize += st.st_size; - unsharedSize += (st.st_nlink - 1) * st.st_size; - continue; - } + if (st.st_nlink != 1) { + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; + continue; + } - printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); + printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); - if (unlink(path.c_str()) == -1) - throw SysError(format("deleting '%1%'") % path); + if (unlink(path.c_str()) == -1) + throw SysError(format("deleting '%1%'") % path); - state.results.bytesFreed += st.st_size; - } + state.results.bytesFreed += st.st_size; + } - struct stat st; - if (stat(linksDir.c_str(), &st) == -1) - throw SysError(format("statting '%1%'") % linksDir); - long long overhead = st.st_blocks * 512ULL; + struct stat st; + if (stat(linksDir.c_str(), &st) == -1) + throw SysError(format("statting '%1%'") % linksDir); + long long overhead = st.st_blocks * 512ULL; - printInfo(format("note: currently hard linking saves %.2f MiB") - % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); + printInfo(format("note: currently hard linking saves %.2f MiB") % + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); } - -void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) -{ - GCState state(results); - state.options = options; - state.gcKeepOutputs = settings.gcKeepOutputs; - state.gcKeepDerivations = settings.gcKeepDerivations; - - /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `keep-outputs' or `keep-derivations' are true - (the garbage collector will recurse into deleting the outputs - or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - state.gcKeepOutputs = false; - state.gcKeepDerivations = false; +void LocalStore::collectGarbage(const GCOptions& options, GCResults& results) { + GCState state(results); + state.options = options; + state.gcKeepOutputs = settings.gcKeepOutputs; + state.gcKeepDerivations = settings.gcKeepDerivations; + + /* Using `--ignore-liveness' with `--delete' can have unintended + consequences if `keep-outputs' or `keep-derivations' are true + (the garbage collector will recurse into deleting the outputs + or derivers, respectively). So disable them. */ + if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + state.gcKeepOutputs = false; + state.gcKeepDerivations = false; + } + + state.shouldDelete = options.action == GCOptions::gcDeleteDead || + options.action == GCOptions::gcDeleteSpecific; + + if (state.shouldDelete) deletePath(reservedPath); + + /* Acquire the global GC root. This prevents + a) New roots from being added. + b) Processes from creating new temporary root files. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); + + /* Find the roots. Since we've grabbed the GC lock, the set of + permanent roots cannot increase now. */ + printError(format("finding garbage collector roots...")); + Roots rootMap; + if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true); + + for (auto& i : rootMap) state.roots.insert(i.first); + + /* Read the temporary roots. This acquires read locks on all + per-process temporary root files. So after this point no paths + can be added to the set of temporary roots. */ + FDs fds; + Roots tempRoots; + findTempRoots(fds, tempRoots, true); + for (auto& root : tempRoots) state.tempRoots.insert(root.first); + state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); + + /* After this point the set of roots or temporary roots cannot + increase, since we hold locks on everything. So everything + that is not reachable from `roots' is garbage. */ + + if (state.shouldDelete) { + if (pathExists(trashDir)) deleteGarbage(state, trashDir); + try { + createDirs(trashDir); + } catch (SysError& e) { + if (e.errNo == ENOSPC) { + printInfo(format("note: can't create trash directory: %1%") % e.msg()); + state.moveToTrash = false; + } } - - state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; - - if (state.shouldDelete) - deletePath(reservedPath); - - /* Acquire the global GC root. This prevents - a) New roots from being added. - b) Processes from creating new temporary root files. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); - - /* Find the roots. Since we've grabbed the GC lock, the set of - permanent roots cannot increase now. */ - printError(format("finding garbage collector roots...")); - Roots rootMap; - if (!options.ignoreLiveness) - findRootsNoTemp(rootMap, true); - - for (auto & i : rootMap) state.roots.insert(i.first); - - /* Read the temporary roots. This acquires read locks on all - per-process temporary root files. So after this point no paths - can be added to the set of temporary roots. */ - FDs fds; - Roots tempRoots; - findTempRoots(fds, tempRoots, true); - for (auto & root : tempRoots) - state.tempRoots.insert(root.first); - state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); - - /* After this point the set of roots or temporary roots cannot - increase, since we hold locks on everything. So everything - that is not reachable from `roots' is garbage. */ - - if (state.shouldDelete) { - if (pathExists(trashDir)) deleteGarbage(state, trashDir); - try { - createDirs(trashDir); - } catch (SysError & e) { - if (e.errNo == ENOSPC) { - printInfo(format("note: can't create trash directory: %1%") % e.msg()); - state.moveToTrash = false; - } - } + } + + /* Now either delete all garbage paths, or just the specified + paths (for gcDeleteSpecific). */ + + if (options.action == GCOptions::gcDeleteSpecific) { + for (auto& i : options.pathsToDelete) { + assertStorePath(i); + tryToDelete(state, i); + if (state.dead.find(i) == state.dead.end()) + throw Error(format("cannot delete path '%1%' since it is still alive") % + i); } - /* Now either delete all garbage paths, or just the specified - paths (for gcDeleteSpecific). */ - - if (options.action == GCOptions::gcDeleteSpecific) { - - for (auto & i : options.pathsToDelete) { - assertStorePath(i); - tryToDelete(state, i); - if (state.dead.find(i) == state.dead.end()) - throw Error(format("cannot delete path '%1%' since it is still alive") % i); - } - - } else if (options.maxFreed > 0) { + } else if (options.maxFreed > 0) { + if (state.shouldDelete) + printError(format("deleting garbage...")); + else + printError(format("determining live/dead paths...")); - if (state.shouldDelete) - printError(format("deleting garbage...")); + try { + AutoCloseDir dir(opendir(realStoreDir.c_str())); + if (!dir) + throw SysError(format("opening directory '%1%'") % realStoreDir); + + /* Read the store and immediately delete all paths that + aren't valid. When using --max-freed etc., deleting + invalid paths is preferred over deleting unreachable + paths, since unreachable paths could become reachable + again. We don't use readDirectory() here so that GCing + can start faster. */ + Paths entries; + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = storeDir + "/" + name; + if (isStorePath(path) && isValidPath(path)) + entries.push_back(path); else - printError(format("determining live/dead paths...")); + tryToDelete(state, path); + } - try { + dir.reset(); - AutoCloseDir dir(opendir(realStoreDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir); - - /* Read the store and immediately delete all paths that - aren't valid. When using --max-freed etc., deleting - invalid paths is preferred over deleting unreachable - paths, since unreachable paths could become reachable - again. We don't use readDirectory() here so that GCing - can start faster. */ - Paths entries; - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { - checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = storeDir + "/" + name; - if (isStorePath(path) && isValidPath(path)) - entries.push_back(path); - else - tryToDelete(state, path); - } - - dir.reset(); - - /* Now delete the unreachable valid paths. Randomise the - order in which we delete entries to make the collector - less biased towards deleting paths that come - alphabetically first (e.g. /nix/store/000...). This - matters when using --max-freed etc. */ - vector<Path> entries_(entries.begin(), entries.end()); - std::mt19937 gen(1); - std::shuffle(entries_.begin(), entries_.end(), gen); - - for (auto & i : entries_) - tryToDelete(state, i); - - } catch (GCLimitReached & e) { - } - } + /* Now delete the unreachable valid paths. Randomise the + order in which we delete entries to make the collector + less biased towards deleting paths that come + alphabetically first (e.g. /nix/store/000...). This + matters when using --max-freed etc. */ + vector<Path> entries_(entries.begin(), entries.end()); + std::mt19937 gen(1); + std::shuffle(entries_.begin(), entries_.end(), gen); - if (state.options.action == GCOptions::gcReturnLive) { - state.results.paths = state.alive; - return; - } + for (auto& i : entries_) tryToDelete(state, i); - if (state.options.action == GCOptions::gcReturnDead) { - state.results.paths = state.dead; - return; + } catch (GCLimitReached& e) { } - - /* Allow other processes to add to the store from here on. */ - fdGCLock = -1; - fds.clear(); - - /* Delete the trash directory. */ - printInfo(format("deleting '%1%'") % trashDir); - deleteGarbage(state, trashDir); - - /* Clean up the links directory. */ - if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { - printError(format("deleting unused links...")); - removeUnusedLinks(state); - } - - /* While we're at it, vacuum the database. */ - //if (options.action == GCOptions::gcDeleteDead) vacuumDB(); + } + + if (state.options.action == GCOptions::gcReturnLive) { + state.results.paths = state.alive; + return; + } + + if (state.options.action == GCOptions::gcReturnDead) { + state.results.paths = state.dead; + return; + } + + /* Allow other processes to add to the store from here on. */ + fdGCLock = -1; + fds.clear(); + + /* Delete the trash directory. */ + printInfo(format("deleting '%1%'") % trashDir); + deleteGarbage(state, trashDir); + + /* Clean up the links directory. */ + if (options.action == GCOptions::gcDeleteDead || + options.action == GCOptions::gcDeleteSpecific) { + printError(format("deleting unused links...")); + removeUnusedLinks(state); + } + + /* While we're at it, vacuum the database. */ + // if (options.action == GCOptions::gcDeleteDead) vacuumDB(); } +void LocalStore::autoGC(bool sync) { + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", ""); -void LocalStore::autoGC(bool sync) -{ - static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", ""); + auto getAvail = [this]() -> uint64_t { + if (!fakeFreeSpaceFile.empty()) + return std::stoll(readFile(fakeFreeSpaceFile)); - auto getAvail = [this]() -> uint64_t { - if (!fakeFreeSpaceFile.empty()) - return std::stoll(readFile(fakeFreeSpaceFile)); + struct statvfs st; + if (statvfs(realStoreDir.c_str(), &st)) + throw SysError("getting filesystem info about '%s'", realStoreDir); - struct statvfs st; - if (statvfs(realStoreDir.c_str(), &st)) - throw SysError("getting filesystem info about '%s'", realStoreDir); + return (uint64_t)st.f_bavail * st.f_bsize; + }; - return (uint64_t) st.f_bavail * st.f_bsize; - }; + std::shared_future<void> future; - std::shared_future<void> future; - - { - auto state(_state.lock()); - - if (state->gcRunning) { - future = state->gcFuture; - debug("waiting for auto-GC to finish"); - goto sync; - } - - auto now = std::chrono::steady_clock::now(); - - if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return; + { + auto state(_state.lock()); - auto avail = getAvail(); + if (state->gcRunning) { + future = state->gcFuture; + debug("waiting for auto-GC to finish"); + goto sync; + } - state->lastGCCheck = now; + auto now = std::chrono::steady_clock::now(); - if (avail >= settings.minFree || avail >= settings.maxFree) return; + if (now < state->lastGCCheck + + std::chrono::seconds(settings.minFreeCheckInterval)) + return; - if (avail > state->availAfterGC * 0.97) return; + auto avail = getAvail(); - state->gcRunning = true; + state->lastGCCheck = now; - std::promise<void> promise; - future = state->gcFuture = promise.get_future().share(); + if (avail >= settings.minFree || avail >= settings.maxFree) return; - std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable { + if (avail > state->availAfterGC * 0.97) return; - try { + state->gcRunning = true; - /* Wake up any threads waiting for the auto-GC to finish. */ - Finally wakeup([&]() { - auto state(_state.lock()); - state->gcRunning = false; - state->lastGCCheck = std::chrono::steady_clock::now(); - promise.set_value(); - }); + std::promise<void> promise; + future = state->gcFuture = promise.get_future().share(); - GCOptions options; - options.maxFreed = settings.maxFree - avail; + std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable { + try { + /* Wake up any threads waiting for the auto-GC to finish. */ + Finally wakeup([&]() { + auto state(_state.lock()); + state->gcRunning = false; + state->lastGCCheck = std::chrono::steady_clock::now(); + promise.set_value(); + }); - printInfo("running auto-GC to free %d bytes", options.maxFreed); + GCOptions options; + options.maxFreed = settings.maxFree - avail; - GCResults results; + printInfo("running auto-GC to free %d bytes", options.maxFreed); - collectGarbage(options, results); + GCResults results; - _state.lock()->availAfterGC = getAvail(); + collectGarbage(options, results); - } catch (...) { - // FIXME: we could propagate the exception to the - // future, but we don't really care. - ignoreException(); - } + _state.lock()->availAfterGC = getAvail(); - }).detach(); - } + } catch (...) { + // FIXME: we could propagate the exception to the + // future, but we don't really care. + ignoreException(); + } + }).detach(); + } - sync: - // Wait for the future outside of the state lock. - if (sync) future.get(); +sync: + // Wait for the future outside of the state lock. + if (sync) future.get(); } - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/globals.cc b/third_party/nix/src/libstore/globals.cc index 1c2c08715a14..62a8a167874b 100644 --- a/third_party/nix/src/libstore/globals.cc +++ b/third_party/nix/src/libstore/globals.cc @@ -1,17 +1,14 @@ #include "globals.hh" -#include "util.hh" -#include "archive.hh" -#include "args.hh" - +#include <dlfcn.h> #include <algorithm> #include <map> #include <thread> -#include <dlfcn.h> - +#include "archive.hh" +#include "args.hh" +#include "util.hh" namespace nix { - /* The default location of the daemon socket, relative to nixStateDir. The socket is in a directory to allow you to control access to the Nix daemon by setting the mode/ownership of the directory @@ -21,9 +18,9 @@ namespace nix { /* chroot-like behavior from Apple's sandbox */ #if __APPLE__ - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" +#define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" #else - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" +#define DEFAULT_ALLOWED_IMPURE_PREFIXES "" #endif Settings settings; @@ -31,157 +28,163 @@ Settings settings; static GlobalConfig::Register r1(&settings); Settings::Settings() - : nixPrefix(NIX_PREFIX) - , nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))) - , nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))) - , nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))) - , nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))) - , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))) - , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))) - , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))) - , nixManDir(canonPath(NIX_MAN_DIR)) - , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) -{ - buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; - - caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "")); - if (caFile == "") { - for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) - if (pathExists(fn)) { - caFile = fn; - break; - } - } - - /* Backwards compatibility. */ - auto s = getEnv("NIX_REMOTE_SYSTEMS"); - if (s != "") { - Strings ss; - for (auto & p : tokenizeString<Strings>(s, ":")) - ss.push_back("@" + p); - builders = concatStringsSep(" ", ss); - } + : nixPrefix(NIX_PREFIX), + nixStore(canonPath( + getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))), + nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))), + nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))), + nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))), + nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))), + nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))), + nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))), + nixManDir(canonPath(NIX_MAN_DIR)), + nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) { + buildUsersGroup = getuid() == 0 ? "nixbld" : ""; + lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; + + caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "")); + if (caFile == "") { + for (auto& fn : + {"/etc/ssl/certs/ca-certificates.crt", + "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) + if (pathExists(fn)) { + caFile = fn; + break; + } + } + + /* Backwards compatibility. */ + auto s = getEnv("NIX_REMOTE_SYSTEMS"); + if (s != "") { + Strings ss; + for (auto& p : tokenizeString<Strings>(s, ":")) ss.push_back("@" + p); + builders = concatStringsSep(" ", ss); + } #if defined(__linux__) && defined(SANDBOX_SHELL) - sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL); + sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL); #endif - allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES); + allowedImpureHostPrefixes = + tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES); } -void loadConfFile() -{ - globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); +void loadConfFile() { + globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); - /* We only want to send overrides to the daemon, i.e. stuff from - ~/.nix/nix.conf or the command line. */ - globalConfig.resetOverriden(); + /* We only want to send overrides to the daemon, i.e. stuff from + ~/.nix/nix.conf or the command line. */ + globalConfig.resetOverriden(); - auto dirs = getConfigDirs(); - // Iterate over them in reverse so that the ones appearing first in the path take priority - for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { - globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); - } + auto dirs = getConfigDirs(); + // Iterate over them in reverse so that the ones appearing first in the path + // take priority + for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { + globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); + } } -unsigned int Settings::getDefaultCores() -{ - return std::max(1U, std::thread::hardware_concurrency()); +unsigned int Settings::getDefaultCores() { + return std::max(1U, std::thread::hardware_concurrency()); } -StringSet Settings::getDefaultSystemFeatures() -{ - /* For backwards compatibility, accept some "features" that are - used in Nixpkgs to route builds to certain machines but don't - actually require anything special on the machines. */ - StringSet features{"nixos-test", "benchmark", "big-parallel"}; +StringSet Settings::getDefaultSystemFeatures() { + /* For backwards compatibility, accept some "features" that are + used in Nixpkgs to route builds to certain machines but don't + actually require anything special on the machines. */ + StringSet features{"nixos-test", "benchmark", "big-parallel"}; - #if __linux__ - if (access("/dev/kvm", R_OK | W_OK) == 0) - features.insert("kvm"); - #endif +#if __linux__ + if (access("/dev/kvm", R_OK | W_OK) == 0) features.insert("kvm"); +#endif - return features; + return features; } const string nixVersion = PACKAGE_VERSION; -template<> void BaseSetting<SandboxMode>::set(const std::string & str) -{ - if (str == "true") value = smEnabled; - else if (str == "relaxed") value = smRelaxed; - else if (str == "false") value = smDisabled; - else throw UsageError("option '%s' has invalid value '%s'", name, str); +template <> +void BaseSetting<SandboxMode>::set(const std::string& str) { + if (str == "true") + value = smEnabled; + else if (str == "relaxed") + value = smRelaxed; + else if (str == "false") + value = smDisabled; + else + throw UsageError("option '%s' has invalid value '%s'", name, str); } -template<> std::string BaseSetting<SandboxMode>::to_string() -{ - if (value == smEnabled) return "true"; - else if (value == smRelaxed) return "relaxed"; - else if (value == smDisabled) return "false"; - else abort(); +template <> +std::string BaseSetting<SandboxMode>::to_string() { + if (value == smEnabled) + return "true"; + else if (value == smRelaxed) + return "relaxed"; + else if (value == smDisabled) + return "false"; + else + abort(); } -template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out) -{ - AbstractSetting::toJSON(out); +template <> +void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder& out) { + AbstractSetting::toJSON(out); } -template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category) -{ - args.mkFlag() - .longName(name) - .description("Enable sandboxing.") - .handler([=](std::vector<std::string> ss) { override(smEnabled); }) - .category(category); - args.mkFlag() - .longName("no-" + name) - .description("Disable sandboxing.") - .handler([=](std::vector<std::string> ss) { override(smDisabled); }) - .category(category); - args.mkFlag() - .longName("relaxed-" + name) - .description("Enable sandboxing, but allow builds to disable it.") - .handler([=](std::vector<std::string> ss) { override(smRelaxed); }) - .category(category); +template <> +void BaseSetting<SandboxMode>::convertToArg(Args& args, + const std::string& category) { + args.mkFlag() + .longName(name) + .description("Enable sandboxing.") + .handler([=](std::vector<std::string> ss) { override(smEnabled); }) + .category(category); + args.mkFlag() + .longName("no-" + name) + .description("Disable sandboxing.") + .handler([=](std::vector<std::string> ss) { override(smDisabled); }) + .category(category); + args.mkFlag() + .longName("relaxed-" + name) + .description("Enable sandboxing, but allow builds to disable it.") + .handler([=](std::vector<std::string> ss) { override(smRelaxed); }) + .category(category); } -void MaxBuildJobsSetting::set(const std::string & str) -{ - if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); - else if (!string2Int(str, value)) - throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); +void MaxBuildJobsSetting::set(const std::string& str) { + if (str == "auto") + value = std::max(1U, std::thread::hardware_concurrency()); + else if (!string2Int(str, value)) + throw UsageError( + "configuration setting '%s' should be 'auto' or an integer", name); } - -void initPlugins() -{ - for (const auto & pluginFile : settings.pluginFiles.get()) { - Paths pluginFiles; - try { - auto ents = readDirectory(pluginFile); - for (const auto & ent : ents) - pluginFiles.emplace_back(pluginFile + "/" + ent.name); - } catch (SysError & e) { - if (e.errNo != ENOTDIR) - throw; - pluginFiles.emplace_back(pluginFile); - } - for (const auto & file : pluginFiles) { - /* handle is purposefully leaked as there may be state in the - DSO needed by the action of the plugin. */ - void *handle = - dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); - if (!handle) - throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); - } +void initPlugins() { + for (const auto& pluginFile : settings.pluginFiles.get()) { + Paths pluginFiles; + try { + auto ents = readDirectory(pluginFile); + for (const auto& ent : ents) + pluginFiles.emplace_back(pluginFile + "/" + ent.name); + } catch (SysError& e) { + if (e.errNo != ENOTDIR) throw; + pluginFiles.emplace_back(pluginFile); } + for (const auto& file : pluginFiles) { + /* handle is purposefully leaked as there may be state in the + DSO needed by the action of the plugin. */ + void* handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw Error("could not dynamically open plugin file '%s': %s", file, + dlerror()); + } + } - /* Since plugins can add settings, try to re-apply previously - unknown settings. */ - globalConfig.reapplyUnknownSettings(); - globalConfig.warnUnknownSettings(); + /* Since plugins can add settings, try to re-apply previously + unknown settings. */ + globalConfig.reapplyUnknownSettings(); + globalConfig.warnUnknownSettings(); } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/globals.hh b/third_party/nix/src/libstore/globals.hh index 4ad9f65197ef..0862ae986482 100644 --- a/third_party/nix/src/libstore/globals.hh +++ b/third_party/nix/src/libstore/globals.hh @@ -1,361 +1,475 @@ #pragma once -#include "types.hh" +#include <sys/types.h> +#include <limits> +#include <map> #include "config.hh" +#include "types.hh" #include "util.hh" -#include <map> -#include <limits> - -#include <sys/types.h> - namespace nix { typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; -struct MaxBuildJobsSetting : public BaseSetting<unsigned int> -{ - MaxBuildJobsSetting(Config * options, - unsigned int def, - const std::string & name, - const std::string & description, - const std::set<std::string> & aliases = {}) - : BaseSetting<unsigned int>(def, name, description, aliases) - { - options->addSetting(this); - } +struct MaxBuildJobsSetting : public BaseSetting<unsigned int> { + MaxBuildJobsSetting(Config* options, unsigned int def, + const std::string& name, const std::string& description, + const std::set<std::string>& aliases = {}) + : BaseSetting<unsigned int>(def, name, description, aliases) { + options->addSetting(this); + } - void set(const std::string & str) override; + void set(const std::string& str) override; }; class Settings : public Config { + unsigned int getDefaultCores(); - unsigned int getDefaultCores(); - - StringSet getDefaultSystemFeatures(); - -public: - - Settings(); - - Path nixPrefix; - - /* The directory where we store sources and derived files. */ - Path nixStore; - - Path nixDataDir; /* !!! fix */ - - /* The directory where we log various operations. */ - Path nixLogDir; - - /* The directory where state is stored. */ - Path nixStateDir; - - /* The directory where configuration files are stored. */ - Path nixConfDir; - - /* The directory where internal helper programs are stored. */ - Path nixLibexecDir; - - /* The directory where the main programs are stored. */ - Path nixBinDir; - - /* The directory where the man pages are stored. */ - Path nixManDir; - - /* File name of the socket the daemon listens to. */ - Path nixDaemonSocketFile; - - Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store", - "The default Nix store to use."}; - - Setting<bool> keepFailed{this, false, "keep-failed", - "Whether to keep temporary directories of failed builds."}; - - Setting<bool> keepGoing{this, false, "keep-going", - "Whether to keep building derivations when another build fails."}; - - Setting<bool> tryFallback{this, false, "fallback", - "Whether to fall back to building when substitution fails.", - {"build-fallback"}}; - - /* Whether to show build log output in real time. */ - bool verboseBuild = true; - - Setting<size_t> logLines{this, 10, "log-lines", - "If verbose-build is false, the number of lines of the tail of " - "the log to show if a build fails."}; - - MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs", - "Maximum number of parallel build jobs. \"auto\" means use number of cores.", - {"build-max-jobs"}}; - - Setting<unsigned int> buildCores{this, getDefaultCores(), "cores", - "Number of CPU cores to utilize in parallel within a build, " - "i.e. by passing this number to Make via '-j'. 0 means that the " - "number of actual CPU cores on the local host ought to be " - "auto-detected.", {"build-cores"}}; - - /* Read-only mode. Don't copy stuff to the store, don't change - the database. */ - bool readOnlyMode = false; - - Setting<std::string> thisSystem{this, SYSTEM, "system", - "The canonical Nix system name."}; - - Setting<time_t> maxSilentTime{this, 0, "max-silent-time", - "The maximum time in seconds that a builer can go without " - "producing any output on stdout/stderr before it is killed. " - "0 means infinity.", - {"build-max-silent-time"}}; - - Setting<time_t> buildTimeout{this, 0, "timeout", - "The maximum duration in seconds that a builder can run. " - "0 means infinity.", {"build-timeout"}}; - - PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", - "The path of the helper program that executes builds to remote machines."}; - - Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders", - "A semicolon-separated list of build machines, in the format of nix.machines."}; - - Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes", - "Whether build machines should use their own substitutes for obtaining " - "build dependencies if possible, rather than waiting for this host to " - "upload them."}; - - Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", - "Amount of reserved disk space for the garbage collector."}; - - Setting<bool> fsyncMetadata{this, true, "fsync-metadata", - "Whether SQLite should use fsync()."}; - - Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal", - "Whether SQLite should use WAL mode."}; - - Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering", - "Whether to call sync() before registering a path as valid."}; - - Setting<bool> useSubstitutes{this, true, "substitute", - "Whether to use substitutes.", - {"build-use-substitutes"}}; + StringSet getDefaultSystemFeatures(); + + public: + Settings(); - Setting<std::string> buildUsersGroup{this, "", "build-users-group", - "The Unix group that contains the build users."}; - - Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26", - "Whether to impersonate a Linux 2.6 machine on newer kernels.", - {"build-impersonate-linux-26"}}; - - Setting<bool> keepLog{this, true, "keep-build-log", - "Whether to store build logs.", - {"build-keep-log"}}; - - Setting<bool> compressLog{this, true, "compress-build-log", - "Whether to compress logs.", - {"build-compress-log"}}; - - Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size", - "Maximum number of bytes a builder can write to stdout/stderr " - "before being killed (0 means no limit).", - {"build-max-log-size"}}; - - /* When buildRepeat > 0 and verboseBuild == true, whether to print - repeated builds (i.e. builds other than the first one) to - stderr. Hack to prevent Hydra logs from being polluted. */ - bool printRepeatedBuilds = true; - - Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", - "How often (in seconds) to poll for locks."}; - - Setting<bool> checkRootReachability{this, false, "gc-check-reachability", - "Whether to check if new GC roots can in fact be found by the " - "garbage collector."}; - - Setting<bool> gcKeepOutputs{this, false, "keep-outputs", - "Whether the garbage collector should keep outputs of live derivations.", - {"gc-keep-outputs"}}; - - Setting<bool> gcKeepDerivations{this, true, "keep-derivations", - "Whether the garbage collector should keep derivers of live paths.", - {"gc-keep-derivations"}}; - - Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store", - "Whether to automatically replace files with identical contents with hard links."}; - - Setting<bool> envKeepDerivations{this, false, "keep-env-derivations", - "Whether to add derivations as a dependency of user environments " - "(to prevent them from being GCed).", - {"env-keep-derivations"}}; - - /* Whether to lock the Nix client and worker to the same CPU. */ - bool lockCPU; - - /* Whether to show a stack trace if Nix evaluation fails. */ - Setting<bool> showTrace{this, false, "show-trace", - "Whether to show a stack trace on evaluation errors."}; - - Setting<SandboxMode> sandboxMode{this, - #if __linux__ - smEnabled - #else - smDisabled - #endif - , "sandbox", - "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", - {"build-use-chroot", "build-use-sandbox"}}; - - Setting<PathSet> sandboxPaths{this, {}, "sandbox-paths", - "The paths to make available inside the build sandbox.", - {"build-chroot-dirs", "build-sandbox-paths"}}; - - Setting<bool> sandboxFallback{this, true, "sandbox-fallback", - "Whether to disable sandboxing when the kernel doesn't allow it."}; - - Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths", - "Additional paths to make available inside the build sandbox.", - {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; - - Setting<size_t> buildRepeat{this, 0, "repeat", - "The number of times to repeat a build in order to verify determinism.", - {"build-repeat"}}; + Path nixPrefix; + + /* The directory where we store sources and derived files. */ + Path nixStore; + + Path nixDataDir; /* !!! fix */ + + /* The directory where we log various operations. */ + Path nixLogDir; + + /* The directory where state is stored. */ + Path nixStateDir; + + /* The directory where configuration files are stored. */ + Path nixConfDir; + + /* The directory where internal helper programs are stored. */ + Path nixLibexecDir; + + /* The directory where the main programs are stored. */ + Path nixBinDir; + + /* The directory where the man pages are stored. */ + Path nixManDir; + + /* File name of the socket the daemon listens to. */ + Path nixDaemonSocketFile; + + Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store", + "The default Nix store to use."}; + + Setting<bool> keepFailed{ + this, false, "keep-failed", + "Whether to keep temporary directories of failed builds."}; + + Setting<bool> keepGoing{ + this, false, "keep-going", + "Whether to keep building derivations when another build fails."}; + + Setting<bool> tryFallback{ + this, + false, + "fallback", + "Whether to fall back to building when substitution fails.", + {"build-fallback"}}; + + /* Whether to show build log output in real time. */ + bool verboseBuild = true; + + Setting<size_t> logLines{ + this, 10, "log-lines", + "If verbose-build is false, the number of lines of the tail of " + "the log to show if a build fails."}; + + MaxBuildJobsSetting maxBuildJobs{this, + 1, + "max-jobs", + "Maximum number of parallel build jobs. " + "\"auto\" means use number of cores.", + {"build-max-jobs"}}; + + Setting<unsigned int> buildCores{ + this, + getDefaultCores(), + "cores", + "Number of CPU cores to utilize in parallel within a build, " + "i.e. by passing this number to Make via '-j'. 0 means that the " + "number of actual CPU cores on the local host ought to be " + "auto-detected.", + {"build-cores"}}; + + /* Read-only mode. Don't copy stuff to the store, don't change + the database. */ + bool readOnlyMode = false; + + Setting<std::string> thisSystem{this, SYSTEM, "system", + "The canonical Nix system name."}; + + Setting<time_t> maxSilentTime{ + this, + 0, + "max-silent-time", + "The maximum time in seconds that a builer can go without " + "producing any output on stdout/stderr before it is killed. " + "0 means infinity.", + {"build-max-silent-time"}}; + + Setting<time_t> buildTimeout{ + this, + 0, + "timeout", + "The maximum duration in seconds that a builder can run. " + "0 means infinity.", + {"build-timeout"}}; + + PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", + "build-hook", + "The path of the helper program that executes builds " + "to remote machines."}; + + Setting<std::string> builders{this, "@" + nixConfDir + "/machines", + "builders", + "A semicolon-separated list of build machines, " + "in the format of nix.machines."}; + + Setting<bool> buildersUseSubstitutes{ + this, false, "builders-use-substitutes", + "Whether build machines should use their own substitutes for obtaining " + "build dependencies if possible, rather than waiting for this host to " + "upload them."}; + + Setting<off_t> reservedSize{ + this, 8 * 1024 * 1024, "gc-reserved-space", + "Amount of reserved disk space for the garbage collector."}; + + Setting<bool> fsyncMetadata{this, true, "fsync-metadata", + "Whether SQLite should use fsync()."}; + + Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal", + "Whether SQLite should use WAL mode."}; + + Setting<bool> syncBeforeRegistering{ + this, false, "sync-before-registering", + "Whether to call sync() before registering a path as valid."}; + + Setting<bool> useSubstitutes{this, + true, + "substitute", + "Whether to use substitutes.", + {"build-use-substitutes"}}; + + Setting<std::string> buildUsersGroup{ + this, "", "build-users-group", + "The Unix group that contains the build users."}; + + Setting<bool> impersonateLinux26{ + this, + false, + "impersonate-linux-26", + "Whether to impersonate a Linux 2.6 machine on newer kernels.", + {"build-impersonate-linux-26"}}; + + Setting<bool> keepLog{this, + true, + "keep-build-log", + "Whether to store build logs.", + {"build-keep-log"}}; + + Setting<bool> compressLog{this, + true, + "compress-build-log", + "Whether to compress logs.", + {"build-compress-log"}}; + + Setting<unsigned long> maxLogSize{ + this, + 0, + "max-build-log-size", + "Maximum number of bytes a builder can write to stdout/stderr " + "before being killed (0 means no limit).", + {"build-max-log-size"}}; + + /* When buildRepeat > 0 and verboseBuild == true, whether to print + repeated builds (i.e. builds other than the first one) to + stderr. Hack to prevent Hydra logs from being polluted. */ + bool printRepeatedBuilds = true; + + Setting<unsigned int> pollInterval{ + this, 5, "build-poll-interval", + "How often (in seconds) to poll for locks."}; + + Setting<bool> checkRootReachability{ + this, false, "gc-check-reachability", + "Whether to check if new GC roots can in fact be found by the " + "garbage collector."}; + + Setting<bool> gcKeepOutputs{ + this, + false, + "keep-outputs", + "Whether the garbage collector should keep outputs of live derivations.", + {"gc-keep-outputs"}}; + + Setting<bool> gcKeepDerivations{ + this, + true, + "keep-derivations", + "Whether the garbage collector should keep derivers of live paths.", + {"gc-keep-derivations"}}; + + Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store", + "Whether to automatically replace files with " + "identical contents with hard links."}; + + Setting<bool> envKeepDerivations{ + this, + false, + "keep-env-derivations", + "Whether to add derivations as a dependency of user environments " + "(to prevent them from being GCed).", + {"env-keep-derivations"}}; + + /* Whether to lock the Nix client and worker to the same CPU. */ + bool lockCPU; + + /* Whether to show a stack trace if Nix evaluation fails. */ + Setting<bool> showTrace{ + this, false, "show-trace", + "Whether to show a stack trace on evaluation errors."}; + + Setting<SandboxMode> sandboxMode { + this, +#if __linux__ + smEnabled +#else + smDisabled +#endif + , + "sandbox", + "Whether to enable sandboxed builds. Can be \"true\", \"false\" or " + "\"relaxed\".", + { + "build-use-chroot", "build-use-sandbox" + } + }; + + Setting<PathSet> sandboxPaths{ + this, + {}, + "sandbox-paths", + "The paths to make available inside the build sandbox.", + {"build-chroot-dirs", "build-sandbox-paths"}}; + + Setting<bool> sandboxFallback{ + this, true, "sandbox-fallback", + "Whether to disable sandboxing when the kernel doesn't allow it."}; + + Setting<PathSet> extraSandboxPaths{ + this, + {}, + "extra-sandbox-paths", + "Additional paths to make available inside the build sandbox.", + {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; + + Setting<size_t> buildRepeat{ + this, + 0, + "repeat", + "The number of times to repeat a build in order to verify determinism.", + {"build-repeat"}}; #if __linux__ - Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size", - "The size of /dev/shm in the build sandbox."}; + Setting<std::string> sandboxShmSize{ + this, "50%", "sandbox-dev-shm-size", + "The size of /dev/shm in the build sandbox."}; - Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir", - "The build directory inside the sandbox."}; + Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir", + "The build directory inside the sandbox."}; #endif - Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps", - "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."}; + Setting<PathSet> allowedImpureHostPrefixes{ + this, + {}, + "allowed-impure-host-deps", + "Which prefixes to allow derivations to ask for access to (primarily for " + "Darwin)."}; #if __APPLE__ - Setting<bool> darwinLogSandboxViolations{this, false, "darwin-log-sandbox-violations", - "Whether to log Darwin sandbox access violations to the system log."}; + Setting<bool> darwinLogSandboxViolations{ + this, false, "darwin-log-sandbox-violations", + "Whether to log Darwin sandbox access violations to the system log."}; #endif - Setting<bool> runDiffHook{this, false, "run-diff-hook", - "Whether to run the program specified by the diff-hook setting " - "repeated builds produce a different result. Typically used to " - "plug in diffoscope."}; - - PathSetting diffHook{this, true, "", "diff-hook", - "A program that prints out the differences between the two paths " - "specified on its command line."}; - - Setting<bool> enforceDeterminism{this, true, "enforce-determinism", - "Whether to fail if repeated builds produce different output."}; - - Setting<Strings> trustedPublicKeys{this, - {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, - "trusted-public-keys", - "Trusted public keys for secure substitution.", - {"binary-cache-public-keys"}}; - - Setting<Strings> secretKeyFiles{this, {}, "secret-key-files", - "Secret keys with which to sign local builds."}; - - Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", - "How long downloaded files are considered up-to-date."}; - - Setting<bool> requireSigs{this, true, "require-sigs", - "Whether to check that any non-content-addressed path added to the " - "Nix store has a valid signature (that is, one signed using a key " - "listed in 'trusted-public-keys'."}; - - Setting<StringSet> extraPlatforms{this, - std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{}, - "extra-platforms", - "Additional platforms that can be built on the local system. " - "These may be supported natively (e.g. armv7 on some aarch64 CPUs " - "or using hacks like qemu-user."}; - - Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), - "system-features", - "Optional features that this system implements (like \"kvm\")."}; - - Setting<Strings> substituters{this, - nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), - "substituters", - "The URIs of substituters (such as https://cache.nixos.org/).", - {"binary-caches"}}; - - // FIXME: provide a way to add to option values. - Setting<Strings> extraSubstituters{this, {}, "extra-substituters", - "Additional URIs of substituters.", - {"extra-binary-caches"}}; - - Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters", - "Disabled substituters that may be enabled via the substituters option by untrusted users.", - {"trusted-binary-caches"}}; - - Setting<Strings> trustedUsers{this, {"root"}, "trusted-users", - "Which users or groups are trusted to ask the daemon to do unsafe things."}; - - Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl", - "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that " - "return an invalid path result"}; - - Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", - "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that " - "return a valid path result."}; - - /* ?Who we trust to use the daemon in safe ways */ - Setting<Strings> allowedUsers{this, {"*"}, "allowed-users", - "Which users or groups are allowed to connect to the daemon."}; - - Setting<bool> printMissing{this, true, "print-missing", - "Whether to print what paths need to be built or downloaded."}; - - Setting<std::string> preBuildHook{this, + Setting<bool> runDiffHook{ + this, false, "run-diff-hook", + "Whether to run the program specified by the diff-hook setting " + "repeated builds produce a different result. Typically used to " + "plug in diffoscope."}; + + PathSetting diffHook{ + this, true, "", "diff-hook", + "A program that prints out the differences between the two paths " + "specified on its command line."}; + + Setting<bool> enforceDeterminism{ + this, true, "enforce-determinism", + "Whether to fail if repeated builds produce different output."}; + + Setting<Strings> trustedPublicKeys{ + this, + {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, + "trusted-public-keys", + "Trusted public keys for secure substitution.", + {"binary-cache-public-keys"}}; + + Setting<Strings> secretKeyFiles{ + this, + {}, + "secret-key-files", + "Secret keys with which to sign local builds."}; + + Setting<unsigned int> tarballTtl{ + this, 60 * 60, "tarball-ttl", + "How long downloaded files are considered up-to-date."}; + + Setting<bool> requireSigs{ + this, true, "require-sigs", + "Whether to check that any non-content-addressed path added to the " + "Nix store has a valid signature (that is, one signed using a key " + "listed in 'trusted-public-keys'."}; + + Setting<StringSet> extraPlatforms{ + this, + std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} + : StringSet{}, + "extra-platforms", + "Additional platforms that can be built on the local system. " + "These may be supported natively (e.g. armv7 on some aarch64 CPUs " + "or using hacks like qemu-user."}; + + Setting<StringSet> systemFeatures{ + this, getDefaultSystemFeatures(), "system-features", + "Optional features that this system implements (like \"kvm\")."}; + + Setting<Strings> substituters{ + this, + nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} + : Strings(), + "substituters", + "The URIs of substituters (such as https://cache.nixos.org/).", + {"binary-caches"}}; + + // FIXME: provide a way to add to option values. + Setting<Strings> extraSubstituters{this, + {}, + "extra-substituters", + "Additional URIs of substituters.", + {"extra-binary-caches"}}; + + Setting<StringSet> trustedSubstituters{ + this, + {}, + "trusted-substituters", + "Disabled substituters that may be enabled via the substituters option " + "by untrusted users.", + {"trusted-binary-caches"}}; + + Setting<Strings> trustedUsers{this, + {"root"}, + "trusted-users", + "Which users or groups are trusted to ask the " + "daemon to do unsafe things."}; + + Setting<unsigned int> ttlNegativeNarInfoCache{ + this, 3600, "narinfo-cache-negative-ttl", + "The TTL in seconds for negative lookups in the disk cache i.e binary " + "cache lookups that " + "return an invalid path result"}; + + Setting<unsigned int> ttlPositiveNarInfoCache{ + this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", + "The TTL in seconds for positive lookups in the disk cache i.e binary " + "cache lookups that " + "return a valid path result."}; + + /* ?Who we trust to use the daemon in safe ways */ + Setting<Strings> allowedUsers{ + this, + {"*"}, + "allowed-users", + "Which users or groups are allowed to connect to the daemon."}; + + Setting<bool> printMissing{ + this, true, "print-missing", + "Whether to print what paths need to be built or downloaded."}; + + Setting<std::string> preBuildHook { + this, #if __APPLE__ nixLibexecDir + "/nix/resolve-system-dependencies", #else "", #endif "pre-build-hook", - "A program to run just before a build to set derivation-specific build settings."}; + "A program to run just before a build to set derivation-specific build " + "settings." + }; - Setting<std::string> postBuildHook{this, "", "post-build-hook", - "A program to run just after each successful build."}; + Setting<std::string> postBuildHook{ + this, "", "post-build-hook", + "A program to run just after each successful build."}; - Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", - "Path to the netrc file used to obtain usernames/passwords for downloads."}; + Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), + "netrc-file", + "Path to the netrc file used to obtain " + "usernames/passwords for downloads."}; - /* Path to the SSL CA file used */ - Path caFile; + /* Path to the SSL CA file used */ + Path caFile; #if __linux__ - Setting<bool> filterSyscalls{this, true, "filter-syscalls", - "Whether to prevent certain dangerous system calls, such as " - "creation of setuid/setgid files or adding ACLs or extended " - "attributes. Only disable this if you're aware of the " - "security implications."}; - - Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges", - "Whether builders can acquire new privileges by calling programs with " - "setuid/setgid bits or with file capabilities."}; + Setting<bool> filterSyscalls{ + this, true, "filter-syscalls", + "Whether to prevent certain dangerous system calls, such as " + "creation of setuid/setgid files or adding ACLs or extended " + "attributes. Only disable this if you're aware of the " + "security implications."}; + + Setting<bool> allowNewPrivileges{ + this, false, "allow-new-privileges", + "Whether builders can acquire new privileges by calling programs with " + "setuid/setgid bits or with file capabilities."}; #endif - Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", - "A list of servers used by builtins.fetchurl to fetch files by hash."}; - - Setting<uint64_t> minFree{this, 0, "min-free", - "Automatically run the garbage collector when free disk space drops below the specified amount."}; - - Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", - "Stop deleting garbage when free disk space is above the specified amount."}; - - Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval", - "Number of seconds between checking free disk space."}; - - Setting<Paths> pluginFiles{this, {}, "plugin-files", - "Plugins to dynamically load at nix initialization time."}; + Setting<Strings> hashedMirrors{ + this, + {"http://tarballs.nixos.org/"}, + "hashed-mirrors", + "A list of servers used by builtins.fetchurl to fetch files by hash."}; + + Setting<uint64_t> minFree{this, 0, "min-free", + "Automatically run the garbage collector when free " + "disk space drops below the specified amount."}; + + Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), + "max-free", + "Stop deleting garbage when free disk space is " + "above the specified amount."}; + + Setting<uint64_t> minFreeCheckInterval{ + this, 5, "min-free-check-interval", + "Number of seconds between checking free disk space."}; + + Setting<Paths> pluginFiles{ + this, + {}, + "plugin-files", + "Plugins to dynamically load at nix initialization time."}; }; - // FIXME: don't use a global variable. extern Settings settings; @@ -367,4 +481,4 @@ void loadConfFile(); extern const string nixVersion; -} +} // namespace nix diff --git a/third_party/nix/src/libstore/http-binary-cache-store.cc b/third_party/nix/src/libstore/http-binary-cache-store.cc index 779f89e68d9c..eb21babd51b8 100644 --- a/third_party/nix/src/libstore/http-binary-cache-store.cc +++ b/third_party/nix/src/libstore/http-binary-cache-store.cc @@ -7,167 +7,150 @@ namespace nix { MakeError(UploadToHTTP, Error); -class HttpBinaryCacheStore : public BinaryCacheStore -{ -private: - - Path cacheUri; - - struct State - { - bool enabled = true; - std::chrono::steady_clock::time_point disabledUntil; - }; - - Sync<State> _state; - -public: - - HttpBinaryCacheStore( - const Params & params, const Path & _cacheUri) - : BinaryCacheStore(params) - , cacheUri(_cacheUri) - { - if (cacheUri.back() == '/') - cacheUri.pop_back(); - - diskCache = getNarInfoDiskCache(); +class HttpBinaryCacheStore : public BinaryCacheStore { + private: + Path cacheUri; + + struct State { + bool enabled = true; + std::chrono::steady_clock::time_point disabledUntil; + }; + + Sync<State> _state; + + public: + HttpBinaryCacheStore(const Params& params, const Path& _cacheUri) + : BinaryCacheStore(params), cacheUri(_cacheUri) { + if (cacheUri.back() == '/') cacheUri.pop_back(); + + diskCache = getNarInfoDiskCache(); + } + + std::string getUri() override { return cacheUri; } + + void init() override { + // FIXME: do this lazily? + if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) { + try { + BinaryCacheStore::init(); + } catch (UploadToHTTP&) { + throw Error("'%s' does not appear to be a binary cache", cacheUri); + } + diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority); } - - std::string getUri() override - { - return cacheUri; - } - - void init() override - { - // FIXME: do this lazily? - if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) { - try { - BinaryCacheStore::init(); - } catch (UploadToHTTP &) { - throw Error("'%s' does not appear to be a binary cache", cacheUri); - } - diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority); - } - } - -protected: - - void maybeDisable() - { - auto state(_state.lock()); - if (state->enabled && settings.tryFallback) { - int t = 60; - printError("disabling binary cache '%s' for %s seconds", getUri(), t); - state->enabled = false; - state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); - } + } + + protected: + void maybeDisable() { + auto state(_state.lock()); + if (state->enabled && settings.tryFallback) { + int t = 60; + printError("disabling binary cache '%s' for %s seconds", getUri(), t); + state->enabled = false; + state->disabledUntil = + std::chrono::steady_clock::now() + std::chrono::seconds(t); } - - void checkEnabled() - { - auto state(_state.lock()); - if (state->enabled) return; - if (std::chrono::steady_clock::now() > state->disabledUntil) { - state->enabled = true; - debug("re-enabling binary cache '%s'", getUri()); - return; - } - throw SubstituterDisabled("substituter '%s' is disabled", getUri()); + } + + void checkEnabled() { + auto state(_state.lock()); + if (state->enabled) return; + if (std::chrono::steady_clock::now() > state->disabledUntil) { + state->enabled = true; + debug("re-enabling binary cache '%s'", getUri()); + return; } - - bool fileExists(const std::string & path) override - { - checkEnabled(); - - try { - DownloadRequest request(cacheUri + "/" + path); - request.head = true; - getDownloader()->download(request); - return true; - } catch (DownloadError & e) { - /* S3 buckets return 403 if a file doesn't exist and the - bucket is unlistable, so treat 403 as 404. */ - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return false; - maybeDisable(); - throw; - } + throw SubstituterDisabled("substituter '%s' is disabled", getUri()); + } + + bool fileExists(const std::string& path) override { + checkEnabled(); + + try { + DownloadRequest request(cacheUri + "/" + path); + request.head = true; + getDownloader()->download(request); + return true; + } catch (DownloadError& e) { + /* S3 buckets return 403 if a file doesn't exist and the + bucket is unlistable, so treat 403 as 404. */ + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return false; + maybeDisable(); + throw; } - - void upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) override - { - auto req = DownloadRequest(cacheUri + "/" + path); - req.data = std::make_shared<string>(data); // FIXME: inefficient - req.mimeType = mimeType; - try { - getDownloader()->download(req); - } catch (DownloadError & e) { - throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); - } + } + + void upsertFile(const std::string& path, const std::string& data, + const std::string& mimeType) override { + auto req = DownloadRequest(cacheUri + "/" + path); + req.data = std::make_shared<string>(data); // FIXME: inefficient + req.mimeType = mimeType; + try { + getDownloader()->download(req); + } catch (DownloadError& e) { + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", + cacheUri, e.msg()); } - - DownloadRequest makeRequest(const std::string & path) - { - DownloadRequest request(cacheUri + "/" + path); - return request; + } + + DownloadRequest makeRequest(const std::string& path) { + DownloadRequest request(cacheUri + "/" + path); + return request; + } + + void getFile(const std::string& path, Sink& sink) override { + checkEnabled(); + auto request(makeRequest(path)); + try { + getDownloader()->download(std::move(request), sink); + } catch (DownloadError& e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + throw NoSuchBinaryCacheFile( + "file '%s' does not exist in binary cache '%s'", path, getUri()); + maybeDisable(); + throw; } - - void getFile(const std::string & path, Sink & sink) override - { - checkEnabled(); - auto request(makeRequest(path)); - try { - getDownloader()->download(std::move(request), sink); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); + } + + void getFile( + const std::string& path, + Callback<std::shared_ptr<std::string>> callback) noexcept override { + checkEnabled(); + + auto request(makeRequest(path)); + + auto callbackPtr = + std::make_shared<decltype(callback)>(std::move(callback)); + + getDownloader()->enqueueDownload( + request, {[callbackPtr, this](std::future<DownloadResult> result) { + try { + (*callbackPtr)(result.get().data); + } catch (DownloadError& e) { + if (e.error == Downloader::NotFound || + e.error == Downloader::Forbidden) + return (*callbackPtr)(std::shared_ptr<std::string>()); maybeDisable(); - throw; - } - } - - void getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback) noexcept override - { - checkEnabled(); - - auto request(makeRequest(path)); - - auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); - - getDownloader()->enqueueDownload(request, - {[callbackPtr, this](std::future<DownloadResult> result) { - try { - (*callbackPtr)(result.get().data); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return (*callbackPtr)(std::shared_ptr<std::string>()); - maybeDisable(); - callbackPtr->rethrow(); - } catch (...) { - callbackPtr->rethrow(); - } - }}); - } - + callbackPtr->rethrow(); + } catch (...) { + callbackPtr->rethrow(); + } + }}); + } }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, 7) != "http://" && - std::string(uri, 0, 8) != "https://" && - (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") - ) return 0; - auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); - store->init(); - return store; -}); - -} - +static RegisterStoreImplementation regStore( + [](const std::string& uri, + const Store::Params& params) -> std::shared_ptr<Store> { + if (std::string(uri, 0, 7) != "http://" && + std::string(uri, 0, 8) != "https://" && + (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || + std::string(uri, 0, 7) != "file://")) + return 0; + auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); + store->init(); + return store; + }); + +} // namespace nix diff --git a/third_party/nix/src/libstore/legacy-ssh-store.cc b/third_party/nix/src/libstore/legacy-ssh-store.cc index d5fbdd25aa47..4dc6c0fbec36 100644 --- a/third_party/nix/src/libstore/legacy-ssh-store.cc +++ b/third_party/nix/src/libstore/legacy-ssh-store.cc @@ -1,293 +1,258 @@ #include "archive.hh" +#include "derivations.hh" #include "pool.hh" #include "remote-store.hh" #include "serve-protocol.hh" +#include "ssh.hh" #include "store-api.hh" #include "worker-protocol.hh" -#include "ssh.hh" -#include "derivations.hh" namespace nix { static std::string uriScheme = "ssh://"; -struct LegacySSHStore : public Store -{ - const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; - const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; - const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; - const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; - const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"}; - - // Hack for getting remote build log output. - const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; - - struct Connection - { - std::unique_ptr<SSHMaster::Connection> sshConn; - FdSink to; - FdSource from; - int remoteVersion; - bool good = true; - }; - - std::string host; - - ref<Pool<Connection>> connections; - - SSHMaster master; - - LegacySSHStore(const string & host, const Params & params) - : Store(params) - , host(host) - , connections(make_ref<Pool<Connection>>( - std::max(1, (int) maxConnections), +struct LegacySSHStore : public Store { + const Setting<int> maxConnections{ + this, 1, "max-connections", + "maximum number of concurrent SSH connections"}; + const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{this, false, "compress", + "whether to compress the connection"}; + const Setting<Path> remoteProgram{ + this, "nix-store", "remote-program", + "path to the nix-store executable on the remote system"}; + const Setting<std::string> remoteStore{ + this, "", "remote-store", "URI of the store on the remote system"}; + + // Hack for getting remote build log output. + const Setting<int> logFD{ + this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + + struct Connection { + std::unique_ptr<SSHMaster::Connection> sshConn; + FdSink to; + FdSource from; + int remoteVersion; + bool good = true; + }; + + std::string host; + + ref<Pool<Connection>> connections; + + SSHMaster master; + + LegacySSHStore(const string& host, const Params& params) + : Store(params), + host(host), + connections(make_ref<Pool<Connection>>( + std::max(1, (int)maxConnections), [this]() { return openConnection(); }, - [](const ref<Connection> & r) { return r->good; } - )) - , master( - host, - sshKey, - // Use SSH master only if using more than 1 connection. - connections->capacity() > 1, - compress, - logFD) - { + [](const ref<Connection>& r) { return r->good; })), + master(host, sshKey, + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, compress, logFD) {} + + ref<Connection> openConnection() { + auto conn = make_ref<Connection>(); + conn->sshConn = master.startCommand( + fmt("%s --serve --write", remoteProgram) + + (remoteStore.get() == "" + ? "" + : " --store " + shellEscape(remoteStore.get()))); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); + + try { + conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; + conn->to.flush(); + + unsigned int magic = readInt(conn->from); + if (magic != SERVE_MAGIC_2) + throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host); + conn->remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) + throw Error("unsupported 'nix-store --serve' protocol version on '%s'", + host); + + } catch (EndOfFile& e) { + throw Error("cannot connect to '%1%'", host); } - ref<Connection> openConnection() - { - auto conn = make_ref<Connection>(); - conn->sshConn = master.startCommand( - fmt("%s --serve --write", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); - conn->to = FdSink(conn->sshConn->in.get()); - conn->from = FdSource(conn->sshConn->out.get()); - - try { - conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; - conn->to.flush(); - - unsigned int magic = readInt(conn->from); - if (magic != SERVE_MAGIC_2) - throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host); - conn->remoteVersion = readInt(conn->from); - if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) - throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); - - } catch (EndOfFile & e) { - throw Error("cannot connect to '%1%'", host); - } - - return conn; - }; - - string getUri() override - { - return uriScheme + host; - } + return conn; + }; - void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override - { - try { - auto conn(connections->get()); + string getUri() override { return uriScheme + host; } - debug("querying remote host '%s' for info on '%s'", host, path); + void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override { + try { + auto conn(connections->get()); - conn->to << cmdQueryPathInfos << PathSet{path}; - conn->to.flush(); + debug("querying remote host '%s' for info on '%s'", host, path); - auto info = std::make_shared<ValidPathInfo>(); - conn->from >> info->path; - if (info->path.empty()) return callback(nullptr); - assert(path == info->path); + conn->to << cmdQueryPathInfos << PathSet{path}; + conn->to.flush(); - PathSet references; - conn->from >> info->deriver; - info->references = readStorePaths<PathSet>(*this, conn->from); - readLongLong(conn->from); // download size - info->narSize = readLongLong(conn->from); + auto info = std::make_shared<ValidPathInfo>(); + conn->from >> info->path; + if (info->path.empty()) return callback(nullptr); + assert(path == info->path); - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { - auto s = readString(conn->from); - info->narHash = s.empty() ? Hash() : Hash(s); - conn->from >> info->ca; - info->sigs = readStrings<StringSet>(conn->from); - } + PathSet references; + conn->from >> info->deriver; + info->references = readStorePaths<PathSet>(*this, conn->from); + readLongLong(conn->from); // download size + info->narSize = readLongLong(conn->from); - auto s = readString(conn->from); - assert(s == ""); + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { + auto s = readString(conn->from); + info->narHash = s.empty() ? Hash() : Hash(s); + conn->from >> info->ca; + info->sigs = readStrings<StringSet>(conn->from); + } - callback(std::move(info)); - } catch (...) { callback.rethrow(); } - } + auto s = readString(conn->from); + assert(s == ""); - void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override - { - debug("adding path '%s' to remote host '%s'", info.path, host); - - auto conn(connections->get()); - - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { - - conn->to - << cmdAddToStoreNar - << info.path - << info.deriver - << info.narHash.to_string(Base16, false) - << info.references - << info.registrationTime - << info.narSize - << info.ultimate - << info.sigs - << info.ca; - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to.flush(); - - } else { - - conn->to - << cmdImportPaths - << 1; - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to - << exportMagic - << info.path - << info.references - << info.deriver - << 0 - << 0; - conn->to.flush(); - - } - - if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s', info.path, host"); + callback(std::move(info)); + } catch (...) { + callback.rethrow(); } - - void narFromPath(const Path & path, Sink & sink) override - { - auto conn(connections->get()); - - conn->to << cmdDumpStorePath << path; - conn->to.flush(); - copyNAR(conn->from, sink); + } + + void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair, + CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) override { + debug("adding path '%s' to remote host '%s'", info.path, host); + + auto conn(connections->get()); + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { + conn->to << cmdAddToStoreNar << info.path << info.deriver + << info.narHash.to_string(Base16, false) << info.references + << info.registrationTime << info.narSize << info.ultimate + << info.sigs << info.ca; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to.flush(); + + } else { + conn->to << cmdImportPaths << 1; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to << exportMagic << info.path << info.references << info.deriver + << 0 << 0; + conn->to.flush(); } - Path queryPathFromHashPart(const string & hashPart) override - { unsupported("queryPathFromHashPart"); } - - Path addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override - { unsupported("addToStore"); } - - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override - { unsupported("addTextToStore"); } - - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { - auto conn(connections->get()); - - conn->to - << cmdBuildDerivation - << drvPath - << drv - << settings.maxSilentTime - << settings.buildTimeout; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) - conn->to - << settings.maxLogSize; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) - conn->to - << settings.buildRepeat - << settings.enforceDeterminism; - - conn->to.flush(); - - BuildResult status; - status.status = (BuildResult::Status) readInt(conn->from); - conn->from >> status.errorMsg; - - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) - conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; - - return status; + if (readInt(conn->from) != 1) + throw Error( + "failed to add path '%s' to remote host '%s', info.path, host"); + } + + void narFromPath(const Path& path, Sink& sink) override { + auto conn(connections->get()); + + conn->to << cmdDumpStorePath << path; + conn->to.flush(); + copyNAR(conn->from, sink); + } + + Path queryPathFromHashPart(const string& hashPart) override { + unsupported("queryPathFromHashPart"); + } + + Path addToStore(const string& name, const Path& srcPath, bool recursive, + HashType hashAlgo, PathFilter& filter, + RepairFlag repair) override { + unsupported("addToStore"); + } + + Path addTextToStore(const string& name, const string& s, + const PathSet& references, RepairFlag repair) override { + unsupported("addTextToStore"); + } + + BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv, + BuildMode buildMode) override { + auto conn(connections->get()); + + conn->to << cmdBuildDerivation << drvPath << drv << settings.maxSilentTime + << settings.buildTimeout; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) + conn->to << settings.maxLogSize; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->to << settings.buildRepeat << settings.enforceDeterminism; + + conn->to.flush(); + + BuildResult status; + status.status = (BuildResult::Status)readInt(conn->from); + conn->from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->from >> status.timesBuilt >> status.isNonDeterministic >> + status.startTime >> status.stopTime; + + return status; + } + + void ensurePath(const Path& path) override { unsupported("ensurePath"); } + + void computeFSClosure(const PathSet& paths, PathSet& out, + bool flipDirection = false, bool includeOutputs = false, + bool includeDerivers = false) override { + if (flipDirection || includeDerivers) { + Store::computeFSClosure(paths, out, flipDirection, includeOutputs, + includeDerivers); + return; } - void ensurePath(const Path & path) override - { unsupported("ensurePath"); } - - void computeFSClosure(const PathSet & paths, - PathSet & out, bool flipDirection = false, - bool includeOutputs = false, bool includeDerivers = false) override - { - if (flipDirection || includeDerivers) { - Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); - return; - } + auto conn(connections->get()); - auto conn(connections->get()); + conn->to << cmdQueryClosure << includeOutputs << paths; + conn->to.flush(); - conn->to - << cmdQueryClosure - << includeOutputs - << paths; - conn->to.flush(); + auto res = readStorePaths<PathSet>(*this, conn->from); - auto res = readStorePaths<PathSet>(*this, conn->from); + out.insert(res.begin(), res.end()); + } - out.insert(res.begin(), res.end()); - } + PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute = + NoSubstitute) override { + auto conn(connections->get()); - PathSet queryValidPaths(const PathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override - { - auto conn(connections->get()); + conn->to << cmdQueryValidPaths << false // lock + << maybeSubstitute << paths; + conn->to.flush(); - conn->to - << cmdQueryValidPaths - << false // lock - << maybeSubstitute - << paths; - conn->to.flush(); + return readStorePaths<PathSet>(*this, conn->from); + } - return readStorePaths<PathSet>(*this, conn->from); - } + void connect() override { auto conn(connections->get()); } - void connect() override - { - auto conn(connections->get()); - } - - unsigned int getProtocol() override - { - auto conn(connections->get()); - return conn->remoteVersion; - } + unsigned int getProtocol() override { + auto conn(connections->get()); + return conn->remoteVersion; + } }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation regStore( + [](const std::string& uri, + const Store::Params& params) -> std::shared_ptr<Store> { + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared<LegacySSHStore>( + std::string(uri, uriScheme.size()), params); + }); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/local-binary-cache-store.cc b/third_party/nix/src/libstore/local-binary-cache-store.cc index b7001795be4d..925fb34de48f 100644 --- a/third_party/nix/src/libstore/local-binary-cache-store.cc +++ b/third_party/nix/src/libstore/local-binary-cache-store.cc @@ -4,100 +4,82 @@ namespace nix { -class LocalBinaryCacheStore : public BinaryCacheStore -{ -private: +class LocalBinaryCacheStore : public BinaryCacheStore { + private: + Path binaryCacheDir; - Path binaryCacheDir; + public: + LocalBinaryCacheStore(const Params& params, const Path& binaryCacheDir) + : BinaryCacheStore(params), binaryCacheDir(binaryCacheDir) {} -public: + void init() override; - LocalBinaryCacheStore( - const Params & params, const Path & binaryCacheDir) - : BinaryCacheStore(params) - , binaryCacheDir(binaryCacheDir) - { - } - - void init() override; - - std::string getUri() override - { - return "file://" + binaryCacheDir; - } + std::string getUri() override { return "file://" + binaryCacheDir; } -protected: + protected: + bool fileExists(const std::string& path) override; - bool fileExists(const std::string & path) override; + void upsertFile(const std::string& path, const std::string& data, + const std::string& mimeType) override; - void upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) override; - - void getFile(const std::string & path, Sink & sink) override - { - try { - readFile(binaryCacheDir + "/" + path, sink); - } catch (SysError & e) { - if (e.errNo == ENOENT) - throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); - } + void getFile(const std::string& path, Sink& sink) override { + try { + readFile(binaryCacheDir + "/" + path, sink); + } catch (SysError& e) { + if (e.errNo == ENOENT) + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", + path); } + } - PathSet queryAllValidPaths() override - { - PathSet paths; - - for (auto & entry : readDirectory(binaryCacheDir)) { - if (entry.name.size() != 40 || - !hasSuffix(entry.name, ".narinfo")) - continue; - paths.insert(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)); - } + PathSet queryAllValidPaths() override { + PathSet paths; - return paths; + for (auto& entry : readDirectory(binaryCacheDir)) { + if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo")) + continue; + paths.insert(storeDir + "/" + + entry.name.substr(0, entry.name.size() - 8)); } + return paths; + } }; -void LocalBinaryCacheStore::init() -{ - createDirs(binaryCacheDir + "/nar"); - BinaryCacheStore::init(); +void LocalBinaryCacheStore::init() { + createDirs(binaryCacheDir + "/nar"); + BinaryCacheStore::init(); } -static void atomicWrite(const Path & path, const std::string & s) -{ - Path tmp = path + ".tmp." + std::to_string(getpid()); - AutoDelete del(tmp, false); - writeFile(tmp, s); - if (rename(tmp.c_str(), path.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % tmp % path); - del.cancel(); +static void atomicWrite(const Path& path, const std::string& s) { + Path tmp = path + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + writeFile(tmp, s); + if (rename(tmp.c_str(), path.c_str())) + throw SysError(format("renaming '%1%' to '%2%'") % tmp % path); + del.cancel(); } -bool LocalBinaryCacheStore::fileExists(const std::string & path) -{ - return pathExists(binaryCacheDir + "/" + path); +bool LocalBinaryCacheStore::fileExists(const std::string& path) { + return pathExists(binaryCacheDir + "/" + path); } -void LocalBinaryCacheStore::upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) -{ - atomicWrite(binaryCacheDir + "/" + path, data); +void LocalBinaryCacheStore::upsertFile(const std::string& path, + const std::string& data, + const std::string& mimeType) { + atomicWrite(binaryCacheDir + "/" + path, data); } -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || - std::string(uri, 0, 7) != "file://") +static RegisterStoreImplementation regStore( + [](const std::string& uri, + const Store::Params& params) -> std::shared_ptr<Store> { + if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || + std::string(uri, 0, 7) != "file://") return 0; - auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7)); - store->init(); - return store; -}); + auto store = + std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7)); + store->init(); + return store; + }); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/local-fs-store.cc b/third_party/nix/src/libstore/local-fs-store.cc index 642e4070d459..0361dbd9774d 100644 --- a/third_party/nix/src/libstore/local-fs-store.cc +++ b/third_party/nix/src/libstore/local-fs-store.cc @@ -1,131 +1,113 @@ #include "archive.hh" -#include "fs-accessor.hh" -#include "store-api.hh" -#include "globals.hh" #include "compression.hh" #include "derivations.hh" +#include "fs-accessor.hh" +#include "globals.hh" +#include "store-api.hh" namespace nix { -LocalFSStore::LocalFSStore(const Params & params) - : Store(params) -{ -} +LocalFSStore::LocalFSStore(const Params& params) : Store(params) {} -struct LocalStoreAccessor : public FSAccessor -{ - ref<LocalFSStore> store; +struct LocalStoreAccessor : public FSAccessor { + ref<LocalFSStore> store; - LocalStoreAccessor(ref<LocalFSStore> store) : store(store) { } + LocalStoreAccessor(ref<LocalFSStore> store) : store(store) {} - Path toRealPath(const Path & path) - { - Path storePath = store->toStorePath(path); - if (!store->isValidPath(storePath)) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); - return store->getRealStoreDir() + std::string(path, store->storeDir.size()); - } + Path toRealPath(const Path& path) { + Path storePath = store->toStorePath(path); + if (!store->isValidPath(storePath)) + throw InvalidPath(format("path '%1%' is not a valid store path") % + storePath); + return store->getRealStoreDir() + std::string(path, store->storeDir.size()); + } - FSAccessor::Stat stat(const Path & path) override - { - auto realPath = toRealPath(path); + FSAccessor::Stat stat(const Path& path) override { + auto realPath = toRealPath(path); - struct stat st; - if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; - throw SysError(format("getting status of '%1%'") % path); - } + struct stat st; + if (lstat(realPath.c_str(), &st)) { + if (errno == ENOENT || errno == ENOTDIR) + return {Type::tMissing, 0, false}; + throw SysError(format("getting status of '%1%'") % path); + } - if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error(format("file '%1%' has unsupported type") % path); + if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) + throw Error(format("file '%1%' has unsupported type") % path); - return { - S_ISREG(st.st_mode) ? Type::tRegular : - S_ISLNK(st.st_mode) ? Type::tSymlink : - Type::tDirectory, - S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, + return {S_ISREG(st.st_mode) + ? Type::tRegular + : S_ISLNK(st.st_mode) ? Type::tSymlink : Type::tDirectory, + S_ISREG(st.st_mode) ? (uint64_t)st.st_size : 0, S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; - } + } - StringSet readDirectory(const Path & path) override - { - auto realPath = toRealPath(path); + StringSet readDirectory(const Path& path) override { + auto realPath = toRealPath(path); - auto entries = nix::readDirectory(realPath); + auto entries = nix::readDirectory(realPath); - StringSet res; - for (auto & entry : entries) - res.insert(entry.name); + StringSet res; + for (auto& entry : entries) res.insert(entry.name); - return res; - } + return res; + } - std::string readFile(const Path & path) override - { - return nix::readFile(toRealPath(path)); - } + std::string readFile(const Path& path) override { + return nix::readFile(toRealPath(path)); + } - std::string readLink(const Path & path) override - { - return nix::readLink(toRealPath(path)); - } + std::string readLink(const Path& path) override { + return nix::readLink(toRealPath(path)); + } }; -ref<FSAccessor> LocalFSStore::getFSAccessor() -{ - return make_ref<LocalStoreAccessor>(ref<LocalFSStore>( - std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); +ref<FSAccessor> LocalFSStore::getFSAccessor() { + return make_ref<LocalStoreAccessor>(ref<LocalFSStore>( + std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); } -void LocalFSStore::narFromPath(const Path & path, Sink & sink) -{ - if (!isValidPath(path)) - throw Error(format("path '%s' is not valid") % path); - dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink); +void LocalFSStore::narFromPath(const Path& path, Sink& sink) { + if (!isValidPath(path)) throw Error(format("path '%s' is not valid") % path); + dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink); } const string LocalFSStore::drvsLogDir = "drvs"; +std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path& path_) { + auto path(path_); + assertStorePath(path); -std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_) -{ - auto path(path_); - - assertStorePath(path); - - - if (!isDerivation(path)) { - try { - path = queryPathInfo(path)->deriver; - } catch (InvalidPath &) { - return nullptr; - } - if (path == "") return nullptr; + if (!isDerivation(path)) { + try { + path = queryPathInfo(path)->deriver; + } catch (InvalidPath&) { + return nullptr; } + if (path == "") return nullptr; + } - string baseName = baseNameOf(path); - - for (int j = 0; j < 2; j++) { + string baseName = baseNameOf(path); - Path logPath = - j == 0 - ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2)) - : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); - Path logBz2Path = logPath + ".bz2"; + for (int j = 0; j < 2; j++) { + Path logPath = j == 0 ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, + string(baseName, 0, 2), string(baseName, 2)) + : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); + Path logBz2Path = logPath + ".bz2"; - if (pathExists(logPath)) - return std::make_shared<std::string>(readFile(logPath)); - - else if (pathExists(logBz2Path)) { - try { - return decompress("bzip2", readFile(logBz2Path)); - } catch (Error &) { } - } + if (pathExists(logPath)) + return std::make_shared<std::string>(readFile(logPath)); + else if (pathExists(logBz2Path)) { + try { + return decompress("bzip2", readFile(logBz2Path)); + } catch (Error&) { + } } + } - return nullptr; + return nullptr; } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/local-store.cc b/third_party/nix/src/libstore/local-store.cc index 84ddd964b469..9da8e9d435c1 100644 --- a/third_party/nix/src/libstore/local-store.cc +++ b/third_party/nix/src/libstore/local-store.cc @@ -1,32 +1,30 @@ #include "local-store.hh" -#include "globals.hh" -#include "archive.hh" -#include "pathlocks.hh" -#include "worker-protocol.hh" -#include "derivations.hh" -#include "nar-info.hh" - -#include <iostream> -#include <algorithm> -#include <cstring> - -#include <sys/types.h> -#include <sys/stat.h> +#include <errno.h> +#include <fcntl.h> +#include <grp.h> +#include <stdio.h> #include <sys/select.h> +#include <sys/stat.h> #include <sys/time.h> +#include <sys/types.h> +#include <time.h> #include <unistd.h> #include <utime.h> -#include <fcntl.h> -#include <errno.h> -#include <stdio.h> -#include <time.h> -#include <grp.h> +#include <algorithm> +#include <cstring> +#include <iostream> +#include "archive.hh" +#include "derivations.hh" +#include "globals.hh" +#include "nar-info.hh" +#include "pathlocks.hh" +#include "worker-protocol.hh" #if __linux__ #include <sched.h> -#include <sys/statvfs.h> -#include <sys/mount.h> #include <sys/ioctl.h> +#include <sys/mount.h> +#include <sys/statvfs.h> #include <sys/xattr.h> #endif @@ -36,1418 +34,1367 @@ #include <sqlite3.h> - namespace nix { - -LocalStore::LocalStore(const Params & params) - : Store(params) - , LocalFSStore(params) - , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", - "physical path to the Nix store"} - , realStoreDir(realStoreDir_) - , dbDir(stateDir + "/db") - , linksDir(realStoreDir + "/.links") - , reservedPath(dbDir + "/reserved") - , schemaPath(dbDir + "/schema") - , trashDir(realStoreDir + "/trash") - , tempRootsDir(stateDir + "/temproots") - , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) -{ - auto state(_state.lock()); - - /* Create missing state directories if they don't already exist. */ - createDirs(realStoreDir); - makeStoreWritable(); - createDirs(linksDir); - Path profilesDir = stateDir + "/profiles"; - createDirs(profilesDir); - createDirs(tempRootsDir); - createDirs(dbDir); - Path gcRootsDir = stateDir + "/gcroots"; - if (!pathExists(gcRootsDir)) { - createDirs(gcRootsDir); - createSymlink(profilesDir, gcRootsDir + "/profiles"); - } - - for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { - createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 0755) == -1) - throw SysError("could not set permissions on '%s' to 755", perUserDir); - } - - createUser(getUserName(), getuid()); - - /* Optionally, create directories and set permissions for a - multi-user install. */ - if (getuid() == 0 && settings.buildUsersGroup != "") { - mode_t perm = 01775; - - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - printError(format("warning: the group '%1%' specified in 'build-users-group' does not exist") - % settings.buildUsersGroup); - else { - struct stat st; - if (stat(realStoreDir.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % realStoreDir); - - if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { - if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) - throw SysError(format("changing ownership of path '%1%'") % realStoreDir); - if (chmod(realStoreDir.c_str(), perm) == -1) - throw SysError(format("changing permissions on path '%1%'") % realStoreDir); - } - } +LocalStore::LocalStore(const Params& params) + : Store(params), + LocalFSStore(params), + realStoreDir_{this, false, + rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + "physical path to the Nix store"}, + realStoreDir(realStoreDir_), + dbDir(stateDir + "/db"), + linksDir(realStoreDir + "/.links"), + reservedPath(dbDir + "/reserved"), + schemaPath(dbDir + "/schema"), + trashDir(realStoreDir + "/trash"), + tempRootsDir(stateDir + "/temproots"), + fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) { + auto state(_state.lock()); + + /* Create missing state directories if they don't already exist. */ + createDirs(realStoreDir); + makeStoreWritable(); + createDirs(linksDir); + Path profilesDir = stateDir + "/profiles"; + createDirs(profilesDir); + createDirs(tempRootsDir); + createDirs(dbDir); + Path gcRootsDir = stateDir + "/gcroots"; + if (!pathExists(gcRootsDir)) { + createDirs(gcRootsDir); + createSymlink(profilesDir, gcRootsDir + "/profiles"); + } + + for (auto& perUserDir : + {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { + createDirs(perUserDir); + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } + + createUser(getUserName(), getuid()); + + /* Optionally, create directories and set permissions for a + multi-user install. */ + if (getuid() == 0 && settings.buildUsersGroup != "") { + mode_t perm = 01775; + + struct group* gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (!gr) + printError(format("warning: the group '%1%' specified in " + "'build-users-group' does not exist") % + settings.buildUsersGroup); + else { + struct stat st; + if (stat(realStoreDir.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % + realStoreDir); + + if (st.st_uid != 0 || st.st_gid != gr->gr_gid || + (st.st_mode & ~S_IFMT) != perm) { + if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) + throw SysError(format("changing ownership of path '%1%'") % + realStoreDir); + if (chmod(realStoreDir.c_str(), perm) == -1) + throw SysError(format("changing permissions on path '%1%'") % + realStoreDir); + } } + } - /* Ensure that the store and its parents are not symlinks. */ - if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") { - Path path = realStoreDir; - struct stat st; - while (path != "/") { - if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of '%1%'") % path); - if (S_ISLNK(st.st_mode)) - throw Error(format( - "the path '%1%' is a symlink; " - "this is not allowed for the Nix store and its parent directories") - % path); - path = dirOf(path); - } + /* Ensure that the store and its parents are not symlinks. */ + if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") { + Path path = realStoreDir; + struct stat st; + while (path != "/") { + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of '%1%'") % path); + if (S_ISLNK(st.st_mode)) + throw Error(format("the path '%1%' is a symlink; " + "this is not allowed for the Nix store and its " + "parent directories") % + path); + path = dirOf(path); } + } - /* We can't open a SQLite database if the disk is full. Since - this prevents the garbage collector from running when it's most - needed, we reserve some dummy space that we can free just - before doing a garbage collection. */ - try { - struct stat st; - if (stat(reservedPath.c_str(), &st) == -1 || - st.st_size != settings.reservedSize) - { - AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600); - int res = -1; + /* We can't open a SQLite database if the disk is full. Since + this prevents the garbage collector from running when it's most + needed, we reserve some dummy space that we can free just + before doing a garbage collection. */ + try { + struct stat st; + if (stat(reservedPath.c_str(), &st) == -1 || + st.st_size != settings.reservedSize) { + AutoCloseFD fd = + open(reservedPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600); + int res = -1; #if HAVE_POSIX_FALLOCATE - res = posix_fallocate(fd.get(), 0, settings.reservedSize); + res = posix_fallocate(fd.get(), 0, settings.reservedSize); #endif - if (res == -1) { - writeFull(fd.get(), string(settings.reservedSize, 'X')); - [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); - } - } - } catch (SysError & e) { /* don't care about errors */ + if (res == -1) { + writeFull(fd.get(), string(settings.reservedSize, 'X')); + [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); + } } - - /* Acquire the big fat lock in shared mode to make sure that no - schema upgrade is in progress. */ - Path globalLockPath = dbDir + "/big-lock"; - globalLock = openLockFile(globalLockPath.c_str(), true); - - if (!lockFile(globalLock.get(), ltRead, false)) { - printError("waiting for the big Nix store lock..."); - lockFile(globalLock.get(), ltRead, true); + } catch (SysError& e) { /* don't care about errors */ + } + + /* Acquire the big fat lock in shared mode to make sure that no + schema upgrade is in progress. */ + Path globalLockPath = dbDir + "/big-lock"; + globalLock = openLockFile(globalLockPath.c_str(), true); + + if (!lockFile(globalLock.get(), ltRead, false)) { + printError("waiting for the big Nix store lock..."); + lockFile(globalLock.get(), ltRead, true); + } + + /* Check the current database schema and if necessary do an + upgrade. */ + int curSchema = getSchema(); + if (curSchema > nixSchemaVersion) + throw Error( + format( + "current Nix store schema is version %1%, but I only support %2%") % + curSchema % nixSchemaVersion); + + else if (curSchema == 0) { /* new store */ + curSchema = nixSchemaVersion; + openDB(*state, true); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + } + + else if (curSchema < nixSchemaVersion) { + if (curSchema < 5) + throw Error( + "Your Nix store has a database in Berkeley DB format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 0.12 first."); + + if (curSchema < 6) + throw Error( + "Your Nix store has a database in flat file format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 1.11 first."); + + if (!lockFile(globalLock.get(), ltWrite, false)) { + printError("waiting for exclusive access to the Nix store..."); + lockFile(globalLock.get(), ltWrite, true); } - /* Check the current database schema and if necessary do an - upgrade. */ - int curSchema = getSchema(); - if (curSchema > nixSchemaVersion) - throw Error(format("current Nix store schema is version %1%, but I only support %2%") - % curSchema % nixSchemaVersion); - - else if (curSchema == 0) { /* new store */ - curSchema = nixSchemaVersion; - openDB(*state, true); - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); - } - - else if (curSchema < nixSchemaVersion) { - if (curSchema < 5) - throw Error( - "Your Nix store has a database in Berkeley DB format,\n" - "which is no longer supported. To convert to the new format,\n" - "please upgrade Nix to version 0.12 first."); - - if (curSchema < 6) - throw Error( - "Your Nix store has a database in flat file format,\n" - "which is no longer supported. To convert to the new format,\n" - "please upgrade Nix to version 1.11 first."); - - if (!lockFile(globalLock.get(), ltWrite, false)) { - printError("waiting for exclusive access to the Nix store..."); - lockFile(globalLock.get(), ltWrite, true); - } - - /* Get the schema version again, because another process may - have performed the upgrade already. */ - curSchema = getSchema(); - - if (curSchema < 7) { upgradeStore7(); } - - openDB(*state, false); - - if (curSchema < 8) { - SQLiteTxn txn(state->db); - state->db.exec("alter table ValidPaths add column ultimate integer"); - state->db.exec("alter table ValidPaths add column sigs text"); - txn.commit(); - } - - if (curSchema < 9) { - SQLiteTxn txn(state->db); - state->db.exec("drop table FailedPaths"); - txn.commit(); - } + /* Get the schema version again, because another process may + have performed the upgrade already. */ + curSchema = getSchema(); - if (curSchema < 10) { - SQLiteTxn txn(state->db); - state->db.exec("alter table ValidPaths add column ca text"); - txn.commit(); - } - - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); - - lockFile(globalLock.get(), ltRead, true); + if (curSchema < 7) { + upgradeStore7(); } - else openDB(*state, false); - - /* Prepare SQL statements. */ - state->stmtRegisterValidPath.create(state->db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); - state->stmtUpdatePathInfo.create(state->db, - "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;"); - state->stmtAddReference.create(state->db, - "insert or replace into Refs (referrer, reference) values (?, ?);"); - state->stmtQueryPathInfo.create(state->db, - "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;"); - state->stmtQueryReferences.create(state->db, - "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - state->stmtQueryReferrers.create(state->db, - "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - state->stmtInvalidatePath.create(state->db, - "delete from ValidPaths where path = ?;"); - state->stmtAddDerivationOutput.create(state->db, - "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - state->stmtQueryValidDerivers.create(state->db, - "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - state->stmtQueryDerivationOutputs.create(state->db, - "select id, path from DerivationOutputs where drv = ?;"); - // Use "path >= ?" with limit 1 rather than "path like '?%'" to - // ensure efficient lookup. - state->stmtQueryPathFromHashPart.create(state->db, - "select path from ValidPaths where path >= ? limit 1;"); - state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths"); -} - + openDB(*state, false); -LocalStore::~LocalStore() -{ - std::shared_future<void> future; - - { - auto state(_state.lock()); - if (state->gcRunning) - future = state->gcFuture; + if (curSchema < 8) { + SQLiteTxn txn(state->db); + state->db.exec("alter table ValidPaths add column ultimate integer"); + state->db.exec("alter table ValidPaths add column sigs text"); + txn.commit(); } - if (future.valid()) { - printError("waiting for auto-GC to finish on exit..."); - future.get(); + if (curSchema < 9) { + SQLiteTxn txn(state->db); + state->db.exec("drop table FailedPaths"); + txn.commit(); } - try { - auto state(_state.lock()); - if (state->fdTempRoots) { - state->fdTempRoots = -1; - unlink(fnTempRoots.c_str()); - } - } catch (...) { - ignoreException(); + if (curSchema < 10) { + SQLiteTxn txn(state->db); + state->db.exec("alter table ValidPaths add column ca text"); + txn.commit(); } + + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + + lockFile(globalLock.get(), ltRead, true); + } + + else + openDB(*state, false); + + /* Prepare SQL statements. */ + state->stmtRegisterValidPath.create( + state->db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, " + "ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); + state->stmtUpdatePathInfo.create( + state->db, + "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca " + "= ? where path = ?;"); + state->stmtAddReference.create( + state->db, + "insert or replace into Refs (referrer, reference) values (?, ?);"); + state->stmtQueryPathInfo.create( + state->db, + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca " + "from ValidPaths where path = ?;"); + state->stmtQueryReferences.create(state->db, + "select path from Refs join ValidPaths on " + "reference = id where referrer = ?;"); + state->stmtQueryReferrers.create( + state->db, + "select path from Refs join ValidPaths on referrer = id where reference " + "= (select id from ValidPaths where path = ?);"); + state->stmtInvalidatePath.create(state->db, + "delete from ValidPaths where path = ?;"); + state->stmtAddDerivationOutput.create( + state->db, + "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, " + "?);"); + state->stmtQueryValidDerivers.create( + state->db, + "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv " + "= v.id where d.path = ?;"); + state->stmtQueryDerivationOutputs.create( + state->db, "select id, path from DerivationOutputs where drv = ?;"); + // Use "path >= ?" with limit 1 rather than "path like '?%'" to + // ensure efficient lookup. + state->stmtQueryPathFromHashPart.create( + state->db, "select path from ValidPaths where path >= ? limit 1;"); + state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths"); } +LocalStore::~LocalStore() { + std::shared_future<void> future; -std::string LocalStore::getUri() -{ - return "local"; -} + { + auto state(_state.lock()); + if (state->gcRunning) future = state->gcFuture; + } + if (future.valid()) { + printError("waiting for auto-GC to finish on exit..."); + future.get(); + } -int LocalStore::getSchema() -{ - int curSchema = 0; - if (pathExists(schemaPath)) { - string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) - throw Error(format("'%1%' is corrupt") % schemaPath); + try { + auto state(_state.lock()); + if (state->fdTempRoots) { + state->fdTempRoots = -1; + unlink(fnTempRoots.c_str()); } - return curSchema; + } catch (...) { + ignoreException(); + } } +std::string LocalStore::getUri() { return "local"; } -void LocalStore::openDB(State & state, bool create) -{ - if (access(dbDir.c_str(), R_OK | W_OK)) - throw SysError(format("Nix database directory '%1%' is not writable") % dbDir); +int LocalStore::getSchema() { + int curSchema = 0; + if (pathExists(schemaPath)) { + string s = readFile(schemaPath); + if (!string2Int(s, curSchema)) + throw Error(format("'%1%' is corrupt") % schemaPath); + } + return curSchema; +} - /* Open the Nix database. */ - string dbPath = dbDir + "/db.sqlite"; - auto & db(state.db); - if (sqlite3_open_v2(dbPath.c_str(), &db.db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open Nix database '%1%'") % dbPath); +void LocalStore::openDB(State& state, bool create) { + if (access(dbDir.c_str(), R_OK | W_OK)) + throw SysError(format("Nix database directory '%1%' is not writable") % + dbDir); + + /* Open the Nix database. */ + string dbPath = dbDir + "/db.sqlite"; + auto& db(state.db); + if (sqlite3_open_v2(dbPath.c_str(), &db.db, + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), + 0) != SQLITE_OK) + throw Error(format("cannot open Nix database '%1%'") % dbPath); #ifdef __CYGWIN__ - /* The cygwin version of sqlite3 has a patch which calls - SetDllDirectory("/usr/bin") on init. It was intended to fix extension - loading, which we don't use, and the effect of SetDllDirectory is - inherited by child processes, and causes libraries to be loaded from - /usr/bin instead of $PATH. This breaks quite a few things (e.g. - checkPhase on openssh), so we set it back to default behaviour. */ - SetDllDirectoryW(L""); + /* The cygwin version of sqlite3 has a patch which calls + SetDllDirectory("/usr/bin") on init. It was intended to fix extension + loading, which we don't use, and the effect of SetDllDirectory is + inherited by child processes, and causes libraries to be loaded from + /usr/bin instead of $PATH. This breaks quite a few things (e.g. + checkPhase on openssh), so we set it back to default behaviour. */ + SetDllDirectoryW(L""); #endif - if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); - - db.exec("pragma foreign_keys = 1"); - - /* !!! check whether sqlite has been built with foreign key - support */ - - /* Whether SQLite should fsync(). "Normal" synchronous mode - should be safe enough. If the user asks for it, don't sync at - all. This can cause database corruption if the system - crashes. */ - string syncMode = settings.fsyncMetadata ? "normal" : "off"; - db.exec("pragma synchronous = " + syncMode); - - /* Set the SQLite journal mode. WAL mode is fastest, so it's the - default. */ - string mode = settings.useSQLiteWAL ? "wal" : "truncate"; - string prevMode; - { - SQLiteStmt stmt; - stmt.create(db, "pragma main.journal_mode;"); - if (sqlite3_step(stmt) != SQLITE_ROW) - throwSQLiteError(db, "querying journal mode"); - prevMode = string((const char *) sqlite3_column_text(stmt, 0)); - } - if (prevMode != mode && - sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting journal mode"); - - /* Increase the auto-checkpoint interval to 40000 pages. This - seems enough to ensure that instantiating the NixOS system - derivation is done in a single fsync(). */ - if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting autocheckpoint interval"); - - /* Initialise the database schema, if necessary. */ - if (create) { - const char * schema = + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(db, "setting timeout"); + + db.exec("pragma foreign_keys = 1"); + + /* !!! check whether sqlite has been built with foreign key + support */ + + /* Whether SQLite should fsync(). "Normal" synchronous mode + should be safe enough. If the user asks for it, don't sync at + all. This can cause database corruption if the system + crashes. */ + string syncMode = settings.fsyncMetadata ? "normal" : "off"; + db.exec("pragma synchronous = " + syncMode); + + /* Set the SQLite journal mode. WAL mode is fastest, so it's the + default. */ + string mode = settings.useSQLiteWAL ? "wal" : "truncate"; + string prevMode; + { + SQLiteStmt stmt; + stmt.create(db, "pragma main.journal_mode;"); + if (sqlite3_step(stmt) != SQLITE_ROW) + throwSQLiteError(db, "querying journal mode"); + prevMode = string((const char*)sqlite3_column_text(stmt, 0)); + } + if (prevMode != mode && + sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, + 0, 0) != SQLITE_OK) + throwSQLiteError(db, "setting journal mode"); + + /* Increase the auto-checkpoint interval to 40000 pages. This + seems enough to ensure that instantiating the NixOS system + derivation is done in a single fsync(). */ + if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, + 0, 0) != SQLITE_OK) + throwSQLiteError(db, "setting autocheckpoint interval"); + + /* Initialise the database schema, if necessary. */ + if (create) { + const char* schema = #include "schema.sql.gen.hh" - ; - db.exec(schema); - } + ; + db.exec(schema); + } } - /* To improve purity, users may want to make the Nix store a read-only bind mount. So make the Nix store writable for this process. */ -void LocalStore::makeStoreWritable() -{ +void LocalStore::makeStoreWritable() { #if __linux__ - if (getuid() != 0) return; - /* Check if /nix/store is on a read-only mount. */ - struct statvfs stat; - if (statvfs(realStoreDir.c_str(), &stat) != 0) - throw SysError("getting info about the Nix store mount point"); - - if (stat.f_flag & ST_RDONLY) { - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - - if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError(format("remounting %1% writable") % realStoreDir); - } + if (getuid() != 0) return; + /* Check if /nix/store is on a read-only mount. */ + struct statvfs stat; + if (statvfs(realStoreDir.c_str(), &stat) != 0) + throw SysError("getting info about the Nix store mount point"); + + if (stat.f_flag & ST_RDONLY) { + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + + if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + throw SysError(format("remounting %1% writable") % realStoreDir); + } #endif } - const time_t mtimeStore = 1; /* 1 second into the epoch */ +static void canonicaliseTimestampAndPermissions(const Path& path, + const struct stat& st) { + if (!S_ISLNK(st.st_mode)) { + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; -static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) -{ - if (!S_ISLNK(st.st_mode)) { - - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) - | 0444 - | (st.st_mode & S_IXUSR ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) - throw SysError(format("changing mode of '%1%' to %2$o") % path % mode); - } - + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) | 0444 | (st.st_mode & S_IXUSR ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) + throw SysError(format("changing mode of '%1%' to %2$o") % path % mode); } - - if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; #if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) + if (lutimes(path.c_str(), times) == -1) + if (errno != ENOSYS || + (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) #else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) #endif - throw SysError(format("changing modification time of '%1%'") % path); - } + throw SysError(format("changing modification time of '%1%'") % path); + } } - -void canonicaliseTimestampAndPermissions(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); - canonicaliseTimestampAndPermissions(path, st); +void canonicaliseTimestampAndPermissions(const Path& path) { + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + canonicaliseTimestampAndPermissions(path, st); } - -static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) -{ - checkInterrupt(); +static void canonicalisePathMetaData_(const Path& path, uid_t fromUid, + InodesSeen& inodesSeen) { + checkInterrupt(); #if __APPLE__ - /* Remove flags, in particular UF_IMMUTABLE which would prevent - the file from being garbage-collected. FIXME: Use - setattrlist() to remove other attributes as well. */ - if (lchflags(path.c_str(), 0)) { - if (errno != ENOTSUP) - throw SysError(format("clearing flags of path '%1%'") % path); - } + /* Remove flags, in particular UF_IMMUTABLE which would prevent + the file from being garbage-collected. FIXME: Use + setattrlist() to remove other attributes as well. */ + if (lchflags(path.c_str(), 0)) { + if (errno != ENOTSUP) + throw SysError(format("clearing flags of path '%1%'") % path); + } #endif - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); - /* Really make sure that the path is of a supported type. */ - if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error(format("file '%1%' has an unsupported type") % path); + /* Really make sure that the path is of a supported type. */ + if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) + throw Error(format("file '%1%' has an unsupported type") % path); #if __linux__ - /* Remove extended attributes / ACLs. */ - ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); - - if (eaSize < 0) { - if (errno != ENOTSUP && errno != ENODATA) - throw SysError("querying extended attributes of '%s'", path); - } else if (eaSize > 0) { - std::vector<char> eaBuf(eaSize); - - if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) - throw SysError("querying extended attributes of '%s'", path); - - for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - /* Ignore SELinux security labels since these cannot be - removed even by root. */ - if (eaName == "security.selinux") continue; - if (lremovexattr(path.c_str(), eaName.c_str()) == -1) - throw SysError("removing extended attribute '%s' from '%s'", eaName, path); - } - } -#endif - - /* Fail if the file is not owned by the build user. This prevents - us from messing up the ownership/permissions of files - hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). - However, ignore files that we chown'ed ourselves previously to - ensure that we don't fail on hard links within the same build - (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { - assert(!S_ISDIR(st.st_mode)); - if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) - throw BuildError(format("invalid ownership on file '%1%'") % path); - mode_t mode = st.st_mode & ~S_IFMT; - assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); - return; + /* Remove extended attributes / ACLs. */ + ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); + + if (eaSize < 0) { + if (errno != ENOTSUP && errno != ENODATA) + throw SysError("querying extended attributes of '%s'", path); + } else if (eaSize > 0) { + std::vector<char> eaBuf(eaSize); + + if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) + throw SysError("querying extended attributes of '%s'", path); + + for (auto& eaName : tokenizeString<Strings>( + std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { + /* Ignore SELinux security labels since these cannot be + removed even by root. */ + if (eaName == "security.selinux") continue; + if (lremovexattr(path.c_str(), eaName.c_str()) == -1) + throw SysError("removing extended attribute '%s' from '%s'", eaName, + path); } + } +#endif - inodesSeen.insert(Inode(st.st_dev, st.st_ino)); - - canonicaliseTimestampAndPermissions(path, st); - - /* Change ownership to the current uid. If it's a symlink, use - lchown if available, otherwise don't bother. Wrong ownership - of a symlink doesn't matter, since the owning user can't change - the symlink and can't delete it because the directory is not - writable. The only exception is top-level paths in the Nix - store (since that directory is group-writable for the Nix build - users group); we check for this case below. */ - if (st.st_uid != geteuid()) { + /* Fail if the file is not owned by the build user. This prevents + us from messing up the ownership/permissions of files + hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). + However, ignore files that we chown'ed ourselves previously to + ensure that we don't fail on hard links within the same build + (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ + if (fromUid != (uid_t)-1 && st.st_uid != fromUid) { + assert(!S_ISDIR(st.st_mode)); + if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) + throw BuildError(format("invalid ownership on file '%1%'") % path); + mode_t mode = st.st_mode & ~S_IFMT; + assert(S_ISLNK(st.st_mode) || + (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && + st.st_mtime == mtimeStore)); + return; + } + + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); + + canonicaliseTimestampAndPermissions(path, st); + + /* Change ownership to the current uid. If it's a symlink, use + lchown if available, otherwise don't bother. Wrong ownership + of a symlink doesn't matter, since the owning user can't change + the symlink and can't delete it because the directory is not + writable. The only exception is top-level paths in the Nix + store (since that directory is group-writable for the Nix build + users group); we check for this case below. */ + if (st.st_uid != geteuid()) { #if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), getegid()) == -1) + if (lchown(path.c_str(), geteuid(), getegid()) == -1) #else - if (!S_ISLNK(st.st_mode) && - chown(path.c_str(), geteuid(), getegid()) == -1) + if (!S_ISLNK(st.st_mode) && chown(path.c_str(), geteuid(), getegid()) == -1) #endif - throw SysError(format("changing owner of '%1%' to %2%") - % path % geteuid()); - } - - if (S_ISDIR(st.st_mode)) { - DirEntries entries = readDirectory(path); - for (auto & i : entries) - canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen); - } + throw SysError(format("changing owner of '%1%' to %2%") % path % + geteuid()); + } + + if (S_ISDIR(st.st_mode)) { + DirEntries entries = readDirectory(path); + for (auto& i : entries) + canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen); + } } +void canonicalisePathMetaData(const Path& path, uid_t fromUid, + InodesSeen& inodesSeen) { + canonicalisePathMetaData_(path, fromUid, inodesSeen); -void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) -{ - canonicalisePathMetaData_(path, fromUid, inodesSeen); - - /* On platforms that don't have lchown(), the top-level path can't - be a symlink, since we can't change its ownership. */ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + /* On platforms that don't have lchown(), the top-level path can't + be a symlink, since we can't change its ownership. */ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); - if (st.st_uid != geteuid()) { - assert(S_ISLNK(st.st_mode)); - throw Error(format("wrong ownership of top-level store path '%1%'") % path); - } + if (st.st_uid != geteuid()) { + assert(S_ISLNK(st.st_mode)); + throw Error(format("wrong ownership of top-level store path '%1%'") % path); + } } - -void canonicalisePathMetaData(const Path & path, uid_t fromUid) -{ - InodesSeen inodesSeen; - canonicalisePathMetaData(path, fromUid, inodesSeen); +void canonicalisePathMetaData(const Path& path, uid_t fromUid) { + InodesSeen inodesSeen; + canonicalisePathMetaData(path, fromUid, inodesSeen); } - -void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv) -{ - string drvName = storePathToName(drvPath); - assert(isDerivation(drvName)); - drvName = string(drvName, 0, drvName.size() - drvExtension.size()); - - if (drv.isFixedOutput()) { - DerivationOutputs::const_iterator out = drv.outputs.find("out"); - if (out == drv.outputs.end()) - throw Error(format("derivation '%1%' does not have an output named 'out'") % drvPath); - - bool recursive; Hash h; - out->second.parseHashInfo(recursive, h); - Path outPath = makeFixedOutputPath(recursive, h, drvName); - - StringPairs::const_iterator j = drv.env.find("out"); - if (out->second.path != outPath || j == drv.env.end() || j->second != outPath) - throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'") - % drvPath % out->second.path % outPath); +void LocalStore::checkDerivationOutputs(const Path& drvPath, + const Derivation& drv) { + string drvName = storePathToName(drvPath); + assert(isDerivation(drvName)); + drvName = string(drvName, 0, drvName.size() - drvExtension.size()); + + if (drv.isFixedOutput()) { + DerivationOutputs::const_iterator out = drv.outputs.find("out"); + if (out == drv.outputs.end()) + throw Error( + format("derivation '%1%' does not have an output named 'out'") % + drvPath); + + bool recursive; + Hash h; + out->second.parseHashInfo(recursive, h); + Path outPath = makeFixedOutputPath(recursive, h, drvName); + + StringPairs::const_iterator j = drv.env.find("out"); + if (out->second.path != outPath || j == drv.env.end() || + j->second != outPath) + throw Error( + format( + "derivation '%1%' has incorrect output '%2%', should be '%3%'") % + drvPath % out->second.path % outPath); + } + + else { + Derivation drvCopy(drv); + for (auto& i : drvCopy.outputs) { + i.second.path = ""; + drvCopy.env[i.first] = ""; } - else { - Derivation drvCopy(drv); - for (auto & i : drvCopy.outputs) { - i.second.path = ""; - drvCopy.env[i.first] = ""; - } - - Hash h = hashDerivationModulo(*this, drvCopy); + Hash h = hashDerivationModulo(*this, drvCopy); - for (auto & i : drv.outputs) { - Path outPath = makeOutputPath(i.first, h, drvName); - StringPairs::const_iterator j = drv.env.find(i.first); - if (i.second.path != outPath || j == drv.env.end() || j->second != outPath) - throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'") - % drvPath % i.second.path % outPath); - } + for (auto& i : drv.outputs) { + Path outPath = makeOutputPath(i.first, h, drvName); + StringPairs::const_iterator j = drv.env.find(i.first); + if (i.second.path != outPath || j == drv.env.end() || + j->second != outPath) + throw Error(format("derivation '%1%' has incorrect output '%2%', " + "should be '%3%'") % + drvPath % i.second.path % outPath); } + } } - -uint64_t LocalStore::addValidPath(State & state, - const ValidPathInfo & info, bool checkOutputs) -{ - if (info.ca != "" && !info.isContentAddressed(*this)) - throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", info.path); - - state.stmtRegisterValidPath.use() - (info.path) - (info.narHash.to_string(Base16)) - (info.registrationTime == 0 ? time(0) : info.registrationTime) - (info.deriver, info.deriver != "") - (info.narSize, info.narSize != 0) - (info.ultimate ? 1 : 0, info.ultimate) - (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (info.ca, !info.ca.empty()) - .exec(); - uint64_t id = sqlite3_last_insert_rowid(state.db); - - /* If this is a derivation, then store the derivation outputs in - the database. This is useful for the garbage collector: it can - efficiently query whether a path is an output of some - derivation. */ - if (isDerivation(info.path)) { - Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(info.path)); - - /* Verify that the output paths in the derivation are correct - (i.e., follow the scheme for computing output paths from - derivations). Note that if this throws an error, then the - DB transaction is rolled back, so the path validity - registration above is undone. */ - if (checkOutputs) checkDerivationOutputs(info.path, drv); - - for (auto & i : drv.outputs) { - state.stmtAddDerivationOutput.use() - (id) - (i.first) - (i.second.path) - .exec(); - } +uint64_t LocalStore::addValidPath(State& state, const ValidPathInfo& info, + bool checkOutputs) { + if (info.ca != "" && !info.isContentAddressed(*this)) + throw Error( + "cannot add path '%s' to the Nix store because it claims to be " + "content-addressed but isn't", + info.path); + + state.stmtRegisterValidPath + .use()(info.path)(info.narHash.to_string(Base16))( + info.registrationTime == 0 ? time(0) : info.registrationTime)( + info.deriver, info.deriver != "")(info.narSize, info.narSize != 0)( + info.ultimate ? 1 : 0, info.ultimate)( + concatStringsSep(" ", info.sigs), !info.sigs.empty())( + info.ca, !info.ca.empty()) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(state.db); + + /* If this is a derivation, then store the derivation outputs in + the database. This is useful for the garbage collector: it can + efficiently query whether a path is an output of some + derivation. */ + if (isDerivation(info.path)) { + Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(info.path)); + + /* Verify that the output paths in the derivation are correct + (i.e., follow the scheme for computing output paths from + derivations). Note that if this throws an error, then the + DB transaction is rolled back, so the path validity + registration above is undone. */ + if (checkOutputs) checkDerivationOutputs(info.path, drv); + + for (auto& i : drv.outputs) { + state.stmtAddDerivationOutput.use()(id)(i.first)(i.second.path).exec(); } + } - { - auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared<ValidPathInfo>(info)); - } + { + auto state_(Store::state.lock()); + state_->pathInfoCache.upsert(storePathToHash(info.path), + std::make_shared<ValidPathInfo>(info)); + } - return id; + return id; } +void LocalStore::queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { + try { + auto info = std::make_shared<ValidPathInfo>(); + info->path = path; -void LocalStore::queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept -{ - try { - auto info = std::make_shared<ValidPathInfo>(); - info->path = path; - - assertStorePath(path); + assertStorePath(path); - callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { - auto state(_state.lock()); + callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { + auto state(_state.lock()); - /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); + /* Get the path info. */ + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); - if (!useQueryPathInfo.next()) - return std::shared_ptr<ValidPathInfo>(); + if (!useQueryPathInfo.next()) return std::shared_ptr<ValidPathInfo>(); - info->id = useQueryPathInfo.getInt(0); + info->id = useQueryPathInfo.getInt(0); - try { - info->narHash = Hash(useQueryPathInfo.getStr(1)); - } catch (BadHash & e) { - throw Error("in valid-path entry for '%s': %s", path, e.what()); - } + try { + info->narHash = Hash(useQueryPathInfo.getStr(1)); + } catch (BadHash& e) { + throw Error("in valid-path entry for '%s': %s", path, e.what()); + } - info->registrationTime = useQueryPathInfo.getInt(2); + info->registrationTime = useQueryPathInfo.getInt(2); - auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); - if (s) info->deriver = s; + auto s = (const char*)sqlite3_column_text(state->stmtQueryPathInfo, 3); + if (s) info->deriver = s; - /* Note that narSize = NULL yields 0. */ - info->narSize = useQueryPathInfo.getInt(4); + /* Note that narSize = NULL yields 0. */ + info->narSize = useQueryPathInfo.getInt(4); - info->ultimate = useQueryPathInfo.getInt(5) == 1; + info->ultimate = useQueryPathInfo.getInt(5) == 1; - s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); - if (s) info->sigs = tokenizeString<StringSet>(s, " "); + s = (const char*)sqlite3_column_text(state->stmtQueryPathInfo, 6); + if (s) info->sigs = tokenizeString<StringSet>(s, " "); - s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = s; + s = (const char*)sqlite3_column_text(state->stmtQueryPathInfo, 7); + if (s) info->ca = s; - /* Get the references. */ - auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); + /* Get the references. */ + auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); - while (useQueryReferences.next()) - info->references.insert(useQueryReferences.getStr(0)); + while (useQueryReferences.next()) + info->references.insert(useQueryReferences.getStr(0)); - return info; - })); + return info; + })); - } catch (...) { callback.rethrow(); } + } catch (...) { + callback.rethrow(); + } } - /* Update path info in the database. */ -void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) -{ - state.stmtUpdatePathInfo.use() - (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16)) - (info.ultimate ? 1 : 0, info.ultimate) - (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (info.ca, !info.ca.empty()) - (info.path) - .exec(); +void LocalStore::updatePathInfo(State& state, const ValidPathInfo& info) { + state.stmtUpdatePathInfo + .use()(info.narSize, info.narSize != 0)(info.narHash.to_string(Base16))( + info.ultimate ? 1 : 0, info.ultimate)( + concatStringsSep(" ", info.sigs), !info.sigs.empty())( + info.ca, !info.ca.empty())(info.path) + .exec(); } - -uint64_t LocalStore::queryValidPathId(State & state, const Path & path) -{ - auto use(state.stmtQueryPathInfo.use()(path)); - if (!use.next()) - throw Error(format("path '%1%' is not valid") % path); - return use.getInt(0); +uint64_t LocalStore::queryValidPathId(State& state, const Path& path) { + auto use(state.stmtQueryPathInfo.use()(path)); + if (!use.next()) throw Error(format("path '%1%' is not valid") % path); + return use.getInt(0); } - -bool LocalStore::isValidPath_(State & state, const Path & path) -{ - return state.stmtQueryPathInfo.use()(path).next(); +bool LocalStore::isValidPath_(State& state, const Path& path) { + return state.stmtQueryPathInfo.use()(path).next(); } - -bool LocalStore::isValidPathUncached(const Path & path) -{ - return retrySQLite<bool>([&]() { - auto state(_state.lock()); - return isValidPath_(*state, path); - }); +bool LocalStore::isValidPathUncached(const Path& path) { + return retrySQLite<bool>([&]() { + auto state(_state.lock()); + return isValidPath_(*state, path); + }); } +PathSet LocalStore::queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute) { + PathSet res; + for (auto& i : paths) + if (isValidPath(i)) res.insert(i); + return res; +} -PathSet LocalStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) -{ +PathSet LocalStore::queryAllValidPaths() { + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + auto use(state->stmtQueryValidPaths.use()); PathSet res; - for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + while (use.next()) res.insert(use.getStr(0)); return res; + }); } +void LocalStore::queryReferrers(State& state, const Path& path, + PathSet& referrers) { + auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); -PathSet LocalStore::queryAllValidPaths() -{ - return retrySQLite<PathSet>([&]() { - auto state(_state.lock()); - auto use(state->stmtQueryValidPaths.use()); - PathSet res; - while (use.next()) res.insert(use.getStr(0)); - return res; - }); + while (useQueryReferrers.next()) + referrers.insert(useQueryReferrers.getStr(0)); } - -void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers) -{ - auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); - - while (useQueryReferrers.next()) - referrers.insert(useQueryReferrers.getStr(0)); -} - - -void LocalStore::queryReferrers(const Path & path, PathSet & referrers) -{ - assertStorePath(path); - return retrySQLite<void>([&]() { - auto state(_state.lock()); - queryReferrers(*state, path, referrers); - }); +void LocalStore::queryReferrers(const Path& path, PathSet& referrers) { + assertStorePath(path); + return retrySQLite<void>([&]() { + auto state(_state.lock()); + queryReferrers(*state, path, referrers); + }); } +PathSet LocalStore::queryValidDerivers(const Path& path) { + assertStorePath(path); -PathSet LocalStore::queryValidDerivers(const Path & path) -{ - assertStorePath(path); - - return retrySQLite<PathSet>([&]() { - auto state(_state.lock()); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); - PathSet derivers; - while (useQueryValidDerivers.next()) - derivers.insert(useQueryValidDerivers.getStr(1)); + PathSet derivers; + while (useQueryValidDerivers.next()) + derivers.insert(useQueryValidDerivers.getStr(1)); - return derivers; - }); + return derivers; + }); } +PathSet LocalStore::queryDerivationOutputs(const Path& path) { + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); -PathSet LocalStore::queryDerivationOutputs(const Path & path) -{ - return retrySQLite<PathSet>([&]() { - auto state(_state.lock()); - - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()( + queryValidPathId(*state, path))); - PathSet outputs; - while (useQueryDerivationOutputs.next()) - outputs.insert(useQueryDerivationOutputs.getStr(1)); + PathSet outputs; + while (useQueryDerivationOutputs.next()) + outputs.insert(useQueryDerivationOutputs.getStr(1)); - return outputs; - }); + return outputs; + }); } +StringSet LocalStore::queryDerivationOutputNames(const Path& path) { + return retrySQLite<StringSet>([&]() { + auto state(_state.lock()); -StringSet LocalStore::queryDerivationOutputNames(const Path & path) -{ - return retrySQLite<StringSet>([&]() { - auto state(_state.lock()); - - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()( + queryValidPathId(*state, path))); - StringSet outputNames; - while (useQueryDerivationOutputs.next()) - outputNames.insert(useQueryDerivationOutputs.getStr(0)); + StringSet outputNames; + while (useQueryDerivationOutputs.next()) + outputNames.insert(useQueryDerivationOutputs.getStr(0)); - return outputNames; - }); + return outputNames; + }); } +Path LocalStore::queryPathFromHashPart(const string& hashPart) { + if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); -Path LocalStore::queryPathFromHashPart(const string & hashPart) -{ - if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); - - Path prefix = storeDir + "/" + hashPart; + Path prefix = storeDir + "/" + hashPart; - return retrySQLite<Path>([&]() -> std::string { - auto state(_state.lock()); + return retrySQLite<Path>([&]() -> std::string { + auto state(_state.lock()); - auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); + auto useQueryPathFromHashPart( + state->stmtQueryPathFromHashPart.use()(prefix)); - if (!useQueryPathFromHashPart.next()) return ""; + if (!useQueryPathFromHashPart.next()) return ""; - const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); - return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; - }); + const char* s = + (const char*)sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); + return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s + : ""; + }); } +PathSet LocalStore::querySubstitutablePaths(const PathSet& paths) { + if (!settings.useSubstitutes) return PathSet(); -PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) -{ - if (!settings.useSubstitutes) return PathSet(); + auto remaining = paths; + PathSet res; - auto remaining = paths; - PathSet res; - - for (auto & sub : getDefaultSubstituters()) { - if (remaining.empty()) break; - if (sub->storeDir != storeDir) continue; - if (!sub->wantMassQuery()) continue; + for (auto& sub : getDefaultSubstituters()) { + if (remaining.empty()) break; + if (sub->storeDir != storeDir) continue; + if (!sub->wantMassQuery()) continue; - auto valid = sub->queryValidPaths(remaining); + auto valid = sub->queryValidPaths(remaining); - PathSet remaining2; - for (auto & path : remaining) - if (valid.count(path)) - res.insert(path); - else - remaining2.insert(path); + PathSet remaining2; + for (auto& path : remaining) + if (valid.count(path)) + res.insert(path); + else + remaining2.insert(path); - std::swap(remaining, remaining2); - } + std::swap(remaining, remaining2); + } - return res; + return res; } - -void LocalStore::querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) -{ - if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - if (sub->storeDir != storeDir) continue; - for (auto & path : paths) { - if (infos.count(path)) continue; - debug(format("checking substituter '%s' for path '%s'") - % sub->getUri() % path); - try { - auto info = sub->queryPathInfo(path); - auto narInfo = std::dynamic_pointer_cast<const NarInfo>( - std::shared_ptr<const ValidPathInfo>(info)); - infos[path] = SubstitutablePathInfo{ - info->deriver, - info->references, - narInfo ? narInfo->fileSize : 0, - info->narSize}; - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - printError(e.what()); - else - throw; - } - } +void LocalStore::querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) { + if (!settings.useSubstitutes) return; + for (auto& sub : getDefaultSubstituters()) { + if (sub->storeDir != storeDir) continue; + for (auto& path : paths) { + if (infos.count(path)) continue; + debug(format("checking substituter '%s' for path '%s'") % sub->getUri() % + path); + try { + auto info = sub->queryPathInfo(path); + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + infos[path] = SubstitutablePathInfo{info->deriver, info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize}; + } catch (InvalidPath&) { + } catch (SubstituterDisabled&) { + } catch (Error& e) { + if (settings.tryFallback) + printError(e.what()); + else + throw; + } } + } } - -void LocalStore::registerValidPath(const ValidPathInfo & info) -{ - ValidPathInfos infos; - infos.push_back(info); - registerValidPaths(infos); +void LocalStore::registerValidPath(const ValidPathInfo& info) { + ValidPathInfos infos; + infos.push_back(info); + registerValidPaths(infos); } +void LocalStore::registerValidPaths(const ValidPathInfos& infos) { + /* SQLite will fsync by default, but the new valid paths may not + be fsync-ed. So some may want to fsync them before registering + the validity, at the expense of some speed of the path + registering operation. */ + if (settings.syncBeforeRegistering) sync(); -void LocalStore::registerValidPaths(const ValidPathInfos & infos) -{ - /* SQLite will fsync by default, but the new valid paths may not - be fsync-ed. So some may want to fsync them before registering - the validity, at the expense of some speed of the path - registering operation. */ - if (settings.syncBeforeRegistering) sync(); - - return retrySQLite<void>([&]() { - auto state(_state.lock()); + return retrySQLite<void>([&]() { + auto state(_state.lock()); - SQLiteTxn txn(state->db); - PathSet paths; + SQLiteTxn txn(state->db); + PathSet paths; - for (auto & i : infos) { - assert(i.narHash.type == htSHA256); - if (isValidPath_(*state, i.path)) - updatePathInfo(*state, i); - else - addValidPath(*state, i, false); - paths.insert(i.path); - } + for (auto& i : infos) { + assert(i.narHash.type == htSHA256); + if (isValidPath_(*state, i.path)) + updatePathInfo(*state, i); + else + addValidPath(*state, i, false); + paths.insert(i.path); + } - for (auto & i : infos) { - auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) - state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); - } + for (auto& i : infos) { + auto referrer = queryValidPathId(*state, i.path); + for (auto& j : i.references) + state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)) + .exec(); + } - /* Check that the derivation outputs are correct. We can't do - this in addValidPath() above, because the references might - not be valid yet. */ - for (auto & i : infos) - if (isDerivation(i.path)) { - // FIXME: inefficient; we already loaded the - // derivation in addValidPath(). - Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(i.path)); - checkDerivationOutputs(i.path, drv); - } - - /* Do a topological sort of the paths. This will throw an - error if a cycle is detected and roll back the - transaction. Cycles can only occur when a derivation - has multiple outputs. */ - topoSortPaths(paths); - - txn.commit(); - }); + /* Check that the derivation outputs are correct. We can't do + this in addValidPath() above, because the references might + not be valid yet. */ + for (auto& i : infos) + if (isDerivation(i.path)) { + // FIXME: inefficient; we already loaded the + // derivation in addValidPath(). + Derivation drv = + readDerivation(realStoreDir + "/" + baseNameOf(i.path)); + checkDerivationOutputs(i.path, drv); + } + + /* Do a topological sort of the paths. This will throw an + error if a cycle is detected and roll back the + transaction. Cycles can only occur when a derivation + has multiple outputs. */ + topoSortPaths(paths); + + txn.commit(); + }); } - /* Invalidate a path. The caller is responsible for checking that there are no referrers. */ -void LocalStore::invalidatePath(State & state, const Path & path) -{ - debug(format("invalidating path '%1%'") % path); +void LocalStore::invalidatePath(State& state, const Path& path) { + debug(format("invalidating path '%1%'") % path); - state.stmtInvalidatePath.use()(path).exec(); + state.stmtInvalidatePath.use()(path).exec(); - /* Note that the foreign key constraints on the Refs table take - care of deleting the references entries for `path'. */ + /* Note that the foreign key constraints on the Refs table take + care of deleting the references entries for `path'. */ - { - auto state_(Store::state.lock()); - state_->pathInfoCache.erase(storePathToHash(path)); - } + { + auto state_(Store::state.lock()); + state_->pathInfoCache.erase(storePathToHash(path)); + } } - -const PublicKeys & LocalStore::getPublicKeys() -{ - auto state(_state.lock()); - if (!state->publicKeys) - state->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys()); - return *state->publicKeys; +const PublicKeys& LocalStore::getPublicKeys() { + auto state(_state.lock()); + if (!state->publicKeys) + state->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys()); + return *state->publicKeys; } +void LocalStore::addToStore(const ValidPathInfo& info, Source& source, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + if (!info.narHash) + throw Error("cannot add path '%s' because it lacks a hash", info.path); -void LocalStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) -{ - if (!info.narHash) - throw Error("cannot add path '%s' because it lacks a hash", info.path); - - if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) - throw Error("cannot add path '%s' because it lacks a valid signature", info.path); - - addTempRoot(info.path); - - if (repair || !isValidPath(info.path)) { - - PathLocks outputLock; + if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) + throw Error("cannot add path '%s' because it lacks a valid signature", + info.path); - Path realPath = realStoreDir + "/" + baseNameOf(info.path); + addTempRoot(info.path); - /* Lock the output path. But don't lock if we're being called - from a build hook (whose parent process already acquired a - lock on this path). */ - if (!locksHeld.count(info.path)) - outputLock.lockPaths({realPath}); + if (repair || !isValidPath(info.path)) { + PathLocks outputLock; - if (repair || !isValidPath(info.path)) { + Path realPath = realStoreDir + "/" + baseNameOf(info.path); - deletePath(realPath); + /* Lock the output path. But don't lock if we're being called + from a build hook (whose parent process already acquired a + lock on this path). */ + if (!locksHeld.count(info.path)) outputLock.lockPaths({realPath}); - /* While restoring the path from the NAR, compute the hash - of the NAR. */ - HashSink hashSink(htSHA256); + if (repair || !isValidPath(info.path)) { + deletePath(realPath); - LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { - size_t n = source.read(data, len); - hashSink(data, n); - return n; - }); + /* While restoring the path from the NAR, compute the hash + of the NAR. */ + HashSink hashSink(htSHA256); - restorePath(realPath, wrapperSource); + LambdaSource wrapperSource( + [&](unsigned char* data, size_t len) -> size_t { + size_t n = source.read(data, len); + hashSink(data, n); + return n; + }); - auto hashResult = hashSink.finish(); + restorePath(realPath, wrapperSource); - if (hashResult.first != info.narHash) - throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - info.path, info.narHash.to_string(), hashResult.first.to_string()); + auto hashResult = hashSink.finish(); - if (hashResult.second != info.narSize) - throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", - info.path, info.narSize, hashResult.second); + if (hashResult.first != info.narHash) + throw Error( + "hash mismatch importing path '%s';\n wanted: %s\n got: %s", + info.path, info.narHash.to_string(), hashResult.first.to_string()); - autoGC(); + if (hashResult.second != info.narSize) + throw Error( + "size mismatch importing path '%s';\n wanted: %s\n got: %s", + info.path, info.narSize, hashResult.second); - canonicalisePathMetaData(realPath, -1); + autoGC(); - optimisePath(realPath); // FIXME: combine with hashPath() + canonicalisePathMetaData(realPath, -1); - registerValidPath(info); - } + optimisePath(realPath); // FIXME: combine with hashPath() - outputLock.setDeletion(true); + registerValidPath(info); } -} - - -Path LocalStore::addToStoreFromDump(const string & dump, const string & name, - bool recursive, HashType hashAlgo, RepairFlag repair) -{ - Hash h = hashString(hashAlgo, dump); - - Path dstPath = makeFixedOutputPath(recursive, h, name); - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - Path realPath = realStoreDir + "/" + baseNameOf(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { + outputLock.setDeletion(true); + } +} - deletePath(realPath); +Path LocalStore::addToStoreFromDump(const string& dump, const string& name, + bool recursive, HashType hashAlgo, + RepairFlag repair) { + Hash h = hashString(hashAlgo, dump); - autoGC(); + Path dstPath = makeFixedOutputPath(recursive, h, name); - if (recursive) { - StringSource source(dump); - restorePath(realPath, source); - } else - writeFile(realPath, dump); + addTempRoot(dstPath); - canonicalisePathMetaData(realPath, -1); + if (repair || !isValidPath(dstPath)) { + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash; - if (recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); + Path realPath = realStoreDir + "/" + baseNameOf(dstPath); - optimisePath(realPath); // FIXME: combine with hashPath() + PathLocks outputLock({realPath}); - ValidPathInfo info; - info.path = dstPath; - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = makeFixedOutputCA(recursive, h); - registerValidPath(info); - } - - outputLock.setDeletion(true); + if (repair || !isValidPath(dstPath)) { + deletePath(realPath); + + autoGC(); + + if (recursive) { + StringSource source(dump); + restorePath(realPath, source); + } else + writeFile(realPath, dump); + + canonicalisePathMetaData(realPath, -1); + + /* Register the SHA-256 hash of the NAR serialisation of + the path in the database. We may just have computed it + above (if called with recursive == true and hashAlgo == + sha256); otherwise, compute it here. */ + HashResult hash; + if (recursive) { + hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); + hash.second = dump.size(); + } else + hash = hashPath(htSHA256, realPath); + + optimisePath(realPath); // FIXME: combine with hashPath() + + ValidPathInfo info; + info.path = dstPath; + info.narHash = hash.first; + info.narSize = hash.second; + info.ca = makeFixedOutputCA(recursive, h); + registerValidPath(info); } - return dstPath; -} - + outputLock.setDeletion(true); + } -Path LocalStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - Path srcPath(absPath(_srcPath)); - - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - if (recursive) - dumpPath(srcPath, sink, filter); - else - sink.s = make_ref<std::string>(readFile(srcPath)); - - return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); + return dstPath; } +Path LocalStore::addToStore(const string& name, const Path& _srcPath, + bool recursive, HashType hashAlgo, + PathFilter& filter, RepairFlag repair) { + Path srcPath(absPath(_srcPath)); + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + if (recursive) + dumpPath(srcPath, sink, filter); + else + sink.s = make_ref<std::string>(readFile(srcPath)); + + return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); +} -Path LocalStore::addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) -{ - auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - Path realPath = realStoreDir + "/" + baseNameOf(dstPath); +Path LocalStore::addTextToStore(const string& name, const string& s, + const PathSet& references, RepairFlag repair) { + auto hash = hashString(htSHA256, s); + auto dstPath = makeTextPath(name, hash, references); - PathLocks outputLock({realPath}); + addTempRoot(dstPath); - if (repair || !isValidPath(dstPath)) { + if (repair || !isValidPath(dstPath)) { + Path realPath = realStoreDir + "/" + baseNameOf(dstPath); - deletePath(realPath); + PathLocks outputLock({realPath}); - autoGC(); + if (repair || !isValidPath(dstPath)) { + deletePath(realPath); - writeFile(realPath, s); + autoGC(); - canonicalisePathMetaData(realPath, -1); + writeFile(realPath, s); - StringSink sink; - dumpString(s, sink); - auto narHash = hashString(htSHA256, *sink.s); + canonicalisePathMetaData(realPath, -1); - optimisePath(realPath); + StringSink sink; + dumpString(s, sink); + auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info; - info.path = dstPath; - info.narHash = narHash; - info.narSize = sink.s->size(); - info.references = references; - info.ca = "text:" + hash.to_string(); - registerValidPath(info); - } + optimisePath(realPath); - outputLock.setDeletion(true); + ValidPathInfo info; + info.path = dstPath; + info.narHash = narHash; + info.narSize = sink.s->size(); + info.references = references; + info.ca = "text:" + hash.to_string(); + registerValidPath(info); } - return dstPath; -} + outputLock.setDeletion(true); + } + return dstPath; +} /* Create a temporary directory in the store that won't be garbage-collected. */ -Path LocalStore::createTempDirInStore() -{ - Path tmpDir; - do { - /* There is a slight possibility that `tmpDir' gets deleted by - the GC between createTempDir() and addTempRoot(), so repeat - until `tmpDir' exists. */ - tmpDir = createTempDir(realStoreDir); - addTempRoot(tmpDir); - } while (!pathExists(tmpDir)); - return tmpDir; +Path LocalStore::createTempDirInStore() { + Path tmpDir; + do { + /* There is a slight possibility that `tmpDir' gets deleted by + the GC between createTempDir() and addTempRoot(), so repeat + until `tmpDir' exists. */ + tmpDir = createTempDir(realStoreDir); + addTempRoot(tmpDir); + } while (!pathExists(tmpDir)); + return tmpDir; } +void LocalStore::invalidatePathChecked(const Path& path) { + assertStorePath(path); -void LocalStore::invalidatePathChecked(const Path & path) -{ - assertStorePath(path); + retrySQLite<void>([&]() { + auto state(_state.lock()); - retrySQLite<void>([&]() { - auto state(_state.lock()); + SQLiteTxn txn(state->db); + + if (isValidPath_(*state, path)) { + PathSet referrers; + queryReferrers(*state, path, referrers); + referrers.erase(path); /* ignore self-references */ + if (!referrers.empty()) + throw PathInUse( + format("cannot delete path '%1%' because it is in use by %2%") % + path % showPaths(referrers)); + invalidatePath(*state, path); + } - SQLiteTxn txn(state->db); + txn.commit(); + }); +} - if (isValidPath_(*state, path)) { - PathSet referrers; queryReferrers(*state, path, referrers); - referrers.erase(path); /* ignore self-references */ - if (!referrers.empty()) - throw PathInUse(format("cannot delete path '%1%' because it is in use by %2%") - % path % showPaths(referrers)); - invalidatePath(*state, path); - } +bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { + printError(format("reading the Nix store...")); - txn.commit(); - }); -} + bool errors = false; + /* Acquire the global GC lock to get a consistent snapshot of + existing and valid paths. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); -bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) -{ - printError(format("reading the Nix store...")); + PathSet store; + for (auto& i : readDirectory(realStoreDir)) store.insert(i.name); - bool errors = false; + /* Check whether all valid paths actually exist. */ + printInfo("checking path existence..."); - /* Acquire the global GC lock to get a consistent snapshot of - existing and valid paths. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); + PathSet validPaths2 = queryAllValidPaths(), validPaths, done; - PathSet store; - for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); + fdGCLock = -1; - /* Check whether all valid paths actually exist. */ - printInfo("checking path existence..."); + for (auto& i : validPaths2) + verifyPath(i, store, done, validPaths, repair, errors); - PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + /* Optionally, check the content hashes (slow). */ + if (checkContents) { + printInfo("checking hashes..."); - fdGCLock = -1; + Hash nullHash(htSHA256); - for (auto & i : validPaths2) - verifyPath(i, store, done, validPaths, repair, errors); + for (auto& i : validPaths) { + try { + auto info = std::const_pointer_cast<ValidPathInfo>( + std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); + + /* Check the content hash (optionally - slow). */ + printMsg(lvlTalkative, format("checking contents of '%1%'") % i); + HashResult current = hashPath(info->narHash.type, toRealPath(i)); - /* Optionally, check the content hashes (slow). */ - if (checkContents) { - printInfo("checking hashes..."); - - Hash nullHash(htSHA256); - - for (auto & i : validPaths) { - try { - auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); - - /* Check the content hash (optionally - slow). */ - printMsg(lvlTalkative, format("checking contents of '%1%'") % i); - HashResult current = hashPath(info->narHash.type, toRealPath(i)); - - if (info->narHash != nullHash && info->narHash != current.first) { - printError(format("path '%1%' was modified! " - "expected hash '%2%', got '%3%'") - % i % info->narHash.to_string() % current.first.to_string()); - if (repair) repairPath(i); else errors = true; - } else { - - bool update = false; - - /* Fill in missing hashes. */ - if (info->narHash == nullHash) { - printError(format("fixing missing hash on '%1%'") % i); - info->narHash = current.first; - update = true; - } - - /* Fill in missing narSize fields (from old stores). */ - if (info->narSize == 0) { - printError(format("updating size field on '%1%' to %2%") % i % current.second); - info->narSize = current.second; - update = true; - } - - if (update) { - auto state(_state.lock()); - updatePathInfo(*state, *info); - } - - } - - } catch (Error & e) { - /* It's possible that the path got GC'ed, so ignore - errors on invalid paths. */ - if (isValidPath(i)) - printError(format("error: %1%") % e.msg()); - else - printError(format("warning: %1%") % e.msg()); - errors = true; - } + if (info->narHash != nullHash && info->narHash != current.first) { + printError(format("path '%1%' was modified! " + "expected hash '%2%', got '%3%'") % + i % info->narHash.to_string() % current.first.to_string()); + if (repair) + repairPath(i); + else + errors = true; + } else { + bool update = false; + + /* Fill in missing hashes. */ + if (info->narHash == nullHash) { + printError(format("fixing missing hash on '%1%'") % i); + info->narHash = current.first; + update = true; + } + + /* Fill in missing narSize fields (from old stores). */ + if (info->narSize == 0) { + printError(format("updating size field on '%1%' to %2%") % i % + current.second); + info->narSize = current.second; + update = true; + } + + if (update) { + auto state(_state.lock()); + updatePathInfo(*state, *info); + } } + + } catch (Error& e) { + /* It's possible that the path got GC'ed, so ignore + errors on invalid paths. */ + if (isValidPath(i)) + printError(format("error: %1%") % e.msg()); + else + printError(format("warning: %1%") % e.msg()); + errors = true; + } } + } - return errors; + return errors; } +void LocalStore::verifyPath(const Path& path, const PathSet& store, + PathSet& done, PathSet& validPaths, + RepairFlag repair, bool& errors) { + checkInterrupt(); -void LocalStore::verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors) -{ - checkInterrupt(); - - if (done.find(path) != done.end()) return; - done.insert(path); - - if (!isStorePath(path)) { - printError(format("path '%1%' is not in the Nix store") % path); - auto state(_state.lock()); - invalidatePath(*state, path); - return; - } + if (done.find(path) != done.end()) return; + done.insert(path); - if (store.find(baseNameOf(path)) == store.end()) { - /* Check any referrers first. If we can invalidate them - first, then we can invalidate this path as well. */ - bool canInvalidate = true; - PathSet referrers; queryReferrers(path, referrers); - for (auto & i : referrers) - if (i != path) { - verifyPath(i, store, done, validPaths, repair, errors); - if (validPaths.find(i) != validPaths.end()) - canInvalidate = false; - } - - if (canInvalidate) { - printError(format("path '%1%' disappeared, removing from database...") % path); - auto state(_state.lock()); - invalidatePath(*state, path); - } else { - printError(format("path '%1%' disappeared, but it still has valid referrers!") % path); - if (repair) - try { - repairPath(path); - } catch (Error & e) { - printError(format("warning: %1%") % e.msg()); - errors = true; - } - else errors = true; + if (!isStorePath(path)) { + printError(format("path '%1%' is not in the Nix store") % path); + auto state(_state.lock()); + invalidatePath(*state, path); + return; + } + + if (store.find(baseNameOf(path)) == store.end()) { + /* Check any referrers first. If we can invalidate them + first, then we can invalidate this path as well. */ + bool canInvalidate = true; + PathSet referrers; + queryReferrers(path, referrers); + for (auto& i : referrers) + if (i != path) { + verifyPath(i, store, done, validPaths, repair, errors); + if (validPaths.find(i) != validPaths.end()) canInvalidate = false; + } + + if (canInvalidate) { + printError(format("path '%1%' disappeared, removing from database...") % + path); + auto state(_state.lock()); + invalidatePath(*state, path); + } else { + printError( + format("path '%1%' disappeared, but it still has valid referrers!") % + path); + if (repair) try { + repairPath(path); + } catch (Error& e) { + printError(format("warning: %1%") % e.msg()); + errors = true; } - - return; + else + errors = true; } - validPaths.insert(path); -} - + return; + } -unsigned int LocalStore::getProtocol() -{ - return PROTOCOL_VERSION; + validPaths.insert(path); } +unsigned int LocalStore::getProtocol() { return PROTOCOL_VERSION; } -#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) +#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && \ + defined(FS_IMMUTABLE_FL) -static void makeMutable(const Path & path) -{ - checkInterrupt(); +static void makeMutable(const Path& path) { + checkInterrupt(); - struct stat st = lstat(path); + struct stat st = lstat(path); - if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; + if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; - if (S_ISDIR(st.st_mode)) { - for (auto & i : readDirectory(path)) - makeMutable(path + "/" + i.name); - } + if (S_ISDIR(st.st_mode)) { + for (auto& i : readDirectory(path)) makeMutable(path + "/" + i.name); + } - /* The O_NOFOLLOW is important to prevent us from changing the - mutable bit on the target of a symlink (which would be a - security hole). */ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); - if (fd == -1) { - if (errno == ELOOP) return; // it's a symlink - throw SysError(format("opening file '%1%'") % path); - } + /* The O_NOFOLLOW is important to prevent us from changing the + mutable bit on the target of a symlink (which would be a + security hole). */ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd == -1) { + if (errno == ELOOP) return; // it's a symlink + throw SysError(format("opening file '%1%'") % path); + } - unsigned int flags = 0, old; + unsigned int flags = 0, old; - /* Silently ignore errors getting/setting the immutable flag so - that we work correctly on filesystems that don't support it. */ - if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return; - old = flags; - flags &= ~FS_IMMUTABLE_FL; - if (old == flags) return; - if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return; + /* Silently ignore errors getting/setting the immutable flag so + that we work correctly on filesystems that don't support it. */ + if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return; + old = flags; + flags &= ~FS_IMMUTABLE_FL; + if (old == flags) return; + if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return; } /* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ -void LocalStore::upgradeStore7() -{ - if (getuid() != 0) return; - printError("removing immutable bits from the Nix store (this may take a while)..."); - makeMutable(realStoreDir); +void LocalStore::upgradeStore7() { + if (getuid() != 0) return; + printError( + "removing immutable bits from the Nix store (this may take a while)..."); + makeMutable(realStoreDir); } #else -void LocalStore::upgradeStore7() -{ -} +void LocalStore::upgradeStore7() {} #endif - -void LocalStore::vacuumDB() -{ - auto state(_state.lock()); - state->db.exec("vacuum"); +void LocalStore::vacuumDB() { + auto state(_state.lock()); + state->db.exec("vacuum"); } +void LocalStore::addSignatures(const Path& storePath, const StringSet& sigs) { + retrySQLite<void>([&]() { + auto state(_state.lock()); -void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs) -{ - retrySQLite<void>([&]() { - auto state(_state.lock()); - - SQLiteTxn txn(state->db); + SQLiteTxn txn(state->db); - auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath))); + auto info = std::const_pointer_cast<ValidPathInfo>( + std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath))); - info->sigs.insert(sigs.begin(), sigs.end()); + info->sigs.insert(sigs.begin(), sigs.end()); - updatePathInfo(*state, *info); + updatePathInfo(*state, *info); - txn.commit(); - }); + txn.commit(); + }); } +void LocalStore::signPathInfo(ValidPathInfo& info) { + // FIXME: keep secret keys in memory. -void LocalStore::signPathInfo(ValidPathInfo & info) -{ - // FIXME: keep secret keys in memory. - - auto secretKeyFiles = settings.secretKeyFiles; + auto secretKeyFiles = settings.secretKeyFiles; - for (auto & secretKeyFile : secretKeyFiles.get()) { - SecretKey secretKey(readFile(secretKeyFile)); - info.sign(secretKey); - } + for (auto& secretKeyFile : secretKeyFiles.get()) { + SecretKey secretKey(readFile(secretKeyFile)); + info.sign(secretKey); + } } - -void LocalStore::createUser(const std::string & userName, uid_t userId) -{ - for (auto & dir : { - fmt("%s/profiles/per-user/%s", stateDir, userName), - fmt("%s/gcroots/per-user/%s", stateDir, userName) - }) { - createDirs(dir); - if (chmod(dir.c_str(), 0755) == -1) - throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, getgid()) == -1) - throw SysError("changing owner of directory '%s'", dir); - } +void LocalStore::createUser(const std::string& userName, uid_t userId) { + for (auto& dir : {fmt("%s/profiles/per-user/%s", stateDir, userName), + fmt("%s/gcroots/per-user/%s", stateDir, userName)}) { + createDirs(dir); + if (chmod(dir.c_str(), 0755) == -1) + throw SysError("changing permissions of directory '%s'", dir); + if (chown(dir.c_str(), userId, getgid()) == -1) + throw SysError("changing owner of directory '%s'", dir); + } } - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/local-store.hh b/third_party/nix/src/libstore/local-store.hh index 379a06af87de..c1cd776a748d 100644 --- a/third_party/nix/src/libstore/local-store.hh +++ b/third_party/nix/src/libstore/local-store.hh @@ -1,309 +1,295 @@ #pragma once -#include "sqlite.hh" - -#include "pathlocks.hh" -#include "store-api.hh" -#include "sync.hh" -#include "util.hh" - #include <chrono> #include <future> #include <string> #include <unordered_set> - +#include "pathlocks.hh" +#include "sqlite.hh" +#include "store-api.hh" +#include "sync.hh" +#include "util.hh" namespace nix { - /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ const int nixSchemaVersion = 10; - struct Derivation; - -struct OptimiseStats -{ - unsigned long filesLinked = 0; - unsigned long long bytesFreed = 0; - unsigned long long blocksFreed = 0; +struct OptimiseStats { + unsigned long filesLinked = 0; + unsigned long long bytesFreed = 0; + unsigned long long blocksFreed = 0; }; +class LocalStore : public LocalFSStore { + private: + /* Lock file used for upgrading. */ + AutoCloseFD globalLock; -class LocalStore : public LocalFSStore -{ -private: - - /* Lock file used for upgrading. */ - AutoCloseFD globalLock; - - struct State - { - /* The SQLite database object. */ - SQLite db; - - /* Some precompiled SQLite statements. */ - SQLiteStmt stmtRegisterValidPath; - SQLiteStmt stmtUpdatePathInfo; - SQLiteStmt stmtAddReference; - SQLiteStmt stmtQueryPathInfo; - SQLiteStmt stmtQueryReferences; - SQLiteStmt stmtQueryReferrers; - SQLiteStmt stmtInvalidatePath; - SQLiteStmt stmtAddDerivationOutput; - SQLiteStmt stmtQueryValidDerivers; - SQLiteStmt stmtQueryDerivationOutputs; - SQLiteStmt stmtQueryPathFromHashPart; - SQLiteStmt stmtQueryValidPaths; - - /* The file to which we write our temporary roots. */ - AutoCloseFD fdTempRoots; + struct State { + /* The SQLite database object. */ + SQLite db; - /* The last time we checked whether to do an auto-GC, or an - auto-GC finished. */ - std::chrono::time_point<std::chrono::steady_clock> lastGCCheck; + /* Some precompiled SQLite statements. */ + SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtUpdatePathInfo; + SQLiteStmt stmtAddReference; + SQLiteStmt stmtQueryPathInfo; + SQLiteStmt stmtQueryReferences; + SQLiteStmt stmtQueryReferrers; + SQLiteStmt stmtInvalidatePath; + SQLiteStmt stmtAddDerivationOutput; + SQLiteStmt stmtQueryValidDerivers; + SQLiteStmt stmtQueryDerivationOutputs; + SQLiteStmt stmtQueryPathFromHashPart; + SQLiteStmt stmtQueryValidPaths; - /* Whether auto-GC is running. If so, get gcFuture to wait for - the GC to finish. */ - bool gcRunning = false; - std::shared_future<void> gcFuture; + /* The file to which we write our temporary roots. */ + AutoCloseFD fdTempRoots; - /* How much disk space was available after the previous - auto-GC. If the current available disk space is below - minFree but not much below availAfterGC, then there is no - point in starting a new GC. */ - uint64_t availAfterGC = std::numeric_limits<uint64_t>::max(); + /* The last time we checked whether to do an auto-GC, or an + auto-GC finished. */ + std::chrono::time_point<std::chrono::steady_clock> lastGCCheck; - std::unique_ptr<PublicKeys> publicKeys; - }; + /* Whether auto-GC is running. If so, get gcFuture to wait for + the GC to finish. */ + bool gcRunning = false; + std::shared_future<void> gcFuture; - Sync<State, std::recursive_mutex> _state; + /* How much disk space was available after the previous + auto-GC. If the current available disk space is below + minFree but not much below availAfterGC, then there is no + point in starting a new GC. */ + uint64_t availAfterGC = std::numeric_limits<uint64_t>::max(); -public: + std::unique_ptr<PublicKeys> publicKeys; + }; - PathSetting realStoreDir_; + Sync<State, std::recursive_mutex> _state; - const Path realStoreDir; - const Path dbDir; - const Path linksDir; - const Path reservedPath; - const Path schemaPath; - const Path trashDir; - const Path tempRootsDir; - const Path fnTempRoots; + public: + PathSetting realStoreDir_; -private: + const Path realStoreDir; + const Path dbDir; + const Path linksDir; + const Path reservedPath; + const Path schemaPath; + const Path trashDir; + const Path tempRootsDir; + const Path fnTempRoots; - Setting<bool> requireSigs{(Store*) this, - settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; + private: + Setting<bool> requireSigs{ + (Store*)this, settings.requireSigs, "require-sigs", + "whether store paths should have a trusted signature on import"}; - const PublicKeys & getPublicKeys(); + const PublicKeys& getPublicKeys(); -public: + public: + // Hack for build-remote.cc. + PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS")); - // Hack for build-remote.cc. - PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS")); + /* Initialise the local store, upgrading the schema if + necessary. */ + LocalStore(const Params& params); - /* Initialise the local store, upgrading the schema if - necessary. */ - LocalStore(const Params & params); + ~LocalStore(); - ~LocalStore(); + /* Implementations of abstract store API methods. */ - /* Implementations of abstract store API methods. */ + std::string getUri() override; - std::string getUri() override; + bool isValidPathUncached(const Path& path) override; - bool isValidPathUncached(const Path & path) override; + PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute = + NoSubstitute) override; - PathSet queryValidPaths(const PathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override; + PathSet queryAllValidPaths() override; - PathSet queryAllValidPaths() override; + void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; - void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; + void queryReferrers(const Path& path, PathSet& referrers) override; - void queryReferrers(const Path & path, PathSet & referrers) override; + PathSet queryValidDerivers(const Path& path) override; - PathSet queryValidDerivers(const Path & path) override; + PathSet queryDerivationOutputs(const Path& path) override; - PathSet queryDerivationOutputs(const Path & path) override; + StringSet queryDerivationOutputNames(const Path& path) override; - StringSet queryDerivationOutputNames(const Path & path) override; + Path queryPathFromHashPart(const string& hashPart) override; - Path queryPathFromHashPart(const string & hashPart) override; + PathSet querySubstitutablePaths(const PathSet& paths) override; - PathSet querySubstitutablePaths(const PathSet & paths) override; + void querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) override; - void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) override; + void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair, + CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) override; - void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; + Path addToStore(const string& name, const Path& srcPath, bool recursive, + HashType hashAlgo, PathFilter& filter, + RepairFlag repair) override; - Path addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override; + /* Like addToStore(), but the contents of the path are contained + in `dump', which is either a NAR serialisation (if recursive == + true) or simply the contents of a regular file (if recursive == + false). */ + Path addToStoreFromDump(const string& dump, const string& name, + bool recursive = true, HashType hashAlgo = htSHA256, + RepairFlag repair = NoRepair); - /* Like addToStore(), but the contents of the path are contained - in `dump', which is either a NAR serialisation (if recursive == - true) or simply the contents of a regular file (if recursive == - false). */ - Path addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair); + Path addTextToStore(const string& name, const string& s, + const PathSet& references, RepairFlag repair) override; - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override; + void buildPaths(const PathSet& paths, BuildMode buildMode) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override; + BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv, + BuildMode buildMode) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override; + void ensurePath(const Path& path) override; - void ensurePath(const Path & path) override; + void addTempRoot(const Path& path) override; - void addTempRoot(const Path & path) override; + void addIndirectRoot(const Path& path) override; - void addIndirectRoot(const Path & path) override; + void syncWithGC() override; - void syncWithGC() override; + private: + typedef std::shared_ptr<AutoCloseFD> FDPtr; + typedef list<FDPtr> FDs; -private: + void findTempRoots(FDs& fds, Roots& roots, bool censor); - typedef std::shared_ptr<AutoCloseFD> FDPtr; - typedef list<FDPtr> FDs; + public: + Roots findRoots(bool censor) override; - void findTempRoots(FDs & fds, Roots & roots, bool censor); + void collectGarbage(const GCOptions& options, GCResults& results) override; -public: + /* Optimise the disk space usage of the Nix store by hard-linking + files with the same contents. */ + void optimiseStore(OptimiseStats& stats); - Roots findRoots(bool censor) override; + void optimiseStore() override; - void collectGarbage(const GCOptions & options, GCResults & results) override; + /* Optimise a single store path. */ + void optimisePath(const Path& path); - /* Optimise the disk space usage of the Nix store by hard-linking - files with the same contents. */ - void optimiseStore(OptimiseStats & stats); + bool verifyStore(bool checkContents, RepairFlag repair) override; - void optimiseStore() override; + /* Register the validity of a path, i.e., that `path' exists, that + the paths referenced by it exists, and in the case of an output + path of a derivation, that it has been produced by a successful + execution of the derivation (or something equivalent). Also + register the hash of the file system contents of the path. The + hash must be a SHA-256 hash. */ + void registerValidPath(const ValidPathInfo& info); - /* Optimise a single store path. */ - void optimisePath(const Path & path); + void registerValidPaths(const ValidPathInfos& infos); - bool verifyStore(bool checkContents, RepairFlag repair) override; + unsigned int getProtocol() override; - /* Register the validity of a path, i.e., that `path' exists, that - the paths referenced by it exists, and in the case of an output - path of a derivation, that it has been produced by a successful - execution of the derivation (or something equivalent). Also - register the hash of the file system contents of the path. The - hash must be a SHA-256 hash. */ - void registerValidPath(const ValidPathInfo & info); + void vacuumDB(); - void registerValidPaths(const ValidPathInfos & infos); + /* Repair the contents of the given path by redownloading it using + a substituter (if available). */ + void repairPath(const Path& path); - unsigned int getProtocol() override; + void addSignatures(const Path& storePath, const StringSet& sigs) override; - void vacuumDB(); + /* If free disk space in /nix/store if below minFree, delete + garbage until it exceeds maxFree. */ + void autoGC(bool sync = true); - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ - void repairPath(const Path & path); + private: + int getSchema(); - void addSignatures(const Path & storePath, const StringSet & sigs) override; + void openDB(State& state, bool create); - /* If free disk space in /nix/store if below minFree, delete - garbage until it exceeds maxFree. */ - void autoGC(bool sync = true); + void makeStoreWritable(); -private: + uint64_t queryValidPathId(State& state, const Path& path); - int getSchema(); + uint64_t addValidPath(State& state, const ValidPathInfo& info, + bool checkOutputs = true); - void openDB(State & state, bool create); + void invalidatePath(State& state, const Path& path); - void makeStoreWritable(); + /* Delete a path from the Nix store. */ + void invalidatePathChecked(const Path& path); - uint64_t queryValidPathId(State & state, const Path & path); + void verifyPath(const Path& path, const PathSet& store, PathSet& done, + PathSet& validPaths, RepairFlag repair, bool& errors); - uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); + void updatePathInfo(State& state, const ValidPathInfo& info); - void invalidatePath(State & state, const Path & path); + void upgradeStore6(); + void upgradeStore7(); + PathSet queryValidPathsOld(); + ValidPathInfo queryPathInfoOld(const Path& path); - /* Delete a path from the Nix store. */ - void invalidatePathChecked(const Path & path); + struct GCState; - void verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors); + void deleteGarbage(GCState& state, const Path& path); - void updatePathInfo(State & state, const ValidPathInfo & info); + void tryToDelete(GCState& state, const Path& path); - void upgradeStore6(); - void upgradeStore7(); - PathSet queryValidPathsOld(); - ValidPathInfo queryPathInfoOld(const Path & path); + bool canReachRoot(GCState& state, PathSet& visited, const Path& path); - struct GCState; + void deletePathRecursive(GCState& state, const Path& path); - void deleteGarbage(GCState & state, const Path & path); + bool isActiveTempFile(const GCState& state, const Path& path, + const string& suffix); - void tryToDelete(GCState & state, const Path & path); + AutoCloseFD openGCLock(LockType lockType); - bool canReachRoot(GCState & state, PathSet & visited, const Path & path); + void findRoots(const Path& path, unsigned char type, Roots& roots); - void deletePathRecursive(GCState & state, const Path & path); + void findRootsNoTemp(Roots& roots, bool censor); - bool isActiveTempFile(const GCState & state, - const Path & path, const string & suffix); + void findRuntimeRoots(Roots& roots, bool censor); - AutoCloseFD openGCLock(LockType lockType); + void removeUnusedLinks(const GCState& state); - void findRoots(const Path & path, unsigned char type, Roots & roots); + Path createTempDirInStore(); - void findRootsNoTemp(Roots & roots, bool censor); + void checkDerivationOutputs(const Path& drvPath, const Derivation& drv); - void findRuntimeRoots(Roots & roots, bool censor); + typedef std::unordered_set<ino_t> InodeHash; - void removeUnusedLinks(const GCState & state); + InodeHash loadInodeHash(); + Strings readDirectoryIgnoringInodes(const Path& path, + const InodeHash& inodeHash); + void optimisePath_(Activity* act, OptimiseStats& stats, const Path& path, + InodeHash& inodeHash); - Path createTempDirInStore(); + // Internal versions that are not wrapped in retry_sqlite. + bool isValidPath_(State& state, const Path& path); + void queryReferrers(State& state, const Path& path, PathSet& referrers); - void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + /* Add signatures to a ValidPathInfo using the secret keys + specified by the ‘secret-key-files’ option. */ + void signPathInfo(ValidPathInfo& info); - typedef std::unordered_set<ino_t> InodeHash; + Path getRealStoreDir() override { return realStoreDir; } - InodeHash loadInodeHash(); - Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); - void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash); + void createUser(const std::string& userName, uid_t userId) override; - // Internal versions that are not wrapped in retry_sqlite. - bool isValidPath_(State & state, const Path & path); - void queryReferrers(State & state, const Path & path, PathSet & referrers); - - /* Add signatures to a ValidPathInfo using the secret keys - specified by the ‘secret-key-files’ option. */ - void signPathInfo(ValidPathInfo & info); - - Path getRealStoreDir() override { return realStoreDir; } - - void createUser(const std::string & userName, uid_t userId) override; - - friend class DerivationGoal; - friend class SubstitutionGoal; + friend class DerivationGoal; + friend class SubstitutionGoal; }; - typedef std::pair<dev_t, ino_t> Inode; typedef set<Inode> InodesSeen; - /* "Fix", or canonicalise, the meta-data of the files in a store path after it has been built. In particular: - the last modification date on each file is set to 1 (i.e., @@ -312,11 +298,12 @@ typedef set<Inode> InodesSeen; without execute permission; setuid bits etc. are cleared) - the owner and group are set to the Nix user and group, if we're running as root. */ -void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen); -void canonicalisePathMetaData(const Path & path, uid_t fromUid); +void canonicalisePathMetaData(const Path& path, uid_t fromUid, + InodesSeen& inodesSeen); +void canonicalisePathMetaData(const Path& path, uid_t fromUid); -void canonicaliseTimestampAndPermissions(const Path & path); +void canonicaliseTimestampAndPermissions(const Path& path); MakeError(PathInUse, Error); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/machines.cc b/third_party/nix/src/libstore/machines.cc index f848582dafd4..526afdbbc0cb 100644 --- a/third_party/nix/src/libstore/machines.cc +++ b/third_party/nix/src/libstore/machines.cc @@ -1,100 +1,93 @@ #include "machines.hh" -#include "util.hh" -#include "globals.hh" - #include <algorithm> +#include "globals.hh" +#include "util.hh" namespace nix { -Machine::Machine(decltype(storeUri) storeUri, - decltype(systemTypes) systemTypes, - decltype(sshKey) sshKey, - decltype(maxJobs) maxJobs, - decltype(speedFactor) speedFactor, - decltype(supportedFeatures) supportedFeatures, - decltype(mandatoryFeatures) mandatoryFeatures, - decltype(sshPublicHostKey) sshPublicHostKey) : - storeUri( - // Backwards compatibility: if the URI is a hostname, - // prepend ssh://. - storeUri.find("://") != std::string::npos - || hasPrefix(storeUri, "local") - || hasPrefix(storeUri, "remote") - || hasPrefix(storeUri, "auto") - || hasPrefix(storeUri, "/") - ? storeUri - : "ssh://" + storeUri), - systemTypes(systemTypes), - sshKey(sshKey), - maxJobs(maxJobs), - speedFactor(std::max(1U, speedFactor)), - supportedFeatures(supportedFeatures), - mandatoryFeatures(mandatoryFeatures), - sshPublicHostKey(sshPublicHostKey) -{} +Machine::Machine(decltype(storeUri) storeUri, decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey) + : storeUri( + // Backwards compatibility: if the URI is a hostname, + // prepend ssh://. + storeUri.find("://") != std::string::npos || + hasPrefix(storeUri, "local") || + hasPrefix(storeUri, "remote") || + hasPrefix(storeUri, "auto") || hasPrefix(storeUri, "/") + ? storeUri + : "ssh://" + storeUri), + systemTypes(systemTypes), + sshKey(sshKey), + maxJobs(maxJobs), + speedFactor(std::max(1U, speedFactor)), + supportedFeatures(supportedFeatures), + mandatoryFeatures(mandatoryFeatures), + sshPublicHostKey(sshPublicHostKey) {} -bool Machine::allSupported(const std::set<string> & features) const { - return std::all_of(features.begin(), features.end(), - [&](const string & feature) { - return supportedFeatures.count(feature) || - mandatoryFeatures.count(feature); - }); +bool Machine::allSupported(const std::set<string>& features) const { + return std::all_of(features.begin(), features.end(), + [&](const string& feature) { + return supportedFeatures.count(feature) || + mandatoryFeatures.count(feature); + }); } -bool Machine::mandatoryMet(const std::set<string> & features) const { - return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), - [&](const string & feature) { - return features.count(feature); - }); +bool Machine::mandatoryMet(const std::set<string>& features) const { + return std::all_of( + mandatoryFeatures.begin(), mandatoryFeatures.end(), + [&](const string& feature) { return features.count(feature); }); } -void parseMachines(const std::string & s, Machines & machines) -{ - for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { - trim(line); - line.erase(std::find(line.begin(), line.end(), '#'), line.end()); - if (line.empty()) continue; +void parseMachines(const std::string& s, Machines& machines) { + for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { + trim(line); + line.erase(std::find(line.begin(), line.end(), '#'), line.end()); + if (line.empty()) continue; - if (line[0] == '@') { - auto file = trim(std::string(line, 1)); - try { - parseMachines(readFile(file), machines); - } catch (const SysError & e) { - if (e.errNo != ENOENT) - throw; - debug("cannot find machines file '%s'", file); - } - continue; - } + if (line[0] == '@') { + auto file = trim(std::string(line, 1)); + try { + parseMachines(readFile(file), machines); + } catch (const SysError& e) { + if (e.errNo != ENOENT) throw; + debug("cannot find machines file '%s'", file); + } + continue; + } - auto tokens = tokenizeString<std::vector<string>>(line); - auto sz = tokens.size(); - if (sz < 1) - throw FormatError("bad machine specification '%s'", line); + auto tokens = tokenizeString<std::vector<string>>(line); + auto sz = tokens.size(); + if (sz < 1) throw FormatError("bad machine specification '%s'", line); - auto isSet = [&](size_t n) { - return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; - }; + auto isSet = [&](size_t n) { + return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; + }; - machines.emplace_back(tokens[0], - isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem}, - isSet(2) ? tokens[2] : "", - isSet(3) ? std::stoull(tokens[3]) : 1LL, - isSet(4) ? std::stoull(tokens[4]) : 1LL, - isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{}, - isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{}, - isSet(7) ? tokens[7] : ""); - } + machines.emplace_back( + tokens[0], + isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") + : std::vector<string>{settings.thisSystem}, + isSet(2) ? tokens[2] : "", isSet(3) ? std::stoull(tokens[3]) : 1LL, + isSet(4) ? std::stoull(tokens[4]) : 1LL, + isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") + : std::set<string>{}, + isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") + : std::set<string>{}, + isSet(7) ? tokens[7] : ""); + } } -Machines getMachines() -{ - static auto machines = [&]() { - Machines machines; - parseMachines(settings.builders, machines); - return machines; - }(); +Machines getMachines() { + static auto machines = [&]() { + Machines machines; + parseMachines(settings.builders, machines); return machines; + }(); + return machines; } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/machines.hh b/third_party/nix/src/libstore/machines.hh index de92eb924e4a..a5013fe4f151 100644 --- a/third_party/nix/src/libstore/machines.hh +++ b/third_party/nix/src/libstore/machines.hh @@ -5,35 +5,32 @@ namespace nix { struct Machine { - - const string storeUri; - const std::vector<string> systemTypes; - const string sshKey; - const unsigned int maxJobs; - const unsigned int speedFactor; - const std::set<string> supportedFeatures; - const std::set<string> mandatoryFeatures; - const std::string sshPublicHostKey; - bool enabled = true; - - bool allSupported(const std::set<string> & features) const; - - bool mandatoryMet(const std::set<string> & features) const; - - Machine(decltype(storeUri) storeUri, - decltype(systemTypes) systemTypes, - decltype(sshKey) sshKey, - decltype(maxJobs) maxJobs, - decltype(speedFactor) speedFactor, - decltype(supportedFeatures) supportedFeatures, - decltype(mandatoryFeatures) mandatoryFeatures, - decltype(sshPublicHostKey) sshPublicHostKey); + const string storeUri; + const std::vector<string> systemTypes; + const string sshKey; + const unsigned int maxJobs; + const unsigned int speedFactor; + const std::set<string> supportedFeatures; + const std::set<string> mandatoryFeatures; + const std::string sshPublicHostKey; + bool enabled = true; + + bool allSupported(const std::set<string>& features) const; + + bool mandatoryMet(const std::set<string>& features) const; + + Machine(decltype(storeUri) storeUri, decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey); }; typedef std::vector<Machine> Machines; -void parseMachines(const std::string & s, Machines & machines); +void parseMachines(const std::string& s, Machines& machines); Machines getMachines(); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/misc.cc b/third_party/nix/src/libstore/misc.cc index dddf134300d6..d1c4dc1550eb 100644 --- a/third_party/nix/src/libstore/misc.cc +++ b/third_party/nix/src/libstore/misc.cc @@ -1,282 +1,266 @@ #include "derivations.hh" -#include "parsed-derivations.hh" #include "globals.hh" #include "local-store.hh" +#include "parsed-derivations.hh" #include "store-api.hh" #include "thread-pool.hh" - namespace nix { +void Store::computeFSClosure(const PathSet& startPaths, PathSet& paths_, + bool flipDirection, bool includeOutputs, + bool includeDerivers) { + struct State { + size_t pending; + PathSet& paths; + std::exception_ptr exc; + }; -void Store::computeFSClosure(const PathSet & startPaths, - PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) -{ - struct State - { - size_t pending; - PathSet & paths; - std::exception_ptr exc; - }; - - Sync<State> state_(State{0, paths_, 0}); - - std::function<void(const Path &)> enqueue; + Sync<State> state_(State{0, paths_, 0}); - std::condition_variable done; + std::function<void(const Path&)> enqueue; - enqueue = [&](const Path & path) -> void { - { - auto state(state_.lock()); - if (state->exc) return; - if (state->paths.count(path)) return; - state->paths.insert(path); - state->pending++; - } + std::condition_variable done; - queryPathInfo(path, {[&, path](std::future<ref<ValidPathInfo>> fut) { - // FIXME: calls to isValidPath() should be async - - try { - auto info = fut.get(); - - if (flipDirection) { + enqueue = [&](const Path& path) -> void { + { + auto state(state_.lock()); + if (state->exc) return; + if (state->paths.count(path)) return; + state->paths.insert(path); + state->pending++; + } - PathSet referrers; - queryReferrers(path, referrers); - for (auto & ref : referrers) - if (ref != path) - enqueue(ref); + queryPathInfo( + path, {[&, path](std::future<ref<ValidPathInfo>> fut) { + // FIXME: calls to isValidPath() should be async - if (includeOutputs) - for (auto & i : queryValidDerivers(path)) - enqueue(i); + try { + auto info = fut.get(); - if (includeDerivers && isDerivation(path)) - for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i) && queryPathInfo(i)->deriver == path) - enqueue(i); + if (flipDirection) { + PathSet referrers; + queryReferrers(path, referrers); + for (auto& ref : referrers) + if (ref != path) enqueue(ref); - } else { + if (includeOutputs) + for (auto& i : queryValidDerivers(path)) enqueue(i); - for (auto & ref : info->references) - if (ref != path) - enqueue(ref); + if (includeDerivers && isDerivation(path)) + for (auto& i : queryDerivationOutputs(path)) + if (isValidPath(i) && queryPathInfo(i)->deriver == path) + enqueue(i); - if (includeOutputs && isDerivation(path)) - for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i)) enqueue(i); + } else { + for (auto& ref : info->references) + if (ref != path) enqueue(ref); - if (includeDerivers && isValidPath(info->deriver)) - enqueue(info->deriver); + if (includeOutputs && isDerivation(path)) + for (auto& i : queryDerivationOutputs(path)) + if (isValidPath(i)) enqueue(i); - } + if (includeDerivers && isValidPath(info->deriver)) + enqueue(info->deriver); + } - { - auto state(state_.lock()); - assert(state->pending); - if (!--state->pending) done.notify_one(); - } + { + auto state(state_.lock()); + assert(state->pending); + if (!--state->pending) done.notify_one(); + } - } catch (...) { - auto state(state_.lock()); - if (!state->exc) state->exc = std::current_exception(); - assert(state->pending); - if (!--state->pending) done.notify_one(); - }; + } catch (...) { + auto state(state_.lock()); + if (!state->exc) state->exc = std::current_exception(); + assert(state->pending); + if (!--state->pending) done.notify_one(); + }; }}); - }; + }; - for (auto & startPath : startPaths) - enqueue(startPath); + for (auto& startPath : startPaths) enqueue(startPath); - { - auto state(state_.lock()); - while (state->pending) state.wait(done); - if (state->exc) std::rethrow_exception(state->exc); - } + { + auto state(state_.lock()); + while (state->pending) state.wait(done); + if (state->exc) std::rethrow_exception(state->exc); + } } - -void Store::computeFSClosure(const Path & startPath, - PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) -{ - computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers); +void Store::computeFSClosure(const Path& startPath, PathSet& paths_, + bool flipDirection, bool includeOutputs, + bool includeDerivers) { + computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, + includeDerivers); } +void Store::queryMissing(const PathSet& targets, PathSet& willBuild_, + PathSet& willSubstitute_, PathSet& unknown_, + unsigned long long& downloadSize_, + unsigned long long& narSize_) { + Activity act(*logger, lvlDebug, actUnknown, + "querying info about missing paths"); -void Store::queryMissing(const PathSet & targets, - PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_, - unsigned long long & downloadSize_, unsigned long long & narSize_) -{ - Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); - - downloadSize_ = narSize_ = 0; - - ThreadPool pool; + downloadSize_ = narSize_ = 0; - struct State - { - PathSet done; - PathSet & unknown, & willSubstitute, & willBuild; - unsigned long long & downloadSize; - unsigned long long & narSize; - }; + ThreadPool pool; - struct DrvState - { - size_t left; - bool done = false; - PathSet outPaths; - DrvState(size_t left) : left(left) { } - }; + struct State { + PathSet done; + PathSet &unknown, &willSubstitute, &willBuild; + unsigned long long& downloadSize; + unsigned long long& narSize; + }; - Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); + struct DrvState { + size_t left; + bool done = false; + PathSet outPaths; + DrvState(size_t left) : left(left) {} + }; - std::function<void(Path)> doPath; + Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, + downloadSize_, narSize_}); - auto mustBuildDrv = [&](const Path & drvPath, const Derivation & drv) { - { - auto state(state_.lock()); - state->willBuild.insert(drvPath); - } + std::function<void(Path)> doPath; - for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second))); - }; - - auto checkOutput = [&]( - const Path & drvPath, ref<Derivation> drv, const Path & outPath, ref<Sync<DrvState>> drvState_) + auto mustBuildDrv = [&](const Path& drvPath, const Derivation& drv) { { - if (drvState_->lock()->done) return; - - SubstitutablePathInfos infos; - querySubstitutablePathInfos({outPath}, infos); - - if (infos.empty()) { - drvState_->lock()->done = true; - mustBuildDrv(drvPath, *drv); - } else { - { - auto drvState(drvState_->lock()); - if (drvState->done) return; - assert(drvState->left); - drvState->left--; - drvState->outPaths.insert(outPath); - if (!drvState->left) { - for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, path)); - } - } - } - }; - - doPath = [&](const Path & path) { + auto state(state_.lock()); + state->willBuild.insert(drvPath); + } - { - auto state(state_.lock()); - if (state->done.count(path)) return; - state->done.insert(path); + for (auto& i : drv.inputDrvs) + pool.enqueue( + std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second))); + }; + + auto checkOutput = [&](const Path& drvPath, ref<Derivation> drv, + const Path& outPath, ref<Sync<DrvState>> drvState_) { + if (drvState_->lock()->done) return; + + SubstitutablePathInfos infos; + querySubstitutablePathInfos({outPath}, infos); + + if (infos.empty()) { + drvState_->lock()->done = true; + mustBuildDrv(drvPath, *drv); + } else { + { + auto drvState(drvState_->lock()); + if (drvState->done) return; + assert(drvState->left); + drvState->left--; + drvState->outPaths.insert(outPath); + if (!drvState->left) { + for (auto& path : drvState->outPaths) + pool.enqueue(std::bind(doPath, path)); } + } + } + }; - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path); - - if (isDerivation(i2.first)) { - if (!isValidPath(i2.first)) { - // FIXME: we could try to substitute the derivation. - auto state(state_.lock()); - state->unknown.insert(path); - return; - } - - Derivation drv = derivationFromPath(i2.first); - ParsedDerivation parsedDrv(i2.first, drv); - - PathSet invalid; - for (auto & j : drv.outputs) - if (wantOutput(j.first, i2.second) - && !isValidPath(j.second.path)) - invalid.insert(j.second.path); - if (invalid.empty()) return; - - if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { - auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); - for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState)); - } else - mustBuildDrv(i2.first, drv); - - } else { - - if (isValidPath(path)) return; + doPath = [&](const Path& path) { + { + auto state(state_.lock()); + if (state->done.count(path)) return; + state->done.insert(path); + } - SubstitutablePathInfos infos; - querySubstitutablePathInfos({path}, infos); + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path); - if (infos.empty()) { - auto state(state_.lock()); - state->unknown.insert(path); - return; - } + if (isDerivation(i2.first)) { + if (!isValidPath(i2.first)) { + // FIXME: we could try to substitute the derivation. + auto state(state_.lock()); + state->unknown.insert(path); + return; + } + + Derivation drv = derivationFromPath(i2.first); + ParsedDerivation parsedDrv(i2.first, drv); + + PathSet invalid; + for (auto& j : drv.outputs) + if (wantOutput(j.first, i2.second) && !isValidPath(j.second.path)) + invalid.insert(j.second.path); + if (invalid.empty()) return; + + if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { + auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); + for (auto& output : invalid) + pool.enqueue(std::bind(checkOutput, i2.first, + make_ref<Derivation>(drv), output, drvState)); + } else + mustBuildDrv(i2.first, drv); + + } else { + if (isValidPath(path)) return; + + SubstitutablePathInfos infos; + querySubstitutablePathInfos({path}, infos); + + if (infos.empty()) { + auto state(state_.lock()); + state->unknown.insert(path); + return; + } - auto info = infos.find(path); - assert(info != infos.end()); + auto info = infos.find(path); + assert(info != infos.end()); - { - auto state(state_.lock()); - state->willSubstitute.insert(path); - state->downloadSize += info->second.downloadSize; - state->narSize += info->second.narSize; - } + { + auto state(state_.lock()); + state->willSubstitute.insert(path); + state->downloadSize += info->second.downloadSize; + state->narSize += info->second.narSize; + } - for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, ref)); - } - }; + for (auto& ref : info->second.references) + pool.enqueue(std::bind(doPath, ref)); + } + }; - for (auto & path : targets) - pool.enqueue(std::bind(doPath, path)); + for (auto& path : targets) pool.enqueue(std::bind(doPath, path)); - pool.process(); + pool.process(); } +Paths Store::topoSortPaths(const PathSet& paths) { + Paths sorted; + PathSet visited, parents; -Paths Store::topoSortPaths(const PathSet & paths) -{ - Paths sorted; - PathSet visited, parents; + std::function<void(const Path& path, const Path* parent)> dfsVisit; - std::function<void(const Path & path, const Path * parent)> dfsVisit; + dfsVisit = [&](const Path& path, const Path* parent) { + if (parents.find(path) != parents.end()) + throw BuildError( + format("cycle detected in the references of '%1%' from '%2%'") % + path % *parent); - dfsVisit = [&](const Path & path, const Path * parent) { - if (parents.find(path) != parents.end()) - throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent); + if (visited.find(path) != visited.end()) return; + visited.insert(path); + parents.insert(path); - if (visited.find(path) != visited.end()) return; - visited.insert(path); - parents.insert(path); - - PathSet references; - try { - references = queryPathInfo(path)->references; - } catch (InvalidPath &) { - } + PathSet references; + try { + references = queryPathInfo(path)->references; + } catch (InvalidPath&) { + } - for (auto & i : references) - /* Don't traverse into paths that don't exist. That can - happen due to substitutes for non-existent paths. */ - if (i != path && paths.find(i) != paths.end()) - dfsVisit(i, &path); + for (auto& i : references) + /* Don't traverse into paths that don't exist. That can + happen due to substitutes for non-existent paths. */ + if (i != path && paths.find(i) != paths.end()) dfsVisit(i, &path); - sorted.push_front(path); - parents.erase(path); - }; + sorted.push_front(path); + parents.erase(path); + }; - for (auto & i : paths) - dfsVisit(i, nullptr); + for (auto& i : paths) dfsVisit(i, nullptr); - return sorted; + return sorted; } - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-accessor.cc b/third_party/nix/src/libstore/nar-accessor.cc index b74480684f2a..f5073ad98076 100644 --- a/third_party/nix/src/libstore/nar-accessor.cc +++ b/third_party/nix/src/libstore/nar-accessor.cc @@ -1,266 +1,242 @@ #include "nar-accessor.hh" -#include "archive.hh" -#include "json.hh" - -#include <map> -#include <stack> #include <algorithm> - +#include <map> #include <nlohmann/json.hpp> +#include <stack> +#include "archive.hh" +#include "json.hh" namespace nix { -struct NarMember -{ - FSAccessor::Type type = FSAccessor::Type::tMissing; +struct NarMember { + FSAccessor::Type type = FSAccessor::Type::tMissing; - bool isExecutable = false; + bool isExecutable = false; - /* If this is a regular file, position of the contents of this - file in the NAR. */ - size_t start = 0, size = 0; + /* If this is a regular file, position of the contents of this + file in the NAR. */ + size_t start = 0, size = 0; - std::string target; + std::string target; - /* If this is a directory, all the children of the directory. */ - std::map<std::string, NarMember> children; + /* If this is a directory, all the children of the directory. */ + std::map<std::string, NarMember> children; }; -struct NarAccessor : public FSAccessor -{ - std::shared_ptr<const std::string> nar; - - GetNarBytes getNarBytes; - - NarMember root; - - struct NarIndexer : ParseSink, StringSource - { - NarAccessor & acc; +struct NarAccessor : public FSAccessor { + std::shared_ptr<const std::string> nar; - std::stack<NarMember *> parents; + GetNarBytes getNarBytes; - std::string currentStart; - bool isExec = false; + NarMember root; - NarIndexer(NarAccessor & acc, const std::string & nar) - : StringSource(nar), acc(acc) - { } + struct NarIndexer : ParseSink, StringSource { + NarAccessor& acc; - void createMember(const Path & path, NarMember member) { - size_t level = std::count(path.begin(), path.end(), '/'); - while (parents.size() > level) parents.pop(); + std::stack<NarMember*> parents; - if (parents.empty()) { - acc.root = std::move(member); - parents.push(&acc.root); - } else { - if (parents.top()->type != FSAccessor::Type::tDirectory) - throw Error("NAR file missing parent directory of path '%s'", path); - auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); - parents.push(&result.first->second); - } - } + std::string currentStart; + bool isExec = false; - void createDirectory(const Path & path) override - { - createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); - } + NarIndexer(NarAccessor& acc, const std::string& nar) + : StringSource(nar), acc(acc) {} - void createRegularFile(const Path & path) override - { - createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); - } + void createMember(const Path& path, NarMember member) { + size_t level = std::count(path.begin(), path.end(), '/'); + while (parents.size() > level) parents.pop(); - void isExecutable() override - { - parents.top()->isExecutable = true; - } - - void preallocateContents(unsigned long long size) override - { - currentStart = string(s, pos, 16); - assert(size <= std::numeric_limits<size_t>::max()); - parents.top()->size = (size_t)size; - parents.top()->start = pos; - } - - void receiveContents(unsigned char * data, unsigned int len) override - { - // Sanity check - if (!currentStart.empty()) { - assert(len < 16 || currentStart == string((char *) data, 16)); - currentStart.clear(); - } - } - - void createSymlink(const Path & path, const string & target) override - { - createMember(path, - NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); - } - }; + if (parents.empty()) { + acc.root = std::move(member); + parents.push(&acc.root); + } else { + if (parents.top()->type != FSAccessor::Type::tDirectory) + throw Error("NAR file missing parent directory of path '%s'", path); + auto result = parents.top()->children.emplace(baseNameOf(path), + std::move(member)); + parents.push(&result.first->second); + } + } - NarAccessor(ref<const std::string> nar) : nar(nar) - { - NarIndexer indexer(*this, *nar); - parseDump(indexer, indexer); + void createDirectory(const Path& path) override { + createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); } - NarAccessor(const std::string & listing, GetNarBytes getNarBytes) - : getNarBytes(getNarBytes) - { - using json = nlohmann::json; - - std::function<void(NarMember &, json &)> recurse; - - recurse = [&](NarMember & member, json & v) { - std::string type = v["type"]; - - if (type == "directory") { - member.type = FSAccessor::Type::tDirectory; - for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { - std::string name = i.key(); - recurse(member.children[name], i.value()); - } - } else if (type == "regular") { - member.type = FSAccessor::Type::tRegular; - member.size = v["size"]; - member.isExecutable = v.value("executable", false); - member.start = v["narOffset"]; - } else if (type == "symlink") { - member.type = FSAccessor::Type::tSymlink; - member.target = v.value("target", ""); - } else return; - }; - - json v = json::parse(listing); - recurse(root, v); + void createRegularFile(const Path& path) override { + createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); } - NarMember * find(const Path & path) - { - Path canon = path == "" ? "" : canonPath(path); - NarMember * current = &root; - auto end = path.end(); - for (auto it = path.begin(); it != end; ) { - // because it != end, the remaining component is non-empty so we need - // a directory - if (current->type != FSAccessor::Type::tDirectory) return nullptr; - - // skip slash (canonPath above ensures that this is always a slash) - assert(*it == '/'); - it += 1; - - // lookup current component - auto next = std::find(it, end, '/'); - auto child = current->children.find(std::string(it, next)); - if (child == current->children.end()) return nullptr; - current = &child->second; - - it = next; - } + void isExecutable() override { parents.top()->isExecutable = true; } - return current; + void preallocateContents(unsigned long long size) override { + currentStart = string(s, pos, 16); + assert(size <= std::numeric_limits<size_t>::max()); + parents.top()->size = (size_t)size; + parents.top()->start = pos; } - NarMember & get(const Path & path) { - auto result = find(path); - if (result == nullptr) - throw Error("NAR file does not contain path '%1%'", path); - return *result; + void receiveContents(unsigned char* data, unsigned int len) override { + // Sanity check + if (!currentStart.empty()) { + assert(len < 16 || currentStart == string((char*)data, 16)); + currentStart.clear(); + } } - Stat stat(const Path & path) override - { - auto i = find(path); - if (i == nullptr) - return {FSAccessor::Type::tMissing, 0, false}; - return {i->type, i->size, i->isExecutable, i->start}; + void createSymlink(const Path& path, const string& target) override { + createMember(path, + NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); } + }; - StringSet readDirectory(const Path & path) override - { - auto i = get(path); - - if (i.type != FSAccessor::Type::tDirectory) - throw Error(format("path '%1%' inside NAR file is not a directory") % path); + NarAccessor(ref<const std::string> nar) : nar(nar) { + NarIndexer indexer(*this, *nar); + parseDump(indexer, indexer); + } - StringSet res; - for (auto & child : i.children) - res.insert(child.first); + NarAccessor(const std::string& listing, GetNarBytes getNarBytes) + : getNarBytes(getNarBytes) { + using json = nlohmann::json; - return res; - } + std::function<void(NarMember&, json&)> recurse; - std::string readFile(const Path & path) override - { - auto i = get(path); - if (i.type != FSAccessor::Type::tRegular) - throw Error(format("path '%1%' inside NAR file is not a regular file") % path); + recurse = [&](NarMember& member, json& v) { + std::string type = v["type"]; - if (getNarBytes) return getNarBytes(i.start, i.size); + if (type == "directory") { + member.type = FSAccessor::Type::tDirectory; + for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { + std::string name = i.key(); + recurse(member.children[name], i.value()); + } + } else if (type == "regular") { + member.type = FSAccessor::Type::tRegular; + member.size = v["size"]; + member.isExecutable = v.value("executable", false); + member.start = v["narOffset"]; + } else if (type == "symlink") { + member.type = FSAccessor::Type::tSymlink; + member.target = v.value("target", ""); + } else + return; + }; - assert(nar); - return std::string(*nar, i.start, i.size); + json v = json::parse(listing); + recurse(root, v); + } + + NarMember* find(const Path& path) { + Path canon = path == "" ? "" : canonPath(path); + NarMember* current = &root; + auto end = path.end(); + for (auto it = path.begin(); it != end;) { + // because it != end, the remaining component is non-empty so we need + // a directory + if (current->type != FSAccessor::Type::tDirectory) return nullptr; + + // skip slash (canonPath above ensures that this is always a slash) + assert(*it == '/'); + it += 1; + + // lookup current component + auto next = std::find(it, end, '/'); + auto child = current->children.find(std::string(it, next)); + if (child == current->children.end()) return nullptr; + current = &child->second; + + it = next; } - std::string readLink(const Path & path) override - { - auto i = get(path); - if (i.type != FSAccessor::Type::tSymlink) - throw Error(format("path '%1%' inside NAR file is not a symlink") % path); - return i.target; - } + return current; + } + + NarMember& get(const Path& path) { + auto result = find(path); + if (result == nullptr) + throw Error("NAR file does not contain path '%1%'", path); + return *result; + } + + Stat stat(const Path& path) override { + auto i = find(path); + if (i == nullptr) return {FSAccessor::Type::tMissing, 0, false}; + return {i->type, i->size, i->isExecutable, i->start}; + } + + StringSet readDirectory(const Path& path) override { + auto i = get(path); + + if (i.type != FSAccessor::Type::tDirectory) + throw Error(format("path '%1%' inside NAR file is not a directory") % + path); + + StringSet res; + for (auto& child : i.children) res.insert(child.first); + + return res; + } + + std::string readFile(const Path& path) override { + auto i = get(path); + if (i.type != FSAccessor::Type::tRegular) + throw Error(format("path '%1%' inside NAR file is not a regular file") % + path); + + if (getNarBytes) return getNarBytes(i.start, i.size); + + assert(nar); + return std::string(*nar, i.start, i.size); + } + + std::string readLink(const Path& path) override { + auto i = get(path); + if (i.type != FSAccessor::Type::tSymlink) + throw Error(format("path '%1%' inside NAR file is not a symlink") % path); + return i.target; + } }; -ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) -{ - return make_ref<NarAccessor>(nar); +ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) { + return make_ref<NarAccessor>(nar); } -ref<FSAccessor> makeLazyNarAccessor(const std::string & listing, - GetNarBytes getNarBytes) -{ - return make_ref<NarAccessor>(listing, getNarBytes); +ref<FSAccessor> makeLazyNarAccessor(const std::string& listing, + GetNarBytes getNarBytes) { + return make_ref<NarAccessor>(listing, getNarBytes); } -void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, - const Path & path, bool recurse) -{ - auto st = accessor->stat(path); +void listNar(JSONPlaceholder& res, ref<FSAccessor> accessor, const Path& path, + bool recurse) { + auto st = accessor->stat(path); - auto obj = res.object(); + auto obj = res.object(); - switch (st.type) { + switch (st.type) { case FSAccessor::Type::tRegular: - obj.attr("type", "regular"); - obj.attr("size", st.fileSize); - if (st.isExecutable) - obj.attr("executable", true); - if (st.narOffset) - obj.attr("narOffset", st.narOffset); - break; + obj.attr("type", "regular"); + obj.attr("size", st.fileSize); + if (st.isExecutable) obj.attr("executable", true); + if (st.narOffset) obj.attr("narOffset", st.narOffset); + break; case FSAccessor::Type::tDirectory: - obj.attr("type", "directory"); - { - auto res2 = obj.object("entries"); - for (auto & name : accessor->readDirectory(path)) { - if (recurse) { - auto res3 = res2.placeholder(name); - listNar(res3, accessor, path + "/" + name, true); - } else - res2.object(name); - } + obj.attr("type", "directory"); + { + auto res2 = obj.object("entries"); + for (auto& name : accessor->readDirectory(path)) { + if (recurse) { + auto res3 = res2.placeholder(name); + listNar(res3, accessor, path + "/" + name, true); + } else + res2.object(name); } - break; + } + break; case FSAccessor::Type::tSymlink: - obj.attr("type", "symlink"); - obj.attr("target", accessor->readLink(path)); - break; + obj.attr("type", "symlink"); + obj.attr("target", accessor->readLink(path)); + break; default: - throw Error("path '%s' does not exist in NAR", path); - } + throw Error("path '%s' does not exist in NAR", path); + } } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-accessor.hh b/third_party/nix/src/libstore/nar-accessor.hh index 2871199de16e..eb6cf0fe736e 100644 --- a/third_party/nix/src/libstore/nar-accessor.hh +++ b/third_party/nix/src/libstore/nar-accessor.hh @@ -1,7 +1,6 @@ #pragma once #include <functional> - #include "fs-accessor.hh" namespace nix { @@ -16,15 +15,14 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar); inside the NAR. */ typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes; -ref<FSAccessor> makeLazyNarAccessor( - const std::string & listing, - GetNarBytes getNarBytes); +ref<FSAccessor> makeLazyNarAccessor(const std::string& listing, + GetNarBytes getNarBytes); class JSONPlaceholder; /* Write a JSON representation of the contents of a NAR (except file contents). */ -void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, - const Path & path, bool recurse); +void listNar(JSONPlaceholder& res, ref<FSAccessor> accessor, const Path& path, + bool recurse); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-info-disk-cache.cc b/third_party/nix/src/libstore/nar-info-disk-cache.cc index 32ad7f2b27ff..2121e89387b2 100644 --- a/third_party/nix/src/libstore/nar-info-disk-cache.cc +++ b/third_party/nix/src/libstore/nar-info-disk-cache.cc @@ -1,13 +1,12 @@ #include "nar-info-disk-cache.hh" -#include "sync.hh" -#include "sqlite.hh" -#include "globals.hh" - #include <sqlite3.h> +#include "globals.hh" +#include "sqlite.hh" +#include "sync.hh" namespace nix { -static const char * schema = R"sql( +static const char* schema = R"sql( create table if not exists BinaryCaches ( id integer primary key autoincrement not null, @@ -45,223 +44,222 @@ create table if not exists LastPurge ( )sql"; -class NarInfoDiskCacheImpl : public NarInfoDiskCache -{ -public: - - /* How often to purge expired entries from the cache. */ - const int purgeInterval = 24 * 3600; - - struct Cache - { - int id; - Path storeDir; - bool wantMassQuery; - int priority; - }; - - struct State - { - SQLite db; - SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache; - std::map<std::string, Cache> caches; - }; - - Sync<State> _state; - - NarInfoDiskCacheImpl() - { - auto state(_state.lock()); - - Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; - createDirs(dirOf(dbPath)); - - state->db = SQLite(dbPath); - - if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(state->db, "setting timeout"); - - // We can always reproduce the cache. - state->db.exec("pragma synchronous = off"); - state->db.exec("pragma main.journal_mode = truncate"); - - state->db.exec(schema); - - state->insertCache.create(state->db, - "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); - - state->queryCache.create(state->db, - "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); - - state->insertNAR.create(state->db, - "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " - "narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); - - state->insertMissingNAR.create(state->db, - "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)"); - - state->queryNAR.create(state->db, - "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); - - /* Periodically purge expired entries from the database. */ - retrySQLite<void>([&]() { - auto now = time(0); - - SQLiteStmt queryLastPurge(state->db, "select value from LastPurge"); - auto queryLastPurge_(queryLastPurge.use()); - - if (!queryLastPurge_.next() || queryLastPurge_.getInt(0) < now - purgeInterval) { - SQLiteStmt(state->db, - "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") - .use() - (now - settings.ttlNegativeNarInfoCache) - (now - settings.ttlPositiveNarInfoCache) - .exec(); - - debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); - - SQLiteStmt(state->db, - "insert or replace into LastPurge(dummy, value) values ('', ?)") - .use()(now).exec(); - } - }); - } - - Cache & getCache(State & state, const std::string & uri) - { - auto i = state.caches.find(uri); - if (i == state.caches.end()) abort(); - return i->second; - } - - void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override - { - retrySQLite<void>([&]() { - auto state(_state.lock()); - - // FIXME: race - - state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); - assert(sqlite3_changes(state->db) == 1); - state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; - }); - } - - bool cacheExists(const std::string & uri, - bool & wantMassQuery, int & priority) override - { - return retrySQLite<bool>([&]() { - auto state(_state.lock()); - - auto i = state->caches.find(uri); - if (i == state->caches.end()) { - auto queryCache(state->queryCache.use()(uri)); - if (!queryCache.next()) return false; - state->caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); - } - - auto & cache(getCache(*state, uri)); - - wantMassQuery = cache.wantMassQuery; - priority = cache.priority; - - return true; - }); - } - - std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( - const std::string & uri, const std::string & hashPart) override - { - return retrySQLite<std::pair<Outcome, std::shared_ptr<NarInfo>>>( - [&]() -> std::pair<Outcome, std::shared_ptr<NarInfo>> { - auto state(_state.lock()); - - auto & cache(getCache(*state, uri)); - - auto now = time(0); - - auto queryNAR(state->queryNAR.use() - (cache.id) - (hashPart) - (now - settings.ttlNegativeNarInfoCache) - (now - settings.ttlPositiveNarInfoCache)); - - if (!queryNAR.next()) - return {oUnknown, 0}; - - if (!queryNAR.getInt(0)) - return {oInvalid, 0}; - - auto narInfo = make_ref<NarInfo>(); - - auto namePart = queryNAR.getStr(1); - narInfo->path = cache.storeDir + "/" + - hashPart + (namePart.empty() ? "" : "-" + namePart); - narInfo->url = queryNAR.getStr(2); - narInfo->compression = queryNAR.getStr(3); - if (!queryNAR.isNull(4)) - narInfo->fileHash = Hash(queryNAR.getStr(4)); - narInfo->fileSize = queryNAR.getInt(5); - narInfo->narHash = Hash(queryNAR.getStr(6)); - narInfo->narSize = queryNAR.getInt(7); - for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) - narInfo->references.insert(cache.storeDir + "/" + r); - if (!queryNAR.isNull(9)) - narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9); - for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) - narInfo->sigs.insert(sig); - narInfo->ca = queryNAR.getStr(11); - - return {oValid, narInfo}; +class NarInfoDiskCacheImpl : public NarInfoDiskCache { + public: + /* How often to purge expired entries from the cache. */ + const int purgeInterval = 24 * 3600; + + struct Cache { + int id; + Path storeDir; + bool wantMassQuery; + int priority; + }; + + struct State { + SQLite db; + SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, + purgeCache; + std::map<std::string, Cache> caches; + }; + + Sync<State> _state; + + NarInfoDiskCacheImpl() { + auto state(_state.lock()); + + Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + + if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(state->db, "setting timeout"); + + // We can always reproduce the cache. + state->db.exec("pragma synchronous = off"); + state->db.exec("pragma main.journal_mode = truncate"); + + state->db.exec(schema); + + state->insertCache.create( + state->db, + "insert or replace into BinaryCaches(url, timestamp, storeDir, " + "wantMassQuery, priority) values (?, ?, ?, ?, ?)"); + + state->queryCache.create(state->db, + "select id, storeDir, wantMassQuery, priority " + "from BinaryCaches where url = ?"); + + state->insertNAR.create( + state->db, + "insert or replace into NARs(cache, hashPart, namePart, url, " + "compression, fileHash, fileSize, narHash, " + "narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, " + "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); + + state->insertMissingNAR.create( + state->db, + "insert or replace into NARs(cache, hashPart, timestamp, present) " + "values (?, ?, ?, 0)"); + + state->queryNAR.create( + state->db, + "select present, namePart, url, compression, fileHash, fileSize, " + "narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? " + "and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 " + "and timestamp > ?))"); + + /* Periodically purge expired entries from the database. */ + retrySQLite<void>([&]() { + auto now = time(0); + + SQLiteStmt queryLastPurge(state->db, "select value from LastPurge"); + auto queryLastPurge_(queryLastPurge.use()); + + if (!queryLastPurge_.next() || + queryLastPurge_.getInt(0) < now - purgeInterval) { + SQLiteStmt(state->db, + "delete from NARs where ((present = 0 and timestamp < ?) or " + "(present = 1 and timestamp < ?))") + .use()(now - settings.ttlNegativeNarInfoCache)( + now - settings.ttlPositiveNarInfoCache) + .exec(); + + debug("deleted %d entries from the NAR info disk cache", + sqlite3_changes(state->db)); + + SQLiteStmt( + state->db, + "insert or replace into LastPurge(dummy, value) values ('', ?)") + .use()(now) + .exec(); + } + }); + } + + Cache& getCache(State& state, const std::string& uri) { + auto i = state.caches.find(uri); + if (i == state.caches.end()) abort(); + return i->second; + } + + void createCache(const std::string& uri, const Path& storeDir, + bool wantMassQuery, int priority) override { + retrySQLite<void>([&]() { + auto state(_state.lock()); + + // FIXME: race + + state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority) + .exec(); + assert(sqlite3_changes(state->db) == 1); + state->caches[uri] = Cache{(int)sqlite3_last_insert_rowid(state->db), + storeDir, wantMassQuery, priority}; + }); + } + + bool cacheExists(const std::string& uri, bool& wantMassQuery, + int& priority) override { + return retrySQLite<bool>([&]() { + auto state(_state.lock()); + + auto i = state->caches.find(uri); + if (i == state->caches.end()) { + auto queryCache(state->queryCache.use()(uri)); + if (!queryCache.next()) return false; + state->caches.emplace( + uri, Cache{(int)queryCache.getInt(0), queryCache.getStr(1), + queryCache.getInt(2) != 0, (int)queryCache.getInt(3)}); + } + + auto& cache(getCache(*state, uri)); + + wantMassQuery = cache.wantMassQuery; + priority = cache.priority; + + return true; + }); + } + + std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( + const std::string& uri, const std::string& hashPart) override { + return retrySQLite<std::pair<Outcome, std::shared_ptr<NarInfo>>>( + [&]() -> std::pair<Outcome, std::shared_ptr<NarInfo>> { + auto state(_state.lock()); + + auto& cache(getCache(*state, uri)); + + auto now = time(0); + + auto queryNAR(state->queryNAR.use()(cache.id)(hashPart)( + now - settings.ttlNegativeNarInfoCache)( + now - settings.ttlPositiveNarInfoCache)); + + if (!queryNAR.next()) return {oUnknown, 0}; + + if (!queryNAR.getInt(0)) return {oInvalid, 0}; + + auto narInfo = make_ref<NarInfo>(); + + auto namePart = queryNAR.getStr(1); + narInfo->path = cache.storeDir + "/" + hashPart + + (namePart.empty() ? "" : "-" + namePart); + narInfo->url = queryNAR.getStr(2); + narInfo->compression = queryNAR.getStr(3); + if (!queryNAR.isNull(4)) narInfo->fileHash = Hash(queryNAR.getStr(4)); + narInfo->fileSize = queryNAR.getInt(5); + narInfo->narHash = Hash(queryNAR.getStr(6)); + narInfo->narSize = queryNAR.getInt(7); + for (auto& r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) + narInfo->references.insert(cache.storeDir + "/" + r); + if (!queryNAR.isNull(9)) + narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9); + for (auto& sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) + narInfo->sigs.insert(sig); + narInfo->ca = queryNAR.getStr(11); + + return {oValid, narInfo}; }); - } - - void upsertNarInfo( - const std::string & uri, const std::string & hashPart, - std::shared_ptr<ValidPathInfo> info) override - { - retrySQLite<void>([&]() { - auto state(_state.lock()); - - auto & cache(getCache(*state, uri)); - - if (info) { - - auto narInfo = std::dynamic_pointer_cast<NarInfo>(info); - - assert(hashPart == storePathToHash(info->path)); - - state->insertNAR.use() - (cache.id) - (hashPart) - (storePathToName(info->path)) - (narInfo ? narInfo->url : "", narInfo != 0) - (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) - (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string()) - (info->narSize) - (concatStringsSep(" ", info->shortRefs())) - (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") - (concatStringsSep(" ", info->sigs)) - (info->ca) - (time(0)).exec(); - - } else { - state->insertMissingNAR.use() - (cache.id) - (hashPart) - (time(0)).exec(); - } - }); - } + } + + void upsertNarInfo(const std::string& uri, const std::string& hashPart, + std::shared_ptr<ValidPathInfo> info) override { + retrySQLite<void>([&]() { + auto state(_state.lock()); + + auto& cache(getCache(*state, uri)); + + if (info) { + auto narInfo = std::dynamic_pointer_cast<NarInfo>(info); + + assert(hashPart == storePathToHash(info->path)); + + state->insertNAR + .use()(cache.id)(hashPart)(storePathToName(info->path))( + narInfo ? narInfo->url : "", narInfo != 0)( + narInfo ? narInfo->compression : "", narInfo != 0)( + narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() + : "", + narInfo && narInfo->fileHash)( + narInfo ? narInfo->fileSize : 0, + narInfo != 0 && narInfo->fileSize)(info->narHash.to_string())( + info->narSize)(concatStringsSep(" ", info->shortRefs()))( + info->deriver != "" ? baseNameOf(info->deriver) : "", + info->deriver != + "")(concatStringsSep(" ", info->sigs))(info->ca)(time(0)) + .exec(); + + } else { + state->insertMissingNAR.use()(cache.id)(hashPart)(time(0)).exec(); + } + }); + } }; -ref<NarInfoDiskCache> getNarInfoDiskCache() -{ - static ref<NarInfoDiskCache> cache = make_ref<NarInfoDiskCacheImpl>(); - return cache; +ref<NarInfoDiskCache> getNarInfoDiskCache() { + static ref<NarInfoDiskCache> cache = make_ref<NarInfoDiskCacheImpl>(); + return cache; } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-info-disk-cache.hh b/third_party/nix/src/libstore/nar-info-disk-cache.hh index 88d909732dbc..65bb773c92f7 100644 --- a/third_party/nix/src/libstore/nar-info-disk-cache.hh +++ b/third_party/nix/src/libstore/nar-info-disk-cache.hh @@ -1,31 +1,30 @@ #pragma once -#include "ref.hh" #include "nar-info.hh" +#include "ref.hh" namespace nix { -class NarInfoDiskCache -{ -public: - typedef enum { oValid, oInvalid, oUnknown } Outcome; +class NarInfoDiskCache { + public: + typedef enum { oValid, oInvalid, oUnknown } Outcome; - virtual void createCache(const std::string & uri, const Path & storeDir, - bool wantMassQuery, int priority) = 0; + virtual void createCache(const std::string& uri, const Path& storeDir, + bool wantMassQuery, int priority) = 0; - virtual bool cacheExists(const std::string & uri, - bool & wantMassQuery, int & priority) = 0; + virtual bool cacheExists(const std::string& uri, bool& wantMassQuery, + int& priority) = 0; - virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( - const std::string & uri, const std::string & hashPart) = 0; + virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( + const std::string& uri, const std::string& hashPart) = 0; - virtual void upsertNarInfo( - const std::string & uri, const std::string & hashPart, - std::shared_ptr<ValidPathInfo> info) = 0; + virtual void upsertNarInfo(const std::string& uri, + const std::string& hashPart, + std::shared_ptr<ValidPathInfo> info) = 0; }; /* Return a singleton cache object that can be used concurrently by multiple threads. */ ref<NarInfoDiskCache> getNarInfoDiskCache(); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-info.cc b/third_party/nix/src/libstore/nar-info.cc index cb568ccdc828..bf195290d040 100644 --- a/third_party/nix/src/libstore/nar-info.cc +++ b/third_party/nix/src/libstore/nar-info.cc @@ -1,116 +1,105 @@ -#include "globals.hh" #include "nar-info.hh" +#include "globals.hh" namespace nix { -NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) -{ - auto corrupt = [&]() { - throw Error(format("NAR info file '%1%' is corrupt") % whence); - }; - - auto parseHashField = [&](const string & s) { - try { - return Hash(s); - } catch (BadHash &) { - corrupt(); - return Hash(); // never reached - } - }; - - size_t pos = 0; - while (pos < s.size()) { - - size_t colon = s.find(':', pos); - if (colon == std::string::npos) corrupt(); - - std::string name(s, pos, colon - pos); - - size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) corrupt(); - - std::string value(s, colon + 2, eol - colon - 2); - - if (name == "StorePath") { - if (!store.isStorePath(value)) corrupt(); - path = value; - } - else if (name == "URL") - url = value; - else if (name == "Compression") - compression = value; - else if (name == "FileHash") - fileHash = parseHashField(value); - else if (name == "FileSize") { - if (!string2Int(value, fileSize)) corrupt(); - } - else if (name == "NarHash") - narHash = parseHashField(value); - else if (name == "NarSize") { - if (!string2Int(value, narSize)) corrupt(); - } - else if (name == "References") { - auto refs = tokenizeString<Strings>(value, " "); - if (!references.empty()) corrupt(); - for (auto & r : refs) { - auto r2 = store.storeDir + "/" + r; - if (!store.isStorePath(r2)) corrupt(); - references.insert(r2); - } - } - else if (name == "Deriver") { - if (value != "unknown-deriver") { - auto p = store.storeDir + "/" + value; - if (!store.isStorePath(p)) corrupt(); - deriver = p; - } - } - else if (name == "System") - system = value; - else if (name == "Sig") - sigs.insert(value); - else if (name == "CA") { - if (!ca.empty()) corrupt(); - ca = value; - } - - pos = eol + 1; +NarInfo::NarInfo(const Store& store, const std::string& s, + const std::string& whence) { + auto corrupt = [&]() { + throw Error(format("NAR info file '%1%' is corrupt") % whence); + }; + + auto parseHashField = [&](const string& s) { + try { + return Hash(s); + } catch (BadHash&) { + corrupt(); + return Hash(); // never reached + } + }; + + size_t pos = 0; + while (pos < s.size()) { + size_t colon = s.find(':', pos); + if (colon == std::string::npos) corrupt(); + + std::string name(s, pos, colon - pos); + + size_t eol = s.find('\n', colon + 2); + if (eol == std::string::npos) corrupt(); + + std::string value(s, colon + 2, eol - colon - 2); + + if (name == "StorePath") { + if (!store.isStorePath(value)) corrupt(); + path = value; + } else if (name == "URL") + url = value; + else if (name == "Compression") + compression = value; + else if (name == "FileHash") + fileHash = parseHashField(value); + else if (name == "FileSize") { + if (!string2Int(value, fileSize)) corrupt(); + } else if (name == "NarHash") + narHash = parseHashField(value); + else if (name == "NarSize") { + if (!string2Int(value, narSize)) corrupt(); + } else if (name == "References") { + auto refs = tokenizeString<Strings>(value, " "); + if (!references.empty()) corrupt(); + for (auto& r : refs) { + auto r2 = store.storeDir + "/" + r; + if (!store.isStorePath(r2)) corrupt(); + references.insert(r2); + } + } else if (name == "Deriver") { + if (value != "unknown-deriver") { + auto p = store.storeDir + "/" + value; + if (!store.isStorePath(p)) corrupt(); + deriver = p; + } + } else if (name == "System") + system = value; + else if (name == "Sig") + sigs.insert(value); + else if (name == "CA") { + if (!ca.empty()) corrupt(); + ca = value; } - if (compression == "") compression = "bzip2"; + pos = eol + 1; + } - if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt(); + if (compression == "") compression = "bzip2"; + + if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt(); } -std::string NarInfo::to_string() const -{ - std::string res; - res += "StorePath: " + path + "\n"; - res += "URL: " + url + "\n"; - assert(compression != ""); - res += "Compression: " + compression + "\n"; - assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32) + "\n"; - res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32) + "\n"; - res += "NarSize: " + std::to_string(narSize) + "\n"; +std::string NarInfo::to_string() const { + std::string res; + res += "StorePath: " + path + "\n"; + res += "URL: " + url + "\n"; + assert(compression != ""); + res += "Compression: " + compression + "\n"; + assert(fileHash.type == htSHA256); + res += "FileHash: " + fileHash.to_string(Base32) + "\n"; + res += "FileSize: " + std::to_string(fileSize) + "\n"; + assert(narHash.type == htSHA256); + res += "NarHash: " + narHash.to_string(Base32) + "\n"; + res += "NarSize: " + std::to_string(narSize) + "\n"; - res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; + res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; - if (!deriver.empty()) - res += "Deriver: " + baseNameOf(deriver) + "\n"; + if (!deriver.empty()) res += "Deriver: " + baseNameOf(deriver) + "\n"; - if (!system.empty()) - res += "System: " + system + "\n"; + if (!system.empty()) res += "System: " + system + "\n"; - for (auto sig : sigs) - res += "Sig: " + sig + "\n"; + for (auto sig : sigs) res += "Sig: " + sig + "\n"; - if (!ca.empty()) - res += "CA: " + ca + "\n"; + if (!ca.empty()) res += "CA: " + ca + "\n"; - return res; + return res; } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-info.hh b/third_party/nix/src/libstore/nar-info.hh index 4995061fbb6d..ce362e703f99 100644 --- a/third_party/nix/src/libstore/nar-info.hh +++ b/third_party/nix/src/libstore/nar-info.hh @@ -1,24 +1,23 @@ #pragma once -#include "types.hh" #include "hash.hh" #include "store-api.hh" +#include "types.hh" namespace nix { -struct NarInfo : ValidPathInfo -{ - std::string url; - std::string compression; - Hash fileHash; - uint64_t fileSize = 0; - std::string system; +struct NarInfo : ValidPathInfo { + std::string url; + std::string compression; + Hash fileHash; + uint64_t fileSize = 0; + std::string system; - NarInfo() { } - NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } - NarInfo(const Store & store, const std::string & s, const std::string & whence); + NarInfo() {} + NarInfo(const ValidPathInfo& info) : ValidPathInfo(info) {} + NarInfo(const Store& store, const std::string& s, const std::string& whence); - std::string to_string() const; + std::string to_string() const; }; -} +} // namespace nix diff --git a/third_party/nix/src/libstore/optimise-store.cc b/third_party/nix/src/libstore/optimise-store.cc index 991512f21795..b43919a2bb40 100644 --- a/third_party/nix/src/libstore/optimise-store.cc +++ b/third_party/nix/src/libstore/optimise-store.cc @@ -1,302 +1,285 @@ -#include "util.hh" -#include "local-store.hh" -#include "globals.hh" - -#include <cstdlib> -#include <cstring> -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> #include <errno.h> #include <stdio.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <cstdlib> +#include <cstring> #include <regex> - +#include "globals.hh" +#include "local-store.hh" +#include "util.hh" namespace nix { - -static void makeWritable(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); - if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("changing writability of '%1%'") % path); +static void makeWritable(const Path& path) { + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("changing writability of '%1%'") % path); } - -struct MakeReadOnly -{ - Path path; - MakeReadOnly(const Path & path) : path(path) { } - ~MakeReadOnly() - { - try { - /* This will make the path read-only. */ - if (path != "") canonicaliseTimestampAndPermissions(path); - } catch (...) { - ignoreException(); - } +struct MakeReadOnly { + Path path; + MakeReadOnly(const Path& path) : path(path) {} + ~MakeReadOnly() { + try { + /* This will make the path read-only. */ + if (path != "") canonicaliseTimestampAndPermissions(path); + } catch (...) { + ignoreException(); } + } }; +LocalStore::InodeHash LocalStore::loadInodeHash() { + debug("loading hash inodes in memory"); + InodeHash inodeHash; -LocalStore::InodeHash LocalStore::loadInodeHash() -{ - debug("loading hash inodes in memory"); - InodeHash inodeHash; - - AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { /* sic */ - checkInterrupt(); - // We don't care if we hit non-hash files, anything goes - inodeHash.insert(dirent->d_ino); - } - if (errno) throw SysError(format("reading directory '%1%'") % linksDir); + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + // We don't care if we hit non-hash files, anything goes + inodeHash.insert(dirent->d_ino); + } + if (errno) throw SysError(format("reading directory '%1%'") % linksDir); - printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); + printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); - return inodeHash; + return inodeHash; } +Strings LocalStore::readDirectoryIgnoringInodes(const Path& path, + const InodeHash& inodeHash) { + Strings names; -Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash) -{ - Strings names; - - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % path); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { /* sic */ - checkInterrupt(); - - if (inodeHash.count(dirent->d_ino)) { - debug(format("'%1%' is already linked") % dirent->d_name); - continue; - } + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - names.push_back(name); + if (inodeHash.count(dirent->d_ino)) { + debug(format("'%1%' is already linked") % dirent->d_name); + continue; } - if (errno) throw SysError(format("reading directory '%1%'") % path); - return names; + string name = dirent->d_name; + if (name == "." || name == "..") continue; + names.push_back(name); + } + if (errno) throw SysError(format("reading directory '%1%'") % path); + + return names; } +void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats, + const Path& path, InodeHash& inodeHash) { + checkInterrupt(); -void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, - const Path & path, InodeHash & inodeHash) -{ - checkInterrupt(); - - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); #if __APPLE__ - /* HFS/macOS has some undocumented security feature disabling hardlinking for - special files within .app dirs. *.app/Contents/PkgInfo and - *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See - https://github.com/NixOS/nix/issues/1443 for more discussion. */ - - if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) - { - debug(format("'%1%' is not allowed to be linked in macOS") % path); - return; - } + /* HFS/macOS has some undocumented security feature disabling hardlinking for + special files within .app dirs. *.app/Contents/PkgInfo and + *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See + https://github.com/NixOS/nix/issues/1443 for more discussion. */ + + if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) { + debug(format("'%1%' is not allowed to be linked in macOS") % path); + return; + } #endif - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectoryIgnoringInodes(path, inodeHash); - for (auto & i : names) - optimisePath_(act, stats, path + "/" + i, inodeHash); - return; - } + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectoryIgnoringInodes(path, inodeHash); + for (auto& i : names) optimisePath_(act, stats, path + "/" + i, inodeHash); + return; + } - /* We can hard link regular files and maybe symlinks. */ - if (!S_ISREG(st.st_mode) + /* We can hard link regular files and maybe symlinks. */ + if (!S_ISREG(st.st_mode) #if CAN_LINK_SYMLINK - && !S_ISLNK(st.st_mode) + && !S_ISLNK(st.st_mode) #endif - ) return; - - /* Sometimes SNAFUs can cause files in the Nix store to be - modified, in particular when running programs as root under - NixOS (example: $fontconfig/var/cache being modified). Skip - those files. FIXME: check the modification time. */ - if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - printError(format("skipping suspicious writable file '%1%'") % path); - return; + ) + return; + + /* Sometimes SNAFUs can cause files in the Nix store to be + modified, in particular when running programs as root under + NixOS (example: $fontconfig/var/cache being modified). Skip + those files. FIXME: check the modification time. */ + if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { + printError(format("skipping suspicious writable file '%1%'") % path); + return; + } + + /* This can still happen on top-level files. */ + if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { + debug(format("'%1%' is already linked, with %2% other file(s)") % path % + (st.st_nlink - 2)); + return; + } + + /* Hash the file. Note that hashPath() returns the hash over the + NAR serialisation, which includes the execute bit on the file. + Thus, executable and non-executable files with the same + contents *won't* be linked (which is good because otherwise the + permissions would be screwed up). + + Also note that if `path' is a symlink, then we're hashing the + contents of the symlink (i.e. the result of readlink()), not + the contents of the target (which may not even exist). */ + Hash hash = hashPath(htSHA256, path).first; + debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); + + /* Check if this is a known hash. */ + Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + +retry: + if (!pathExists(linkPath)) { + /* Nope, create a hard link in the links directory. */ + if (link(path.c_str(), linkPath.c_str()) == 0) { + inodeHash.insert(st.st_ino); + return; } - /* This can still happen on top-level files. */ - if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { - debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2)); + switch (errno) { + case EEXIST: + /* Fall through if another process created ‘linkPath’ before + we did. */ + break; + + case ENOSPC: + /* On ext4, that probably means the directory index is + full. When that happens, it's fine to ignore it: we + just effectively disable deduplication of this + file. */ + printInfo("cannot link '%s' to '%s': %s", linkPath, path, + strerror(errno)); return; - } - /* Hash the file. Note that hashPath() returns the hash over the - NAR serialisation, which includes the execute bit on the file. - Thus, executable and non-executable files with the same - contents *won't* be linked (which is good because otherwise the - permissions would be screwed up). - - Also note that if `path' is a symlink, then we're hashing the - contents of the symlink (i.e. the result of readlink()), not - the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); - - /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(Base32, false); - - retry: - if (!pathExists(linkPath)) { - /* Nope, create a hard link in the links directory. */ - if (link(path.c_str(), linkPath.c_str()) == 0) { - inodeHash.insert(st.st_ino); - return; - } - - switch (errno) { - case EEXIST: - /* Fall through if another process created ‘linkPath’ before - we did. */ - break; - - case ENOSPC: - /* On ext4, that probably means the directory index is - full. When that happens, it's fine to ignore it: we - just effectively disable deduplication of this - file. */ - printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno)); - return; - - default: - throw SysError("cannot link '%1%' to '%2%'", linkPath, path); - } + default: + throw SysError("cannot link '%1%' to '%2%'", linkPath, path); } - - /* Yes! We've seen a file with the same contents. Replace the - current file with a hard link to that file. */ - struct stat stLink; - if (lstat(linkPath.c_str(), &stLink)) - throw SysError(format("getting attributes of path '%1%'") % linkPath); - - if (st.st_ino == stLink.st_ino) { - debug(format("'%1%' is already linked to '%2%'") % path % linkPath); - return; - } - - if (st.st_size != stLink.st_size) { - printError(format("removing corrupted link '%1%'") % linkPath); - unlink(linkPath.c_str()); - goto retry; - } - - printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); - - /* Make the containing directory writable, but only if it's not - the store itself (we don't want or need to mess with its - permissions). */ - bool mustToggle = dirOf(path) != realStoreDir; - if (mustToggle) makeWritable(dirOf(path)); - - /* When we're done, make the directory read-only again and reset - its timestamp back to 0. */ - MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); - - Path tempLink = (format("%1%/.tmp-link-%2%-%3%") - % realStoreDir % getpid() % random()).str(); - - if (link(linkPath.c_str(), tempLink.c_str()) == -1) { - if (errno == EMLINK) { - /* Too many links to the same file (>= 32000 on most file - systems). This is likely to happen with empty files. - Just shrug and ignore. */ - if (st.st_size) - printInfo(format("'%1%' has maximum number of links") % linkPath); - return; - } - throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); + } + + /* Yes! We've seen a file with the same contents. Replace the + current file with a hard link to that file. */ + struct stat stLink; + if (lstat(linkPath.c_str(), &stLink)) + throw SysError(format("getting attributes of path '%1%'") % linkPath); + + if (st.st_ino == stLink.st_ino) { + debug(format("'%1%' is already linked to '%2%'") % path % linkPath); + return; + } + + if (st.st_size != stLink.st_size) { + printError(format("removing corrupted link '%1%'") % linkPath); + unlink(linkPath.c_str()); + goto retry; + } + + printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); + + /* Make the containing directory writable, but only if it's not + the store itself (we don't want or need to mess with its + permissions). */ + bool mustToggle = dirOf(path) != realStoreDir; + if (mustToggle) makeWritable(dirOf(path)); + + /* When we're done, make the directory read-only again and reset + its timestamp back to 0. */ + MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); + + Path tempLink = + (format("%1%/.tmp-link-%2%-%3%") % realStoreDir % getpid() % random()) + .str(); + + if (link(linkPath.c_str(), tempLink.c_str()) == -1) { + if (errno == EMLINK) { + /* Too many links to the same file (>= 32000 on most file + systems). This is likely to happen with empty files. + Just shrug and ignore. */ + if (st.st_size) + printInfo(format("'%1%' has maximum number of links") % linkPath); + return; } - - /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { - if (unlink(tempLink.c_str()) == -1) - printError(format("unable to unlink '%1%'") % tempLink); - if (errno == EMLINK) { - /* Some filesystems generate too many links on the rename, - rather than on the original link. (Probably it - temporarily increases the st_nlink field before - decreasing it again.) */ - debug("'%s' has reached maximum number of links", linkPath); - return; - } - throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path); + throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); + } + + /* Atomically replace the old file with the new hard link. */ + if (rename(tempLink.c_str(), path.c_str()) == -1) { + if (unlink(tempLink.c_str()) == -1) + printError(format("unable to unlink '%1%'") % tempLink); + if (errno == EMLINK) { + /* Some filesystems generate too many links on the rename, + rather than on the original link. (Probably it + temporarily increases the st_nlink field before + decreasing it again.) */ + debug("'%s' has reached maximum number of links", linkPath); + return; } + throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path); + } - stats.filesLinked++; - stats.bytesFreed += st.st_size; - stats.blocksFreed += st.st_blocks; + stats.filesLinked++; + stats.bytesFreed += st.st_size; + stats.blocksFreed += st.st_blocks; - if (act) - act->result(resFileLinked, st.st_size, st.st_blocks); + if (act) act->result(resFileLinked, st.st_size, st.st_blocks); } +void LocalStore::optimiseStore(OptimiseStats& stats) { + Activity act(*logger, actOptimiseStore); -void LocalStore::optimiseStore(OptimiseStats & stats) -{ - Activity act(*logger, actOptimiseStore); - - PathSet paths = queryAllValidPaths(); - InodeHash inodeHash = loadInodeHash(); + PathSet paths = queryAllValidPaths(); + InodeHash inodeHash = loadInodeHash(); - act.progress(0, paths.size()); + act.progress(0, paths.size()); - uint64_t done = 0; + uint64_t done = 0; - for (auto & i : paths) { - addTempRoot(i); - if (!isValidPath(i)) continue; /* path was GC'ed, probably */ - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i)); - optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash); - } - done++; - act.progress(done, paths.size()); + for (auto& i : paths) { + addTempRoot(i); + if (!isValidPath(i)) continue; /* path was GC'ed, probably */ + { + Activity act(*logger, lvlTalkative, actUnknown, + fmt("optimising path '%s'", i)); + optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash); } + done++; + act.progress(done, paths.size()); + } } -static string showBytes(unsigned long long bytes) -{ - return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); +static string showBytes(unsigned long long bytes) { + return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); } -void LocalStore::optimiseStore() -{ - OptimiseStats stats; +void LocalStore::optimiseStore() { + OptimiseStats stats; - optimiseStore(stats); + optimiseStore(stats); - printInfo( - format("%1% freed by hard-linking %2% files") - % showBytes(stats.bytesFreed) - % stats.filesLinked); + printInfo(format("%1% freed by hard-linking %2% files") % + showBytes(stats.bytesFreed) % stats.filesLinked); } -void LocalStore::optimisePath(const Path & path) -{ - OptimiseStats stats; - InodeHash inodeHash; +void LocalStore::optimisePath(const Path& path) { + OptimiseStats stats; + InodeHash inodeHash; - if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash); + if (settings.autoOptimiseStore) + optimisePath_(nullptr, stats, path, inodeHash); } - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/parsed-derivations.cc b/third_party/nix/src/libstore/parsed-derivations.cc index 87be8a24ead6..6a8c235a0c17 100644 --- a/third_party/nix/src/libstore/parsed-derivations.cc +++ b/third_party/nix/src/libstore/parsed-derivations.cc @@ -2,115 +2,115 @@ namespace nix { -ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) - : drvPath(drvPath), drv(drv) -{ - /* Parse the __json attribute, if any. */ - auto jsonAttr = drv.env.find("__json"); - if (jsonAttr != drv.env.end()) { - try { - structuredAttrs = nlohmann::json::parse(jsonAttr->second); - } catch (std::exception & e) { - throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); - } +ParsedDerivation::ParsedDerivation(const Path& drvPath, BasicDerivation& drv) + : drvPath(drvPath), drv(drv) { + /* Parse the __json attribute, if any. */ + auto jsonAttr = drv.env.find("__json"); + if (jsonAttr != drv.env.end()) { + try { + structuredAttrs = nlohmann::json::parse(jsonAttr->second); + } catch (std::exception& e) { + throw Error("cannot process __json attribute of '%s': %s", drvPath, + e.what()); } + } } -std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const -{ - if (structuredAttrs) { - auto i = structuredAttrs->find(name); - if (i == structuredAttrs->end()) - return {}; - else { - if (!i->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath); - return i->get<std::string>(); - } - } else { - auto i = drv.env.find(name); - if (i == drv.env.end()) - return {}; - else - return i->second; +std::optional<std::string> ParsedDerivation::getStringAttr( + const std::string& name) const { + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a string", name, + drvPath); + return i->get<std::string>(); } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return i->second; + } } -bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const -{ - if (structuredAttrs) { - auto i = structuredAttrs->find(name); - if (i == structuredAttrs->end()) - return def; - else { - if (!i->is_boolean()) - throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath); - return i->get<bool>(); - } - } else { - auto i = drv.env.find(name); - if (i == drv.env.end()) - return def; - else - return i->second == "1"; +bool ParsedDerivation::getBoolAttr(const std::string& name, bool def) const { + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return def; + else { + if (!i->is_boolean()) + throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, + drvPath); + return i->get<bool>(); } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return def; + else + return i->second == "1"; + } } -std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const -{ - if (structuredAttrs) { - auto i = structuredAttrs->find(name); - if (i == structuredAttrs->end()) - return {}; - else { - if (!i->is_array()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); - res.push_back(j->get<std::string>()); - } - return res; - } - } else { - auto i = drv.env.find(name); - if (i == drv.env.end()) - return {}; - else - return tokenizeString<Strings>(i->second); +std::optional<Strings> ParsedDerivation::getStringsAttr( + const std::string& name) const { + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_array()) + throw Error( + "attribute '%s' of derivation '%s' must be a list of strings", name, + drvPath); + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error( + "attribute '%s' of derivation '%s' must be a list of strings", + name, drvPath); + res.push_back(j->get<std::string>()); + } + return res; } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return tokenizeString<Strings>(i->second); + } } -StringSet ParsedDerivation::getRequiredSystemFeatures() const -{ - StringSet res; - for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) - res.insert(i); - return res; +StringSet ParsedDerivation::getRequiredSystemFeatures() const { + StringSet res; + for (auto& i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) + res.insert(i); + return res; } -bool ParsedDerivation::canBuildLocally() const -{ - if (drv.platform != settings.thisSystem.get() - && !settings.extraPlatforms.get().count(drv.platform) - && !drv.isBuiltin()) - return false; +bool ParsedDerivation::canBuildLocally() const { + if (drv.platform != settings.thisSystem.get() && + !settings.extraPlatforms.get().count(drv.platform) && !drv.isBuiltin()) + return false; - for (auto & feature : getRequiredSystemFeatures()) - if (!settings.systemFeatures.get().count(feature)) return false; + for (auto& feature : getRequiredSystemFeatures()) + if (!settings.systemFeatures.get().count(feature)) return false; - return true; + return true; } -bool ParsedDerivation::willBuildLocally() const -{ - return getBoolAttr("preferLocalBuild") && canBuildLocally(); +bool ParsedDerivation::willBuildLocally() const { + return getBoolAttr("preferLocalBuild") && canBuildLocally(); } -bool ParsedDerivation::substitutesAllowed() const -{ - return getBoolAttr("allowSubstitutes", true); +bool ParsedDerivation::substitutesAllowed() const { + return getBoolAttr("allowSubstitutes", true); } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/parsed-derivations.hh b/third_party/nix/src/libstore/parsed-derivations.hh index 9bde4b4dcfc7..70424b2ce187 100644 --- a/third_party/nix/src/libstore/parsed-derivations.hh +++ b/third_party/nix/src/libstore/parsed-derivations.hh @@ -1,37 +1,33 @@ -#include "derivations.hh" - #include <nlohmann/json.hpp> +#include "derivations.hh" namespace nix { -class ParsedDerivation -{ - Path drvPath; - BasicDerivation & drv; - std::optional<nlohmann::json> structuredAttrs; - -public: +class ParsedDerivation { + Path drvPath; + BasicDerivation& drv; + std::optional<nlohmann::json> structuredAttrs; - ParsedDerivation(const Path & drvPath, BasicDerivation & drv); + public: + ParsedDerivation(const Path& drvPath, BasicDerivation& drv); - const std::optional<nlohmann::json> & getStructuredAttrs() const - { - return structuredAttrs; - } + const std::optional<nlohmann::json>& getStructuredAttrs() const { + return structuredAttrs; + } - std::optional<std::string> getStringAttr(const std::string & name) const; + std::optional<std::string> getStringAttr(const std::string& name) const; - bool getBoolAttr(const std::string & name, bool def = false) const; + bool getBoolAttr(const std::string& name, bool def = false) const; - std::optional<Strings> getStringsAttr(const std::string & name) const; + std::optional<Strings> getStringsAttr(const std::string& name) const; - StringSet getRequiredSystemFeatures() const; + StringSet getRequiredSystemFeatures() const; - bool canBuildLocally() const; + bool canBuildLocally() const; - bool willBuildLocally() const; + bool willBuildLocally() const; - bool substitutesAllowed() const; + bool substitutesAllowed() const; }; -} +} // namespace nix diff --git a/third_party/nix/src/libstore/pathlocks.cc b/third_party/nix/src/libstore/pathlocks.cc index 2635e3940af8..7fa32e21a655 100644 --- a/third_party/nix/src/libstore/pathlocks.cc +++ b/third_party/nix/src/libstore/pathlocks.cc @@ -1,178 +1,156 @@ #include "pathlocks.hh" -#include "util.hh" -#include "sync.hh" - -#include <cerrno> -#include <cstdlib> - #include <fcntl.h> -#include <sys/types.h> -#include <sys/stat.h> #include <sys/file.h> - +#include <sys/stat.h> +#include <sys/types.h> +#include <cerrno> +#include <cstdlib> +#include "sync.hh" +#include "util.hh" namespace nix { +AutoCloseFD openLockFile(const Path& path, bool create) { + AutoCloseFD fd; -AutoCloseFD openLockFile(const Path & path, bool create) -{ - AutoCloseFD fd; - - fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); - if (!fd && (create || errno != ENOENT)) - throw SysError(format("opening lock file '%1%'") % path); + fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); + if (!fd && (create || errno != ENOENT)) + throw SysError(format("opening lock file '%1%'") % path); - return fd; + return fd; } - -void deleteLockFile(const Path & path, int fd) -{ - /* Get rid of the lock file. Have to be careful not to introduce - races. Write a (meaningless) token to the file to indicate to - other processes waiting on this lock that the lock is stale - (deleted). */ - unlink(path.c_str()); - writeFull(fd, "d"); - /* Note that the result of unlink() is ignored; removing the lock - file is an optimisation, not a necessity. */ +void deleteLockFile(const Path& path, int fd) { + /* Get rid of the lock file. Have to be careful not to introduce + races. Write a (meaningless) token to the file to indicate to + other processes waiting on this lock that the lock is stale + (deleted). */ + unlink(path.c_str()); + writeFull(fd, "d"); + /* Note that the result of unlink() is ignored; removing the lock + file is an optimisation, not a necessity. */ } - -bool lockFile(int fd, LockType lockType, bool wait) -{ - int type; - if (lockType == ltRead) type = LOCK_SH; - else if (lockType == ltWrite) type = LOCK_EX; - else if (lockType == ltNone) type = LOCK_UN; - else abort(); - - if (wait) { - while (flock(fd, type) != 0) { - checkInterrupt(); - if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); - else - return false; - } - } else { - while (flock(fd, type | LOCK_NB) != 0) { - checkInterrupt(); - if (errno == EWOULDBLOCK) return false; - if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); - } +bool lockFile(int fd, LockType lockType, bool wait) { + int type; + if (lockType == ltRead) + type = LOCK_SH; + else if (lockType == ltWrite) + type = LOCK_EX; + else if (lockType == ltNone) + type = LOCK_UN; + else + abort(); + + if (wait) { + while (flock(fd, type) != 0) { + checkInterrupt(); + if (errno != EINTR) + throw SysError(format("acquiring/releasing lock")); + else + return false; } + } else { + while (flock(fd, type | LOCK_NB) != 0) { + checkInterrupt(); + if (errno == EWOULDBLOCK) return false; + if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); + } + } - return true; + return true; } +PathLocks::PathLocks() : deletePaths(false) {} -PathLocks::PathLocks() - : deletePaths(false) -{ +PathLocks::PathLocks(const PathSet& paths, const string& waitMsg) + : deletePaths(false) { + lockPaths(paths, waitMsg); } +bool PathLocks::lockPaths(const PathSet& paths, const string& waitMsg, + bool wait) { + assert(fds.empty()); -PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) - : deletePaths(false) -{ - lockPaths(paths, waitMsg); -} + /* Note that `fds' is built incrementally so that the destructor + will only release those locks that we have already acquired. */ + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ + for (auto& path : paths) { + checkInterrupt(); + Path lockPath = path + ".lock"; -bool PathLocks::lockPaths(const PathSet & paths, - const string & waitMsg, bool wait) -{ - assert(fds.empty()); - - /* Note that `fds' is built incrementally so that the destructor - will only release those locks that we have already acquired. */ - - /* Acquire the lock for each path in sorted order. This ensures - that locks are always acquired in the same order, thus - preventing deadlocks. */ - for (auto & path : paths) { - checkInterrupt(); - Path lockPath = path + ".lock"; - - debug(format("locking path '%1%'") % path); - - AutoCloseFD fd; - - while (1) { - - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); - - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - return false; - } - } - - debug(format("lock acquired on '%1%'") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file '%1%'") % lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file '%1%' has become stale") % lockPath); - else - break; - } + debug(format("locking path '%1%'") % path); - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); - } + AutoCloseFD fd; - return true; -} + while (1) { + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); + + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; + } + } + + debug(format("lock acquired on '%1%'") % lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError(format("statting lock file '%1%'") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file '%1%' has become stale") % lockPath); + else + break; + } + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); + } -PathLocks::~PathLocks() -{ - try { - unlock(); - } catch (...) { - ignoreException(); - } + return true; } +PathLocks::~PathLocks() { + try { + unlock(); + } catch (...) { + ignoreException(); + } +} -void PathLocks::unlock() -{ - for (auto & i : fds) { - if (deletePaths) deleteLockFile(i.second, i.first); +void PathLocks::unlock() { + for (auto& i : fds) { + if (deletePaths) deleteLockFile(i.second, i.first); - if (close(i.first) == -1) - printError( - format("error (ignored): cannot close lock file on '%1%'") % i.second); + if (close(i.first) == -1) + printError(format("error (ignored): cannot close lock file on '%1%'") % + i.second); - debug(format("lock released on '%1%'") % i.second); - } + debug(format("lock released on '%1%'") % i.second); + } - fds.clear(); + fds.clear(); } - -void PathLocks::setDeletion(bool deletePaths) -{ - this->deletePaths = deletePaths; +void PathLocks::setDeletion(bool deletePaths) { + this->deletePaths = deletePaths; } - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/pathlocks.hh b/third_party/nix/src/libstore/pathlocks.hh index 411da022295d..90184989cde3 100644 --- a/third_party/nix/src/libstore/pathlocks.hh +++ b/third_party/nix/src/libstore/pathlocks.hh @@ -7,32 +7,29 @@ namespace nix { /* Open (possibly create) a lock file and return the file descriptor. -1 is returned if create is false and the lock could not be opened because it doesn't exist. Any other error throws an exception. */ -AutoCloseFD openLockFile(const Path & path, bool create); +AutoCloseFD openLockFile(const Path& path, bool create); /* Delete an open lock file. */ -void deleteLockFile(const Path & path, int fd); +void deleteLockFile(const Path& path, int fd); enum LockType { ltRead, ltWrite, ltNone }; bool lockFile(int fd, LockType lockType, bool wait); -class PathLocks -{ -private: - typedef std::pair<int, Path> FDPair; - list<FDPair> fds; - bool deletePaths; - -public: - PathLocks(); - PathLocks(const PathSet & paths, - const string & waitMsg = ""); - bool lockPaths(const PathSet & _paths, - const string & waitMsg = "", - bool wait = true); - ~PathLocks(); - void unlock(); - void setDeletion(bool deletePaths); +class PathLocks { + private: + typedef std::pair<int, Path> FDPair; + list<FDPair> fds; + bool deletePaths; + + public: + PathLocks(); + PathLocks(const PathSet& paths, const string& waitMsg = ""); + bool lockPaths(const PathSet& _paths, const string& waitMsg = "", + bool wait = true); + ~PathLocks(); + void unlock(); + void setDeletion(bool deletePaths); }; -} +} // namespace nix diff --git a/third_party/nix/src/libstore/profiles.cc b/third_party/nix/src/libstore/profiles.cc index 4c6af567ae6f..614153d89dd3 100644 --- a/third_party/nix/src/libstore/profiles.cc +++ b/third_party/nix/src/libstore/profiles.cc @@ -1,259 +1,226 @@ #include "profiles.hh" -#include "store-api.hh" -#include "util.hh" - -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> #include <errno.h> #include <stdio.h> - +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include "store-api.hh" +#include "util.hh" namespace nix { - -static bool cmpGensByNumber(const Generation & a, const Generation & b) -{ - return a.number < b.number; +static bool cmpGensByNumber(const Generation& a, const Generation& b) { + return a.number < b.number; } - /* Parse a generation name of the format `<profilename>-<number>-link'. */ -static int parseName(const string & profileName, const string & name) -{ - if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1; - string s = string(name, profileName.size() + 1); - string::size_type p = s.find("-link"); - if (p == string::npos) return -1; - int n; - if (string2Int(string(s, 0, p), n) && n >= 0) - return n; - else - return -1; +static int parseName(const string& profileName, const string& name) { + if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1; + string s = string(name, profileName.size() + 1); + string::size_type p = s.find("-link"); + if (p == string::npos) return -1; + int n; + if (string2Int(string(s, 0, p), n) && n >= 0) + return n; + else + return -1; } +Generations findGenerations(Path profile, int& curGen) { + Generations gens; + Path profileDir = dirOf(profile); + string profileName = baseNameOf(profile); -Generations findGenerations(Path profile, int & curGen) -{ - Generations gens; - - Path profileDir = dirOf(profile); - string profileName = baseNameOf(profile); - - for (auto & i : readDirectory(profileDir)) { - int n; - if ((n = parseName(profileName, i.name)) != -1) { - Generation gen; - gen.path = profileDir + "/" + i.name; - gen.number = n; - struct stat st; - if (lstat(gen.path.c_str(), &st) != 0) - throw SysError(format("statting '%1%'") % gen.path); - gen.creationTime = st.st_mtime; - gens.push_back(gen); - } + for (auto& i : readDirectory(profileDir)) { + int n; + if ((n = parseName(profileName, i.name)) != -1) { + Generation gen; + gen.path = profileDir + "/" + i.name; + gen.number = n; + struct stat st; + if (lstat(gen.path.c_str(), &st) != 0) + throw SysError(format("statting '%1%'") % gen.path); + gen.creationTime = st.st_mtime; + gens.push_back(gen); } + } - gens.sort(cmpGensByNumber); + gens.sort(cmpGensByNumber); - curGen = pathExists(profile) - ? parseName(profileName, readLink(profile)) - : -1; + curGen = pathExists(profile) ? parseName(profileName, readLink(profile)) : -1; - return gens; + return gens; } - -static void makeName(const Path & profile, unsigned int num, - Path & outLink) -{ - Path prefix = (format("%1%-%2%") % profile % num).str(); - outLink = prefix + "-link"; +static void makeName(const Path& profile, unsigned int num, Path& outLink) { + Path prefix = (format("%1%-%2%") % profile % num).str(); + outLink = prefix + "-link"; } +Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) { + /* The new generation number should be higher than old the + previous ones. */ + int dummy; + Generations gens = findGenerations(profile, dummy); -Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) -{ - /* The new generation number should be higher than old the - previous ones. */ - int dummy; - Generations gens = findGenerations(profile, dummy); + unsigned int num; + if (gens.size() > 0) { + Generation last = gens.back(); - unsigned int num; - if (gens.size() > 0) { - Generation last = gens.back(); + if (readLink(last.path) == outPath) { + /* We only create a new generation symlink if it differs + from the last one. - if (readLink(last.path) == outPath) { - /* We only create a new generation symlink if it differs - from the last one. - - This helps keeping gratuitous installs/rebuilds from piling - up uncontrolled numbers of generations, cluttering up the - UI like grub. */ - return last.path; - } - - num = gens.back().number; - } else { - num = 0; + This helps keeping gratuitous installs/rebuilds from piling + up uncontrolled numbers of generations, cluttering up the + UI like grub. */ + return last.path; } - /* Create the new generation. Note that addPermRoot() blocks if - the garbage collector is running to prevent the stuff we've - built from moving from the temporary roots (which the GC knows) - to the permanent roots (of which the GC would have a stale - view). If we didn't do it this way, the GC might remove the - user environment etc. we've just built. */ - Path generation; - makeName(profile, num + 1, generation); - store->addPermRoot(outPath, generation, false, true); - - return generation; -} + num = gens.back().number; + } else { + num = 0; + } + /* Create the new generation. Note that addPermRoot() blocks if + the garbage collector is running to prevent the stuff we've + built from moving from the temporary roots (which the GC knows) + to the permanent roots (of which the GC would have a stale + view). If we didn't do it this way, the GC might remove the + user environment etc. we've just built. */ + Path generation; + makeName(profile, num + 1, generation); + store->addPermRoot(outPath, generation, false, true); -static void removeFile(const Path & path) -{ - if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink '%1%'") % path); + return generation; } - -void deleteGeneration(const Path & profile, unsigned int gen) -{ - Path generation; - makeName(profile, gen, generation); - removeFile(generation); +static void removeFile(const Path& path) { + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink '%1%'") % path); } - -static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun) -{ - if (dryRun) - printInfo(format("would remove generation %1%") % gen); - else { - printInfo(format("removing generation %1%") % gen); - deleteGeneration(profile, gen); - } +void deleteGeneration(const Path& profile, unsigned int gen) { + Path generation; + makeName(profile, gen, generation); + removeFile(generation); } +static void deleteGeneration2(const Path& profile, unsigned int gen, + bool dryRun) { + if (dryRun) + printInfo(format("would remove generation %1%") % gen); + else { + printInfo(format("removing generation %1%") % gen); + deleteGeneration(profile, gen); + } +} -void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun) -{ - PathLocks lock; - lockProfile(lock, profile); +void deleteGenerations(const Path& profile, + const std::set<unsigned int>& gensToDelete, + bool dryRun) { + PathLocks lock; + lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + int curGen; + Generations gens = findGenerations(profile, curGen); - if (gensToDelete.find(curGen) != gensToDelete.end()) - throw Error(format("cannot delete current generation of profile %1%'") % profile); + if (gensToDelete.find(curGen) != gensToDelete.end()) + throw Error(format("cannot delete current generation of profile %1%'") % + profile); - for (auto & i : gens) { - if (gensToDelete.find(i.number) == gensToDelete.end()) continue; - deleteGeneration2(profile, i.number, dryRun); - } + for (auto& i : gens) { + if (gensToDelete.find(i.number) == gensToDelete.end()) continue; + deleteGeneration2(profile, i.number, dryRun); + } } -void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun) -{ - PathLocks lock; - lockProfile(lock, profile); - - int curGen; - bool fromCurGen = false; - Generations gens = findGenerations(profile, curGen); - for (auto i = gens.rbegin(); i != gens.rend(); ++i) { - if (i->number == curGen) { - fromCurGen = true; - max--; - continue; - } - if (fromCurGen) { - if (max) { - max--; - continue; - } - deleteGeneration2(profile, i->number, dryRun); - } +void deleteGenerationsGreaterThan(const Path& profile, int max, bool dryRun) { + PathLocks lock; + lockProfile(lock, profile); + + int curGen; + bool fromCurGen = false; + Generations gens = findGenerations(profile, curGen); + for (auto i = gens.rbegin(); i != gens.rend(); ++i) { + if (i->number == curGen) { + fromCurGen = true; + max--; + continue; } + if (fromCurGen) { + if (max) { + max--; + continue; + } + deleteGeneration2(profile, i->number, dryRun); + } + } } -void deleteOldGenerations(const Path & profile, bool dryRun) -{ - PathLocks lock; - lockProfile(lock, profile); +void deleteOldGenerations(const Path& profile, bool dryRun) { + PathLocks lock; + lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + int curGen; + Generations gens = findGenerations(profile, curGen); - for (auto & i : gens) - if (i.number != curGen) - deleteGeneration2(profile, i.number, dryRun); + for (auto& i : gens) + if (i.number != curGen) deleteGeneration2(profile, i.number, dryRun); } +void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun) { + PathLocks lock; + lockProfile(lock, profile); -void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) -{ - PathLocks lock; - lockProfile(lock, profile); - - int curGen; - Generations gens = findGenerations(profile, curGen); - - bool canDelete = false; - for (auto i = gens.rbegin(); i != gens.rend(); ++i) - if (canDelete) { - assert(i->creationTime < t); - if (i->number != curGen) - deleteGeneration2(profile, i->number, dryRun); - } else if (i->creationTime < t) { - /* We may now start deleting generations, but we don't - delete this generation yet, because this generation was - still the one that was active at the requested point in - time. */ - canDelete = true; - } -} + int curGen; + Generations gens = findGenerations(profile, curGen); + bool canDelete = false; + for (auto i = gens.rbegin(); i != gens.rend(); ++i) + if (canDelete) { + assert(i->creationTime < t); + if (i->number != curGen) deleteGeneration2(profile, i->number, dryRun); + } else if (i->creationTime < t) { + /* We may now start deleting generations, but we don't + delete this generation yet, because this generation was + still the one that was active at the requested point in + time. */ + canDelete = true; + } +} -void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun) -{ - time_t curTime = time(0); - string strDays = string(timeSpec, 0, timeSpec.size() - 1); - int days; +void deleteGenerationsOlderThan(const Path& profile, const string& timeSpec, + bool dryRun) { + time_t curTime = time(0); + string strDays = string(timeSpec, 0, timeSpec.size() - 1); + int days; - if (!string2Int(strDays, days) || days < 1) - throw Error(format("invalid number of days specifier '%1%'") % timeSpec); + if (!string2Int(strDays, days) || days < 1) + throw Error(format("invalid number of days specifier '%1%'") % timeSpec); - time_t oldTime = curTime - days * 24 * 3600; + time_t oldTime = curTime - days * 24 * 3600; - deleteGenerationsOlderThan(profile, oldTime, dryRun); + deleteGenerationsOlderThan(profile, oldTime, dryRun); } +void switchLink(Path link, Path target) { + /* Hacky. */ + if (dirOf(target) == dirOf(link)) target = baseNameOf(target); -void switchLink(Path link, Path target) -{ - /* Hacky. */ - if (dirOf(target) == dirOf(link)) target = baseNameOf(target); - - replaceSymlink(target, link); + replaceSymlink(target, link); } - -void lockProfile(PathLocks & lock, const Path & profile) -{ - lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str()); - lock.setDeletion(true); +void lockProfile(PathLocks& lock, const Path& profile) { + lock.lockPaths({profile}, + (format("waiting for lock on profile '%1%'") % profile).str()); + lock.setDeletion(true); } - -string optimisticLockProfile(const Path & profile) -{ - return pathExists(profile) ? readLink(profile) : ""; +string optimisticLockProfile(const Path& profile) { + return pathExists(profile) ? readLink(profile) : ""; } - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/profiles.hh b/third_party/nix/src/libstore/profiles.hh index 5fa1533de311..c23fe1f8598d 100644 --- a/third_party/nix/src/libstore/profiles.hh +++ b/third_party/nix/src/libstore/profiles.hh @@ -1,57 +1,49 @@ #pragma once -#include "types.hh" -#include "pathlocks.hh" - #include <time.h> - +#include "pathlocks.hh" +#include "types.hh" namespace nix { - -struct Generation -{ - int number; - Path path; - time_t creationTime; - Generation() - { - number = -1; - } - operator bool() const - { - return number != -1; - } +struct Generation { + int number; + Path path; + time_t creationTime; + Generation() { number = -1; } + operator bool() const { return number != -1; } }; typedef list<Generation> Generations; - /* Returns the list of currently present generations for the specified profile, sorted by generation number. */ -Generations findGenerations(Path profile, int & curGen); +Generations findGenerations(Path profile, int& curGen); class LocalFSStore; Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath); -void deleteGeneration(const Path & profile, unsigned int gen); +void deleteGeneration(const Path& profile, unsigned int gen); -void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun); +void deleteGenerations(const Path& profile, + const std::set<unsigned int>& gensToDelete, bool dryRun); -void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun); +void deleteGenerationsGreaterThan(const Path& profile, const int max, + bool dryRun); -void deleteOldGenerations(const Path & profile, bool dryRun); +void deleteOldGenerations(const Path& profile, bool dryRun); -void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun); +void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun); -void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun); +void deleteGenerationsOlderThan(const Path& profile, const string& timeSpec, + bool dryRun); void switchLink(Path link, Path target); /* Ensure exclusive access to a profile. Any command that modifies the profile first acquires this lock. */ -void lockProfile(PathLocks & lock, const Path & profile); +void lockProfile(PathLocks& lock, const Path& profile); /* Optimistic locking is used by long-running operations like `nix-env -i'. Instead of acquiring the exclusive lock for the entire @@ -62,6 +54,6 @@ void lockProfile(PathLocks & lock, const Path & profile); generally cheap, since the build results are still in the Nix store. Most of the time, only the user environment has to be rebuilt. */ -string optimisticLockProfile(const Path & profile); +string optimisticLockProfile(const Path& profile); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/references.cc b/third_party/nix/src/libstore/references.cc index 5b7eb1f846af..df3ee73fde84 100644 --- a/third_party/nix/src/libstore/references.cc +++ b/third_party/nix/src/libstore/references.cc @@ -1,122 +1,110 @@ #include "references.hh" +#include <cstdlib> +#include <map> +#include "archive.hh" #include "hash.hh" #include "util.hh" -#include "archive.hh" - -#include <map> -#include <cstdlib> - namespace nix { - static unsigned int refLength = 32; /* characters */ - -static void search(const unsigned char * s, size_t len, - StringSet & hashes, StringSet & seen) -{ - static bool initialised = false; - static bool isBase32[256]; - if (!initialised) { - for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; - for (unsigned int i = 0; i < base32Chars.size(); ++i) - isBase32[(unsigned char) base32Chars[i]] = true; - initialised = true; - } - - for (size_t i = 0; i + refLength <= len; ) { - int j; - bool match = true; - for (j = refLength - 1; j >= 0; --j) - if (!isBase32[(unsigned char) s[i + j]]) { - i += j + 1; - match = false; - break; - } - if (!match) continue; - string ref((const char *) s + i, refLength); - if (hashes.find(ref) != hashes.end()) { - debug(format("found reference to '%1%' at offset '%2%'") - % ref % i); - seen.insert(ref); - hashes.erase(ref); - } - ++i; +static void search(const unsigned char* s, size_t len, StringSet& hashes, + StringSet& seen) { + static bool initialised = false; + static bool isBase32[256]; + if (!initialised) { + for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; + for (unsigned int i = 0; i < base32Chars.size(); ++i) + isBase32[(unsigned char)base32Chars[i]] = true; + initialised = true; + } + + for (size_t i = 0; i + refLength <= len;) { + int j; + bool match = true; + for (j = refLength - 1; j >= 0; --j) + if (!isBase32[(unsigned char)s[i + j]]) { + i += j + 1; + match = false; + break; + } + if (!match) continue; + string ref((const char*)s + i, refLength); + if (hashes.find(ref) != hashes.end()) { + debug(format("found reference to '%1%' at offset '%2%'") % ref % i); + seen.insert(ref); + hashes.erase(ref); } + ++i; + } } +struct RefScanSink : Sink { + HashSink hashSink; + StringSet hashes; + StringSet seen; -struct RefScanSink : Sink -{ - HashSink hashSink; - StringSet hashes; - StringSet seen; + string tail; - string tail; + RefScanSink() : hashSink(htSHA256) {} - RefScanSink() : hashSink(htSHA256) { } - - void operator () (const unsigned char * data, size_t len); + void operator()(const unsigned char* data, size_t len); }; +void RefScanSink::operator()(const unsigned char* data, size_t len) { + hashSink(data, len); -void RefScanSink::operator () (const unsigned char * data, size_t len) -{ - hashSink(data, len); - - /* It's possible that a reference spans the previous and current - fragment, so search in the concatenation of the tail of the - previous fragment and the start of the current fragment. */ - string s = tail + string((const char *) data, len > refLength ? refLength : len); - search((const unsigned char *) s.data(), s.size(), hashes, seen); + /* It's possible that a reference spans the previous and current + fragment, so search in the concatenation of the tail of the + previous fragment and the start of the current fragment. */ + string s = + tail + string((const char*)data, len > refLength ? refLength : len); + search((const unsigned char*)s.data(), s.size(), hashes, seen); - search(data, len, hashes, seen); + search(data, len, hashes, seen); - size_t tailLen = len <= refLength ? len : refLength; - tail = - string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + - string((const char *) data + len - tailLen, tailLen); + size_t tailLen = len <= refLength ? len : refLength; + tail = string(tail, tail.size() < refLength - tailLen + ? 0 + : tail.size() - (refLength - tailLen)) + + string((const char*)data + len - tailLen, tailLen); } - -PathSet scanForReferences(const string & path, - const PathSet & refs, HashResult & hash) -{ - RefScanSink sink; - std::map<string, Path> backMap; - - /* For efficiency (and a higher hit rate), just search for the - hash part of the file name. (This assumes that all references - have the form `HASH-bla'). */ - for (auto & i : refs) { - string baseName = baseNameOf(i); - string::size_type pos = baseName.find('-'); - if (pos == string::npos) - throw Error(format("bad reference '%1%'") % i); - string s = string(baseName, 0, pos); - assert(s.size() == refLength); - assert(backMap.find(s) == backMap.end()); - // parseHash(htSHA256, s); - sink.hashes.insert(s); - backMap[s] = i; - } - - /* Look for the hashes in the NAR dump of the path. */ - dumpPath(path, sink); - - /* Map the hashes found back to their store paths. */ - PathSet found; - for (auto & i : sink.seen) { - std::map<string, Path>::iterator j; - if ((j = backMap.find(i)) == backMap.end()) abort(); - found.insert(j->second); - } - - hash = sink.hashSink.finish(); - - return found; +PathSet scanForReferences(const string& path, const PathSet& refs, + HashResult& hash) { + RefScanSink sink; + std::map<string, Path> backMap; + + /* For efficiency (and a higher hit rate), just search for the + hash part of the file name. (This assumes that all references + have the form `HASH-bla'). */ + for (auto& i : refs) { + string baseName = baseNameOf(i); + string::size_type pos = baseName.find('-'); + if (pos == string::npos) throw Error(format("bad reference '%1%'") % i); + string s = string(baseName, 0, pos); + assert(s.size() == refLength); + assert(backMap.find(s) == backMap.end()); + // parseHash(htSHA256, s); + sink.hashes.insert(s); + backMap[s] = i; + } + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + /* Map the hashes found back to their store paths. */ + PathSet found; + for (auto& i : sink.seen) { + std::map<string, Path>::iterator j; + if ((j = backMap.find(i)) == backMap.end()) abort(); + found.insert(j->second); + } + + hash = sink.hashSink.finish(); + + return found; } - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/references.hh b/third_party/nix/src/libstore/references.hh index 013809d122f3..2229150e3359 100644 --- a/third_party/nix/src/libstore/references.hh +++ b/third_party/nix/src/libstore/references.hh @@ -1,11 +1,11 @@ #pragma once -#include "types.hh" #include "hash.hh" +#include "types.hh" namespace nix { -PathSet scanForReferences(const Path & path, const PathSet & refs, - HashResult & hash); - +PathSet scanForReferences(const Path& path, const PathSet& refs, + HashResult& hash); + } diff --git a/third_party/nix/src/libstore/remote-fs-accessor.cc b/third_party/nix/src/libstore/remote-fs-accessor.cc index 5233fb2c239b..044b9ab5f931 100644 --- a/third_party/nix/src/libstore/remote-fs-accessor.cc +++ b/third_party/nix/src/libstore/remote-fs-accessor.cc @@ -1,129 +1,120 @@ #include "remote-fs-accessor.hh" -#include "nar-accessor.hh" -#include "json.hh" - -#include <sys/types.h> -#include <sys/stat.h> #include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> +#include "json.hh" +#include "nar-accessor.hh" namespace nix { -RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir) - : store(store) - , cacheDir(cacheDir) -{ - if (cacheDir != "") - createDirs(cacheDir); +RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path& cacheDir) + : store(store), cacheDir(cacheDir) { + if (cacheDir != "") createDirs(cacheDir); } -Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) -{ - assert(cacheDir != ""); - return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext); +Path RemoteFSAccessor::makeCacheFile(const Path& storePath, + const std::string& ext) { + assert(cacheDir != ""); + return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext); } -void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, - ref<FSAccessor> narAccessor) -{ - nars.emplace(storePath, narAccessor); +void RemoteFSAccessor::addToCache(const Path& storePath, const std::string& nar, + ref<FSAccessor> narAccessor) { + nars.emplace(storePath, narAccessor); - if (cacheDir != "") { - try { - std::ostringstream str; - JSONPlaceholder jsonRoot(str); - listNar(jsonRoot, narAccessor, "", true); - writeFile(makeCacheFile(storePath, "ls"), str.str()); + if (cacheDir != "") { + try { + std::ostringstream str; + JSONPlaceholder jsonRoot(str); + listNar(jsonRoot, narAccessor, "", true); + writeFile(makeCacheFile(storePath, "ls"), str.str()); - /* FIXME: do this asynchronously. */ - writeFile(makeCacheFile(storePath, "nar"), nar); + /* FIXME: do this asynchronously. */ + writeFile(makeCacheFile(storePath, "nar"), nar); - } catch (...) { - ignoreException(); - } + } catch (...) { + ignoreException(); } + } } -std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) -{ - auto path = canonPath(path_); - - auto storePath = store->toStorePath(path); - std::string restPath = std::string(path, storePath.size()); +std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path& path_) { + auto path = canonPath(path_); - if (!store->isValidPath(storePath)) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); + auto storePath = store->toStorePath(path); + std::string restPath = std::string(path, storePath.size()); - auto i = nars.find(storePath); - if (i != nars.end()) return {i->second, restPath}; + if (!store->isValidPath(storePath)) + throw InvalidPath(format("path '%1%' is not a valid store path") % + storePath); - StringSink sink; - std::string listing; - Path cacheFile; + auto i = nars.find(storePath); + if (i != nars.end()) return {i->second, restPath}; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { + StringSink sink; + std::string listing; + Path cacheFile; - try { - listing = nix::readFile(makeCacheFile(storePath, "ls")); + if (cacheDir != "" && + pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { + try { + listing = nix::readFile(makeCacheFile(storePath, "ls")); - auto narAccessor = makeLazyNarAccessor(listing, - [cacheFile](uint64_t offset, uint64_t length) { + auto narAccessor = makeLazyNarAccessor( + listing, [cacheFile](uint64_t offset, uint64_t length) { + AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError("opening NAR cache file '%s'", cacheFile); - AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening NAR cache file '%s'", cacheFile); + if (lseek(fd.get(), offset, SEEK_SET) != (off_t)offset) + throw SysError("seeking in '%s'", cacheFile); - if (lseek(fd.get(), offset, SEEK_SET) != (off_t) offset) - throw SysError("seeking in '%s'", cacheFile); + std::string buf(length, 0); + readFull(fd.get(), (unsigned char*)buf.data(), length); - std::string buf(length, 0); - readFull(fd.get(), (unsigned char *) buf.data(), length); + return buf; + }); - return buf; - }); + nars.emplace(storePath, narAccessor); + return {narAccessor, restPath}; - nars.emplace(storePath, narAccessor); - return {narAccessor, restPath}; - - } catch (SysError &) { } + } catch (SysError&) { + } - try { - *sink.s = nix::readFile(cacheFile); + try { + *sink.s = nix::readFile(cacheFile); - auto narAccessor = makeNarAccessor(sink.s); - nars.emplace(storePath, narAccessor); - return {narAccessor, restPath}; + auto narAccessor = makeNarAccessor(sink.s); + nars.emplace(storePath, narAccessor); + return {narAccessor, restPath}; - } catch (SysError &) { } + } catch (SysError&) { } + } - store->narFromPath(storePath, sink); - auto narAccessor = makeNarAccessor(sink.s); - addToCache(storePath, *sink.s, narAccessor); - return {narAccessor, restPath}; + store->narFromPath(storePath, sink); + auto narAccessor = makeNarAccessor(sink.s); + addToCache(storePath, *sink.s, narAccessor); + return {narAccessor, restPath}; } -FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) -{ - auto res = fetch(path); - return res.first->stat(res.second); +FSAccessor::Stat RemoteFSAccessor::stat(const Path& path) { + auto res = fetch(path); + return res.first->stat(res.second); } -StringSet RemoteFSAccessor::readDirectory(const Path & path) -{ - auto res = fetch(path); - return res.first->readDirectory(res.second); +StringSet RemoteFSAccessor::readDirectory(const Path& path) { + auto res = fetch(path); + return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path) -{ - auto res = fetch(path); - return res.first->readFile(res.second); +std::string RemoteFSAccessor::readFile(const Path& path) { + auto res = fetch(path); + return res.first->readFile(res.second); } -std::string RemoteFSAccessor::readLink(const Path & path) -{ - auto res = fetch(path); - return res.first->readLink(res.second); +std::string RemoteFSAccessor::readLink(const Path& path) { + auto res = fetch(path); + return res.first->readLink(res.second); } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/remote-fs-accessor.hh b/third_party/nix/src/libstore/remote-fs-accessor.hh index 4afb3be95736..08985de5291f 100644 --- a/third_party/nix/src/libstore/remote-fs-accessor.hh +++ b/third_party/nix/src/libstore/remote-fs-accessor.hh @@ -6,35 +6,33 @@ namespace nix { -class RemoteFSAccessor : public FSAccessor -{ - ref<Store> store; +class RemoteFSAccessor : public FSAccessor { + ref<Store> store; - std::map<Path, ref<FSAccessor>> nars; + std::map<Path, ref<FSAccessor>> nars; - Path cacheDir; + Path cacheDir; - std::pair<ref<FSAccessor>, Path> fetch(const Path & path_); + std::pair<ref<FSAccessor>, Path> fetch(const Path& path_); - friend class BinaryCacheStore; + friend class BinaryCacheStore; - Path makeCacheFile(const Path & storePath, const std::string & ext); + Path makeCacheFile(const Path& storePath, const std::string& ext); - void addToCache(const Path & storePath, const std::string & nar, - ref<FSAccessor> narAccessor); + void addToCache(const Path& storePath, const std::string& nar, + ref<FSAccessor> narAccessor); -public: + public: + RemoteFSAccessor(ref<Store> store, + const /* FIXME: use std::optional */ Path& cacheDir = ""); - RemoteFSAccessor(ref<Store> store, - const /* FIXME: use std::optional */ Path & cacheDir = ""); + Stat stat(const Path& path) override; - Stat stat(const Path & path) override; + StringSet readDirectory(const Path& path) override; - StringSet readDirectory(const Path & path) override; + std::string readFile(const Path& path) override; - std::string readFile(const Path & path) override; - - std::string readLink(const Path & path) override; + std::string readLink(const Path& path) override; }; -} +} // namespace nix diff --git a/third_party/nix/src/libstore/remote-store.cc b/third_party/nix/src/libstore/remote-store.cc index e21f3449b0ef..802316dcf351 100644 --- a/third_party/nix/src/libstore/remote-store.cc +++ b/third_party/nix/src/libstore/remote-store.cc @@ -1,817 +1,715 @@ -#include "serialise.hh" -#include "util.hh" #include "remote-store.hh" -#include "worker-protocol.hh" -#include "archive.hh" -#include "affinity.hh" -#include "globals.hh" -#include "derivations.hh" -#include "pool.hh" -#include "finally.hh" - -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/socket.h> -#include <sys/un.h> #include <errno.h> #include <fcntl.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/un.h> #include <unistd.h> - #include <cstring> +#include "affinity.hh" +#include "archive.hh" +#include "derivations.hh" +#include "finally.hh" +#include "globals.hh" +#include "pool.hh" +#include "serialise.hh" +#include "util.hh" +#include "worker-protocol.hh" namespace nix { - -Path readStorePath(Store & store, Source & from) -{ - Path path = readString(from); - store.assertStorePath(path); - return path; -} - - -template<class T> T readStorePaths(Store & store, Source & from) -{ - T paths = readStrings<T>(from); - for (auto & i : paths) store.assertStorePath(i); - return paths; -} - -template PathSet readStorePaths(Store & store, Source & from); -template Paths readStorePaths(Store & store, Source & from); - -/* TODO: Separate these store impls into different files, give them better names */ -RemoteStore::RemoteStore(const Params & params) - : Store(params) - , connections(make_ref<Pool<Connection>>( - std::max(1, (int) maxConnections), - [this]() { return openConnectionWrapper(); }, - [this](const ref<Connection> & r) { - return - r->to.good() - && r->from.good() - && std::chrono::duration_cast<std::chrono::seconds>( - std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; - } - )) -{ -} - - -ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper() -{ - if (failed) - throw Error("opening a connection to remote store '%s' previously failed", getUri()); - try { - return openConnection(); - } catch (...) { - failed = true; - throw; +Path readStorePath(Store& store, Source& from) { + Path path = readString(from); + store.assertStorePath(path); + return path; +} + +template <class T> +T readStorePaths(Store& store, Source& from) { + T paths = readStrings<T>(from); + for (auto& i : paths) store.assertStorePath(i); + return paths; +} + +template PathSet readStorePaths(Store& store, Source& from); +template Paths readStorePaths(Store& store, Source& from); + +/* TODO: Separate these store impls into different files, give them better names + */ +RemoteStore::RemoteStore(const Params& params) + : Store(params), + connections(make_ref<Pool<Connection>>( + std::max(1, (int)maxConnections), + [this]() { return openConnectionWrapper(); }, + [this](const ref<Connection>& r) { + return r->to.good() && r->from.good() && + std::chrono::duration_cast<std::chrono::seconds>( + std::chrono::steady_clock::now() - r->startTime) + .count() < maxConnectionAge; + })) {} + +ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper() { + if (failed) + throw Error("opening a connection to remote store '%s' previously failed", + getUri()); + try { + return openConnection(); + } catch (...) { + failed = true; + throw; + } +} + +UDSRemoteStore::UDSRemoteStore(const Params& params) + : Store(params), LocalFSStore(params), RemoteStore(params) {} + +UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params& params) + : Store(params), + LocalFSStore(params), + RemoteStore(params), + path(socket_path) {} + +std::string UDSRemoteStore::getUri() { + if (path) { + return std::string("unix://") + *path; + } else { + return "daemon"; + } +} + +ref<RemoteStore::Connection> UDSRemoteStore::openConnection() { + auto conn = make_ref<Connection>(); + + /* Connect to a daemon that does the privileged work for us. */ + conn->fd = socket(PF_UNIX, + SOCK_STREAM +#ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC +#endif + , + 0); + if (!conn->fd) throw SysError("cannot create Unix domain socket"); + closeOnExec(conn->fd.get()); + + string socketPath = path ? *path : settings.nixDaemonSocketFile; + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (socketPath.size() + 1 >= sizeof(addr.sun_path)) + throw Error(format("socket path '%1%' is too long") % socketPath); + strcpy(addr.sun_path, socketPath.c_str()); + + if (::connect(conn->fd.get(), (struct sockaddr*)&addr, sizeof(addr)) == -1) + throw SysError(format("cannot connect to daemon at '%1%'") % socketPath); + + conn->from.fd = conn->fd.get(); + conn->to.fd = conn->fd.get(); + + conn->startTime = std::chrono::steady_clock::now(); + + initConnection(*conn); + + return conn; +} + +void RemoteStore::initConnection(Connection& conn) { + /* Send the magic greeting, check for the reply. */ + try { + conn.to << WORKER_MAGIC_1; + conn.to.flush(); + unsigned int magic = readInt(conn.from); + if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); + + conn.from >> conn.daemonVersion; + if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != + GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + throw Error("Nix daemon protocol version not supported"); + if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) + throw Error("the Nix daemon version is too old"); + conn.to << PROTOCOL_VERSION; + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { + int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; + if (cpu != -1) + conn.to << 1 << cpu; + else + conn.to << 0; } -} - - -UDSRemoteStore::UDSRemoteStore(const Params & params) - : Store(params) - , LocalFSStore(params) - , RemoteStore(params) -{ -} - - -UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params) - : Store(params) - , LocalFSStore(params) - , RemoteStore(params) - , path(socket_path) -{ -} - - -std::string UDSRemoteStore::getUri() -{ - if (path) { - return std::string("unix://") + *path; - } else { - return "daemon"; - } -} - - -ref<RemoteStore::Connection> UDSRemoteStore::openConnection() -{ - auto conn = make_ref<Connection>(); - - /* Connect to a daemon that does the privileged work for us. */ - conn->fd = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!conn->fd) - throw SysError("cannot create Unix domain socket"); - closeOnExec(conn->fd.get()); - - string socketPath = path ? *path : settings.nixDaemonSocketFile; - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPath.size() + 1 >= sizeof(addr.sun_path)) - throw Error(format("socket path '%1%' is too long") % socketPath); - strcpy(addr.sun_path, socketPath.c_str()); - if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at '%1%'") % socketPath); - - conn->from.fd = conn->fd.get(); - conn->to.fd = conn->fd.get(); - - conn->startTime = std::chrono::steady_clock::now(); - - initConnection(*conn); - - return conn; -} - - -void RemoteStore::initConnection(Connection & conn) -{ - /* Send the magic greeting, check for the reply. */ - try { - conn.to << WORKER_MAGIC_1; - conn.to.flush(); - unsigned int magic = readInt(conn.from); - if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); - - conn.from >> conn.daemonVersion; - if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) - throw Error("Nix daemon protocol version not supported"); - if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) - throw Error("the Nix daemon version is too old"); - conn.to << PROTOCOL_VERSION; - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { - int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; - if (cpu != -1) - conn.to << 1 << cpu; - else - conn.to << 0; - } - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) - conn.to << false; - - auto ex = conn.processStderr(); - if (ex) std::rethrow_exception(ex); - } - catch (Error & e) { - throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what()); - } - - setOptions(conn); -} - - -void RemoteStore::setOptions(Connection & conn) -{ - conn.to << wopSetOptions - << settings.keepFailed - << settings.keepGoing - << settings.tryFallback - << verbosity - << settings.maxBuildJobs - << settings.maxSilentTime - << true - << (settings.verboseBuild ? lvlError : lvlVomit) - << 0 // obsolete log type - << 0 /* obsolete print build trace */ - << settings.buildCores - << settings.useSubstitutes; - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { - std::map<std::string, Config::SettingInfo> overrides; - globalConfig.getSettings(overrides, true); - overrides.erase(settings.keepFailed.name); - overrides.erase(settings.keepGoing.name); - overrides.erase(settings.tryFallback.name); - overrides.erase(settings.maxBuildJobs.name); - overrides.erase(settings.maxSilentTime.name); - overrides.erase(settings.buildCores.name); - overrides.erase(settings.useSubstitutes.name); - overrides.erase(settings.showTrace.name); - conn.to << overrides.size(); - for (auto & i : overrides) - conn.to << i.first << i.second.value; - } + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) conn.to << false; auto ex = conn.processStderr(); if (ex) std::rethrow_exception(ex); + } catch (Error& e) { + throw Error("cannot open connection to remote store '%s': %s", getUri(), + e.what()); + } + + setOptions(conn); +} + +void RemoteStore::setOptions(Connection& conn) { + conn.to << wopSetOptions << settings.keepFailed << settings.keepGoing + << settings.tryFallback << verbosity << settings.maxBuildJobs + << settings.maxSilentTime << true + << (settings.verboseBuild ? lvlError : lvlVomit) + << 0 // obsolete log type + << 0 /* obsolete print build trace */ + << settings.buildCores << settings.useSubstitutes; + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { + std::map<std::string, Config::SettingInfo> overrides; + globalConfig.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); + overrides.erase(settings.showTrace.name); + conn.to << overrides.size(); + for (auto& i : overrides) conn.to << i.first << i.second.value; + } + + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); } - /* A wrapper around Pool<RemoteStore::Connection>::Handle that marks the connection as bad (causing it to be closed) if a non-daemon exception is thrown before the handle is closed. Such an exception causes a deviation from the expected protocol and therefore a desynchronization between the client and daemon. */ -struct ConnectionHandle -{ - Pool<RemoteStore::Connection>::Handle handle; - bool daemonException = false; +struct ConnectionHandle { + Pool<RemoteStore::Connection>::Handle handle; + bool daemonException = false; - ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle) - : handle(std::move(handle)) - { } + ConnectionHandle(Pool<RemoteStore::Connection>::Handle&& handle) + : handle(std::move(handle)) {} - ConnectionHandle(ConnectionHandle && h) - : handle(std::move(h.handle)) - { } + ConnectionHandle(ConnectionHandle&& h) : handle(std::move(h.handle)) {} - ~ConnectionHandle() - { - if (!daemonException && std::uncaught_exceptions()) { - handle.markBad(); - debug("closing daemon connection because of an exception"); - } + ~ConnectionHandle() { + if (!daemonException && std::uncaught_exceptions()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); } + } - RemoteStore::Connection * operator -> () { return &*handle; } + RemoteStore::Connection* operator->() { return &*handle; } - void processStderr(Sink * sink = 0, Source * source = 0) - { - auto ex = handle->processStderr(sink, source); - if (ex) { - daemonException = true; - std::rethrow_exception(ex); - } + void processStderr(Sink* sink = 0, Source* source = 0) { + auto ex = handle->processStderr(sink, source); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); } + } }; - -ConnectionHandle RemoteStore::getConnection() -{ - return ConnectionHandle(connections->get()); +ConnectionHandle RemoteStore::getConnection() { + return ConnectionHandle(connections->get()); } - -bool RemoteStore::isValidPathUncached(const Path & path) -{ - auto conn(getConnection()); - conn->to << wopIsValidPath << path; - conn.processStderr(); - return readInt(conn->from); -} - - -PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) -{ - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - PathSet res; - for (auto & i : paths) - if (isValidPath(i)) res.insert(i); - return res; - } else { - conn->to << wopQueryValidPaths << paths; - conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); - } +bool RemoteStore::isValidPathUncached(const Path& path) { + auto conn(getConnection()); + conn->to << wopIsValidPath << path; + conn.processStderr(); + return readInt(conn->from); } - -PathSet RemoteStore::queryAllValidPaths() -{ - auto conn(getConnection()); - conn->to << wopQueryAllValidPaths; +PathSet RemoteStore::queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute) { + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + PathSet res; + for (auto& i : paths) + if (isValidPath(i)) res.insert(i); + return res; + } else { + conn->to << wopQueryValidPaths << paths; conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); + } } - -PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) -{ - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - PathSet res; - for (auto & i : paths) { - conn->to << wopHasSubstitutes << i; - conn.processStderr(); - if (readInt(conn->from)) res.insert(i); - } - return res; - } else { - conn->to << wopQuerySubstitutablePaths << paths; - conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); - } +PathSet RemoteStore::queryAllValidPaths() { + auto conn(getConnection()); + conn->to << wopQueryAllValidPaths; + conn.processStderr(); + return readStorePaths<PathSet>(*this, conn->from); } - -void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) -{ - if (paths.empty()) return; - - auto conn(getConnection()); - - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - - for (auto & i : paths) { - SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << i; - conn.processStderr(); - unsigned int reply = readInt(conn->from); - if (reply == 0) continue; - info.deriver = readString(conn->from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(*this, conn->from); - info.downloadSize = readLongLong(conn->from); - info.narSize = readLongLong(conn->from); - infos[i] = info; - } - - } else { - - conn->to << wopQuerySubstitutablePathInfos << paths; - conn.processStderr(); - size_t count = readNum<size_t>(conn->from); - for (size_t n = 0; n < count; n++) { - Path path = readStorePath(*this, conn->from); - SubstitutablePathInfo & info(infos[path]); - info.deriver = readString(conn->from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(*this, conn->from); - info.downloadSize = readLongLong(conn->from); - info.narSize = readLongLong(conn->from); - } - +PathSet RemoteStore::querySubstitutablePaths(const PathSet& paths) { + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + PathSet res; + for (auto& i : paths) { + conn->to << wopHasSubstitutes << i; + conn.processStderr(); + if (readInt(conn->from)) res.insert(i); } -} - - -void RemoteStore::queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept -{ - try { - std::shared_ptr<ValidPathInfo> info; - { - auto conn(getConnection()); - conn->to << wopQueryPathInfo << path; - try { - conn.processStderr(); - } catch (Error & e) { - // Ugly backwards compatibility hack. - if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.what()); - throw; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { - bool valid; conn->from >> valid; - if (!valid) throw InvalidPath(format("path '%s' is not valid") % path); - } - info = std::make_shared<ValidPathInfo>(); - info->path = path; - info->deriver = readString(conn->from); - if (info->deriver != "") assertStorePath(info->deriver); - info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths<PathSet>(*this, conn->from); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings<StringSet>(conn->from); - conn->from >> info->ca; - } - } - callback(std::move(info)); - } catch (...) { callback.rethrow(); } -} - - -void RemoteStore::queryReferrers(const Path & path, - PathSet & referrers) -{ - auto conn(getConnection()); - conn->to << wopQueryReferrers << path; - conn.processStderr(); - PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from); - referrers.insert(referrers2.begin(), referrers2.end()); -} - - -PathSet RemoteStore::queryValidDerivers(const Path & path) -{ - auto conn(getConnection()); - conn->to << wopQueryValidDerivers << path; - conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); -} - - -PathSet RemoteStore::queryDerivationOutputs(const Path & path) -{ - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputs << path; + return res; + } else { + conn->to << wopQuerySubstitutablePaths << paths; conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); -} - - -PathSet RemoteStore::queryDerivationOutputNames(const Path & path) -{ - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputNames << path; - conn.processStderr(); - return readStrings<PathSet>(conn->from); -} - - -Path RemoteStore::queryPathFromHashPart(const string & hashPart) -{ - auto conn(getConnection()); - conn->to << wopQueryPathFromHashPart << hashPart; - conn.processStderr(); - Path path = readString(conn->from); - if (!path.empty()) assertStorePath(path); - return path; -} - - -void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) -{ - auto conn(getConnection()); - - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { - conn->to << wopImportPaths; - - auto source2 = sinkToSource([&](Sink & sink) { - sink << 1 // == path follows - ; - copyNAR(source, sink); - sink - << exportMagic - << info.path - << info.references - << info.deriver - << 0 // == no legacy signature - << 0 // == no path follows - ; - }); - - conn.processStderr(0, source2.get()); - - auto importedPaths = readStorePaths<PathSet>(*this, conn->from); - assert(importedPaths.size() <= 1); + } +} + +void RemoteStore::querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) { + if (paths.empty()) return; + + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + for (auto& i : paths) { + SubstitutablePathInfo info; + conn->to << wopQuerySubstitutablePathInfo << i; + conn.processStderr(); + unsigned int reply = readInt(conn->from); + if (reply == 0) continue; + info.deriver = readString(conn->from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths<PathSet>(*this, conn->from); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); + infos[i] = info; } - else { - conn->to << wopAddToStoreNar - << info.path << info.deriver << info.narHash.to_string(Base16, false) - << info.references << info.registrationTime << info.narSize - << info.ultimate << info.sigs << info.ca - << repair << !checkSigs; - bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; - if (!tunnel) copyNAR(source, conn->to); - conn.processStderr(0, tunnel ? &source : nullptr); + } else { + conn->to << wopQuerySubstitutablePathInfos << paths; + conn.processStderr(); + size_t count = readNum<size_t>(conn->from); + for (size_t n = 0; n < count; n++) { + Path path = readStorePath(*this, conn->from); + SubstitutablePathInfo& info(infos[path]); + info.deriver = readString(conn->from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths<PathSet>(*this, conn->from); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); } + } } - -Path RemoteStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - - Path srcPath(absPath(_srcPath)); - - conn->to << wopAddToStore << name - << ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */ - << (recursive ? 1 : 0) - << printHashType(hashAlgo); - - try { - conn->to.written = 0; - conn->to.warn = true; - connections->incCapacity(); - { - Finally cleanup([&]() { connections->decCapacity(); }); - dumpPath(srcPath, conn->to, filter); - } - conn->to.warn = false; +void RemoteStore::queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { + try { + std::shared_ptr<ValidPathInfo> info; + { + auto conn(getConnection()); + conn->to << wopQueryPathInfo << path; + try { conn.processStderr(); - } catch (SysError & e) { - /* Daemon closed while we were sending the path. Probably OOM - or I/O error. */ - if (e.errNo == EPIPE) - try { - conn.processStderr(); - } catch (EndOfFile & e) { } + } catch (Error& e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) + throw InvalidPath(e.what()); throw; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { + bool valid; + conn->from >> valid; + if (!valid) throw InvalidPath(format("path '%s' is not valid") % path); + } + info = std::make_shared<ValidPathInfo>(); + info->path = path; + info->deriver = readString(conn->from); + if (info->deriver != "") assertStorePath(info->deriver); + info->narHash = Hash(readString(conn->from), htSHA256); + info->references = readStorePaths<PathSet>(*this, conn->from); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings<StringSet>(conn->from); + conn->from >> info->ca; + } } - - return readStorePath(*this, conn->from); -} - - -Path RemoteStore::addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) -{ - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - conn->to << wopAddTextToStore << name << s << references; - - conn.processStderr(); - return readStorePath(*this, conn->from); -} - - -void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) -{ - auto conn(getConnection()); - conn->to << wopBuildPaths; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { - conn->to << drvPaths; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) - conn->to << buildMode; - else - /* Old daemons did not take a 'buildMode' parameter, so we - need to validate it here on the client side. */ - if (buildMode != bmNormal) - throw Error("repairing or checking is not supported when building through the Nix daemon"); - } else { - /* For backwards compatibility with old daemons, strip output - identifiers. */ - PathSet drvPaths2; - for (auto & i : drvPaths) - drvPaths2.insert(string(i, 0, i.find('!'))); - conn->to << drvPaths2; + callback(std::move(info)); + } catch (...) { + callback.rethrow(); + } +} + +void RemoteStore::queryReferrers(const Path& path, PathSet& referrers) { + auto conn(getConnection()); + conn->to << wopQueryReferrers << path; + conn.processStderr(); + PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from); + referrers.insert(referrers2.begin(), referrers2.end()); +} + +PathSet RemoteStore::queryValidDerivers(const Path& path) { + auto conn(getConnection()); + conn->to << wopQueryValidDerivers << path; + conn.processStderr(); + return readStorePaths<PathSet>(*this, conn->from); +} + +PathSet RemoteStore::queryDerivationOutputs(const Path& path) { + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputs << path; + conn.processStderr(); + return readStorePaths<PathSet>(*this, conn->from); +} + +PathSet RemoteStore::queryDerivationOutputNames(const Path& path) { + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputNames << path; + conn.processStderr(); + return readStrings<PathSet>(conn->from); +} + +Path RemoteStore::queryPathFromHashPart(const string& hashPart) { + auto conn(getConnection()); + conn->to << wopQueryPathFromHashPart << hashPart; + conn.processStderr(); + Path path = readString(conn->from); + if (!path.empty()) assertStorePath(path); + return path; +} + +void RemoteStore::addToStore(const ValidPathInfo& info, Source& source, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { + conn->to << wopImportPaths; + + auto source2 = sinkToSource([&](Sink& sink) { + sink << 1 // == path follows + ; + copyNAR(source, sink); + sink << exportMagic << info.path << info.references << info.deriver + << 0 // == no legacy signature + << 0 // == no path follows + ; + }); + + conn.processStderr(0, source2.get()); + + auto importedPaths = readStorePaths<PathSet>(*this, conn->from); + assert(importedPaths.size() <= 1); + } + + else { + conn->to << wopAddToStoreNar << info.path << info.deriver + << info.narHash.to_string(Base16, false) << info.references + << info.registrationTime << info.narSize << info.ultimate + << info.sigs << info.ca << repair << !checkSigs; + bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; + if (!tunnel) copyNAR(source, conn->to); + conn.processStderr(0, tunnel ? &source : nullptr); + } +} + +Path RemoteStore::addToStore(const string& name, const Path& _srcPath, + bool recursive, HashType hashAlgo, + PathFilter& filter, RepairFlag repair) { + if (repair) + throw Error( + "repairing is not supported when building through the Nix daemon"); + + auto conn(getConnection()); + + Path srcPath(absPath(_srcPath)); + + conn->to << wopAddToStore << name + << ((hashAlgo == htSHA256 && recursive) + ? 0 + : 1) /* backwards compatibility hack */ + << (recursive ? 1 : 0) << printHashType(hashAlgo); + + try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + dumpPath(srcPath, conn->to, filter); } + conn->to.warn = false; conn.processStderr(); - readInt(conn->from); -} - - -BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - auto conn(getConnection()); - conn->to << wopBuildDerivation << drvPath << drv << buildMode; - conn.processStderr(); - BuildResult res; - unsigned int status; - conn->from >> status >> res.errorMsg; - res.status = (BuildResult::Status) status; - return res; + } catch (SysError& e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) try { + conn.processStderr(); + } catch (EndOfFile& e) { + } + throw; + } + + return readStorePath(*this, conn->from); +} + +Path RemoteStore::addTextToStore(const string& name, const string& s, + const PathSet& references, RepairFlag repair) { + if (repair) + throw Error( + "repairing is not supported when building through the Nix daemon"); + + auto conn(getConnection()); + conn->to << wopAddTextToStore << name << s << references; + + conn.processStderr(); + return readStorePath(*this, conn->from); +} + +void RemoteStore::buildPaths(const PathSet& drvPaths, BuildMode buildMode) { + auto conn(getConnection()); + conn->to << wopBuildPaths; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { + conn->to << drvPaths; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + conn->to << buildMode; + else + /* Old daemons did not take a 'buildMode' parameter, so we + need to validate it here on the client side. */ + if (buildMode != bmNormal) + throw Error( + "repairing or checking is not supported when building through the " + "Nix daemon"); + } else { + /* For backwards compatibility with old daemons, strip output + identifiers. */ + PathSet drvPaths2; + for (auto& i : drvPaths) drvPaths2.insert(string(i, 0, i.find('!'))); + conn->to << drvPaths2; + } + conn.processStderr(); + readInt(conn->from); +} + +BuildResult RemoteStore::buildDerivation(const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) { + auto conn(getConnection()); + conn->to << wopBuildDerivation << drvPath << drv << buildMode; + conn.processStderr(); + BuildResult res; + unsigned int status; + conn->from >> status >> res.errorMsg; + res.status = (BuildResult::Status)status; + return res; +} + +void RemoteStore::ensurePath(const Path& path) { + auto conn(getConnection()); + conn->to << wopEnsurePath << path; + conn.processStderr(); + readInt(conn->from); +} + +void RemoteStore::addTempRoot(const Path& path) { + auto conn(getConnection()); + conn->to << wopAddTempRoot << path; + conn.processStderr(); + readInt(conn->from); +} + +void RemoteStore::addIndirectRoot(const Path& path) { + auto conn(getConnection()); + conn->to << wopAddIndirectRoot << path; + conn.processStderr(); + readInt(conn->from); +} + +void RemoteStore::syncWithGC() { + auto conn(getConnection()); + conn->to << wopSyncWithGC; + conn.processStderr(); + readInt(conn->from); +} + +Roots RemoteStore::findRoots(bool censor) { + auto conn(getConnection()); + conn->to << wopFindRoots; + conn.processStderr(); + size_t count = readNum<size_t>(conn->from); + Roots result; + while (count--) { + Path link = readString(conn->from); + Path target = readStorePath(*this, conn->from); + result[target].emplace(link); + } + return result; +} + +void RemoteStore::collectGarbage(const GCOptions& options, GCResults& results) { + auto conn(getConnection()); + + conn->to << wopCollectGarbage << options.action << options.pathsToDelete + << options.ignoreLiveness + << options.maxFreed + /* removed options */ + << 0 << 0 << 0; + + conn.processStderr(); + + results.paths = readStrings<PathSet>(conn->from); + results.bytesFreed = readLongLong(conn->from); + readLongLong(conn->from); // obsolete + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.clear(); + } +} + +void RemoteStore::optimiseStore() { + auto conn(getConnection()); + conn->to << wopOptimiseStore; + conn.processStderr(); + readInt(conn->from); +} + +bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) { + auto conn(getConnection()); + conn->to << wopVerifyStore << checkContents << repair; + conn.processStderr(); + return readInt(conn->from); +} + +void RemoteStore::addSignatures(const Path& storePath, const StringSet& sigs) { + auto conn(getConnection()); + conn->to << wopAddSignatures << storePath << sigs; + conn.processStderr(); + readInt(conn->from); } - -void RemoteStore::ensurePath(const Path & path) -{ +void RemoteStore::queryMissing(const PathSet& targets, PathSet& willBuild, + PathSet& willSubstitute, PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize) { + { auto conn(getConnection()); - conn->to << wopEnsurePath << path; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) + // Don't hold the connection handle in the fallback case + // to prevent a deadlock. + goto fallback; + conn->to << wopQueryMissing << targets; conn.processStderr(); - readInt(conn->from); -} - + willBuild = readStorePaths<PathSet>(*this, conn->from); + willSubstitute = readStorePaths<PathSet>(*this, conn->from); + unknown = readStorePaths<PathSet>(*this, conn->from); + conn->from >> downloadSize >> narSize; + return; + } -void RemoteStore::addTempRoot(const Path & path) -{ - auto conn(getConnection()); - conn->to << wopAddTempRoot << path; - conn.processStderr(); - readInt(conn->from); +fallback: + return Store::queryMissing(targets, willBuild, willSubstitute, unknown, + downloadSize, narSize); } +void RemoteStore::connect() { auto conn(getConnection()); } -void RemoteStore::addIndirectRoot(const Path & path) -{ - auto conn(getConnection()); - conn->to << wopAddIndirectRoot << path; - conn.processStderr(); - readInt(conn->from); +unsigned int RemoteStore::getProtocol() { + auto conn(connections->get()); + return conn->daemonVersion; } +void RemoteStore::flushBadConnections() { connections->flushBad(); } -void RemoteStore::syncWithGC() -{ - auto conn(getConnection()); - conn->to << wopSyncWithGC; - conn.processStderr(); - readInt(conn->from); -} - - -Roots RemoteStore::findRoots(bool censor) -{ - auto conn(getConnection()); - conn->to << wopFindRoots; - conn.processStderr(); - size_t count = readNum<size_t>(conn->from); - Roots result; - while (count--) { - Path link = readString(conn->from); - Path target = readStorePath(*this, conn->from); - result[target].emplace(link); +RemoteStore::Connection::~Connection() { + try { + to.flush(); + } catch (...) { + ignoreException(); + } +} + +static Logger::Fields readFields(Source& from) { + Logger::Fields fields; + size_t size = readInt(from); + for (size_t n = 0; n < size; n++) { + auto type = (decltype(Logger::Field::type))readInt(from); + if (type == Logger::Field::tInt) + fields.push_back(readNum<uint64_t>(from)); + else if (type == Logger::Field::tString) + fields.push_back(readString(from)); + else + throw Error("got unsupported field type %x from Nix daemon", (int)type); + } + return fields; +} + +std::exception_ptr RemoteStore::Connection::processStderr(Sink* sink, + Source* source) { + to.flush(); + + while (true) { + auto msg = readNum<uint64_t>(from); + + if (msg == STDERR_WRITE) { + string s = readString(from); + if (!sink) throw Error("no sink"); + (*sink)(s); } - return result; -} - - -void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) -{ - auto conn(getConnection()); - - conn->to - << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness - << options.maxFreed - /* removed options */ - << 0 << 0 << 0; - - conn.processStderr(); - - results.paths = readStrings<PathSet>(conn->from); - results.bytesFreed = readLongLong(conn->from); - readLongLong(conn->from); // obsolete - { - auto state_(Store::state.lock()); - state_->pathInfoCache.clear(); + else if (msg == STDERR_READ) { + if (!source) throw Error("no source"); + size_t len = readNum<size_t>(from); + auto buf = std::make_unique<unsigned char[]>(len); + writeString(buf.get(), source->read(buf.get(), len), to); + to.flush(); } -} - - -void RemoteStore::optimiseStore() -{ - auto conn(getConnection()); - conn->to << wopOptimiseStore; - conn.processStderr(); - readInt(conn->from); -} - - -bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) -{ - auto conn(getConnection()); - conn->to << wopVerifyStore << checkContents << repair; - conn.processStderr(); - return readInt(conn->from); -} - -void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) -{ - auto conn(getConnection()); - conn->to << wopAddSignatures << storePath << sigs; - conn.processStderr(); - readInt(conn->from); -} - - -void RemoteStore::queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) -{ - { - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) - // Don't hold the connection handle in the fallback case - // to prevent a deadlock. - goto fallback; - conn->to << wopQueryMissing << targets; - conn.processStderr(); - willBuild = readStorePaths<PathSet>(*this, conn->from); - willSubstitute = readStorePaths<PathSet>(*this, conn->from); - unknown = readStorePaths<PathSet>(*this, conn->from); - conn->from >> downloadSize >> narSize; - return; + else if (msg == STDERR_ERROR) { + string error = readString(from); + unsigned int status = readInt(from); + return std::make_exception_ptr(Error(status, error)); } - fallback: - return Store::queryMissing(targets, willBuild, willSubstitute, - unknown, downloadSize, narSize); -} - - -void RemoteStore::connect() -{ - auto conn(getConnection()); -} - - -unsigned int RemoteStore::getProtocol() -{ - auto conn(connections->get()); - return conn->daemonVersion; -} - - -void RemoteStore::flushBadConnections() -{ - connections->flushBad(); -} - - -RemoteStore::Connection::~Connection() -{ - try { - to.flush(); - } catch (...) { - ignoreException(); + else if (msg == STDERR_NEXT) + printError(chomp(readString(from))); + + else if (msg == STDERR_START_ACTIVITY) { + auto act = readNum<ActivityId>(from); + auto lvl = (Verbosity)readInt(from); + auto type = (ActivityType)readInt(from); + auto s = readString(from); + auto fields = readFields(from); + auto parent = readNum<ActivityId>(from); + logger->startActivity(act, lvl, type, s, fields, parent); } -} - -static Logger::Fields readFields(Source & from) -{ - Logger::Fields fields; - size_t size = readInt(from); - for (size_t n = 0; n < size; n++) { - auto type = (decltype(Logger::Field::type)) readInt(from); - if (type == Logger::Field::tInt) - fields.push_back(readNum<uint64_t>(from)); - else if (type == Logger::Field::tString) - fields.push_back(readString(from)); - else - throw Error("got unsupported field type %x from Nix daemon", (int) type); + else if (msg == STDERR_STOP_ACTIVITY) { + auto act = readNum<ActivityId>(from); + logger->stopActivity(act); } - return fields; -} + else if (msg == STDERR_RESULT) { + auto act = readNum<ActivityId>(from); + auto type = (ResultType)readInt(from); + auto fields = readFields(from); + logger->result(act, type, fields); + } -std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source) -{ - to.flush(); + else if (msg == STDERR_LAST) + break; - while (true) { - - auto msg = readNum<uint64_t>(from); - - if (msg == STDERR_WRITE) { - string s = readString(from); - if (!sink) throw Error("no sink"); - (*sink)(s); - } - - else if (msg == STDERR_READ) { - if (!source) throw Error("no source"); - size_t len = readNum<size_t>(from); - auto buf = std::make_unique<unsigned char[]>(len); - writeString(buf.get(), source->read(buf.get(), len), to); - to.flush(); - } - - else if (msg == STDERR_ERROR) { - string error = readString(from); - unsigned int status = readInt(from); - return std::make_exception_ptr(Error(status, error)); - } - - else if (msg == STDERR_NEXT) - printError(chomp(readString(from))); - - else if (msg == STDERR_START_ACTIVITY) { - auto act = readNum<ActivityId>(from); - auto lvl = (Verbosity) readInt(from); - auto type = (ActivityType) readInt(from); - auto s = readString(from); - auto fields = readFields(from); - auto parent = readNum<ActivityId>(from); - logger->startActivity(act, lvl, type, s, fields, parent); - } - - else if (msg == STDERR_STOP_ACTIVITY) { - auto act = readNum<ActivityId>(from); - logger->stopActivity(act); - } - - else if (msg == STDERR_RESULT) { - auto act = readNum<ActivityId>(from); - auto type = (ResultType) readInt(from); - auto fields = readFields(from); - logger->result(act, type, fields); - } - - else if (msg == STDERR_LAST) - break; - - else - throw Error("got unknown message type %x from Nix daemon", msg); - } + else + throw Error("got unknown message type %x from Nix daemon", msg); + } - return nullptr; + return nullptr; } static std::string uriScheme = "unix://"; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation regStore( + [](const std::string& uri, + const Store::Params& params) -> std::shared_ptr<Store> { + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared<UDSRemoteStore>( + std::string(uri, uriScheme.size()), params); + }); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/remote-store.hh b/third_party/nix/src/libstore/remote-store.hh index 1f375dd71520..dd37b83f84a5 100644 --- a/third_party/nix/src/libstore/remote-store.hh +++ b/third_party/nix/src/libstore/remote-store.hh @@ -2,160 +2,151 @@ #include <limits> #include <string> - #include "store-api.hh" - namespace nix { - class Pipe; class Pid; struct FdSink; struct FdSource; -template<typename T> class Pool; +template <typename T> +class Pool; struct ConnectionHandle; - /* FIXME: RemoteStore is a misnomer - should be something like DaemonStore. */ -class RemoteStore : public virtual Store -{ -public: - - const Setting<int> maxConnections{(Store*) this, 1, - "max-connections", "maximum number of concurrent connections to the Nix daemon"}; - - const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(), - "max-connection-age", "number of seconds to reuse a connection"}; +class RemoteStore : public virtual Store { + public: + const Setting<int> maxConnections{ + (Store*)this, 1, "max-connections", + "maximum number of concurrent connections to the Nix daemon"}; - virtual bool sameMachine() = 0; + const Setting<unsigned int> maxConnectionAge{ + (Store*)this, std::numeric_limits<unsigned int>::max(), + "max-connection-age", "number of seconds to reuse a connection"}; - RemoteStore(const Params & params); + virtual bool sameMachine() = 0; - /* Implementations of abstract store API methods. */ + RemoteStore(const Params& params); - bool isValidPathUncached(const Path & path) override; + /* Implementations of abstract store API methods. */ - PathSet queryValidPaths(const PathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override; + bool isValidPathUncached(const Path& path) override; - PathSet queryAllValidPaths() override; + PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute = + NoSubstitute) override; - void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; + PathSet queryAllValidPaths() override; - void queryReferrers(const Path & path, PathSet & referrers) override; + void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; - PathSet queryValidDerivers(const Path & path) override; + void queryReferrers(const Path& path, PathSet& referrers) override; - PathSet queryDerivationOutputs(const Path & path) override; + PathSet queryValidDerivers(const Path& path) override; - StringSet queryDerivationOutputNames(const Path & path) override; + PathSet queryDerivationOutputs(const Path& path) override; - Path queryPathFromHashPart(const string & hashPart) override; + StringSet queryDerivationOutputNames(const Path& path) override; - PathSet querySubstitutablePaths(const PathSet & paths) override; + Path queryPathFromHashPart(const string& hashPart) override; - void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) override; + PathSet querySubstitutablePaths(const PathSet& paths) override; - void addToStore(const ValidPathInfo & info, Source & nar, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; + void querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) override; - Path addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; + void addToStore(const ValidPathInfo& info, Source& nar, RepairFlag repair, + CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) override; - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override; + Path addToStore(const string& name, const Path& srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter& filter = defaultPathFilter, + RepairFlag repair = NoRepair) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override; + Path addTextToStore(const string& name, const string& s, + const PathSet& references, RepairFlag repair) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override; + void buildPaths(const PathSet& paths, BuildMode buildMode) override; - void ensurePath(const Path & path) override; + BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv, + BuildMode buildMode) override; - void addTempRoot(const Path & path) override; + void ensurePath(const Path& path) override; - void addIndirectRoot(const Path & path) override; + void addTempRoot(const Path& path) override; - void syncWithGC() override; + void addIndirectRoot(const Path& path) override; - Roots findRoots(bool censor) override; + void syncWithGC() override; - void collectGarbage(const GCOptions & options, GCResults & results) override; + Roots findRoots(bool censor) override; - void optimiseStore() override; + void collectGarbage(const GCOptions& options, GCResults& results) override; - bool verifyStore(bool checkContents, RepairFlag repair) override; + void optimiseStore() override; - void addSignatures(const Path & storePath, const StringSet & sigs) override; + bool verifyStore(bool checkContents, RepairFlag repair) override; - void queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override; + void addSignatures(const Path& storePath, const StringSet& sigs) override; - void connect() override; + void queryMissing(const PathSet& targets, PathSet& willBuild, + PathSet& willSubstitute, PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize) override; - unsigned int getProtocol() override; + void connect() override; - void flushBadConnections(); + unsigned int getProtocol() override; -protected: + void flushBadConnections(); - struct Connection - { - AutoCloseFD fd; - FdSink to; - FdSource from; - unsigned int daemonVersion; - std::chrono::time_point<std::chrono::steady_clock> startTime; + protected: + struct Connection { + AutoCloseFD fd; + FdSink to; + FdSource from; + unsigned int daemonVersion; + std::chrono::time_point<std::chrono::steady_clock> startTime; - virtual ~Connection(); + virtual ~Connection(); - std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0); - }; + std::exception_ptr processStderr(Sink* sink = 0, Source* source = 0); + }; - ref<Connection> openConnectionWrapper(); + ref<Connection> openConnectionWrapper(); - virtual ref<Connection> openConnection() = 0; + virtual ref<Connection> openConnection() = 0; - void initConnection(Connection & conn); + void initConnection(Connection& conn); - ref<Pool<Connection>> connections; + ref<Pool<Connection>> connections; - virtual void setOptions(Connection & conn); + virtual void setOptions(Connection& conn); - ConnectionHandle getConnection(); + ConnectionHandle getConnection(); - friend struct ConnectionHandle; - -private: - - std::atomic_bool failed{false}; + friend struct ConnectionHandle; + private: + std::atomic_bool failed{false}; }; -class UDSRemoteStore : public LocalFSStore, public RemoteStore -{ -public: - - UDSRemoteStore(const Params & params); - UDSRemoteStore(std::string path, const Params & params); +class UDSRemoteStore : public LocalFSStore, public RemoteStore { + public: + UDSRemoteStore(const Params& params); + UDSRemoteStore(std::string path, const Params& params); - std::string getUri() override; + std::string getUri() override; - bool sameMachine() - { return true; } + bool sameMachine() { return true; } -private: - - ref<RemoteStore::Connection> openConnection() override; - std::optional<std::string> path; + private: + ref<RemoteStore::Connection> openConnection() override; + std::optional<std::string> path; }; - -} +} // namespace nix diff --git a/third_party/nix/src/libstore/s3-binary-cache-store.cc b/third_party/nix/src/libstore/s3-binary-cache-store.cc index cd547a964850..8730c2dd4df2 100644 --- a/third_party/nix/src/libstore/s3-binary-cache-store.cc +++ b/third_party/nix/src/libstore/s3-binary-cache-store.cc @@ -1,14 +1,6 @@ #if ENABLE_S3 -#include "s3.hh" #include "s3-binary-cache-store.hh" -#include "nar-info.hh" -#include "nar-info-disk-cache.hh" -#include "globals.hh" -#include "compression.hh" -#include "download.hh" -#include "istringstream_nocopy.hh" - #include <aws/core/Aws.h> #include <aws/core/VersionConfig.h> #include <aws/core/auth/AWSCredentialsProvider.h> @@ -24,408 +16,403 @@ #include <aws/s3/model/ListObjectsRequest.h> #include <aws/s3/model/PutObjectRequest.h> #include <aws/transfer/TransferManager.h> +#include "compression.hh" +#include "download.hh" +#include "globals.hh" +#include "istringstream_nocopy.hh" +#include "nar-info-disk-cache.hh" +#include "nar-info.hh" +#include "s3.hh" using namespace Aws::Transfer; namespace nix { -struct S3Error : public Error -{ - Aws::S3::S3Errors err; - S3Error(Aws::S3::S3Errors err, const FormatOrString & fs) - : Error(fs), err(err) { }; +struct S3Error : public Error { + Aws::S3::S3Errors err; + S3Error(Aws::S3::S3Errors err, const FormatOrString& fs) + : Error(fs), err(err){}; }; /* Helper: given an Outcome<R, E>, return R in case of success, or throw an exception in case of an error. */ -template<typename R, typename E> -R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome) -{ - if (!outcome.IsSuccess()) - throw S3Error( - outcome.GetError().GetErrorType(), - fs.s + ": " + outcome.GetError().GetMessage()); - return outcome.GetResultWithOwnership(); +template <typename R, typename E> +R&& checkAws(const FormatOrString& fs, Aws::Utils::Outcome<R, E>&& outcome) { + if (!outcome.IsSuccess()) + throw S3Error(outcome.GetError().GetErrorType(), + fs.s + ": " + outcome.GetError().GetMessage()); + return outcome.GetResultWithOwnership(); } -class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem -{ - using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem; +class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem { + using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem; - void ProcessFormattedStatement(Aws::String && statement) override - { - debug("AWS: %s", chomp(statement)); - } + void ProcessFormattedStatement(Aws::String&& statement) override { + debug("AWS: %s", chomp(statement)); + } }; -static void initAWS() -{ - static std::once_flag flag; - std::call_once(flag, []() { - Aws::SDKOptions options; - - /* We install our own OpenSSL locking function (see - shared.cc), so don't let aws-sdk-cpp override it. */ - options.cryptoOptions.initAndCleanupOpenSSL = false; - - if (verbosity >= lvlDebug) { - options.loggingOptions.logLevel = - verbosity == lvlDebug - ? Aws::Utils::Logging::LogLevel::Debug - : Aws::Utils::Logging::LogLevel::Trace; - options.loggingOptions.logger_create_fn = [options]() { - return std::make_shared<AwsLogger>(options.loggingOptions.logLevel); - }; - } +static void initAWS() { + static std::once_flag flag; + std::call_once(flag, []() { + Aws::SDKOptions options; + + /* We install our own OpenSSL locking function (see + shared.cc), so don't let aws-sdk-cpp override it. */ + options.cryptoOptions.initAndCleanupOpenSSL = false; + + if (verbosity >= lvlDebug) { + options.loggingOptions.logLevel = + verbosity == lvlDebug ? Aws::Utils::Logging::LogLevel::Debug + : Aws::Utils::Logging::LogLevel::Trace; + options.loggingOptions.logger_create_fn = [options]() { + return std::make_shared<AwsLogger>(options.loggingOptions.logLevel); + }; + } - Aws::InitAPI(options); - }); + Aws::InitAPI(options); + }); } -S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint) - : config(makeConfig(region, scheme, endpoint)) - , client(make_ref<Aws::S3::S3Client>( - profile == "" - ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( - std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>()) - : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( - std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())), - *config, - // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 +S3Helper::S3Helper(const string& profile, const string& region, + const string& scheme, const string& endpoint) + : config(makeConfig(region, scheme, endpoint)), + client(make_ref<Aws::S3::S3Client>( + profile == "" + ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( + std::make_shared< + Aws::Auth::DefaultAWSCredentialsProviderChain>()) + : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( + std::make_shared< + Aws::Auth::ProfileConfigFileAWSCredentialsProvider>( + profile.c_str())), + *config, +// FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 #if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 - false, + false, #else - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, #endif - endpoint.empty())) -{ + endpoint.empty())) { } /* Log AWS retries. */ -class RetryStrategy : public Aws::Client::DefaultRetryStrategy -{ - bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override - { - auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); - if (retry) - printError("AWS error '%s' (%s), will retry in %d ms", - error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); - return retry; - } +class RetryStrategy : public Aws::Client::DefaultRetryStrategy { + bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, + long attemptedRetries) const override { + auto retry = + Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); + if (retry) + printError("AWS error '%s' (%s), will retry in %d ms", + error.GetExceptionName(), error.GetMessage(), + CalculateDelayBeforeNextRetry(error, attemptedRetries)); + return retry; + } }; -ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint) -{ - initAWS(); - auto res = make_ref<Aws::Client::ClientConfiguration>(); - res->region = region; - if (!scheme.empty()) { - res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); - } - if (!endpoint.empty()) { - res->endpointOverride = endpoint; - } - res->requestTimeoutMs = 600 * 1000; - res->connectTimeoutMs = 5 * 1000; - res->retryStrategy = std::make_shared<RetryStrategy>(); - res->caFile = settings.caFile; - return res; +ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig( + const string& region, const string& scheme, const string& endpoint) { + initAWS(); + auto res = make_ref<Aws::Client::ClientConfiguration>(); + res->region = region; + if (!scheme.empty()) { + res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); + } + if (!endpoint.empty()) { + res->endpointOverride = endpoint; + } + res->requestTimeoutMs = 600 * 1000; + res->connectTimeoutMs = 5 * 1000; + res->retryStrategy = std::make_shared<RetryStrategy>(); + res->caFile = settings.caFile; + return res; } -S3Helper::DownloadResult S3Helper::getObject( - const std::string & bucketName, const std::string & key) -{ - debug("fetching 's3://%s/%s'...", bucketName, key); +S3Helper::DownloadResult S3Helper::getObject(const std::string& bucketName, + const std::string& key) { + debug("fetching 's3://%s/%s'...", bucketName, key); - auto request = - Aws::S3::Model::GetObjectRequest() - .WithBucket(bucketName) - .WithKey(key); - - request.SetResponseStreamFactory([&]() { - return Aws::New<std::stringstream>("STRINGSTREAM"); - }); + auto request = + Aws::S3::Model::GetObjectRequest().WithBucket(bucketName).WithKey(key); - DownloadResult res; + request.SetResponseStreamFactory( + [&]() { return Aws::New<std::stringstream>("STRINGSTREAM"); }); - auto now1 = std::chrono::steady_clock::now(); + DownloadResult res; - try { + auto now1 = std::chrono::steady_clock::now(); - auto result = checkAws(fmt("AWS error fetching '%s'", key), - client->GetObject(request)); + try { + auto result = checkAws(fmt("AWS error fetching '%s'", key), + client->GetObject(request)); - res.data = decompress(result.GetContentEncoding(), - dynamic_cast<std::stringstream &>(result.GetBody()).str()); + res.data = + decompress(result.GetContentEncoding(), + dynamic_cast<std::stringstream&>(result.GetBody()).str()); - } catch (S3Error & e) { - if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; - } + } catch (S3Error& e) { + if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; + } - auto now2 = std::chrono::steady_clock::now(); + auto now2 = std::chrono::steady_clock::now(); - res.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); + res.durationMs = + std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) + .count(); - return res; + return res; } -struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore -{ - const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."}; - const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; - const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; - const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; - const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; - const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; - const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; - const Setting<bool> multipartUpload{ - this, false, "multipart-upload", "whether to use multi-part uploads"}; - const Setting<uint64_t> bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; - - std::string bucketName; - - Stats stats; - - S3Helper s3Helper; - - S3BinaryCacheStoreImpl( - const Params & params, const std::string & bucketName) - : S3BinaryCacheStore(params) - , bucketName(bucketName) - , s3Helper(profile, region, scheme, endpoint) - { - diskCache = getNarInfoDiskCache(); - } - - std::string getUri() override - { - return "s3://" + bucketName; - } - - void init() override - { - if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { - - BinaryCacheStore::init(); - - diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); - } +struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { + const Setting<std::string> profile{ + this, "", "profile", "The name of the AWS configuration profile to use."}; + const Setting<std::string> region{ + this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting<std::string> scheme{ + this, "", "scheme", + "The scheme to use for S3 requests, https by default."}; + const Setting<std::string> endpoint{ + this, "", "endpoint", + "An optional override of the endpoint to use when talking to S3."}; + const Setting<std::string> narinfoCompression{ + this, "", "narinfo-compression", "compression method for .narinfo files"}; + const Setting<std::string> lsCompression{this, "", "ls-compression", + "compression method for .ls files"}; + const Setting<std::string> logCompression{ + this, "", "log-compression", "compression method for log/* files"}; + const Setting<bool> multipartUpload{this, false, "multipart-upload", + "whether to use multi-part uploads"}; + const Setting<uint64_t> bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", + "size (in bytes) of each part in multi-part uploads"}; + + std::string bucketName; + + Stats stats; + + S3Helper s3Helper; + + S3BinaryCacheStoreImpl(const Params& params, const std::string& bucketName) + : S3BinaryCacheStore(params), + bucketName(bucketName), + s3Helper(profile, region, scheme, endpoint) { + diskCache = getNarInfoDiskCache(); + } + + std::string getUri() override { return "s3://" + bucketName; } + + void init() override { + if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { + BinaryCacheStore::init(); + + diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); } + } - const Stats & getS3Stats() override - { - return stats; - } + const Stats& getS3Stats() override { return stats; } - /* This is a specialisation of isValidPath() that optimistically - fetches the .narinfo file, rather than first checking for its - existence via a HEAD request. Since .narinfos are small, doing - a GET is unlikely to be slower than HEAD. */ - bool isValidPathUncached(const Path & storePath) override - { - try { - queryPathInfo(storePath); - return true; - } catch (InvalidPath & e) { - return false; - } + /* This is a specialisation of isValidPath() that optimistically + fetches the .narinfo file, rather than first checking for its + existence via a HEAD request. Since .narinfos are small, doing + a GET is unlikely to be slower than HEAD. */ + bool isValidPathUncached(const Path& storePath) override { + try { + queryPathInfo(storePath); + return true; + } catch (InvalidPath& e) { + return false; } - - bool fileExists(const std::string & path) override - { - stats.head++; - - auto res = s3Helper.client->HeadObject( - Aws::S3::Model::HeadObjectRequest() - .WithBucket(bucketName) - .WithKey(path)); - - if (!res.IsSuccess()) { - auto & error = res.GetError(); - if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND - || error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY - // If bucket listing is disabled, 404s turn into 403s - || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) - return false; - throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage()); - } - - return true; + } + + bool fileExists(const std::string& path) override { + stats.head++; + + auto res = s3Helper.client->HeadObject(Aws::S3::Model::HeadObjectRequest() + .WithBucket(bucketName) + .WithKey(path)); + + if (!res.IsSuccess()) { + auto& error = res.GetError(); + if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND || + error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY + // If bucket listing is disabled, 404s turn into 403s + || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) + return false; + throw Error(format("AWS error fetching '%s': %s") % path % + error.GetMessage()); } - std::shared_ptr<TransferManager> transferManager; - std::once_flag transferManagerCreated; - - void uploadFile(const std::string & path, const std::string & data, - const std::string & mimeType, - const std::string & contentEncoding) - { - auto stream = std::make_shared<istringstream_nocopy>(data); - - auto maxThreads = std::thread::hardware_concurrency(); - - static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> - executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads); - - std::call_once(transferManagerCreated, [&]() - { - if (multipartUpload) { - TransferManagerConfiguration transferConfig(executor.get()); - - transferConfig.s3Client = s3Helper.client; - transferConfig.bufferSize = bufferSize; - - transferConfig.uploadProgressCallback = - [](const TransferManager *transferManager, - const std::shared_ptr<const TransferHandle> - &transferHandle) - { - //FIXME: find a way to properly abort the multipart upload. - //checkInterrupt(); - debug("upload progress ('%s'): '%d' of '%d' bytes", - transferHandle->GetKey(), - transferHandle->GetBytesTransferred(), - transferHandle->GetBytesTotalSize()); - }; - - transferManager = TransferManager::Create(transferConfig); - } - }); - - auto now1 = std::chrono::steady_clock::now(); - - if (transferManager) { - - if (contentEncoding != "") - throw Error("setting a content encoding is not supported with S3 multi-part uploads"); - - std::shared_ptr<TransferHandle> transferHandle = - transferManager->UploadFile( - stream, bucketName, path, mimeType, - Aws::Map<Aws::String, Aws::String>(), - nullptr /*, contentEncoding */); + return true; + } - transferHandle->WaitUntilFinished(); + std::shared_ptr<TransferManager> transferManager; + std::once_flag transferManagerCreated; - if (transferHandle->GetStatus() == TransferStatus::FAILED) - throw Error("AWS error: failed to upload 's3://%s/%s': %s", - bucketName, path, transferHandle->GetLastError().GetMessage()); + void uploadFile(const std::string& path, const std::string& data, + const std::string& mimeType, + const std::string& contentEncoding) { + auto stream = std::make_shared<istringstream_nocopy>(data); - if (transferHandle->GetStatus() != TransferStatus::COMPLETED) - throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", - bucketName, path); + auto maxThreads = std::thread::hardware_concurrency(); - } else { + static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> + executor = + std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>( + maxThreads); - auto request = - Aws::S3::Model::PutObjectRequest() - .WithBucket(bucketName) - .WithKey(path); + std::call_once(transferManagerCreated, [&]() { + if (multipartUpload) { + TransferManagerConfiguration transferConfig(executor.get()); - request.SetContentType(mimeType); + transferConfig.s3Client = s3Helper.client; + transferConfig.bufferSize = bufferSize; - if (contentEncoding != "") - request.SetContentEncoding(contentEncoding); + transferConfig.uploadProgressCallback = + [](const TransferManager* transferManager, + const std::shared_ptr<const TransferHandle>& transferHandle) { + // FIXME: find a way to properly abort the multipart upload. + // checkInterrupt(); + debug("upload progress ('%s'): '%d' of '%d' bytes", + transferHandle->GetKey(), + transferHandle->GetBytesTransferred(), + transferHandle->GetBytesTotalSize()); + }; - auto stream = std::make_shared<istringstream_nocopy>(data); + transferManager = TransferManager::Create(transferConfig); + } + }); - request.SetBody(stream); + auto now1 = std::chrono::steady_clock::now(); - auto result = checkAws(fmt("AWS error uploading '%s'", path), - s3Helper.client->PutObject(request)); - } + if (transferManager) { + if (contentEncoding != "") + throw Error( + "setting a content encoding is not supported with S3 multi-part " + "uploads"); - auto now2 = std::chrono::steady_clock::now(); + std::shared_ptr<TransferHandle> transferHandle = + transferManager->UploadFile(stream, bucketName, path, mimeType, + Aws::Map<Aws::String, Aws::String>(), + nullptr /*, contentEncoding */); - auto duration = - std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) - .count(); + transferHandle->WaitUntilFinished(); - printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") % - bucketName % path % data.size() % duration); + if (transferHandle->GetStatus() == TransferStatus::FAILED) + throw Error("AWS error: failed to upload 's3://%s/%s': %s", bucketName, + path, transferHandle->GetLastError().GetMessage()); - stats.putTimeMs += duration; - stats.putBytes += data.size(); - stats.put++; - } + if (transferHandle->GetStatus() != TransferStatus::COMPLETED) + throw Error( + "AWS error: transfer status of 's3://%s/%s' in unexpected state", + bucketName, path); - void upsertFile(const std::string & path, const std::string & data, - const std::string & mimeType) override - { - if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) - uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression); - else if (lsCompression != "" && hasSuffix(path, ".ls")) - uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); - else if (logCompression != "" && hasPrefix(path, "log/")) - uploadFile(path, *compress(logCompression, data), mimeType, logCompression); - else - uploadFile(path, data, mimeType, ""); - } + } else { + auto request = Aws::S3::Model::PutObjectRequest() + .WithBucket(bucketName) + .WithKey(path); - void getFile(const std::string & path, Sink & sink) override - { - stats.get++; + request.SetContentType(mimeType); - // FIXME: stream output to sink. - auto res = s3Helper.getObject(bucketName, path); + if (contentEncoding != "") request.SetContentEncoding(contentEncoding); - stats.getBytes += res.data ? res.data->size() : 0; - stats.getTimeMs += res.durationMs; + auto stream = std::make_shared<istringstream_nocopy>(data); - if (res.data) { - printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", - bucketName, path, res.data->size(), res.durationMs); + request.SetBody(stream); - sink((unsigned char *) res.data->data(), res.data->size()); - } else - throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); + auto result = checkAws(fmt("AWS error uploading '%s'", path), + s3Helper.client->PutObject(request)); } - PathSet queryAllValidPaths() override - { - PathSet paths; - std::string marker; - - do { - debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker); - - auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName, - s3Helper.client->ListObjects( - Aws::S3::Model::ListObjectsRequest() - .WithBucket(bucketName) - .WithDelimiter("/") - .WithMarker(marker))); - - auto & contents = res.GetContents(); - - debug(format("got %d keys, next marker '%s'") - % contents.size() % res.GetNextMarker()); - - for (auto object : contents) { - auto & key = object.GetKey(); - if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; - paths.insert(storeDir + "/" + key.substr(0, key.size() - 8)); - } - - marker = res.GetNextMarker(); - } while (!marker.empty()); - - return paths; - } + auto now2 = std::chrono::steady_clock::now(); + auto duration = + std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) + .count(); + + printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") % + bucketName % path % data.size() % duration); + + stats.putTimeMs += duration; + stats.putBytes += data.size(); + stats.put++; + } + + void upsertFile(const std::string& path, const std::string& data, + const std::string& mimeType) override { + if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) + uploadFile(path, *compress(narinfoCompression, data), mimeType, + narinfoCompression); + else if (lsCompression != "" && hasSuffix(path, ".ls")) + uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); + else if (logCompression != "" && hasPrefix(path, "log/")) + uploadFile(path, *compress(logCompression, data), mimeType, + logCompression); + else + uploadFile(path, data, mimeType, ""); + } + + void getFile(const std::string& path, Sink& sink) override { + stats.get++; + + // FIXME: stream output to sink. + auto res = s3Helper.getObject(bucketName, path); + + stats.getBytes += res.data ? res.data->size() : 0; + stats.getTimeMs += res.durationMs; + + if (res.data) { + printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", bucketName, + path, res.data->size(), res.durationMs); + + sink((unsigned char*)res.data->data(), res.data->size()); + } else + throw NoSuchBinaryCacheFile( + "file '%s' does not exist in binary cache '%s'", path, getUri()); + } + + PathSet queryAllValidPaths() override { + PathSet paths; + std::string marker; + + do { + debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % + marker); + + auto res = checkAws( + format("AWS error listing bucket '%s'") % bucketName, + s3Helper.client->ListObjects(Aws::S3::Model::ListObjectsRequest() + .WithBucket(bucketName) + .WithDelimiter("/") + .WithMarker(marker))); + + auto& contents = res.GetContents(); + + debug(format("got %d keys, next marker '%s'") % contents.size() % + res.GetNextMarker()); + + for (auto object : contents) { + auto& key = object.GetKey(); + if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; + paths.insert(storeDir + "/" + key.substr(0, key.size() - 8)); + } + + marker = res.GetNextMarker(); + } while (!marker.empty()); + + return paths; + } }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, 5) != "s3://") return 0; - auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5)); - store->init(); - return store; -}); +static RegisterStoreImplementation regStore( + [](const std::string& uri, + const Store::Params& params) -> std::shared_ptr<Store> { + if (std::string(uri, 0, 5) != "s3://") return 0; + auto store = + std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5)); + store->init(); + return store; + }); -} +} // namespace nix #endif diff --git a/third_party/nix/src/libstore/s3-binary-cache-store.hh b/third_party/nix/src/libstore/s3-binary-cache-store.hh index 4d43fe4d23d8..b9ed04ed112f 100644 --- a/third_party/nix/src/libstore/s3-binary-cache-store.hh +++ b/third_party/nix/src/libstore/s3-binary-cache-store.hh @@ -1,33 +1,26 @@ #pragma once -#include "binary-cache-store.hh" - #include <atomic> +#include "binary-cache-store.hh" namespace nix { -class S3BinaryCacheStore : public BinaryCacheStore -{ -protected: - - S3BinaryCacheStore(const Params & params) - : BinaryCacheStore(params) - { } - -public: - - struct Stats - { - std::atomic<uint64_t> put{0}; - std::atomic<uint64_t> putBytes{0}; - std::atomic<uint64_t> putTimeMs{0}; - std::atomic<uint64_t> get{0}; - std::atomic<uint64_t> getBytes{0}; - std::atomic<uint64_t> getTimeMs{0}; - std::atomic<uint64_t> head{0}; - }; - - virtual const Stats & getS3Stats() = 0; +class S3BinaryCacheStore : public BinaryCacheStore { + protected: + S3BinaryCacheStore(const Params& params) : BinaryCacheStore(params) {} + + public: + struct Stats { + std::atomic<uint64_t> put{0}; + std::atomic<uint64_t> putBytes{0}; + std::atomic<uint64_t> putTimeMs{0}; + std::atomic<uint64_t> get{0}; + std::atomic<uint64_t> getBytes{0}; + std::atomic<uint64_t> getTimeMs{0}; + std::atomic<uint64_t> head{0}; + }; + + virtual const Stats& getS3Stats() = 0; }; -} +} // namespace nix diff --git a/third_party/nix/src/libstore/s3.hh b/third_party/nix/src/libstore/s3.hh index ef5f23d0f253..09935fabed47 100644 --- a/third_party/nix/src/libstore/s3.hh +++ b/third_party/nix/src/libstore/s3.hh @@ -4,30 +4,39 @@ #include "ref.hh" -namespace Aws { namespace Client { class ClientConfiguration; } } -namespace Aws { namespace S3 { class S3Client; } } +namespace Aws { +namespace Client { +class ClientConfiguration; +} +} // namespace Aws +namespace Aws { +namespace S3 { +class S3Client; +} +} // namespace Aws namespace nix { -struct S3Helper -{ - ref<Aws::Client::ClientConfiguration> config; - ref<Aws::S3::S3Client> client; +struct S3Helper { + ref<Aws::Client::ClientConfiguration> config; + ref<Aws::S3::S3Client> client; - S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint); + S3Helper(const std::string& profile, const std::string& region, + const std::string& scheme, const std::string& endpoint); - ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); + ref<Aws::Client::ClientConfiguration> makeConfig(const std::string& region, + const std::string& scheme, + const std::string& endpoint); - struct DownloadResult - { - std::shared_ptr<std::string> data; - unsigned int durationMs; - }; + struct DownloadResult { + std::shared_ptr<std::string> data; + unsigned int durationMs; + }; - DownloadResult getObject( - const std::string & bucketName, const std::string & key); + DownloadResult getObject(const std::string& bucketName, + const std::string& key); }; -} +} // namespace nix #endif diff --git a/third_party/nix/src/libstore/serve-protocol.hh b/third_party/nix/src/libstore/serve-protocol.hh index 9fae6d5349f1..a07a7ef97425 100644 --- a/third_party/nix/src/libstore/serve-protocol.hh +++ b/third_party/nix/src/libstore/serve-protocol.hh @@ -6,19 +6,19 @@ namespace nix { #define SERVE_MAGIC_2 0x5452eecb #define SERVE_PROTOCOL_VERSION 0x205 -#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) -#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) +#define GET_PROTOCOL_MAJOR(x) ((x)&0xff00) +#define GET_PROTOCOL_MINOR(x) ((x)&0x00ff) typedef enum { - cmdQueryValidPaths = 1, - cmdQueryPathInfos = 2, - cmdDumpStorePath = 3, - cmdImportPaths = 4, - cmdExportPaths = 5, - cmdBuildPaths = 6, - cmdQueryClosure = 7, - cmdBuildDerivation = 8, - cmdAddToStoreNar = 9, + cmdQueryValidPaths = 1, + cmdQueryPathInfos = 2, + cmdDumpStorePath = 3, + cmdImportPaths = 4, + cmdExportPaths = 5, + cmdBuildPaths = 6, + cmdQueryClosure = 7, + cmdBuildDerivation = 8, + cmdAddToStoreNar = 9, } ServeCommand; -} +} // namespace nix diff --git a/third_party/nix/src/libstore/sqlite.cc b/third_party/nix/src/libstore/sqlite.cc index a061d64f36d8..2f4bfb654c65 100644 --- a/third_party/nix/src/libstore/sqlite.cc +++ b/third_party/nix/src/libstore/sqlite.cc @@ -1,198 +1,172 @@ #include "sqlite.hh" -#include "util.hh" - #include <sqlite3.h> - #include <atomic> +#include "util.hh" namespace nix { -[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs) -{ - int err = sqlite3_errcode(db); - int exterr = sqlite3_extended_errcode(db); +[[noreturn]] void throwSQLiteError(sqlite3* db, const FormatOrString& fs) { + int err = sqlite3_errcode(db); + int exterr = sqlite3_extended_errcode(db); - auto path = sqlite3_db_filename(db, nullptr); - if (!path) path = "(in-memory)"; + auto path = sqlite3_db_filename(db, nullptr); + if (!path) path = "(in-memory)"; - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - throw SQLiteBusy( - err == SQLITE_PROTOCOL + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + throw SQLiteBusy( + err == SQLITE_PROTOCOL ? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path) : fmt("SQLite database '%s' is busy", path)); - } - else - throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); + } else + throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); } -SQLite::SQLite(const Path & path) -{ - if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) - throw Error(format("cannot open SQLite database '%s'") % path); +SQLite::SQLite(const Path& path) { + if (sqlite3_open_v2(path.c_str(), &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + 0) != SQLITE_OK) + throw Error(format("cannot open SQLite database '%s'") % path); } -SQLite::~SQLite() -{ - try { - if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); - } catch (...) { - ignoreException(); - } +SQLite::~SQLite() { + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throwSQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } } -void SQLite::exec(const std::string & stmt) -{ - retrySQLite<void>([&]() { - if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt); - }); +void SQLite::exec(const std::string& stmt) { + retrySQLite<void>([&]() { + if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt); + }); } -void SQLiteStmt::create(sqlite3 * db, const string & sql) -{ - checkInterrupt(); - assert(!stmt); - if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, fmt("creating statement '%s'", sql)); - this->db = db; - this->sql = sql; +void SQLiteStmt::create(sqlite3* db, const string& sql) { + checkInterrupt(); + assert(!stmt); + if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK) + throwSQLiteError(db, fmt("creating statement '%s'", sql)); + this->db = db; + this->sql = sql; } -SQLiteStmt::~SQLiteStmt() -{ - try { - if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, fmt("finalizing statement '%s'", sql)); - } catch (...) { - ignoreException(); - } +SQLiteStmt::~SQLiteStmt() { + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throwSQLiteError(db, fmt("finalizing statement '%s'", sql)); + } catch (...) { + ignoreException(); + } } -SQLiteStmt::Use::Use(SQLiteStmt & stmt) - : stmt(stmt) -{ - assert(stmt.stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); +SQLiteStmt::Use::Use(SQLiteStmt& stmt) : stmt(stmt) { + assert(stmt.stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); } -SQLiteStmt::Use::~Use() -{ - sqlite3_reset(stmt); -} +SQLiteStmt::Use::~Use() { sqlite3_reset(stmt); } -SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) -{ - if (notNull) { - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); - } else - bind(); - return *this; +SQLiteStmt::Use& SQLiteStmt::Use::operator()(const std::string& value, + bool notNull) { + if (notNull) { + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, + SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; } -SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) -{ - if (notNull) { - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); - } else - bind(); - return *this; +SQLiteStmt::Use& SQLiteStmt::Use::operator()(int64_t value, bool notNull) { + if (notNull) { + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; } -SQLiteStmt::Use & SQLiteStmt::Use::bind() -{ - if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); - return *this; +SQLiteStmt::Use& SQLiteStmt::Use::bind() { + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + return *this; } -int SQLiteStmt::Use::step() -{ - return sqlite3_step(stmt); -} +int SQLiteStmt::Use::step() { return sqlite3_step(stmt); } -void SQLiteStmt::Use::exec() -{ - int r = step(); - assert(r != SQLITE_ROW); - if (r != SQLITE_DONE) - throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql)); +void SQLiteStmt::Use::exec() { + int r = step(); + assert(r != SQLITE_ROW); + if (r != SQLITE_DONE) + throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql)); } -bool SQLiteStmt::Use::next() -{ - int r = step(); - if (r != SQLITE_DONE && r != SQLITE_ROW) - throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql)); - return r == SQLITE_ROW; +bool SQLiteStmt::Use::next() { + int r = step(); + if (r != SQLITE_DONE && r != SQLITE_ROW) + throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql)); + return r == SQLITE_ROW; } -std::string SQLiteStmt::Use::getStr(int col) -{ - auto s = (const char *) sqlite3_column_text(stmt, col); - assert(s); - return s; +std::string SQLiteStmt::Use::getStr(int col) { + auto s = (const char*)sqlite3_column_text(stmt, col); + assert(s); + return s; } -int64_t SQLiteStmt::Use::getInt(int col) -{ - // FIXME: detect nulls? - return sqlite3_column_int64(stmt, col); +int64_t SQLiteStmt::Use::getInt(int col) { + // FIXME: detect nulls? + return sqlite3_column_int64(stmt, col); } -bool SQLiteStmt::Use::isNull(int col) -{ - return sqlite3_column_type(stmt, col) == SQLITE_NULL; +bool SQLiteStmt::Use::isNull(int col) { + return sqlite3_column_type(stmt, col) == SQLITE_NULL; } -SQLiteTxn::SQLiteTxn(sqlite3 * db) -{ - this->db = db; - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); - active = true; +SQLiteTxn::SQLiteTxn(sqlite3* db) { + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "starting transaction"); + active = true; } -void SQLiteTxn::commit() -{ - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); - active = false; +void SQLiteTxn::commit() { + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "committing transaction"); + active = false; } -SQLiteTxn::~SQLiteTxn() -{ - try { - if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); - } catch (...) { - ignoreException(); - } +SQLiteTxn::~SQLiteTxn() { + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } } -void handleSQLiteBusy(const SQLiteBusy & e) -{ - static std::atomic<time_t> lastWarned{0}; +void handleSQLiteBusy(const SQLiteBusy& e) { + static std::atomic<time_t> lastWarned{0}; - time_t now = time(0); + time_t now = time(0); - if (now > lastWarned + 10) { - lastWarned = now; - printError("warning: %s", e.what()); - } + if (now > lastWarned + 10) { + lastWarned = now; + printError("warning: %s", e.what()); + } - /* Sleep for a while since retrying the transaction right away - is likely to fail again. */ - checkInterrupt(); - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ + checkInterrupt(); + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, 0); } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/sqlite.hh b/third_party/nix/src/libstore/sqlite.hh index 115679b84159..633a12189a6c 100644 --- a/third_party/nix/src/libstore/sqlite.hh +++ b/third_party/nix/src/libstore/sqlite.hh @@ -2,7 +2,6 @@ #include <functional> #include <string> - #include "types.hh" class sqlite3; @@ -11,104 +10,99 @@ class sqlite3_stmt; namespace nix { /* RAII wrapper to close a SQLite database automatically. */ -struct SQLite -{ - sqlite3 * db = 0; - SQLite() { } - SQLite(const Path & path); - SQLite(const SQLite & from) = delete; - SQLite& operator = (const SQLite & from) = delete; - SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } - ~SQLite(); - operator sqlite3 * () { return db; } - - void exec(const std::string & stmt); +struct SQLite { + sqlite3* db = 0; + SQLite() {} + SQLite(const Path& path); + SQLite(const SQLite& from) = delete; + SQLite& operator=(const SQLite& from) = delete; + SQLite& operator=(SQLite&& from) { + db = from.db; + from.db = 0; + return *this; + } + ~SQLite(); + operator sqlite3*() { return db; } + + void exec(const std::string& stmt); }; /* RAII wrapper to create and destroy SQLite prepared statements. */ -struct SQLiteStmt -{ - sqlite3 * db = 0; - sqlite3_stmt * stmt = 0; - std::string sql; - SQLiteStmt() { } - SQLiteStmt(sqlite3 * db, const std::string & sql) { create(db, sql); } - void create(sqlite3 * db, const std::string & s); - ~SQLiteStmt(); - operator sqlite3_stmt * () { return stmt; } - - /* Helper for binding / executing statements. */ - class Use - { - friend struct SQLiteStmt; - private: - SQLiteStmt & stmt; - unsigned int curArg = 1; - Use(SQLiteStmt & stmt); - - public: - - ~Use(); - - /* Bind the next parameter. */ - Use & operator () (const std::string & value, bool notNull = true); - Use & operator () (int64_t value, bool notNull = true); - Use & bind(); // null - - int step(); - - /* Execute a statement that does not return rows. */ - void exec(); - - /* For statements that return 0 or more rows. Returns true iff - a row is available. */ - bool next(); - - std::string getStr(int col); - int64_t getInt(int col); - bool isNull(int col); - }; - - Use use() - { - return Use(*this); - } +struct SQLiteStmt { + sqlite3* db = 0; + sqlite3_stmt* stmt = 0; + std::string sql; + SQLiteStmt() {} + SQLiteStmt(sqlite3* db, const std::string& sql) { create(db, sql); } + void create(sqlite3* db, const std::string& s); + ~SQLiteStmt(); + operator sqlite3_stmt*() { return stmt; } + + /* Helper for binding / executing statements. */ + class Use { + friend struct SQLiteStmt; + + private: + SQLiteStmt& stmt; + unsigned int curArg = 1; + Use(SQLiteStmt& stmt); + + public: + ~Use(); + + /* Bind the next parameter. */ + Use& operator()(const std::string& value, bool notNull = true); + Use& operator()(int64_t value, bool notNull = true); + Use& bind(); // null + + int step(); + + /* Execute a statement that does not return rows. */ + void exec(); + + /* For statements that return 0 or more rows. Returns true iff + a row is available. */ + bool next(); + + std::string getStr(int col); + int64_t getInt(int col); + bool isNull(int col); + }; + + Use use() { return Use(*this); } }; /* RAII helper that ensures transactions are aborted unless explicitly committed. */ -struct SQLiteTxn -{ - bool active = false; - sqlite3 * db; +struct SQLiteTxn { + bool active = false; + sqlite3* db; - SQLiteTxn(sqlite3 * db); + SQLiteTxn(sqlite3* db); - void commit(); + void commit(); - ~SQLiteTxn(); + ~SQLiteTxn(); }; - MakeError(SQLiteError, Error); MakeError(SQLiteBusy, SQLiteError); -[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs); +[[noreturn]] void throwSQLiteError(sqlite3* db, const FormatOrString& fs); -void handleSQLiteBusy(const SQLiteBusy & e); +void handleSQLiteBusy(const SQLiteBusy& e); /* Convenience function for retrying a SQLite transaction when the database is busy. */ -template<typename T> -T retrySQLite(std::function<T()> fun) -{ - while (true) { - try { - return fun(); - } catch (SQLiteBusy & e) { - handleSQLiteBusy(e); - } +template <typename T> +T retrySQLite(std::function<T()> fun) { + while (true) { + try { + return fun(); + } catch (SQLiteBusy& e) { + handleSQLiteBusy(e); } + } } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/ssh-store.cc b/third_party/nix/src/libstore/ssh-store.cc index 93e11738991f..f11acc7b80ae 100644 --- a/third_party/nix/src/libstore/ssh-store.cc +++ b/third_party/nix/src/libstore/ssh-store.cc @@ -1,100 +1,84 @@ -#include "store-api.hh" -#include "remote-store.hh" -#include "remote-fs-accessor.hh" #include "archive.hh" -#include "worker-protocol.hh" #include "pool.hh" +#include "remote-fs-accessor.hh" +#include "remote-store.hh" #include "ssh.hh" +#include "store-api.hh" +#include "worker-protocol.hh" namespace nix { static std::string uriScheme = "ssh-ng://"; -class SSHStore : public RemoteStore -{ -public: - - const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"}; +class SSHStore : public RemoteStore { + public: + const Setting<Path> sshKey{(Store*)this, "", "ssh-key", + "path to an SSH private key"}; + const Setting<bool> compress{(Store*)this, false, "compress", + "whether to compress the connection"}; - SSHStore(const std::string & host, const Params & params) - : Store(params) - , RemoteStore(params) - , host(host) - , master( - host, - sshKey, - // Use SSH master only if using more than 1 connection. - connections->capacity() > 1, - compress) - { - } + SSHStore(const std::string& host, const Params& params) + : Store(params), + RemoteStore(params), + host(host), + master(host, sshKey, + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, compress) {} - std::string getUri() override - { - return uriScheme + host; - } + std::string getUri() override { return uriScheme + host; } - bool sameMachine() - { return false; } + bool sameMachine() { return false; } - void narFromPath(const Path & path, Sink & sink) override; + void narFromPath(const Path& path, Sink& sink) override; - ref<FSAccessor> getFSAccessor() override; + ref<FSAccessor> getFSAccessor() override; -private: + private: + struct Connection : RemoteStore::Connection { + std::unique_ptr<SSHMaster::Connection> sshConn; + }; - struct Connection : RemoteStore::Connection - { - std::unique_ptr<SSHMaster::Connection> sshConn; - }; + ref<RemoteStore::Connection> openConnection() override; - ref<RemoteStore::Connection> openConnection() override; + std::string host; - std::string host; + SSHMaster master; - SSHMaster master; - - void setOptions(RemoteStore::Connection & conn) override - { - /* TODO Add a way to explicitly ask for some options to be - forwarded. One option: A way to query the daemon for its - settings, and then a series of params to SSHStore like - forward-cores or forward-overridden-cores that only - override the requested settings. - */ - }; + void setOptions(RemoteStore::Connection& conn) override{ + /* TODO Add a way to explicitly ask for some options to be + forwarded. One option: A way to query the daemon for its + settings, and then a series of params to SSHStore like + forward-cores or forward-overridden-cores that only + override the requested settings. + */ + }; }; -void SSHStore::narFromPath(const Path & path, Sink & sink) -{ - auto conn(connections->get()); - conn->to << wopNarFromPath << path; - conn->processStderr(); - copyNAR(conn->from, sink); +void SSHStore::narFromPath(const Path& path, Sink& sink) { + auto conn(connections->get()); + conn->to << wopNarFromPath << path; + conn->processStderr(); + copyNAR(conn->from, sink); } -ref<FSAccessor> SSHStore::getFSAccessor() -{ - return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); +ref<FSAccessor> SSHStore::getFSAccessor() { + return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); } -ref<RemoteStore::Connection> SSHStore::openConnection() -{ - auto conn = make_ref<Connection>(); - conn->sshConn = master.startCommand("nix-daemon --stdio"); - conn->to = FdSink(conn->sshConn->in.get()); - conn->from = FdSource(conn->sshConn->out.get()); - initConnection(*conn); - return conn; +ref<RemoteStore::Connection> SSHStore::openConnection() { + auto conn = make_ref<Connection>(); + conn->sshConn = master.startCommand("nix-daemon --stdio"); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); + initConnection(*conn); + return conn; } -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params); +static RegisterStoreImplementation regStore([](const std::string& uri, + const Store::Params& params) + -> std::shared_ptr<Store> { + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params); }); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/ssh.cc b/third_party/nix/src/libstore/ssh.cc index dea16e533ef5..72ab1cfcf13b 100644 --- a/third_party/nix/src/libstore/ssh.cc +++ b/third_party/nix/src/libstore/ssh.cc @@ -2,64 +2,60 @@ namespace nix { -SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD) - : host(host) - , fakeSSH(host == "localhost") - , keyFile(keyFile) - , useMaster(useMaster && !fakeSSH) - , compress(compress) - , logFD(logFD) -{ - if (host == "" || hasPrefix(host, "-")) - throw Error("invalid SSH host name '%s'", host); +SSHMaster::SSHMaster(const std::string& host, const std::string& keyFile, + bool useMaster, bool compress, int logFD) + : host(host), + fakeSSH(host == "localhost"), + keyFile(keyFile), + useMaster(useMaster && !fakeSSH), + compress(compress), + logFD(logFD) { + if (host == "" || hasPrefix(host, "-")) + throw Error("invalid SSH host name '%s'", host); } -void SSHMaster::addCommonSSHOpts(Strings & args) -{ - for (auto & i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS"))) - args.push_back(i); - if (!keyFile.empty()) - args.insert(args.end(), {"-i", keyFile}); - if (compress) - args.push_back("-C"); +void SSHMaster::addCommonSSHOpts(Strings& args) { + for (auto& i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS"))) + args.push_back(i); + if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile}); + if (compress) args.push_back("-C"); } -std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command) -{ - Path socketPath = startMaster(); +std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand( + const std::string& command) { + Path socketPath = startMaster(); - Pipe in, out; - in.create(); - out.create(); + Pipe in, out; + in.create(); + out.create(); - auto conn = std::make_unique<Connection>(); - ProcessOptions options; - options.dieWithParent = false; + auto conn = std::make_unique<Connection>(); + ProcessOptions options; + options.dieWithParent = false; - conn->sshPid = startProcess([&]() { + conn->sshPid = startProcess( + [&]() { restoreSignals(); close(in.writeSide.get()); close(out.readSide.get()); if (dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("duping over stdin"); + throw SysError("duping over stdin"); if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("duping over stdout"); + throw SysError("duping over stdout"); if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1) - throw SysError("duping over stderr"); + throw SysError("duping over stderr"); Strings args; if (fakeSSH) { - args = { "bash", "-c" }; + args = {"bash", "-c"}; } else { - args = { "ssh", host.c_str(), "-x", "-a" }; - addCommonSSHOpts(args); - if (socketPath != "") - args.insert(args.end(), {"-S", socketPath}); - if (verbosity >= lvlChatty) - args.push_back("-v"); + args = {"ssh", host.c_str(), "-x", "-a"}; + addCommonSSHOpts(args); + if (socketPath != "") args.insert(args.end(), {"-S", socketPath}); + if (verbosity >= lvlChatty) args.push_back("-v"); } args.push_back(command); @@ -67,68 +63,70 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string // could not exec ssh/bash throw SysError("unable to execute '%s'", args.front()); - }, options); + }, + options); + in.readSide = -1; + out.writeSide = -1; - in.readSide = -1; - out.writeSide = -1; + conn->out = std::move(out.readSide); + conn->in = std::move(in.writeSide); - conn->out = std::move(out.readSide); - conn->in = std::move(in.writeSide); - - return conn; + return conn; } -Path SSHMaster::startMaster() -{ - if (!useMaster) return ""; +Path SSHMaster::startMaster() { + if (!useMaster) return ""; - auto state(state_.lock()); + auto state(state_.lock()); - if (state->sshMaster != -1) return state->socketPath; + if (state->sshMaster != -1) return state->socketPath; - state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700)); + state->tmpDir = + std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700)); - state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; + state->socketPath = (Path)*state->tmpDir + "/ssh.sock"; - Pipe out; - out.create(); + Pipe out; + out.create(); - ProcessOptions options; - options.dieWithParent = false; + ProcessOptions options; + options.dieWithParent = false; - state->sshMaster = startProcess([&]() { + state->sshMaster = startProcess( + [&]() { restoreSignals(); close(out.readSide.get()); if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("duping over stdout"); - - Strings args = - { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath - , "-o", "LocalCommand=echo started" - , "-o", "PermitLocalCommand=yes" - }; - if (verbosity >= lvlChatty) - args.push_back("-v"); + throw SysError("duping over stdout"); + + Strings args = {"ssh", host.c_str(), + "-M", "-N", + "-S", state->socketPath, + "-o", "LocalCommand=echo started", + "-o", "PermitLocalCommand=yes"}; + if (verbosity >= lvlChatty) args.push_back("-v"); addCommonSSHOpts(args); execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); throw SysError("unable to execute '%s'", args.front()); - }, options); + }, + options); - out.writeSide = -1; + out.writeSide = -1; - std::string reply; - try { - reply = readLine(out.readSide.get()); - } catch (EndOfFile & e) { } + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile& e) { + } - if (reply != "started") - throw Error("failed to start SSH master connection to '%s'", host); + if (reply != "started") + throw Error("failed to start SSH master connection to '%s'", host); - return state->socketPath; + return state->socketPath; } -} +} // namespace nix diff --git a/third_party/nix/src/libstore/ssh.hh b/third_party/nix/src/libstore/ssh.hh index 4f0f0bd29f9f..7cdc69e0abe0 100644 --- a/third_party/nix/src/libstore/ssh.hh +++ b/third_party/nix/src/libstore/ssh.hh @@ -1,45 +1,41 @@ #pragma once -#include "util.hh" #include "sync.hh" +#include "util.hh" namespace nix { -class SSHMaster -{ -private: - - const std::string host; - bool fakeSSH; - const std::string keyFile; - const bool useMaster; - const bool compress; - const int logFD; - - struct State - { - Pid sshMaster; - std::unique_ptr<AutoDelete> tmpDir; - Path socketPath; - }; +class SSHMaster { + private: + const std::string host; + bool fakeSSH; + const std::string keyFile; + const bool useMaster; + const bool compress; + const int logFD; - Sync<State> state_; + struct State { + Pid sshMaster; + std::unique_ptr<AutoDelete> tmpDir; + Path socketPath; + }; - void addCommonSSHOpts(Strings & args); + Sync<State> state_; -public: + void addCommonSSHOpts(Strings& args); - SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1); + public: + SSHMaster(const std::string& host, const std::string& keyFile, bool useMaster, + bool compress, int logFD = -1); - struct Connection - { - Pid sshPid; - AutoCloseFD out, in; - }; + struct Connection { + Pid sshPid; + AutoCloseFD out, in; + }; - std::unique_ptr<Connection> startCommand(const std::string & command); + std::unique_ptr<Connection> startCommand(const std::string& command); - Path startMaster(); + Path startMaster(); }; -} +} // namespace nix diff --git a/third_party/nix/src/libstore/store-api.cc b/third_party/nix/src/libstore/store-api.cc index 5f63c53b562d..d8ee68904480 100644 --- a/third_party/nix/src/libstore/store-api.cc +++ b/third_party/nix/src/libstore/store-api.cc @@ -1,117 +1,98 @@ +#include "store-api.hh" +#include <future> #include "crypto.hh" +#include "derivations.hh" #include "globals.hh" -#include "store-api.hh" -#include "util.hh" +#include "json.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" -#include "json.hh" -#include "derivations.hh" - -#include <future> - +#include "util.hh" namespace nix { - -bool Store::isInStore(const Path & path) const -{ - return isInDir(path, storeDir); -} - - -bool Store::isStorePath(const Path & path) const -{ - return isInStore(path) - && path.size() >= storeDir.size() + 1 + storePathHashLen - && path.find('/', storeDir.size() + 1) == Path::npos; +bool Store::isInStore(const Path& path) const { + return isInDir(path, storeDir); } - -void Store::assertStorePath(const Path & path) const -{ - if (!isStorePath(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); +bool Store::isStorePath(const Path& path) const { + return isInStore(path) && + path.size() >= storeDir.size() + 1 + storePathHashLen && + path.find('/', storeDir.size() + 1) == Path::npos; } - -Path Store::toStorePath(const Path & path) const -{ - if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); - Path::size_type slash = path.find('/', storeDir.size() + 1); - if (slash == Path::npos) - return path; - else - return Path(path, 0, slash); +void Store::assertStorePath(const Path& path) const { + if (!isStorePath(path)) + throw Error(format("path '%1%' is not in the Nix store") % path); } - -Path Store::followLinksToStore(const Path & _path) const -{ - Path path = absPath(_path); - while (!isInStore(path)) { - if (!isLink(path)) break; - string target = readLink(path); - path = absPath(target, dirOf(path)); - } - if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); +Path Store::toStorePath(const Path& path) const { + if (!isInStore(path)) + throw Error(format("path '%1%' is not in the Nix store") % path); + Path::size_type slash = path.find('/', storeDir.size() + 1); + if (slash == Path::npos) return path; + else + return Path(path, 0, slash); +} + +Path Store::followLinksToStore(const Path& _path) const { + Path path = absPath(_path); + while (!isInStore(path)) { + if (!isLink(path)) break; + string target = readLink(path); + path = absPath(target, dirOf(path)); + } + if (!isInStore(path)) + throw Error(format("path '%1%' is not in the Nix store") % path); + return path; +} + +Path Store::followLinksToStorePath(const Path& path) const { + return toStorePath(followLinksToStore(path)); +} + +string storePathToName(const Path& path) { + auto base = baseNameOf(path); + assert(base.size() == storePathHashLen || + (base.size() > storePathHashLen && base[storePathHashLen] == '-')); + return base.size() == storePathHashLen ? "" + : string(base, storePathHashLen + 1); +} + +string storePathToHash(const Path& path) { + auto base = baseNameOf(path); + assert(base.size() >= storePathHashLen); + return string(base, 0, storePathHashLen); +} + +void checkStoreName(const string& name) { + string validChars = "+-._?="; + + auto baseError = + format( + "The path name '%2%' is invalid: %3%. " + "Path names are alphanumeric and can include the symbols %1% " + "and must not begin with a period. " + "Note: If '%2%' is a source file and you cannot rename it on " + "disk, builtins.path { name = ... } can be used to give it an " + "alternative name.") % + validChars % name; + + /* Disallow names starting with a dot for possible security + reasons (e.g., "." and ".."). */ + if (string(name, 0, 1) == ".") + throw Error(baseError % "it is illegal to start the name with a period"); + /* Disallow names longer than 211 characters. ext4’s max is 256, + but we need extra space for the hash and .chroot extensions. */ + if (name.length() > 211) + throw Error(baseError % "name must be less than 212 characters"); + for (auto& i : name) + if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || + (i >= '0' && i <= '9') || validChars.find(i) != string::npos)) { + throw Error(baseError % (format("the '%1%' character is invalid") % i)); + } } - -Path Store::followLinksToStorePath(const Path & path) const -{ - return toStorePath(followLinksToStore(path)); -} - - -string storePathToName(const Path & path) -{ - auto base = baseNameOf(path); - assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-')); - return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1); -} - - -string storePathToHash(const Path & path) -{ - auto base = baseNameOf(path); - assert(base.size() >= storePathHashLen); - return string(base, 0, storePathHashLen); -} - - -void checkStoreName(const string & name) -{ - string validChars = "+-._?="; - - auto baseError = format("The path name '%2%' is invalid: %3%. " - "Path names are alphanumeric and can include the symbols %1% " - "and must not begin with a period. " - "Note: If '%2%' is a source file and you cannot rename it on " - "disk, builtins.path { name = ... } can be used to give it an " - "alternative name.") % validChars % name; - - /* Disallow names starting with a dot for possible security - reasons (e.g., "." and ".."). */ - if (string(name, 0, 1) == ".") - throw Error(baseError % "it is illegal to start the name with a period"); - /* Disallow names longer than 211 characters. ext4’s max is 256, - but we need extra space for the hash and .chroot extensions. */ - if (name.length() > 211) - throw Error(baseError % "name must be less than 212 characters"); - for (auto & i : name) - if (!((i >= 'A' && i <= 'Z') || - (i >= 'a' && i <= 'z') || - (i >= '0' && i <= '9') || - validChars.find(i) != string::npos)) - { - throw Error(baseError % (format("the '%1%' character is invalid") % i)); - } -} - - /* Store paths have the following form: <store>/<h>-<name> @@ -182,802 +163,732 @@ void checkStoreName(const string & name) "source:". */ +Path Store::makeStorePath(const string& type, const Hash& hash, + const string& name) const { + /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ + string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name; -Path Store::makeStorePath(const string & type, - const Hash & hash, const string & name) const -{ - /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name; + checkStoreName(name); - checkStoreName(name); - - return storeDir + "/" - + compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) - + "-" + name; + return storeDir + "/" + + compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) + + "-" + name; } - -Path Store::makeOutputPath(const string & id, - const Hash & hash, const string & name) const -{ - return makeStorePath("output:" + id, hash, - name + (id == "out" ? "" : "-" + id)); +Path Store::makeOutputPath(const string& id, const Hash& hash, + const string& name) const { + return makeStorePath("output:" + id, hash, + name + (id == "out" ? "" : "-" + id)); } - -Path Store::makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name) const -{ - return hash.type == htSHA256 && recursive - ? makeStorePath("source", hash, name) - : makeStorePath("output:out", hashString(htSHA256, - "fixed:out:" + (recursive ? (string) "r:" : "") + - hash.to_string(Base16) + ":"), - name); +Path Store::makeFixedOutputPath(bool recursive, const Hash& hash, + const string& name) const { + return hash.type == htSHA256 && recursive + ? makeStorePath("source", hash, name) + : makeStorePath( + "output:out", + hashString(htSHA256, + "fixed:out:" + (recursive ? (string) "r:" : "") + + hash.to_string(Base16) + ":"), + name); } - -Path Store::makeTextPath(const string & name, const Hash & hash, - const PathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - string type = "text"; - for (auto & i : references) { - type += ":"; - type += i; - } - return makeStorePath(type, hash, name); -} - - -std::pair<Path, Hash> Store::computeStorePathForPath(const string & name, - const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const -{ - Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - Path dstPath = makeFixedOutputPath(recursive, h, name); - return std::pair<Path, Hash>(dstPath, h); +Path Store::makeTextPath(const string& name, const Hash& hash, + const PathSet& references) const { + assert(hash.type == htSHA256); + /* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ + string type = "text"; + for (auto& i : references) { + type += ":"; + type += i; + } + return makeStorePath(type, hash, name); } - -Path Store::computeStorePathForText(const string & name, const string & s, - const PathSet & references) const -{ - return makeTextPath(name, hashString(htSHA256, s), references); +std::pair<Path, Hash> Store::computeStorePathForPath(const string& name, + const Path& srcPath, + bool recursive, + HashType hashAlgo, + PathFilter& filter) const { + Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first + : hashFile(hashAlgo, srcPath); + Path dstPath = makeFixedOutputPath(recursive, h, name); + return std::pair<Path, Hash>(dstPath, h); } - -Store::Store(const Params & params) - : Config(params) - , state({(size_t) pathInfoCacheSize}) -{ +Path Store::computeStorePathForText(const string& name, const string& s, + const PathSet& references) const { + return makeTextPath(name, hashString(htSHA256, s), references); } +Store::Store(const Params& params) + : Config(params), state({(size_t)pathInfoCacheSize}) {} -std::string Store::getUri() -{ - return ""; -} - +std::string Store::getUri() { return ""; } -bool Store::isValidPath(const Path & storePath) -{ - assertStorePath(storePath); +bool Store::isValidPath(const Path& storePath) { + assertStorePath(storePath); - auto hashPart = storePathToHash(storePath); + auto hashPart = storePathToHash(storePath); - { - auto state_(state.lock()); - auto res = state_->pathInfoCache.get(hashPart); - if (res) { - stats.narInfoReadAverted++; - return *res != 0; - } + { + auto state_(state.lock()); + auto res = state_->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + return *res != 0; } - - if (diskCache) { - auto res = diskCache->lookupNarInfo(getUri(), hashPart); - if (res.first != NarInfoDiskCache::oUnknown) { - stats.narInfoReadAverted++; - auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, - res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); - return res.first == NarInfoDiskCache::oValid; - } + } + + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + auto state_(state.lock()); + state_->pathInfoCache.upsert( + hashPart, res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + return res.first == NarInfoDiskCache::oValid; } + } - bool valid = isValidPathUncached(storePath); + bool valid = isValidPathUncached(storePath); - if (diskCache && !valid) - // FIXME: handle valid = true case. - diskCache->upsertNarInfo(getUri(), hashPart, 0); + if (diskCache && !valid) + // FIXME: handle valid = true case. + diskCache->upsertNarInfo(getUri(), hashPart, 0); - return valid; + return valid; } - /* Default implementation for stores that only implement queryPathInfoUncached(). */ -bool Store::isValidPathUncached(const Path & path) -{ - try { - queryPathInfo(path); - return true; - } catch (InvalidPath &) { - return false; - } +bool Store::isValidPathUncached(const Path& path) { + try { + queryPathInfo(path); + return true; + } catch (InvalidPath&) { + return false; + } } +ref<const ValidPathInfo> Store::queryPathInfo(const Path& storePath) { + std::promise<ref<ValidPathInfo>> promise; -ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) -{ - std::promise<ref<ValidPathInfo>> promise; - - queryPathInfo(storePath, - {[&](std::future<ref<ValidPathInfo>> result) { - try { - promise.set_value(result.get()); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }}); + queryPathInfo(storePath, {[&](std::future<ref<ValidPathInfo>> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); - return promise.get_future().get(); + return promise.get_future().get(); } +void Store::queryPathInfo(const Path& storePath, + Callback<ref<ValidPathInfo>> callback) noexcept { + std::string hashPart; -void Store::queryPathInfo(const Path & storePath, - Callback<ref<ValidPathInfo>> callback) noexcept -{ - std::string hashPart; + try { + assertStorePath(storePath); - try { - assertStorePath(storePath); + hashPart = storePathToHash(storePath); - hashPart = storePathToHash(storePath); + { + auto res = state.lock()->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + if (!*res) + throw InvalidPath(format("path '%s' is not valid") % storePath); + return callback(ref<ValidPathInfo>(*res)); + } + } + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; { - auto res = state.lock()->pathInfoCache.get(hashPart); - if (res) { - stats.narInfoReadAverted++; - if (!*res) - throw InvalidPath(format("path '%s' is not valid") % storePath); - return callback(ref<ValidPathInfo>(*res)); - } - } - - if (diskCache) { - auto res = diskCache->lookupNarInfo(getUri(), hashPart); - if (res.first != NarInfoDiskCache::oUnknown) { - stats.narInfoReadAverted++; - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, - res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); - if (res.first == NarInfoDiskCache::oInvalid || - (res.second->path != storePath && storePathToName(storePath) != "")) - throw InvalidPath(format("path '%s' is not valid") % storePath); - } - return callback(ref<ValidPathInfo>(res.second)); - } + auto state_(state.lock()); + state_->pathInfoCache.upsert( + hashPart, + res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + if (res.first == NarInfoDiskCache::oInvalid || + (res.second->path != storePath && + storePathToName(storePath) != "")) + throw InvalidPath(format("path '%s' is not valid") % storePath); } + return callback(ref<ValidPathInfo>(res.second)); + } + } - } catch (...) { return callback.rethrow(); } - - auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); - - queryPathInfoUncached(storePath, - {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<ValidPathInfo>> fut) { - - try { - auto info = fut.get(); - - if (diskCache) - diskCache->upsertNarInfo(getUri(), hashPart, info); - - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, info); - } - - if (!info - || (info->path != storePath && storePathToName(storePath) != "")) - { - stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePath); - } - - (*callbackPtr)(ref<ValidPathInfo>(info)); - } catch (...) { callbackPtr->rethrow(); } - }}); -} + } catch (...) { + return callback.rethrow(); + } + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); -PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) -{ - struct State - { - size_t left; - PathSet valid; - std::exception_ptr exc; - }; - - Sync<State> state_(State{paths.size(), PathSet()}); + queryPathInfoUncached( + storePath, {[this, storePath, hashPart, callbackPtr]( + std::future<std::shared_ptr<ValidPathInfo>> fut) { + try { + auto info = fut.get(); - std::condition_variable wakeup; - ThreadPool pool; + if (diskCache) diskCache->upsertNarInfo(getUri(), hashPart, info); - auto doQuery = [&](const Path & path ) { - checkInterrupt(); - queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) { - auto state(state_.lock()); - try { - auto info = fut.get(); - state->valid.insert(path); - } catch (InvalidPath &) { - } catch (...) { - state->exc = std::current_exception(); - } - assert(state->left); - if (!--state->left) - wakeup.notify_one(); + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, info); + } + + if (!info || + (info->path != storePath && storePathToName(storePath) != "")) { + stats.narInfoMissing++; + throw InvalidPath("path '%s' is not valid", storePath); + } + + (*callbackPtr)(ref<ValidPathInfo>(info)); + } catch (...) { + callbackPtr->rethrow(); + } + }}); +} + +PathSet Store::queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute) { + struct State { + size_t left; + PathSet valid; + std::exception_ptr exc; + }; + + Sync<State> state_(State{paths.size(), PathSet()}); + + std::condition_variable wakeup; + ThreadPool pool; + + auto doQuery = [&](const Path& path) { + checkInterrupt(); + queryPathInfo( + path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) { + auto state(state_.lock()); + try { + auto info = fut.get(); + state->valid.insert(path); + } catch (InvalidPath&) { + } catch (...) { + state->exc = std::current_exception(); + } + assert(state->left); + if (!--state->left) wakeup.notify_one(); }}); - }; + }; - for (auto & path : paths) - pool.enqueue(std::bind(doQuery, path)); + for (auto& path : paths) pool.enqueue(std::bind(doQuery, path)); - pool.process(); + pool.process(); - while (true) { - auto state(state_.lock()); - if (!state->left) { - if (state->exc) std::rethrow_exception(state->exc); - return state->valid; - } - state.wait(wakeup); + while (true) { + auto state(state_.lock()); + if (!state->left) { + if (state->exc) std::rethrow_exception(state->exc); + return state->valid; } + state.wait(wakeup); + } } - /* Return a string accepted by decodeValidPathInfo() that registers the specified paths as valid. Note: it's the responsibility of the caller to provide a closure. */ -string Store::makeValidityRegistration(const PathSet & paths, - bool showDerivers, bool showHash) -{ - string s = ""; +string Store::makeValidityRegistration(const PathSet& paths, bool showDerivers, + bool showHash) { + string s = ""; - for (auto & i : paths) { - s += i + "\n"; + for (auto& i : paths) { + s += i + "\n"; - auto info = queryPathInfo(i); + auto info = queryPathInfo(i); - if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; - s += (format("%1%\n") % info->narSize).str(); - } - - Path deriver = showDerivers ? info->deriver : ""; - s += deriver + "\n"; - - s += (format("%1%\n") % info->references.size()).str(); - - for (auto & j : info->references) - s += j + "\n"; + if (showHash) { + s += info->narHash.to_string(Base16, false) + "\n"; + s += (format("%1%\n") % info->narSize).str(); } - return s; -} - + Path deriver = showDerivers ? info->deriver : ""; + s += deriver + "\n"; -void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid) -{ - auto jsonList = jsonOut.list(); + s += (format("%1%\n") % info->references.size()).str(); - for (auto storePath : storePaths) { - auto jsonPath = jsonList.object(); - jsonPath.attr("path", storePath); + for (auto& j : info->references) s += j + "\n"; + } - try { - auto info = queryPathInfo(storePath); - storePath = info->path; - - jsonPath - .attr("narHash", info->narHash.to_string()) - .attr("narSize", info->narSize); + return s; +} - { - auto jsonRefs = jsonPath.list("references"); - for (auto & ref : info->references) - jsonRefs.elem(ref); - } +void Store::pathInfoToJSON(JSONPlaceholder& jsonOut, const PathSet& storePaths, + bool includeImpureInfo, bool showClosureSize, + AllowInvalidFlag allowInvalid) { + auto jsonList = jsonOut.list(); - if (info->ca != "") - jsonPath.attr("ca", info->ca); + for (auto storePath : storePaths) { + auto jsonPath = jsonList.object(); + jsonPath.attr("path", storePath); - std::pair<uint64_t, uint64_t> closureSizes; + try { + auto info = queryPathInfo(storePath); + storePath = info->path; - if (showClosureSize) { - closureSizes = getClosureSize(storePath); - jsonPath.attr("closureSize", closureSizes.first); - } + jsonPath.attr("narHash", info->narHash.to_string()) + .attr("narSize", info->narSize); - if (includeImpureInfo) { + { + auto jsonRefs = jsonPath.list("references"); + for (auto& ref : info->references) jsonRefs.elem(ref); + } - if (info->deriver != "") - jsonPath.attr("deriver", info->deriver); + if (info->ca != "") jsonPath.attr("ca", info->ca); - if (info->registrationTime) - jsonPath.attr("registrationTime", info->registrationTime); + std::pair<uint64_t, uint64_t> closureSizes; - if (info->ultimate) - jsonPath.attr("ultimate", info->ultimate); + if (showClosureSize) { + closureSizes = getClosureSize(storePath); + jsonPath.attr("closureSize", closureSizes.first); + } - if (!info->sigs.empty()) { - auto jsonSigs = jsonPath.list("signatures"); - for (auto & sig : info->sigs) - jsonSigs.elem(sig); - } + if (includeImpureInfo) { + if (info->deriver != "") jsonPath.attr("deriver", info->deriver); - auto narInfo = std::dynamic_pointer_cast<const NarInfo>( - std::shared_ptr<const ValidPathInfo>(info)); + if (info->registrationTime) + jsonPath.attr("registrationTime", info->registrationTime); - if (narInfo) { - if (!narInfo->url.empty()) - jsonPath.attr("url", narInfo->url); - if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); - if (narInfo->fileSize) - jsonPath.attr("downloadSize", narInfo->fileSize); - if (showClosureSize) - jsonPath.attr("closureDownloadSize", closureSizes.second); - } - } + if (info->ultimate) jsonPath.attr("ultimate", info->ultimate); - } catch (InvalidPath &) { - jsonPath.attr("valid", false); + if (!info->sigs.empty()) { + auto jsonSigs = jsonPath.list("signatures"); + for (auto& sig : info->sigs) jsonSigs.elem(sig); } - } -} - -std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath) -{ - uint64_t totalNarSize = 0, totalDownloadSize = 0; - PathSet closure; - computeFSClosure(storePath, closure, false, false); - for (auto & p : closure) { - auto info = queryPathInfo(p); - totalNarSize += info->narSize; auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); - if (narInfo) - totalDownloadSize += narInfo->fileSize; - } - return {totalNarSize, totalDownloadSize}; -} + if (narInfo) { + if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); + if (narInfo->fileHash) + jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); + if (narInfo->fileSize) + jsonPath.attr("downloadSize", narInfo->fileSize); + if (showClosureSize) + jsonPath.attr("closureDownloadSize", closureSizes.second); + } + } -const Store::Stats & Store::getStats() -{ - { - auto state_(state.lock()); - stats.pathInfoCacheSize = state_->pathInfoCache.size(); + } catch (InvalidPath&) { + jsonPath.attr("valid", false); } - return stats; + } } - -void Store::buildPaths(const PathSet & paths, BuildMode buildMode) -{ - for (auto & path : paths) - if (isDerivation(path)) - unsupported("buildPaths"); - - if (queryValidPaths(paths).size() != paths.size()) - unsupported("buildPaths"); +std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path& storePath) { + uint64_t totalNarSize = 0, totalDownloadSize = 0; + PathSet closure; + computeFSClosure(storePath, closure, false, false); + for (auto& p : closure) { + auto info = queryPathInfo(p); + totalNarSize += info->narSize; + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + if (narInfo) totalDownloadSize += narInfo->fileSize; + } + return {totalNarSize, totalDownloadSize}; } +const Store::Stats& Store::getStats() { + { + auto state_(state.lock()); + stats.pathInfoCacheSize = state_->pathInfoCache.size(); + } + return stats; +} -void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) -{ - auto srcUri = srcStore->getUri(); - auto dstUri = dstStore->getUri(); - - Activity act(*logger, lvlInfo, actCopyPath, - srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", storePath, dstUri) - : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", storePath, srcUri) - : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), - {storePath, srcUri, dstUri}); - PushActivity pact(act.id); - - auto info = srcStore->queryPathInfo(storePath); - - uint64_t total = 0; - - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); - auto info2 = make_ref<ValidPathInfo>(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; - info = info2; - - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; - } +void Store::buildPaths(const PathSet& paths, BuildMode buildMode) { + for (auto& path : paths) + if (isDerivation(path)) unsupported("buildPaths"); - if (info->ultimate) { - auto info2 = make_ref<ValidPathInfo>(*info); - info2->ultimate = false; - info = info2; - } + if (queryValidPaths(paths).size() != paths.size()) unsupported("buildPaths"); +} - auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - total += len; - act.progress(total, info->narSize); +void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, + const Path& storePath, RepairFlag repair, + CheckSigsFlag checkSigs) { + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); + + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, + srcUri, dstUri), + {storePath, srcUri, dstUri}); + PushActivity pact(act.id); + + auto info = srcStore->queryPathInfo(storePath); + + uint64_t total = 0; + + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref<ValidPathInfo>(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; + info = info2; + + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } + + if (info->ultimate) { + auto info2 = make_ref<ValidPathInfo>(*info); + info2->ultimate = false; + info = info2; + } + + auto source = sinkToSource( + [&](Sink& sink) { + LambdaSink wrapperSink([&](const unsigned char* data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); }); srcStore->narFromPath({storePath}, wrapperSink); - }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); - }); + }, + [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", + storePath, srcStore->getUri()); + }); - dstStore->addToStore(*info, *source, repair, checkSigs); + dstStore->addToStore(*info, *source, repair, checkSigs); } +void copyPaths(ref<Store> srcStore, ref<Store> dstStore, + const PathSet& storePaths, RepairFlag repair, + CheckSigsFlag checkSigs, SubstituteFlag substitute) { + PathSet valid = dstStore->queryValidPaths(storePaths, substitute); -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths, - RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) -{ - PathSet valid = dstStore->queryValidPaths(storePaths, substitute); + PathSet missing; + for (auto& path : storePaths) + if (!valid.count(path)) missing.insert(path); - PathSet missing; - for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(path); + if (missing.empty()) return; - if (missing.empty()) return; + Activity act(*logger, lvlInfo, actCopyPaths, + fmt("copying %d paths", missing.size())); - Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + std::atomic<size_t> nrDone{0}; + std::atomic<size_t> nrFailed{0}; + std::atomic<uint64_t> bytesExpected{0}; + std::atomic<uint64_t> nrRunning{0}; - std::atomic<size_t> nrDone{0}; - std::atomic<size_t> nrFailed{0}; - std::atomic<uint64_t> bytesExpected{0}; - std::atomic<uint64_t> nrRunning{0}; + auto showProgress = [&]() { + act.progress(nrDone, missing.size(), nrRunning, nrFailed); + }; - auto showProgress = [&]() { - act.progress(nrDone, missing.size(), nrRunning, nrFailed); - }; + ThreadPool pool; - ThreadPool pool; + processGraph<Path>( + pool, PathSet(missing.begin(), missing.end()), - processGraph<Path>(pool, - PathSet(missing.begin(), missing.end()), + [&](const Path& storePath) { + if (dstStore->isValidPath(storePath)) { + nrDone++; + showProgress(); + return PathSet(); + } - [&](const Path & storePath) { - if (dstStore->isValidPath(storePath)) { - nrDone++; - showProgress(); - return PathSet(); - } + auto info = srcStore->queryPathInfo(storePath); - auto info = srcStore->queryPathInfo(storePath); - - bytesExpected += info->narSize; - act.setExpected(actCopyPath, bytesExpected); - - return info->references; - }, - - [&](const Path & storePath) { - checkInterrupt(); - - if (!dstStore->isValidPath(storePath)) { - MaintainCount<decltype(nrRunning)> mc(nrRunning); - showProgress(); - try { - copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); - } catch (Error &e) { - nrFailed++; - if (!settings.keepGoing) - throw e; - logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what()); - showProgress(); - return; - } - } + bytesExpected += info->narSize; + act.setExpected(actCopyPath, bytesExpected); + + return info->references; + }, + + [&](const Path& storePath) { + checkInterrupt(); - nrDone++; + if (!dstStore->isValidPath(storePath)) { + MaintainCount<decltype(nrRunning)> mc(nrRunning); + showProgress(); + try { + copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + } catch (Error& e) { + nrFailed++; + if (!settings.keepGoing) throw e; + logger->log(lvlError, + format("could not copy %s: %s") % storePath % e.what()); showProgress(); - }); -} + return; + } + } + nrDone++; + showProgress(); + }); +} void copyClosure(ref<Store> srcStore, ref<Store> dstStore, - const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, - SubstituteFlag substitute) -{ - PathSet closure; - srcStore->computeFSClosure({storePaths}, closure); - copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); -} - - -ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) -{ - ValidPathInfo info; - getline(str, info.path); - if (str.eof()) { info.path = ""; return info; } - if (hashGiven) { - string s; - getline(str, s); - info.narHash = Hash(s, htSHA256); - getline(str, s); - if (!string2Int(s, info.narSize)) throw Error("number expected"); - } - getline(str, info.deriver); - string s; int n; - getline(str, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { - getline(str, s); - info.references.insert(s); - } - if (!str || str.eof()) throw Error("missing input"); + const PathSet& storePaths, RepairFlag repair, + CheckSigsFlag checkSigs, SubstituteFlag substitute) { + PathSet closure; + srcStore->computeFSClosure({storePaths}, closure); + copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); +} + +ValidPathInfo decodeValidPathInfo(std::istream& str, bool hashGiven) { + ValidPathInfo info; + getline(str, info.path); + if (str.eof()) { + info.path = ""; return info; -} - - -string showPaths(const PathSet & paths) -{ + } + if (hashGiven) { string s; - for (auto & i : paths) { - if (s.size() != 0) s += ", "; - s += "'" + i + "'"; - } - return s; + getline(str, s); + info.narHash = Hash(s, htSHA256); + getline(str, s); + if (!string2Int(s, info.narSize)) throw Error("number expected"); + } + getline(str, info.deriver); + string s; + int n; + getline(str, s); + if (!string2Int(s, n)) throw Error("number expected"); + while (n--) { + getline(str, s); + info.references.insert(s); + } + if (!str || str.eof()) throw Error("missing input"); + return info; } - -std::string ValidPathInfo::fingerprint() const -{ - if (narSize == 0 || !narHash) - throw Error(format("cannot calculate fingerprint of path '%s' because its size/hash is not known") - % path); - return - "1;" + path + ";" - + narHash.to_string(Base32) + ";" - + std::to_string(narSize) + ";" - + concatStringsSep(",", references); +string showPaths(const PathSet& paths) { + string s; + for (auto& i : paths) { + if (s.size() != 0) s += ", "; + s += "'" + i + "'"; + } + return s; } - -void ValidPathInfo::sign(const SecretKey & secretKey) -{ - sigs.insert(secretKey.signDetached(fingerprint())); +std::string ValidPathInfo::fingerprint() const { + if (narSize == 0 || !narHash) + throw Error(format("cannot calculate fingerprint of path '%s' because its " + "size/hash is not known") % + path); + return "1;" + path + ";" + narHash.to_string(Base32) + ";" + + std::to_string(narSize) + ";" + concatStringsSep(",", references); } - -bool ValidPathInfo::isContentAddressed(const Store & store) const -{ - auto warn = [&]() { - printError(format("warning: path '%s' claims to be content-addressed but isn't") % path); - }; - - if (hasPrefix(ca, "text:")) { - Hash hash(std::string(ca, 5)); - if (store.makeTextPath(storePathToName(path), hash, references) == path) - return true; - else - warn(); - } - - else if (hasPrefix(ca, "fixed:")) { - bool recursive = ca.compare(6, 2, "r:") == 0; - Hash hash(std::string(ca, recursive ? 8 : 6)); - if (references.empty() && - store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) - return true; - else - warn(); - } - - return false; +void ValidPathInfo::sign(const SecretKey& secretKey) { + sigs.insert(secretKey.signDetached(fingerprint())); } +bool ValidPathInfo::isContentAddressed(const Store& store) const { + auto warn = [&]() { + printError( + format("warning: path '%s' claims to be content-addressed but isn't") % + path); + }; -size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const -{ - if (isContentAddressed(store)) return maxSigs; + if (hasPrefix(ca, "text:")) { + Hash hash(std::string(ca, 5)); + if (store.makeTextPath(storePathToName(path), hash, references) == path) + return true; + else + warn(); + } + + else if (hasPrefix(ca, "fixed:")) { + bool recursive = ca.compare(6, 2, "r:") == 0; + Hash hash(std::string(ca, recursive ? 8 : 6)); + if (references.empty() && + store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == + path) + return true; + else + warn(); + } - size_t good = 0; - for (auto & sig : sigs) - if (checkSignature(publicKeys, sig)) - good++; - return good; + return false; } +size_t ValidPathInfo::checkSignatures(const Store& store, + const PublicKeys& publicKeys) const { + if (isContentAddressed(store)) return maxSigs; -bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const -{ - return verifyDetached(fingerprint(), sig, publicKeys); + size_t good = 0; + for (auto& sig : sigs) + if (checkSignature(publicKeys, sig)) good++; + return good; } - -Strings ValidPathInfo::shortRefs() const -{ - Strings refs; - for (auto & r : references) - refs.push_back(baseNameOf(r)); - return refs; +bool ValidPathInfo::checkSignature(const PublicKeys& publicKeys, + const std::string& sig) const { + return verifyDetached(fingerprint(), sig, publicKeys); } - -std::string makeFixedOutputCA(bool recursive, const Hash & hash) -{ - return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string(); +Strings ValidPathInfo::shortRefs() const { + Strings refs; + for (auto& r : references) refs.push_back(baseNameOf(r)); + return refs; } - -void Store::addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) -{ - addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor); +std::string makeFixedOutputCA(bool recursive, const Hash& hash) { + return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string(); } -void Store::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) -{ - StringSource source(*nar); - addToStore(info, source, repair, checkSigs, accessor); +void Store::addToStore(const ValidPathInfo& info, Source& narSource, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, + accessor); } +void Store::addToStore(const ValidPathInfo& info, const ref<std::string>& nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + StringSource source(*nar); + addToStore(info, source, repair, checkSigs, accessor); } +} // namespace nix #include "local-store.hh" #include "remote-store.hh" - namespace nix { - -RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; +RegisterStoreImplementation::Implementations* + RegisterStoreImplementation::implementations = 0; /* Split URI into protocol+hierarchy part and its parameter set. */ -std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_) -{ - auto uri(uri_); - Store::Params params; - auto q = uri.find('?'); - if (q != std::string::npos) { - for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { - auto e = s.find('='); - if (e != std::string::npos) { - auto value = s.substr(e + 1); - std::string decoded; - for (size_t i = 0; i < value.size(); ) { - if (value[i] == '%') { - if (i + 2 >= value.size()) - throw Error("invalid URI parameter '%s'", value); - try { - decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); - i += 3; - } catch (...) { - throw Error("invalid URI parameter '%s'", value); - } - } else - decoded += value[i++]; - } - params[s.substr(0, e)] = decoded; +std::pair<std::string, Store::Params> splitUriAndParams( + const std::string& uri_) { + auto uri(uri_); + Store::Params params; + auto q = uri.find('?'); + if (q != std::string::npos) { + for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { + auto e = s.find('='); + if (e != std::string::npos) { + auto value = s.substr(e + 1); + std::string decoded; + for (size_t i = 0; i < value.size();) { + if (value[i] == '%') { + if (i + 2 >= value.size()) + throw Error("invalid URI parameter '%s'", value); + try { + decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); + i += 3; + } catch (...) { + throw Error("invalid URI parameter '%s'", value); } + } else + decoded += value[i++]; } - uri = uri_.substr(0, q); + params[s.substr(0, e)] = decoded; + } } - return {uri, params}; -} - -ref<Store> openStore(const std::string & uri_, - const Store::Params & extraParams) -{ - auto [uri, uriParams] = splitUriAndParams(uri_); - auto params = extraParams; - params.insert(uriParams.begin(), uriParams.end()); - - for (auto fun : *RegisterStoreImplementation::implementations) { - auto store = fun(uri, params); - if (store) { - store->warnUnknownSettings(); - return ref<Store>(store); - } + uri = uri_.substr(0, q); + } + return {uri, params}; +} + +ref<Store> openStore(const std::string& uri_, + const Store::Params& extraParams) { + auto [uri, uriParams] = splitUriAndParams(uri_); + auto params = extraParams; + params.insert(uriParams.begin(), uriParams.end()); + + for (auto fun : *RegisterStoreImplementation::implementations) { + auto store = fun(uri, params); + if (store) { + store->warnUnknownSettings(); + return ref<Store>(store); } + } - throw Error("don't know how to open Nix store '%s'", uri); -} - - -StoreType getStoreType(const std::string & uri, const std::string & stateDir) -{ - if (uri == "daemon") { - return tDaemon; - } else if (uri == "local" || hasPrefix(uri, "/")) { - return tLocal; - } else if (uri == "" || uri == "auto") { - if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return tLocal; - else if (pathExists(settings.nixDaemonSocketFile)) - return tDaemon; - else - return tLocal; - } else { - return tOther; - } + throw Error("don't know how to open Nix store '%s'", uri); } - -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { - case tDaemon: - return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); - case tLocal: { - Store::Params params2 = params; - if (hasPrefix(uri, "/")) - params2["root"] = uri; - return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); - } - default: - return nullptr; +StoreType getStoreType(const std::string& uri, const std::string& stateDir) { + if (uri == "daemon") { + return tDaemon; + } else if (uri == "local" || hasPrefix(uri, "/")) { + return tLocal; + } else if (uri == "" || uri == "auto") { + if (access(stateDir.c_str(), R_OK | W_OK) == 0) + return tLocal; + else if (pathExists(settings.nixDaemonSocketFile)) + return tDaemon; + else + return tLocal; + } else { + return tOther; + } +} + +static RegisterStoreImplementation regStore([](const std::string& uri, + const Store::Params& params) + -> std::shared_ptr<Store> { + switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { + case tDaemon: + return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); + case tLocal: { + Store::Params params2 = params; + if (hasPrefix(uri, "/")) params2["root"] = uri; + return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); } + default: + return nullptr; + } }); +std::list<ref<Store>> getDefaultSubstituters() { + static auto stores([]() { + std::list<ref<Store>> stores; -std::list<ref<Store>> getDefaultSubstituters() -{ - static auto stores([]() { - std::list<ref<Store>> stores; - - StringSet done; + StringSet done; - auto addStore = [&](const std::string & uri) { - if (done.count(uri)) return; - done.insert(uri); - try { - stores.push_back(openStore(uri)); - } catch (Error & e) { - printError("warning: %s", e.what()); - } - }; + auto addStore = [&](const std::string& uri) { + if (done.count(uri)) return; + done.insert(uri); + try { + stores.push_back(openStore(uri)); + } catch (Error& e) { + printError("warning: %s", e.what()); + } + }; - for (auto uri : settings.substituters.get()) - addStore(uri); + for (auto uri : settings.substituters.get()) addStore(uri); - for (auto uri : settings.extraSubstituters.get()) - addStore(uri); + for (auto uri : settings.extraSubstituters.get()) addStore(uri); - stores.sort([](ref<Store> & a, ref<Store> & b) { - return a->getPriority() < b->getPriority(); - }); - - return stores; - } ()); + stores.sort([](ref<Store>& a, ref<Store>& b) { + return a->getPriority() < b->getPriority(); + }); return stores; -} - + }()); + return stores; } + +} // namespace nix diff --git a/third_party/nix/src/libstore/store-api.hh b/third_party/nix/src/libstore/store-api.hh index a751b4662a79..a1106dbfeb53 100644 --- a/third_party/nix/src/libstore/store-api.hh +++ b/third_party/nix/src/libstore/store-api.hh @@ -1,730 +1,700 @@ #pragma once -#include "hash.hh" -#include "serialise.hh" -#include "crypto.hh" -#include "lru-cache.hh" -#include "sync.hh" -#include "globals.hh" -#include "config.hh" - #include <atomic> #include <limits> #include <map> -#include <unordered_map> -#include <unordered_set> #include <memory> #include <string> - +#include <unordered_map> +#include <unordered_set> +#include "config.hh" +#include "crypto.hh" +#include "globals.hh" +#include "hash.hh" +#include "lru-cache.hh" +#include "serialise.hh" +#include "sync.hh" namespace nix { - MakeError(SubstError, Error) -MakeError(BuildError, Error) /* denotes a permanent build failure */ -MakeError(InvalidPath, Error) -MakeError(Unsupported, Error) -MakeError(SubstituteGone, Error) -MakeError(SubstituterDisabled, Error) + MakeError(BuildError, Error) /* denotes a permanent build failure */ + MakeError(InvalidPath, Error) MakeError(Unsupported, Error) + MakeError(SubstituteGone, Error) MakeError(SubstituterDisabled, Error) - -struct BasicDerivation; + struct BasicDerivation; struct Derivation; class FSAccessor; class NarInfoDiskCache; class Store; class JSONPlaceholder; - enum RepairFlag : bool { NoRepair = false, Repair = true }; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; - /* Size of the hash part of store paths, in base-32 characters. */ -const size_t storePathHashLen = 32; // i.e. 160 bits +const size_t storePathHashLen = 32; // i.e. 160 bits /* Magic header of exportPath() output (obsolete). */ const uint32_t exportMagic = 0x4558494e; - typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots; +struct GCOptions { + /* Garbage collector operation: -struct GCOptions -{ - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. + - `gcReturnLive': return the set of paths reachable from + (i.e. in the closure of) the roots. - - `gcReturnDead': return the set of paths not reachable from - the roots. + - `gcReturnDead': return the set of paths not reachable from + the roots. - - `gcDeleteDead': actually delete the latter set. + - `gcDeleteDead': actually delete the latter set. - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ - typedef enum { - gcReturnLive, - gcReturnDead, - gcDeleteDead, - gcDeleteSpecific, - } GCAction; + - `gcDeleteSpecific': delete the paths listed in + `pathsToDelete', insofar as they are not reachable. + */ + typedef enum { + gcReturnLive, + gcReturnDead, + gcDeleteDead, + gcDeleteSpecific, + } GCAction; - GCAction action{gcDeleteDead}; + GCAction action{gcDeleteDead}; - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ - bool ignoreLiveness{false}; + /* If `ignoreLiveness' is set, then reachability from the roots is + ignored (dangerous!). However, the paths must still be + unreferenced *within* the store (i.e., there can be no other + store paths that depend on them). */ + bool ignoreLiveness{false}; - /* For `gcDeleteSpecific', the paths to delete. */ - PathSet pathsToDelete; + /* For `gcDeleteSpecific', the paths to delete. */ + PathSet pathsToDelete; - /* Stop after at least `maxFreed' bytes have been freed. */ - unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()}; + /* Stop after at least `maxFreed' bytes have been freed. */ + unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()}; }; +struct GCResults { + /* Depending on the action, the GC roots, or the paths that would + be or have been deleted. */ + PathSet paths; -struct GCResults -{ - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ - PathSet paths; - - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ - unsigned long long bytesFreed = 0; + /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the + number of bytes that would be or was freed. */ + unsigned long long bytesFreed = 0; }; - -struct SubstitutablePathInfo -{ - Path deriver; - PathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ +struct SubstitutablePathInfo { + Path deriver; + PathSet references; + unsigned long long downloadSize; /* 0 = unknown or inapplicable */ + unsigned long long narSize; /* 0 = unknown */ }; typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos; - -struct ValidPathInfo -{ - Path path; - Path deriver; - Hash narHash; - PathSet references; - time_t registrationTime = 0; - uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only - - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ - bool ultimate = false; - - StringSet sigs; // note: not necessarily verified - - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a - Boolean, and the store path would be computed from - ‘storePathToName(path)’, ‘narHash’ and ‘references’. However, - 1) we've accumulated several types of content-addressed paths - over the years; and 2) fixed-output derivations support - multiple hash algorithms and serialisation methods (flat file - vs NAR). Thus, ‘ca’ has one of the following forms: - - * ‘text:sha256:<sha256 hash of file contents>’: For paths - computed by makeTextPath() / addTextToStore(). - - * ‘fixed:<r?>:<ht>:<h>’: For paths computed by - makeFixedOutputPath() / addToStore(). - */ - std::string ca; - - bool operator == (const ValidPathInfo & i) const - { - return - path == i.path - && narHash == i.narHash - && references == i.references; - } - - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ - std::string fingerprint() const; - - void sign(const SecretKey & secretKey); - - /* Return true iff the path is verifiably content-addressed. */ - bool isContentAddressed(const Store & store) const; - - static const size_t maxSigs = std::numeric_limits<size_t>::max(); - - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ - size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - - /* Verify a single signature. */ - bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; - - Strings shortRefs() const; - - virtual ~ValidPathInfo() { } +struct ValidPathInfo { + Path path; + Path deriver; + Hash narHash; + PathSet references; + time_t registrationTime = 0; + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only + + /* Whether the path is ultimately trusted, that is, it's a + derivation output that was built locally. */ + bool ultimate = false; + + StringSet sigs; // note: not necessarily verified + + /* If non-empty, an assertion that the path is content-addressed, + i.e., that the store path is computed from a cryptographic hash + of the contents of the path, plus some other bits of data like + the "name" part of the path. Such a path doesn't need + signatures, since we don't have to trust anybody's claim that + the path is the output of a particular derivation. (In the + extensional store model, we have to trust that the *contents* + of an output path of a derivation were actually produced by + that derivation. In the intensional model, we have to trust + that a particular output path was produced by a derivation; the + path then implies the contents.) + + Ideally, the content-addressability assertion would just be a + Boolean, and the store path would be computed from + ‘storePathToName(path)’, ‘narHash’ and ‘references’. However, + 1) we've accumulated several types of content-addressed paths + over the years; and 2) fixed-output derivations support + multiple hash algorithms and serialisation methods (flat file + vs NAR). Thus, ‘ca’ has one of the following forms: + + * ‘text:sha256:<sha256 hash of file contents>’: For paths + computed by makeTextPath() / addTextToStore(). + + * ‘fixed:<r?>:<ht>:<h>’: For paths computed by + makeFixedOutputPath() / addToStore(). + */ + std::string ca; + + bool operator==(const ValidPathInfo& i) const { + return path == i.path && narHash == i.narHash && references == i.references; + } + + /* Return a fingerprint of the store path to be used in binary + cache signatures. It contains the store path, the base-32 + SHA-256 hash of the NAR serialisation of the path, the size of + the NAR, and the sorted references. The size field is strictly + speaking superfluous, but might prevent endless/excessive data + attacks. */ + std::string fingerprint() const; + + void sign(const SecretKey& secretKey); + + /* Return true iff the path is verifiably content-addressed. */ + bool isContentAddressed(const Store& store) const; + + static const size_t maxSigs = std::numeric_limits<size_t>::max(); + + /* Return the number of signatures on this .narinfo that were + produced by one of the specified keys, or maxSigs if the path + is content-addressed. */ + size_t checkSignatures(const Store& store, + const PublicKeys& publicKeys) const; + + /* Verify a single signature. */ + bool checkSignature(const PublicKeys& publicKeys, + const std::string& sig) const; + + Strings shortRefs() const; + + virtual ~ValidPathInfo() {} }; typedef list<ValidPathInfo> ValidPathInfos; - enum BuildMode { bmNormal, bmRepair, bmCheck }; - -struct BuildResult -{ - /* Note: don't remove status codes, and only add new status codes - at the end of the list, to prevent client/server - incompatibilities in the nix-store --serve protocol. */ - enum Status { - Built = 0, - Substituted, - AlreadyValid, - PermanentFailure, - InputRejected, - OutputRejected, - TransientFailure, // possibly transient - CachedFailure, // no longer used - TimedOut, - MiscFailure, - DependencyFailed, - LogLimitExceeded, - NotDeterministic, - } status = MiscFailure; - std::string errorMsg; - - /* How many times this build was performed. */ - unsigned int timesBuilt = 0; - - /* If timesBuilt > 1, whether some builds did not produce the same - result. (Note that 'isNonDeterministic = false' does not mean - the build is deterministic, just that we don't have evidence of - non-determinism.) */ - bool isNonDeterministic = false; - - /* The start/stop times of the build (or one of the rounds, if it - was repeated). */ - time_t startTime = 0, stopTime = 0; - - bool success() { - return status == Built || status == Substituted || status == AlreadyValid; - } +struct BuildResult { + /* Note: don't remove status codes, and only add new status codes + at the end of the list, to prevent client/server + incompatibilities in the nix-store --serve protocol. */ + enum Status { + Built = 0, + Substituted, + AlreadyValid, + PermanentFailure, + InputRejected, + OutputRejected, + TransientFailure, // possibly transient + CachedFailure, // no longer used + TimedOut, + MiscFailure, + DependencyFailed, + LogLimitExceeded, + NotDeterministic, + } status = MiscFailure; + std::string errorMsg; + + /* How many times this build was performed. */ + unsigned int timesBuilt = 0; + + /* If timesBuilt > 1, whether some builds did not produce the same + result. (Note that 'isNonDeterministic = false' does not mean + the build is deterministic, just that we don't have evidence of + non-determinism.) */ + bool isNonDeterministic = false; + + /* The start/stop times of the build (or one of the rounds, if it + was repeated). */ + time_t startTime = 0, stopTime = 0; + + bool success() { + return status == Built || status == Substituted || status == AlreadyValid; + } }; +class Store : public std::enable_shared_from_this<Store>, public Config { + public: + typedef std::map<std::string, std::string> Params; -class Store : public std::enable_shared_from_this<Store>, public Config -{ -public: - - typedef std::map<std::string, std::string> Params; - - const PathSetting storeDir_{this, false, settings.nixStore, - "store", "path to the Nix store"}; - const Path storeDir = storeDir_; - - const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; - - const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; - -protected: - - struct State - { - LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache; - }; - - Sync<State> state; - - std::shared_ptr<NarInfoDiskCache> diskCache; - - Store(const Params & params); - -public: + const PathSetting storeDir_{this, false, settings.nixStore, "store", + "path to the Nix store"}; + const Path storeDir = storeDir_; - virtual ~Store() { } + const Setting<int> pathInfoCacheSize{ + this, 65536, "path-info-cache-size", + "size of the in-memory store path information cache"}; - virtual std::string getUri() = 0; + const Setting<bool> isTrusted{ + this, false, "trusted", + "whether paths from this store can be used as substitutes even when they " + "lack trusted signatures"}; - /* Return true if ‘path’ is in the Nix store (but not the Nix - store itself). */ - bool isInStore(const Path & path) const; + protected: + struct State { + LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache; + }; - /* Return true if ‘path’ is a store path, i.e. a direct child of - the Nix store. */ - bool isStorePath(const Path & path) const; + Sync<State> state; - /* Throw an exception if ‘path’ is not a store path. */ - void assertStorePath(const Path & path) const; + std::shared_ptr<NarInfoDiskCache> diskCache; - /* Chop off the parts after the top-level store name, e.g., - /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ - Path toStorePath(const Path & path) const; + Store(const Params& params); - /* Follow symlinks until we end up with a path in the Nix store. */ - Path followLinksToStore(const Path & path) const; - - /* Same as followLinksToStore(), but apply toStorePath() to the - result. */ - Path followLinksToStorePath(const Path & path) const; - - /* Constructs a unique store path name. */ - Path makeStorePath(const string & type, - const Hash & hash, const string & name) const; - - Path makeOutputPath(const string & id, - const Hash & hash, const string & name) const; - - Path makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name) const; - - Path makeTextPath(const string & name, const Hash & hash, - const PathSet & references) const; - - /* This is the preparatory part of addToStore(); it computes the - store path to which srcPath is to be copied. Returns the store - path and the cryptographic hash of the contents of srcPath. */ - std::pair<Path, Hash> computeStorePathForPath(const string & name, - const Path & srcPath, bool recursive = true, - HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; - - /* Preparatory part of addTextToStore(). - - !!! Computation of the path should take the references given to - addTextToStore() into account, otherwise we have a (relatively - minor) security hole: a caller can register a source file with - bogus references. If there are too many references, the path may - not be garbage collected when it has to be (not really a problem, - the caller could create a root anyway), or it may be garbage - collected when it shouldn't be (more serious). - - Hashing the references would solve this (bogus references would - simply yield a different store path, so other users wouldn't be - affected), but it has some backwards compatibility issues (the - hashing scheme changes), so I'm not doing that for now. */ - Path computeStorePathForText(const string & name, const string & s, - const PathSet & references) const; - - /* Check whether a path is valid. */ - bool isValidPath(const Path & path); - -protected: - - virtual bool isValidPathUncached(const Path & path); - -public: - - /* Query which of the given paths is valid. Optionally, try to - substitute missing paths. */ - virtual PathSet queryValidPaths(const PathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute); - - /* Query the set of all valid paths. Note that for some store - backends, the name part of store paths may be omitted - (i.e. you'll get /nix/store/<hash> rather than - /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the - full store path. */ - virtual PathSet queryAllValidPaths() - { unsupported("queryAllValidPaths"); } - - /* Query information about a valid path. It is permitted to omit - the name part of the store path. */ - ref<const ValidPathInfo> queryPathInfo(const Path & path); - - /* Asynchronous version of queryPathInfo(). */ - void queryPathInfo(const Path & path, - Callback<ref<ValidPathInfo>> callback) noexcept; - -protected: - - virtual void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0; - -public: - - /* Queries the set of incoming FS references for a store path. - The result is not cleared. */ - virtual void queryReferrers(const Path & path, PathSet & referrers) - { unsupported("queryReferrers"); } - - /* Return all currently valid derivations that have `path' as an - output. (Note that the result of `queryDeriver()' is the - derivation that was actually used to produce `path', which may - not exist anymore.) */ - virtual PathSet queryValidDerivers(const Path & path) { return {}; }; - - /* Query the outputs of the derivation denoted by `path'. */ - virtual PathSet queryDerivationOutputs(const Path & path) - { unsupported("queryDerivationOutputs"); } - - /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const Path & path) - { unsupported("queryDerivationOutputNames"); } - - /* Query the full store path given the hash part of a valid store - path, or "" if the path doesn't exist. */ - virtual Path queryPathFromHashPart(const string & hashPart) = 0; - - /* Query which of the given paths have substitutes. */ - virtual PathSet querySubstitutablePaths(const PathSet & paths) { return {}; }; - - /* Query substitute info (i.e. references, derivers and download - sizes) of a set of paths. If a path does not have substitute - info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) { return; }; - - virtual bool wantMassQuery() { return false; } - - /* Import a path into the store. */ - virtual void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0); - - // FIXME: remove - virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0); - - /* Copy the contents of a path to the store and register the - validity the resulting path. The resulting path is returned. - The function object `filter' can be used to exclude files (see - libutil/archive.hh). */ - virtual Path addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; - - /* Like addToStore, but the contents written to the output path is - a regular file containing the given string. */ - virtual Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair = NoRepair) = 0; - - /* Write a NAR dump of a store path. */ - virtual void narFromPath(const Path & path, Sink & sink) = 0; - - /* For each path, if it's a derivation, build it. Building a - derivation means ensuring that the output paths are valid. If - they are already valid, this is a no-op. Otherwise, validity - can be reached in two ways. First, if the output paths is - substitutable, then build the path that way. Second, the - output paths can be created by running the builder, after - recursively building any sub-derivations. For inputs that are - not derivations, substitute them. */ - virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal); - - /* Build a single non-materialized derivation (i.e. not from an - on-disk .drv file). Note that ‘drvPath’ is only used for - informational purposes. */ - virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) = 0; - - /* Ensure that a path is valid. If it is not currently valid, it - may be made valid by running a substitute (if defined for the - path). */ - virtual void ensurePath(const Path & path) = 0; - - /* Add a store path as a temporary root of the garbage collector. - The root disappears as soon as we exit. */ - virtual void addTempRoot(const Path & path) - { unsupported("addTempRoot"); } - - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ - virtual void addIndirectRoot(const Path & path) - { unsupported("addIndirectRoot"); } - - /* Acquire the global GC lock, then immediately release it. This - function must be called after registering a new permanent root, - but before exiting. Otherwise, it is possible that a running - garbage collector doesn't see the new root and deletes the - stuff we've just built. By acquiring the lock briefly, we - ensure that either: - - - The collector is already running, and so we block until the - collector is finished. The collector will know about our - *temporary* locks, which should include whatever it is we - want to register as a permanent lock. - - - The collector isn't running, or it's just started but hasn't - acquired the GC lock yet. In that case we get and release - the lock right away, then exit. The collector scans the - permanent root and sees our's. - - In either case the permanent root is seen by the collector. */ - virtual void syncWithGC() { }; - - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. If - 'censor' is true, privacy-sensitive information about roots - found in /proc is censored. */ - virtual Roots findRoots(bool censor) - { unsupported("findRoots"); } - - /* Perform a garbage collection. */ - virtual void collectGarbage(const GCOptions & options, GCResults & results) - { unsupported("collectGarbage"); } - - /* Return a string representing information about the path that - can be loaded into the database using `nix-store --load-db' or - `nix-store --register-validity'. */ - string makeValidityRegistration(const PathSet & paths, - bool showDerivers, bool showHash); - - /* Write a JSON representation of store path metadata, such as the - hash and the references. If ‘includeImpureInfo’ is true, - variable elements such as the registration time are - included. If ‘showClosureSize’ is true, the closure size of - each path is included. */ - void pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - AllowInvalidFlag allowInvalid = DisallowInvalid); - - /* Return the size of the closure of the specified path, that is, - the sum of the size of the NAR serialisation of each path in - the closure. */ - std::pair<uint64_t, uint64_t> getClosureSize(const Path & storePath); - - /* Optimise the disk space usage of the Nix store by hard-linking files - with the same contents. */ - virtual void optimiseStore() { }; - - /* Check the integrity of the Nix store. Returns true if errors - remain. */ - virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; - - /* Return an object to access files in the Nix store. */ - virtual ref<FSAccessor> getFSAccessor() - { unsupported("getFSAccessor"); } - - /* Add signatures to the specified store path. The signatures are - not verified. */ - virtual void addSignatures(const Path & storePath, const StringSet & sigs) - { unsupported("addSignatures"); } - - /* Utility functions. */ - - /* Read a derivation, after ensuring its existence through - ensurePath(). */ - Derivation derivationFromPath(const Path & drvPath); - - /* Place in `out' the set of all store paths in the file system - closure of `storePath'; that is, all paths than can be directly - or indirectly reached from it. `out' is not cleared. If - `flipDirection' is true, the set of paths that can reach - `storePath' is returned; that is, the closures under the - `referrers' relation instead of the `references' relation is - returned. */ - virtual void computeFSClosure(const PathSet & paths, - PathSet & out, bool flipDirection = false, - bool includeOutputs = false, bool includeDerivers = false); - - void computeFSClosure(const Path & path, - PathSet & out, bool flipDirection = false, - bool includeOutputs = false, bool includeDerivers = false); - - /* Given a set of paths that are to be built, return the set of - derivations that will be built, and the set of output paths - that will be substituted. */ - virtual void queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize); - - /* Sort a set of paths topologically under the references - relation. If p refers to q, then p precedes q in this list. */ - Paths topoSortPaths(const PathSet & paths); - - /* Export multiple paths in the format expected by ‘nix-store - --import’. */ - void exportPaths(const Paths & paths, Sink & sink); - - void exportPath(const Path & path, Sink & sink); - - /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. Optionally, the contents of the NARs are - preloaded into the specified FS accessor to speed up subsequent - access. */ - Paths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, - CheckSigsFlag checkSigs = CheckSigs); - - struct Stats - { - std::atomic<uint64_t> narInfoRead{0}; - std::atomic<uint64_t> narInfoReadAverted{0}; - std::atomic<uint64_t> narInfoMissing{0}; - std::atomic<uint64_t> narInfoWrite{0}; - std::atomic<uint64_t> pathInfoCacheSize{0}; - std::atomic<uint64_t> narRead{0}; - std::atomic<uint64_t> narReadBytes{0}; - std::atomic<uint64_t> narReadCompressedBytes{0}; - std::atomic<uint64_t> narWrite{0}; - std::atomic<uint64_t> narWriteAverted{0}; - std::atomic<uint64_t> narWriteBytes{0}; - std::atomic<uint64_t> narWriteCompressedBytes{0}; - std::atomic<uint64_t> narWriteCompressionTimeMs{0}; - }; - - const Stats & getStats(); - - /* Return the build log of the specified store path, if available, - or null otherwise. */ - virtual std::shared_ptr<std::string> getBuildLog(const Path & path) - { return nullptr; } - - /* Hack to allow long-running processes like hydra-queue-runner to - occasionally flush their path info cache. */ - void clearPathInfoCache() - { - state.lock()->pathInfoCache.clear(); - } - - /* Establish a connection to the store, for store types that have - a notion of connection. Otherwise this is a no-op. */ - virtual void connect() { }; - - /* Get the protocol version of this store or it's connection. */ - virtual unsigned int getProtocol() - { - return 0; - }; - - /* Get the priority of the store, used to order substituters. In - particular, binary caches can specify a priority field in their - "nix-cache-info" file. Lower value means higher priority. */ - virtual int getPriority() { return 0; } - - virtual Path toRealPath(const Path & storePath) - { - return storePath; - } - - virtual void createUser(const std::string & userName, uid_t userId) - { } - -protected: - - Stats stats; - - /* Unsupported methods. */ - [[noreturn]] void unsupported(const std::string & op) - { - throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri()); - } + public: + virtual ~Store() {} + + virtual std::string getUri() = 0; + /* Return true if ‘path’ is in the Nix store (but not the Nix + store itself). */ + bool isInStore(const Path& path) const; + + /* Return true if ‘path’ is a store path, i.e. a direct child of + the Nix store. */ + bool isStorePath(const Path& path) const; + + /* Throw an exception if ‘path’ is not a store path. */ + void assertStorePath(const Path& path) const; + + /* Chop off the parts after the top-level store name, e.g., + /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ + Path toStorePath(const Path& path) const; + + /* Follow symlinks until we end up with a path in the Nix store. */ + Path followLinksToStore(const Path& path) const; + + /* Same as followLinksToStore(), but apply toStorePath() to the + result. */ + Path followLinksToStorePath(const Path& path) const; + + /* Constructs a unique store path name. */ + Path makeStorePath(const string& type, const Hash& hash, + const string& name) const; + + Path makeOutputPath(const string& id, const Hash& hash, + const string& name) const; + + Path makeFixedOutputPath(bool recursive, const Hash& hash, + const string& name) const; + + Path makeTextPath(const string& name, const Hash& hash, + const PathSet& references) const; + + /* This is the preparatory part of addToStore(); it computes the + store path to which srcPath is to be copied. Returns the store + path and the cryptographic hash of the contents of srcPath. */ + std::pair<Path, Hash> computeStorePathForPath( + const string& name, const Path& srcPath, bool recursive = true, + HashType hashAlgo = htSHA256, + PathFilter& filter = defaultPathFilter) const; + + /* Preparatory part of addTextToStore(). + + !!! Computation of the path should take the references given to + addTextToStore() into account, otherwise we have a (relatively + minor) security hole: a caller can register a source file with + bogus references. If there are too many references, the path may + not be garbage collected when it has to be (not really a problem, + the caller could create a root anyway), or it may be garbage + collected when it shouldn't be (more serious). + + Hashing the references would solve this (bogus references would + simply yield a different store path, so other users wouldn't be + affected), but it has some backwards compatibility issues (the + hashing scheme changes), so I'm not doing that for now. */ + Path computeStorePathForText(const string& name, const string& s, + const PathSet& references) const; + + /* Check whether a path is valid. */ + bool isValidPath(const Path& path); + + protected: + virtual bool isValidPathUncached(const Path& path); + + public: + /* Query which of the given paths is valid. Optionally, try to + substitute missing paths. */ + virtual PathSet queryValidPaths( + const PathSet& paths, SubstituteFlag maybeSubstitute = NoSubstitute); + + /* Query the set of all valid paths. Note that for some store + backends, the name part of store paths may be omitted + (i.e. you'll get /nix/store/<hash> rather than + /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the + full store path. */ + virtual PathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } + + /* Query information about a valid path. It is permitted to omit + the name part of the store path. */ + ref<const ValidPathInfo> queryPathInfo(const Path& path); + + /* Asynchronous version of queryPathInfo(). */ + void queryPathInfo(const Path& path, + Callback<ref<ValidPathInfo>> callback) noexcept; + + protected: + virtual void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0; + + public: + /* Queries the set of incoming FS references for a store path. + The result is not cleared. */ + virtual void queryReferrers(const Path& path, PathSet& referrers) { + unsupported("queryReferrers"); + } + + /* Return all currently valid derivations that have `path' as an + output. (Note that the result of `queryDeriver()' is the + derivation that was actually used to produce `path', which may + not exist anymore.) */ + virtual PathSet queryValidDerivers(const Path& path) { return {}; }; + + /* Query the outputs of the derivation denoted by `path'. */ + virtual PathSet queryDerivationOutputs(const Path& path) { + unsupported("queryDerivationOutputs"); + } + + /* Query the output names of the derivation denoted by `path'. */ + virtual StringSet queryDerivationOutputNames(const Path& path) { + unsupported("queryDerivationOutputNames"); + } + + /* Query the full store path given the hash part of a valid store + path, or "" if the path doesn't exist. */ + virtual Path queryPathFromHashPart(const string& hashPart) = 0; + + /* Query which of the given paths have substitutes. */ + virtual PathSet querySubstitutablePaths(const PathSet& paths) { return {}; }; + + /* Query substitute info (i.e. references, derivers and download + sizes) of a set of paths. If a path does not have substitute + info, it's omitted from the resulting ‘infos’ map. */ + virtual void querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) { + return; + }; + + virtual bool wantMassQuery() { return false; } + + /* Import a path into the store. */ + virtual void addToStore(const ValidPathInfo& info, Source& narSource, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr<FSAccessor> accessor = 0); + + // FIXME: remove + virtual void addToStore(const ValidPathInfo& info, + const ref<std::string>& nar, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr<FSAccessor> accessor = 0); + + /* Copy the contents of a path to the store and register the + validity the resulting path. The resulting path is returned. + The function object `filter' can be used to exclude files (see + libutil/archive.hh). */ + virtual Path addToStore(const string& name, const Path& srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter& filter = defaultPathFilter, + RepairFlag repair = NoRepair) = 0; + + /* Like addToStore, but the contents written to the output path is + a regular file containing the given string. */ + virtual Path addTextToStore(const string& name, const string& s, + const PathSet& references, + RepairFlag repair = NoRepair) = 0; + + /* Write a NAR dump of a store path. */ + virtual void narFromPath(const Path& path, Sink& sink) = 0; + + /* For each path, if it's a derivation, build it. Building a + derivation means ensuring that the output paths are valid. If + they are already valid, this is a no-op. Otherwise, validity + can be reached in two ways. First, if the output paths is + substitutable, then build the path that way. Second, the + output paths can be created by running the builder, after + recursively building any sub-derivations. For inputs that are + not derivations, substitute them. */ + virtual void buildPaths(const PathSet& paths, BuildMode buildMode = bmNormal); + + /* Build a single non-materialized derivation (i.e. not from an + on-disk .drv file). Note that ‘drvPath’ is only used for + informational purposes. */ + virtual BuildResult buildDerivation(const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode = bmNormal) = 0; + + /* Ensure that a path is valid. If it is not currently valid, it + may be made valid by running a substitute (if defined for the + path). */ + virtual void ensurePath(const Path& path) = 0; + + /* Add a store path as a temporary root of the garbage collector. + The root disappears as soon as we exit. */ + virtual void addTempRoot(const Path& path) { unsupported("addTempRoot"); } + + /* Add an indirect root, which is merely a symlink to `path' from + /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed + to be a symlink to a store path. The garbage collector will + automatically remove the indirect root when it finds that + `path' has disappeared. */ + virtual void addIndirectRoot(const Path& path) { + unsupported("addIndirectRoot"); + } + + /* Acquire the global GC lock, then immediately release it. This + function must be called after registering a new permanent root, + but before exiting. Otherwise, it is possible that a running + garbage collector doesn't see the new root and deletes the + stuff we've just built. By acquiring the lock briefly, we + ensure that either: + + - The collector is already running, and so we block until the + collector is finished. The collector will know about our + *temporary* locks, which should include whatever it is we + want to register as a permanent lock. + + - The collector isn't running, or it's just started but hasn't + acquired the GC lock yet. In that case we get and release + the lock right away, then exit. The collector scans the + permanent root and sees our's. + + In either case the permanent root is seen by the collector. */ + virtual void syncWithGC(){}; + + /* Find the roots of the garbage collector. Each root is a pair + (link, storepath) where `link' is the path of the symlink + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) { unsupported("findRoots"); } + + /* Perform a garbage collection. */ + virtual void collectGarbage(const GCOptions& options, GCResults& results) { + unsupported("collectGarbage"); + } + + /* Return a string representing information about the path that + can be loaded into the database using `nix-store --load-db' or + `nix-store --register-validity'. */ + string makeValidityRegistration(const PathSet& paths, bool showDerivers, + bool showHash); + + /* Write a JSON representation of store path metadata, such as the + hash and the references. If ‘includeImpureInfo’ is true, + variable elements such as the registration time are + included. If ‘showClosureSize’ is true, the closure size of + each path is included. */ + void pathInfoToJSON(JSONPlaceholder& jsonOut, const PathSet& storePaths, + bool includeImpureInfo, bool showClosureSize, + AllowInvalidFlag allowInvalid = DisallowInvalid); + + /* Return the size of the closure of the specified path, that is, + the sum of the size of the NAR serialisation of each path in + the closure. */ + std::pair<uint64_t, uint64_t> getClosureSize(const Path& storePath); + + /* Optimise the disk space usage of the Nix store by hard-linking files + with the same contents. */ + virtual void optimiseStore(){}; + + /* Check the integrity of the Nix store. Returns true if errors + remain. */ + virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { + return false; + }; + + /* Return an object to access files in the Nix store. */ + virtual ref<FSAccessor> getFSAccessor() { unsupported("getFSAccessor"); } + + /* Add signatures to the specified store path. The signatures are + not verified. */ + virtual void addSignatures(const Path& storePath, const StringSet& sigs) { + unsupported("addSignatures"); + } + + /* Utility functions. */ + + /* Read a derivation, after ensuring its existence through + ensurePath(). */ + Derivation derivationFromPath(const Path& drvPath); + + /* Place in `out' the set of all store paths in the file system + closure of `storePath'; that is, all paths than can be directly + or indirectly reached from it. `out' is not cleared. If + `flipDirection' is true, the set of paths that can reach + `storePath' is returned; that is, the closures under the + `referrers' relation instead of the `references' relation is + returned. */ + virtual void computeFSClosure(const PathSet& paths, PathSet& out, + bool flipDirection = false, + bool includeOutputs = false, + bool includeDerivers = false); + + void computeFSClosure(const Path& path, PathSet& out, + bool flipDirection = false, bool includeOutputs = false, + bool includeDerivers = false); + + /* Given a set of paths that are to be built, return the set of + derivations that will be built, and the set of output paths + that will be substituted. */ + virtual void queryMissing(const PathSet& targets, PathSet& willBuild, + PathSet& willSubstitute, PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize); + + /* Sort a set of paths topologically under the references + relation. If p refers to q, then p precedes q in this list. */ + Paths topoSortPaths(const PathSet& paths); + + /* Export multiple paths in the format expected by ‘nix-store + --import’. */ + void exportPaths(const Paths& paths, Sink& sink); + + void exportPath(const Path& path, Sink& sink); + + /* Import a sequence of NAR dumps created by exportPaths() into + the Nix store. Optionally, the contents of the NARs are + preloaded into the specified FS accessor to speed up subsequent + access. */ + Paths importPaths(Source& source, std::shared_ptr<FSAccessor> accessor, + CheckSigsFlag checkSigs = CheckSigs); + + struct Stats { + std::atomic<uint64_t> narInfoRead{0}; + std::atomic<uint64_t> narInfoReadAverted{0}; + std::atomic<uint64_t> narInfoMissing{0}; + std::atomic<uint64_t> narInfoWrite{0}; + std::atomic<uint64_t> pathInfoCacheSize{0}; + std::atomic<uint64_t> narRead{0}; + std::atomic<uint64_t> narReadBytes{0}; + std::atomic<uint64_t> narReadCompressedBytes{0}; + std::atomic<uint64_t> narWrite{0}; + std::atomic<uint64_t> narWriteAverted{0}; + std::atomic<uint64_t> narWriteBytes{0}; + std::atomic<uint64_t> narWriteCompressedBytes{0}; + std::atomic<uint64_t> narWriteCompressionTimeMs{0}; + }; + + const Stats& getStats(); + + /* Return the build log of the specified store path, if available, + or null otherwise. */ + virtual std::shared_ptr<std::string> getBuildLog(const Path& path) { + return nullptr; + } + + /* Hack to allow long-running processes like hydra-queue-runner to + occasionally flush their path info cache. */ + void clearPathInfoCache() { state.lock()->pathInfoCache.clear(); } + + /* Establish a connection to the store, for store types that have + a notion of connection. Otherwise this is a no-op. */ + virtual void connect(){}; + + /* Get the protocol version of this store or it's connection. */ + virtual unsigned int getProtocol() { return 0; }; + + /* Get the priority of the store, used to order substituters. In + particular, binary caches can specify a priority field in their + "nix-cache-info" file. Lower value means higher priority. */ + virtual int getPriority() { return 0; } + + virtual Path toRealPath(const Path& storePath) { return storePath; } + + virtual void createUser(const std::string& userName, uid_t userId) {} + + protected: + Stats stats; + + /* Unsupported methods. */ + [[noreturn]] void unsupported(const std::string& op) { + throw Unsupported("operation '%s' is not supported by store '%s'", op, + getUri()); + } }; - -class LocalFSStore : public virtual Store -{ -public: - - // FIXME: the (Store*) cast works around a bug in gcc that causes - // it to emit the call to the Option constructor. Clang works fine - // either way. - const PathSetting rootDir{(Store*) this, true, "", - "root", "directory prefixed to all other paths"}; - const PathSetting stateDir{(Store*) this, false, - rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, - "state", "directory where Nix will store state"}; - const PathSetting logDir{(Store*) this, false, - rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, - "log", "directory where Nix will store state"}; - - const static string drvsLogDir; - - LocalFSStore(const Params & params); - - void narFromPath(const Path & path, Sink & sink) override; - ref<FSAccessor> getFSAccessor() override; - - /* Register a permanent GC root. */ - Path addPermRoot(const Path & storePath, - const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); - - virtual Path getRealStoreDir() { return storeDir; } - - Path toRealPath(const Path & storePath) override - { - assert(isInStore(storePath)); - return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); - } - - std::shared_ptr<std::string> getBuildLog(const Path & path) override; +class LocalFSStore : public virtual Store { + public: + // FIXME: the (Store*) cast works around a bug in gcc that causes + // it to emit the call to the Option constructor. Clang works fine + // either way. + const PathSetting rootDir{(Store*)this, true, "", "root", + "directory prefixed to all other paths"}; + const PathSetting stateDir{ + (Store*)this, false, + rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, "state", + "directory where Nix will store state"}; + const PathSetting logDir{ + (Store*)this, false, + rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log", + "directory where Nix will store state"}; + + const static string drvsLogDir; + + LocalFSStore(const Params& params); + + void narFromPath(const Path& path, Sink& sink) override; + ref<FSAccessor> getFSAccessor() override; + + /* Register a permanent GC root. */ + Path addPermRoot(const Path& storePath, const Path& gcRoot, bool indirect, + bool allowOutsideRootsDir = false); + + virtual Path getRealStoreDir() { return storeDir; } + + Path toRealPath(const Path& storePath) override { + assert(isInStore(storePath)); + return getRealStoreDir() + "/" + + std::string(storePath, storeDir.size() + 1); + } + + std::shared_ptr<std::string> getBuildLog(const Path& path) override; }; - /* Extract the name part of the given store path. */ -string storePathToName(const Path & path); +string storePathToName(const Path& path); /* Extract the hash part of the given store path. */ -string storePathToHash(const Path & path); +string storePathToHash(const Path& path); /* Check whether ‘name’ is a valid store path name part, i.e. contains only the characters [a-zA-Z0-9\+\-\.\_\?\=] and doesn't start with a dot. */ -void checkStoreName(const string & name); - +void checkStoreName(const string& name); /* Copy a path from one store to another. */ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); - + const Path& storePath, RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs); /* Copy store paths from one store to another. The paths may be copied in parallel. They are copied in a topologically sorted order (i.e. if A is a reference of B, then A is copied before B), but the set of store paths is not automatically closed; use copyClosure() for that. */ -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths, - RepairFlag repair = NoRepair, - CheckSigsFlag checkSigs = CheckSigs, - SubstituteFlag substitute = NoSubstitute); - +void copyPaths(ref<Store> srcStore, ref<Store> dstStore, + const PathSet& storePaths, RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); /* Copy the closure of the specified paths from one store to another. */ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, - const PathSet & storePaths, - RepairFlag repair = NoRepair, - CheckSigsFlag checkSigs = CheckSigs, - SubstituteFlag substitute = NoSubstitute); - + const PathSet& storePaths, RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); /* Remove the temporary roots file for this process. Any temporary root becomes garbage after this point unless it has been registered as a (permanent) root. */ void removeTempRoots(); - /* Return a Store object to access the Nix store denoted by ‘uri’ (slight misnomer...). Supported values are: @@ -754,58 +724,44 @@ void removeTempRoots(); You can pass parameters to the store implementation by appending ‘?key=value&key=value&...’ to the URI. */ -ref<Store> openStore(const std::string & uri = settings.storeUri.get(), - const Store::Params & extraParams = Store::Params()); - +ref<Store> openStore(const std::string& uri = settings.storeUri.get(), + const Store::Params& extraParams = Store::Params()); -enum StoreType { - tDaemon, - tLocal, - tOther -}; +enum StoreType { tDaemon, tLocal, tOther }; - -StoreType getStoreType(const std::string & uri = settings.storeUri.get(), - const std::string & stateDir = settings.nixStateDir); +StoreType getStoreType(const std::string& uri = settings.storeUri.get(), + const std::string& stateDir = settings.nixStateDir); /* Return the default substituter stores, defined by the ‘substituters’ option and various legacy options. */ std::list<ref<Store>> getDefaultSubstituters(); - /* Store implementation registration. */ -typedef std::function<std::shared_ptr<Store>( - const std::string & uri, const Store::Params & params)> OpenStore; - -struct RegisterStoreImplementation -{ - typedef std::vector<OpenStore> Implementations; - static Implementations * implementations; - - RegisterStoreImplementation(OpenStore fun) - { - if (!implementations) implementations = new Implementations; - implementations->push_back(fun); - } +typedef std::function<std::shared_ptr<Store>(const std::string& uri, + const Store::Params& params)> + OpenStore; + +struct RegisterStoreImplementation { + typedef std::vector<OpenStore> Implementations; + static Implementations* implementations; + + RegisterStoreImplementation(OpenStore fun) { + if (!implementations) implementations = new Implementations; + implementations->push_back(fun); + } }; - - /* Display a set of paths in human-readable form (i.e., between quotes and separated by commas). */ -string showPaths(const PathSet & paths); - - -ValidPathInfo decodeValidPathInfo(std::istream & str, - bool hashGiven = false); +string showPaths(const PathSet& paths); +ValidPathInfo decodeValidPathInfo(std::istream& str, bool hashGiven = false); /* Compute the content-addressability assertion (ValidPathInfo::ca) for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(bool recursive, const Hash & hash); - +std::string makeFixedOutputCA(bool recursive, const Hash& hash); /* Split URI into protocol+hierarchy part and its parameter set. */ -std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); +std::pair<std::string, Store::Params> splitUriAndParams(const std::string& uri); -} +} // namespace nix diff --git a/third_party/nix/src/libstore/worker-protocol.hh b/third_party/nix/src/libstore/worker-protocol.hh index 5ebdfaf134d6..970d494acee1 100644 --- a/third_party/nix/src/libstore/worker-protocol.hh +++ b/third_party/nix/src/libstore/worker-protocol.hh @@ -2,68 +2,64 @@ namespace nix { - #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f #define PROTOCOL_VERSION 0x115 -#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) -#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) - +#define GET_PROTOCOL_MAJOR(x) ((x)&0xff00) +#define GET_PROTOCOL_MINOR(x) ((x)&0x00ff) typedef enum { - wopIsValidPath = 1, - wopHasSubstitutes = 3, - wopQueryPathHash = 4, // obsolete - wopQueryReferences = 5, // obsolete - wopQueryReferrers = 6, - wopAddToStore = 7, - wopAddTextToStore = 8, - wopBuildPaths = 9, - wopEnsurePath = 10, - wopAddTempRoot = 11, - wopAddIndirectRoot = 12, - wopSyncWithGC = 13, - wopFindRoots = 14, - wopExportPath = 16, // obsolete - wopQueryDeriver = 18, // obsolete - wopSetOptions = 19, - wopCollectGarbage = 20, - wopQuerySubstitutablePathInfo = 21, - wopQueryDerivationOutputs = 22, - wopQueryAllValidPaths = 23, - wopQueryFailedPaths = 24, - wopClearFailedPaths = 25, - wopQueryPathInfo = 26, - wopImportPaths = 27, // obsolete - wopQueryDerivationOutputNames = 28, - wopQueryPathFromHashPart = 29, - wopQuerySubstitutablePathInfos = 30, - wopQueryValidPaths = 31, - wopQuerySubstitutablePaths = 32, - wopQueryValidDerivers = 33, - wopOptimiseStore = 34, - wopVerifyStore = 35, - wopBuildDerivation = 36, - wopAddSignatures = 37, - wopNarFromPath = 38, - wopAddToStoreNar = 39, - wopQueryMissing = 40, + wopIsValidPath = 1, + wopHasSubstitutes = 3, + wopQueryPathHash = 4, // obsolete + wopQueryReferences = 5, // obsolete + wopQueryReferrers = 6, + wopAddToStore = 7, + wopAddTextToStore = 8, + wopBuildPaths = 9, + wopEnsurePath = 10, + wopAddTempRoot = 11, + wopAddIndirectRoot = 12, + wopSyncWithGC = 13, + wopFindRoots = 14, + wopExportPath = 16, // obsolete + wopQueryDeriver = 18, // obsolete + wopSetOptions = 19, + wopCollectGarbage = 20, + wopQuerySubstitutablePathInfo = 21, + wopQueryDerivationOutputs = 22, + wopQueryAllValidPaths = 23, + wopQueryFailedPaths = 24, + wopClearFailedPaths = 25, + wopQueryPathInfo = 26, + wopImportPaths = 27, // obsolete + wopQueryDerivationOutputNames = 28, + wopQueryPathFromHashPart = 29, + wopQuerySubstitutablePathInfos = 30, + wopQueryValidPaths = 31, + wopQuerySubstitutablePaths = 32, + wopQueryValidDerivers = 33, + wopOptimiseStore = 34, + wopVerifyStore = 35, + wopBuildDerivation = 36, + wopAddSignatures = 37, + wopNarFromPath = 38, + wopAddToStoreNar = 39, + wopQueryMissing = 40, } WorkerOp; - -#define STDERR_NEXT 0x6f6c6d67 -#define STDERR_READ 0x64617461 // data needed from source -#define STDERR_WRITE 0x64617416 // data for sink -#define STDERR_LAST 0x616c7473 +#define STDERR_NEXT 0x6f6c6d67 +#define STDERR_READ 0x64617461 // data needed from source +#define STDERR_WRITE 0x64617416 // data for sink +#define STDERR_LAST 0x616c7473 #define STDERR_ERROR 0x63787470 #define STDERR_START_ACTIVITY 0x53545254 -#define STDERR_STOP_ACTIVITY 0x53544f50 -#define STDERR_RESULT 0x52534c54 - - -Path readStorePath(Store & store, Source & from); -template<class T> T readStorePaths(Store & store, Source & from); +#define STDERR_STOP_ACTIVITY 0x53544f50 +#define STDERR_RESULT 0x52534c54 +Path readStorePath(Store& store, Source& from); +template <class T> +T readStorePaths(Store& store, Source& from); -} +} // namespace nix diff --git a/third_party/nix/src/libutil/affinity.cc b/third_party/nix/src/libutil/affinity.cc index 98f8287ada67..faee4926d579 100644 --- a/third_party/nix/src/libutil/affinity.cc +++ b/third_party/nix/src/libutil/affinity.cc @@ -1,6 +1,6 @@ +#include "affinity.hh" #include "types.hh" #include "util.hh" -#include "affinity.hh" #if __linux__ #include <sched.h> @@ -8,48 +8,40 @@ namespace nix { - #if __linux__ static bool didSaveAffinity = false; static cpu_set_t savedAffinity; #endif - -void setAffinityTo(int cpu) -{ +void setAffinityTo(int cpu) { #if __linux__ - if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; - didSaveAffinity = true; - debug(format("locking this thread to CPU %1%") % cpu); - cpu_set_t newAffinity; - CPU_ZERO(&newAffinity); - CPU_SET(cpu, &newAffinity); - if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) - printError(format("failed to lock thread to CPU %1%") % cpu); + if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; + didSaveAffinity = true; + debug(format("locking this thread to CPU %1%") % cpu); + cpu_set_t newAffinity; + CPU_ZERO(&newAffinity); + CPU_SET(cpu, &newAffinity); + if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) + printError(format("failed to lock thread to CPU %1%") % cpu); #endif } - -int lockToCurrentCPU() -{ +int lockToCurrentCPU() { #if __linux__ - int cpu = sched_getcpu(); - if (cpu != -1) setAffinityTo(cpu); - return cpu; + int cpu = sched_getcpu(); + if (cpu != -1) setAffinityTo(cpu); + return cpu; #else - return -1; + return -1; #endif } - -void restoreAffinity() -{ +void restoreAffinity() { #if __linux__ - if (!didSaveAffinity) return; - if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) - printError("failed to restore affinity %1%"); + if (!didSaveAffinity) return; + if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) + printError("failed to restore affinity %1%"); #endif } - -} +} // namespace nix diff --git a/third_party/nix/src/libutil/affinity.hh b/third_party/nix/src/libutil/affinity.hh index c1bd28e1367a..5e5ef9b0de0d 100644 --- a/third_party/nix/src/libutil/affinity.hh +++ b/third_party/nix/src/libutil/affinity.hh @@ -6,4 +6,4 @@ void setAffinityTo(int cpu); int lockToCurrentCPU(); void restoreAffinity(); -} +} // namespace nix diff --git a/third_party/nix/src/libutil/archive.cc b/third_party/nix/src/libutil/archive.cc index 3aa120270970..9ae5c76db751 100644 --- a/third_party/nix/src/libutil/archive.cc +++ b/third_party/nix/src/libutil/archive.cc @@ -1,32 +1,31 @@ -#include <cerrno> -#include <algorithm> -#include <vector> -#include <map> - -#include <strings.h> // for strcasecmp - -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> +#include "archive.hh" #include <dirent.h> #include <fcntl.h> - -#include "archive.hh" -#include "util.hh" +#include <strings.h> // for strcasecmp +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <algorithm> +#include <cerrno> +#include <map> +#include <vector> #include "config.hh" +#include "util.hh" namespace nix { -struct ArchiveSettings : Config -{ - Setting<bool> useCaseHack{this, - #if __APPLE__ - true, - #else - false, - #endif +struct ArchiveSettings : Config { + Setting<bool> useCaseHack { + this, +#if __APPLE__ + true, +#else + false, +#endif "use-case-hack", - "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; + "Whether to enable a Darwin-specific hack for dealing with file name " + "collisions." + }; }; static ArchiveSettings archiveSettings; @@ -37,105 +36,105 @@ const std::string narVersionMagic1 = "nix-archive-1"; static string caseHackSuffix = "~nix~case~hack~"; -PathFilter defaultPathFilter = [](const Path &) { return true; }; - +PathFilter defaultPathFilter = [](const Path&) { return true; }; -static void dumpContents(const Path & path, size_t size, - Sink & sink) -{ - sink << "contents" << size; +static void dumpContents(const Path& path, size_t size, Sink& sink) { + sink << "contents" << size; - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError(format("opening file '%1%'") % path); + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError(format("opening file '%1%'") % path); - std::vector<unsigned char> buf(65536); - size_t left = size; + std::vector<unsigned char> buf(65536); + size_t left = size; - while (left > 0) { - auto n = std::min(left, buf.size()); - readFull(fd.get(), buf.data(), n); - left -= n; - sink(buf.data(), n); - } + while (left > 0) { + auto n = std::min(left, buf.size()); + readFull(fd.get(), buf.data(), n); + left -= n; + sink(buf.data(), n); + } - writePadding(size, sink); + writePadding(size, sink); } - -static void dump(const Path & path, Sink & sink, PathFilter & filter) -{ - checkInterrupt(); - - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); - - sink << "("; - - if (S_ISREG(st.st_mode)) { - sink << "type" << "regular"; - if (st.st_mode & S_IXUSR) - sink << "executable" << ""; - dumpContents(path, (size_t) st.st_size, sink); - } - - else if (S_ISDIR(st.st_mode)) { - sink << "type" << "directory"; - - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map<string, string> unhacked; - for (auto & i : readDirectory(path)) - if (archiveSettings.useCaseHack) { - string name(i.name); - size_t pos = i.name.find(caseHackSuffix); - if (pos != string::npos) { - debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name)); - name.erase(pos); - } - if (unhacked.find(name) != unhacked.end()) - throw Error(format("file name collision in between '%1%' and '%2%'") - % (path + "/" + unhacked[name]) % (path + "/" + i.name)); - unhacked[name] = i.name; - } else - unhacked[i.name] = i.name; - - for (auto & i : unhacked) - if (filter(path + "/" + i.first)) { - sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + "/" + i.second, sink, filter); - sink << ")"; - } - } - - else if (S_ISLNK(st.st_mode)) - sink << "type" << "symlink" << "target" << readLink(path); - - else throw Error(format("file '%1%' has an unsupported type") % path); - - sink << ")"; +static void dump(const Path& path, Sink& sink, PathFilter& filter) { + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + + sink << "("; + + if (S_ISREG(st.st_mode)) { + sink << "type" + << "regular"; + if (st.st_mode & S_IXUSR) + sink << "executable" + << ""; + dumpContents(path, (size_t)st.st_size, sink); + } + + else if (S_ISDIR(st.st_mode)) { + sink << "type" + << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map<string, string> unhacked; + for (auto& i : readDirectory(path)) + if (archiveSettings.useCaseHack) { + string name(i.name); + size_t pos = i.name.find(caseHackSuffix); + if (pos != string::npos) { + debug(format("removing case hack suffix from '%1%'") % + (path + "/" + i.name)); + name.erase(pos); + } + if (unhacked.find(name) != unhacked.end()) + throw Error(format("file name collision in between '%1%' and '%2%'") % + (path + "/" + unhacked[name]) % (path + "/" + i.name)); + unhacked[name] = i.name; + } else + unhacked[i.name] = i.name; + + for (auto& i : unhacked) + if (filter(path + "/" + i.first)) { + sink << "entry" + << "(" + << "name" << i.first << "node"; + dump(path + "/" + i.second, sink, filter); + sink << ")"; + } + } + + else if (S_ISLNK(st.st_mode)) + sink << "type" + << "symlink" + << "target" << readLink(path); + + else + throw Error(format("file '%1%' has an unsupported type") % path); + + sink << ")"; } - -void dumpPath(const Path & path, Sink & sink, PathFilter & filter) -{ - sink << narVersionMagic1; - dump(path, sink, filter); +void dumpPath(const Path& path, Sink& sink, PathFilter& filter) { + sink << narVersionMagic1; + dump(path, sink, filter); } - -void dumpString(const std::string & s, Sink & sink) -{ - sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; +void dumpString(const std::string& s, Sink& sink) { + sink << narVersionMagic1 << "(" + << "type" + << "regular" + << "contents" << s << ")"; } - -static SerialisationError badArchive(string s) -{ - return SerialisationError("bad archive: " + s); +static SerialisationError badArchive(string s) { + return SerialisationError("bad archive: " + s); } - #if 0 static void skipGeneric(Source & source) { @@ -146,233 +145,212 @@ static void skipGeneric(Source & source) } #endif +static void parseContents(ParseSink& sink, Source& source, const Path& path) { + unsigned long long size = readLongLong(source); -static void parseContents(ParseSink & sink, Source & source, const Path & path) -{ - unsigned long long size = readLongLong(source); + sink.preallocateContents(size); - sink.preallocateContents(size); + unsigned long long left = size; + std::vector<unsigned char> buf(65536); - unsigned long long left = size; - std::vector<unsigned char> buf(65536); + while (left) { + checkInterrupt(); + auto n = buf.size(); + if ((unsigned long long)n > left) n = left; + source(buf.data(), n); + sink.receiveContents(buf.data(), n); + left -= n; + } + + readPadding(size, source); +} - while (left) { - checkInterrupt(); - auto n = buf.size(); - if ((unsigned long long)n > left) n = left; - source(buf.data(), n); - sink.receiveContents(buf.data(), n); - left -= n; - } +struct CaseInsensitiveCompare { + bool operator()(const string& a, const string& b) const { + return strcasecmp(a.c_str(), b.c_str()) < 0; + } +}; - readPadding(size, source); -} +static void parse(ParseSink& sink, Source& source, const Path& path) { + string s; + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); -struct CaseInsensitiveCompare -{ - bool operator() (const string & a, const string & b) const - { - return strcasecmp(a.c_str(), b.c_str()) < 0; - } -}; + enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; + std::map<Path, int, CaseInsensitiveCompare> names; -static void parse(ParseSink & sink, Source & source, const Path & path) -{ - string s; + while (1) { + checkInterrupt(); s = readString(source); - if (s != "(") throw badArchive("expected open tag"); - enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; + if (s == ")") { + break; + } - std::map<Path, int, CaseInsensitiveCompare> names; + else if (s == "type") { + if (type != tpUnknown) throw badArchive("multiple type fields"); + string t = readString(source); - while (1) { - checkInterrupt(); + if (t == "regular") { + type = tpRegular; + sink.createRegularFile(path); + } - s = readString(source); + else if (t == "directory") { + sink.createDirectory(path); + type = tpDirectory; + } - if (s == ")") { - break; - } - - else if (s == "type") { - if (type != tpUnknown) - throw badArchive("multiple type fields"); - string t = readString(source); + else if (t == "symlink") { + type = tpSymlink; + } - if (t == "regular") { - type = tpRegular; - sink.createRegularFile(path); - } + else + throw badArchive("unknown file type " + t); - else if (t == "directory") { - sink.createDirectory(path); - type = tpDirectory; - } + } - else if (t == "symlink") { - type = tpSymlink; - } + else if (s == "contents" && type == tpRegular) { + parseContents(sink, source, path); + } - else throw badArchive("unknown file type " + t); + else if (s == "executable" && type == tpRegular) { + auto s = readString(source); + if (s != "") throw badArchive("executable marker has non-empty value"); + sink.isExecutable(); + } - } + else if (s == "entry" && type == tpDirectory) { + string name, prevName; - else if (s == "contents" && type == tpRegular) { - parseContents(sink, source, path); - } + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); - else if (s == "executable" && type == tpRegular) { - auto s = readString(source); - if (s != "") throw badArchive("executable marker has non-empty value"); - sink.isExecutable(); - } + while (1) { + checkInterrupt(); - else if (s == "entry" && type == tpDirectory) { - string name, prevName; - - s = readString(source); - if (s != "(") throw badArchive("expected open tag"); - - while (1) { - checkInterrupt(); - - s = readString(source); - - if (s == ")") { - break; - } else if (s == "name") { - name = readString(source); - if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos) - throw Error(format("NAR contains invalid file name '%1%'") % name); - if (name <= prevName) - throw Error("NAR directory is not sorted"); - prevName = name; - if (archiveSettings.useCaseHack) { - auto i = names.find(name); - if (i != names.end()) { - debug(format("case collision between '%1%' and '%2%'") % i->first % name); - name += caseHackSuffix; - name += std::to_string(++i->second); - } else - names[name] = 0; - } - } else if (s == "node") { - if (s.empty()) throw badArchive("entry name missing"); - parse(sink, source, path + "/" + name); - } else - throw badArchive("unknown field " + s); - } - } + s = readString(source); - else if (s == "target" && type == tpSymlink) { - string target = readString(source); - sink.createSymlink(path, target); - } + if (s == ")") { + break; + } else if (s == "name") { + name = readString(source); + if (name.empty() || name == "." || name == ".." || + name.find('/') != string::npos || + name.find((char)0) != string::npos) + throw Error(format("NAR contains invalid file name '%1%'") % name); + if (name <= prevName) throw Error("NAR directory is not sorted"); + prevName = name; + if (archiveSettings.useCaseHack) { + auto i = names.find(name); + if (i != names.end()) { + debug(format("case collision between '%1%' and '%2%'") % + i->first % name); + name += caseHackSuffix; + name += std::to_string(++i->second); + } else + names[name] = 0; + } + } else if (s == "node") { + if (s.empty()) throw badArchive("entry name missing"); + parse(sink, source, path + "/" + name); + } else + throw badArchive("unknown field " + s); + } + } - else - throw badArchive("unknown field " + s); + else if (s == "target" && type == tpSymlink) { + string target = readString(source); + sink.createSymlink(path, target); } -} + else + throw badArchive("unknown field " + s); + } +} -void parseDump(ParseSink & sink, Source & source) -{ - string version; - try { - version = readString(source, narVersionMagic1.size()); - } catch (SerialisationError & e) { - /* This generally means the integer at the start couldn't be - decoded. Ignore and throw the exception below. */ - } - if (version != narVersionMagic1) - throw badArchive("input doesn't look like a Nix archive"); - parse(sink, source, ""); +void parseDump(ParseSink& sink, Source& source) { + string version; + try { + version = readString(source, narVersionMagic1.size()); + } catch (SerialisationError& e) { + /* This generally means the integer at the start couldn't be + decoded. Ignore and throw the exception below. */ + } + if (version != narVersionMagic1) + throw badArchive("input doesn't look like a Nix archive"); + parse(sink, source, ""); } +struct RestoreSink : ParseSink { + Path dstPath; + AutoCloseFD fd; -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError(format("creating directory '%1%'") % p); - }; - - void createRegularFile(const Path & path) - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError(format("creating file '%1%'") % p); - } + void createDirectory(const Path& path) { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError(format("creating directory '%1%'") % p); + }; - void isExecutable() - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } + void createRegularFile(const Path& path) { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError(format("creating file '%1%'") % p); + } + + void isExecutable() { + struct stat st; + if (fstat(fd.get(), &st) == -1) throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } - void preallocateContents(unsigned long long len) - { + void preallocateContents(unsigned long long len) { #if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError(format("preallocating file of %1% bytes") % len); - } -#endif + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError(format("preallocating file of %1% bytes") % len); } +#endif + } - void receiveContents(unsigned char * data, unsigned int len) - { - writeFull(fd.get(), data, len); - } + void receiveContents(unsigned char* data, unsigned int len) { + writeFull(fd.get(), data, len); + } - void createSymlink(const Path & path, const string & target) - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } + void createSymlink(const Path& path, const string& target) { + Path p = dstPath + path; + nix::createSymlink(target, p); + } }; - -void restorePath(const Path & path, Source & source) -{ - RestoreSink sink; - sink.dstPath = path; - parseDump(sink, source); +void restorePath(const Path& path, Source& source) { + RestoreSink sink; + sink.dstPath = path; + parseDump(sink, source); } +void copyNAR(Source& source, Sink& sink) { + // FIXME: if 'source' is the output of dumpPath() followed by EOF, + // we should just forward all data directly without parsing. -void copyNAR(Source & source, Sink & sink) -{ - // FIXME: if 'source' is the output of dumpPath() followed by EOF, - // we should just forward all data directly without parsing. - - ParseSink parseSink; /* null sink; just parse the NAR */ + ParseSink parseSink; /* null sink; just parse the NAR */ - LambdaSource wrapper([&](unsigned char * data, size_t len) { - auto n = source.read(data, len); - sink(data, n); - return n; - }); + LambdaSource wrapper([&](unsigned char* data, size_t len) { + auto n = source.read(data, len); + sink(data, n); + return n; + }); - parseDump(parseSink, wrapper); + parseDump(parseSink, wrapper); } - -} +} // namespace nix diff --git a/third_party/nix/src/libutil/archive.hh b/third_party/nix/src/libutil/archive.hh index 25be426c1a4d..9a656edae4a3 100644 --- a/third_party/nix/src/libutil/archive.hh +++ b/third_party/nix/src/libutil/archive.hh @@ -1,12 +1,10 @@ #pragma once -#include "types.hh" #include "serialise.hh" - +#include "types.hh" namespace nix { - /* dumpPath creates a Nix archive of the specified path. The format is as follows: @@ -44,41 +42,36 @@ namespace nix { `+' denotes string concatenation. */ +void dumpPath(const Path& path, Sink& sink, + PathFilter& filter = defaultPathFilter); -void dumpPath(const Path & path, Sink & sink, - PathFilter & filter = defaultPathFilter); - -void dumpString(const std::string & s, Sink & sink); +void dumpString(const std::string& s, Sink& sink); /* FIXME: fix this API, it sucks. */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; +struct ParseSink { + virtual void createDirectory(const Path& path){}; - virtual void createRegularFile(const Path & path) { }; - virtual void isExecutable() { }; - virtual void preallocateContents(unsigned long long size) { }; - virtual void receiveContents(unsigned char * data, unsigned int len) { }; + virtual void createRegularFile(const Path& path){}; + virtual void isExecutable(){}; + virtual void preallocateContents(unsigned long long size){}; + virtual void receiveContents(unsigned char* data, unsigned int len){}; - virtual void createSymlink(const Path & path, const string & target) { }; + virtual void createSymlink(const Path& path, const string& target){}; }; -struct TeeSink : ParseSink -{ - TeeSource source; +struct TeeSink : ParseSink { + TeeSource source; - TeeSink(Source & source) : source(source) { } + TeeSink(Source& source) : source(source) {} }; -void parseDump(ParseSink & sink, Source & source); +void parseDump(ParseSink& sink, Source& source); -void restorePath(const Path & path, Source & source); +void restorePath(const Path& path, Source& source); /* Read a NAR from 'source' and write it to 'sink'. */ -void copyNAR(Source & source, Sink & sink); - +void copyNAR(Source& source, Sink& sink); extern const std::string narVersionMagic1; - -} +} // namespace nix diff --git a/third_party/nix/src/libutil/args.cc b/third_party/nix/src/libutil/args.cc index 7af2a1bf731a..b5bfb0e6a746 100644 --- a/third_party/nix/src/libutil/args.cc +++ b/third_party/nix/src/libutil/args.cc @@ -3,201 +3,184 @@ namespace nix { -Args::FlagMaker Args::mkFlag() -{ - return FlagMaker(*this); -} +Args::FlagMaker Args::mkFlag() { return FlagMaker(*this); } -Args::FlagMaker::~FlagMaker() -{ - assert(flag->longName != ""); - args.longFlags[flag->longName] = flag; - if (flag->shortName) args.shortFlags[flag->shortName] = flag; +Args::FlagMaker::~FlagMaker() { + assert(flag->longName != ""); + args.longFlags[flag->longName] = flag; + if (flag->shortName) args.shortFlags[flag->shortName] = flag; } -void Args::parseCmdline(const Strings & _cmdline) -{ - Strings pendingArgs; - bool dashDash = false; - - Strings cmdline(_cmdline); - - for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { - - auto arg = *pos; - - /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', - `-j3` -> `-j 3`). */ - if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { - *pos = (string) "-" + arg[1]; - auto next = pos; ++next; - for (unsigned int j = 2; j < arg.length(); j++) - if (isalpha(arg[j])) - cmdline.insert(next, (string) "-" + arg[j]); - else { - cmdline.insert(next, string(arg, j)); - break; - } - arg = *pos; - } - - if (!dashDash && arg == "--") { - dashDash = true; - ++pos; - } - else if (!dashDash && std::string(arg, 0, 1) == "-") { - if (!processFlag(pos, cmdline.end())) - throw UsageError(format("unrecognised flag '%1%'") % arg); - } +void Args::parseCmdline(const Strings& _cmdline) { + Strings pendingArgs; + bool dashDash = false; + + Strings cmdline(_cmdline); + + for (auto pos = cmdline.begin(); pos != cmdline.end();) { + auto arg = *pos; + + /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', + `-j3` -> `-j 3`). */ + if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && + isalpha(arg[1])) { + *pos = (string) "-" + arg[1]; + auto next = pos; + ++next; + for (unsigned int j = 2; j < arg.length(); j++) + if (isalpha(arg[j])) + cmdline.insert(next, (string) "-" + arg[j]); else { - pendingArgs.push_back(*pos++); - if (processArgs(pendingArgs, false)) - pendingArgs.clear(); + cmdline.insert(next, string(arg, j)); + break; } + arg = *pos; } - processArgs(pendingArgs, true); + if (!dashDash && arg == "--") { + dashDash = true; + ++pos; + } else if (!dashDash && std::string(arg, 0, 1) == "-") { + if (!processFlag(pos, cmdline.end())) + throw UsageError(format("unrecognised flag '%1%'") % arg); + } else { + pendingArgs.push_back(*pos++); + if (processArgs(pendingArgs, false)) pendingArgs.clear(); + } + } + + processArgs(pendingArgs, true); } -void Args::printHelp(const string & programName, std::ostream & out) -{ - std::cout << "Usage: " << programName << " <FLAGS>..."; - for (auto & exp : expectedArgs) { - std::cout << renderLabels({exp.label}); - // FIXME: handle arity > 1 - if (exp.arity == 0) std::cout << "..."; - if (exp.optional) std::cout << "?"; - } - std::cout << "\n"; +void Args::printHelp(const string& programName, std::ostream& out) { + std::cout << "Usage: " << programName << " <FLAGS>..."; + for (auto& exp : expectedArgs) { + std::cout << renderLabels({exp.label}); + // FIXME: handle arity > 1 + if (exp.arity == 0) std::cout << "..."; + if (exp.optional) std::cout << "?"; + } + std::cout << "\n"; - auto s = description(); - if (s != "") - std::cout << "\nSummary: " << s << ".\n"; + auto s = description(); + if (s != "") std::cout << "\nSummary: " << s << ".\n"; - if (longFlags.size()) { - std::cout << "\n"; - std::cout << "Flags:\n"; - printFlags(out); - } + if (longFlags.size()) { + std::cout << "\n"; + std::cout << "Flags:\n"; + printFlags(out); + } } -void Args::printFlags(std::ostream & out) -{ - Table2 table; - for (auto & flag : longFlags) { - if (hiddenCategories.count(flag.second->category)) continue; - table.push_back(std::make_pair( - (flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ") - + "--" + flag.first + renderLabels(flag.second->labels), - flag.second->description)); - } - printTable(out, table); +void Args::printFlags(std::ostream& out) { + Table2 table; + for (auto& flag : longFlags) { + if (hiddenCategories.count(flag.second->category)) continue; + table.push_back(std::make_pair( + (flag.second->shortName + ? std::string("-") + flag.second->shortName + ", " + : " ") + + "--" + flag.first + renderLabels(flag.second->labels), + flag.second->description)); + } + printTable(out, table); } -bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) -{ - assert(pos != end); - - auto process = [&](const std::string & name, const Flag & flag) -> bool { - ++pos; - std::vector<std::string> args; - for (size_t n = 0 ; n < flag.arity; ++n) { - if (pos == end) { - if (flag.arity == ArityAny) break; - throw UsageError(format("flag '%1%' requires %2% argument(s)") - % name % flag.arity); - } - args.push_back(*pos++); - } - flag.handler(std::move(args)); - return true; - }; - - if (string(*pos, 0, 2) == "--") { - auto i = longFlags.find(string(*pos, 2)); - if (i == longFlags.end()) return false; - return process("--" + i->first, *i->second); +bool Args::processFlag(Strings::iterator& pos, Strings::iterator end) { + assert(pos != end); + + auto process = [&](const std::string& name, const Flag& flag) -> bool { + ++pos; + std::vector<std::string> args; + for (size_t n = 0; n < flag.arity; ++n) { + if (pos == end) { + if (flag.arity == ArityAny) break; + throw UsageError(format("flag '%1%' requires %2% argument(s)") % name % + flag.arity); + } + args.push_back(*pos++); } - - if (string(*pos, 0, 1) == "-" && pos->size() == 2) { - auto c = (*pos)[1]; - auto i = shortFlags.find(c); - if (i == shortFlags.end()) return false; - return process(std::string("-") + c, *i->second); - } - - return false; + flag.handler(std::move(args)); + return true; + }; + + if (string(*pos, 0, 2) == "--") { + auto i = longFlags.find(string(*pos, 2)); + if (i == longFlags.end()) return false; + return process("--" + i->first, *i->second); + } + + if (string(*pos, 0, 1) == "-" && pos->size() == 2) { + auto c = (*pos)[1]; + auto i = shortFlags.find(c); + if (i == shortFlags.end()) return false; + return process(std::string("-") + c, *i->second); + } + + return false; } -bool Args::processArgs(const Strings & args, bool finish) -{ - if (expectedArgs.empty()) { - if (!args.empty()) - throw UsageError(format("unexpected argument '%1%'") % args.front()); - return true; - } +bool Args::processArgs(const Strings& args, bool finish) { + if (expectedArgs.empty()) { + if (!args.empty()) + throw UsageError(format("unexpected argument '%1%'") % args.front()); + return true; + } - auto & exp = expectedArgs.front(); + auto& exp = expectedArgs.front(); - bool res = false; + bool res = false; - if ((exp.arity == 0 && finish) || - (exp.arity > 0 && args.size() == exp.arity)) - { - std::vector<std::string> ss; - for (auto & s : args) ss.push_back(s); - exp.handler(std::move(ss)); - expectedArgs.pop_front(); - res = true; - } + if ((exp.arity == 0 && finish) || + (exp.arity > 0 && args.size() == exp.arity)) { + std::vector<std::string> ss; + for (auto& s : args) ss.push_back(s); + exp.handler(std::move(ss)); + expectedArgs.pop_front(); + res = true; + } - if (finish && !expectedArgs.empty() && !expectedArgs.front().optional) - throw UsageError("more arguments are required"); + if (finish && !expectedArgs.empty() && !expectedArgs.front().optional) + throw UsageError("more arguments are required"); - return res; + return res; } -Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht) -{ - arity(1); - label("type"); - description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')"); - handler([ht](std::string s) { - *ht = parseHashType(s); - if (*ht == htUnknown) - throw UsageError("unknown hash type '%1%'", s); - }); - return *this; +Args::FlagMaker& Args::FlagMaker::mkHashTypeFlag(HashType* ht) { + arity(1); + label("type"); + description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')"); + handler([ht](std::string s) { + *ht = parseHashType(s); + if (*ht == htUnknown) throw UsageError("unknown hash type '%1%'", s); + }); + return *this; } -Strings argvToStrings(int argc, char * * argv) -{ - Strings args; - argc--; argv++; - while (argc--) args.push_back(*argv++); - return args; +Strings argvToStrings(int argc, char** argv) { + Strings args; + argc--; + argv++; + while (argc--) args.push_back(*argv++); + return args; } -std::string renderLabels(const Strings & labels) -{ - std::string res; - for (auto label : labels) { - for (auto & c : label) c = std::toupper(c); - res += " <" + label + ">"; - } - return res; +std::string renderLabels(const Strings& labels) { + std::string res; + for (auto label : labels) { + for (auto& c : label) c = std::toupper(c); + res += " <" + label + ">"; + } + return res; } -void printTable(std::ostream & out, const Table2 & table) -{ - size_t max = 0; - for (auto & row : table) - max = std::max(max, row.first.size()); - for (auto & row : table) { - out << " " << row.first - << std::string(max - row.first.size() + 2, ' ') - << row.second << "\n"; - } +void printTable(std::ostream& out, const Table2& table) { + size_t max = 0; + for (auto& row : table) max = std::max(max, row.first.size()); + for (auto& row : table) { + out << " " << row.first << std::string(max - row.first.size() + 2, ' ') + << row.second << "\n"; + } } -} +} // namespace nix diff --git a/third_party/nix/src/libutil/args.hh b/third_party/nix/src/libutil/args.hh index ad5fcca39418..dc41a6849b40 100644 --- a/third_party/nix/src/libutil/args.hh +++ b/third_party/nix/src/libutil/args.hh @@ -3,7 +3,6 @@ #include <iostream> #include <map> #include <memory> - #include "util.hh" namespace nix { @@ -12,190 +11,206 @@ MakeError(UsageError, Error); enum HashType : char; -class Args -{ -public: +class Args { + public: + /* Parse the command line, throwing a UsageError if something goes + wrong. */ + void parseCmdline(const Strings& cmdline); - /* Parse the command line, throwing a UsageError if something goes - wrong. */ - void parseCmdline(const Strings & cmdline); + virtual void printHelp(const string& programName, std::ostream& out); - virtual void printHelp(const string & programName, std::ostream & out); + virtual std::string description() { return ""; } - virtual std::string description() { return ""; } + protected: + static const size_t ArityAny = std::numeric_limits<size_t>::max(); -protected: + /* Flags. */ + struct Flag { + typedef std::shared_ptr<Flag> ptr; + std::string longName; + char shortName = 0; + std::string description; + Strings labels; + size_t arity = 0; + std::function<void(std::vector<std::string>)> handler; + std::string category; + }; - static const size_t ArityAny = std::numeric_limits<size_t>::max(); + std::map<std::string, Flag::ptr> longFlags; + std::map<char, Flag::ptr> shortFlags; - /* Flags. */ - struct Flag - { - typedef std::shared_ptr<Flag> ptr; - std::string longName; - char shortName = 0; - std::string description; - Strings labels; - size_t arity = 0; - std::function<void(std::vector<std::string>)> handler; - std::string category; - }; + virtual bool processFlag(Strings::iterator& pos, Strings::iterator end); + + virtual void printFlags(std::ostream& out); + + /* Positional arguments. */ + struct ExpectedArg { + std::string label; + size_t arity; // 0 = any + bool optional; + std::function<void(std::vector<std::string>)> handler; + }; - std::map<std::string, Flag::ptr> longFlags; - std::map<char, Flag::ptr> shortFlags; + std::list<ExpectedArg> expectedArgs; - virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); + virtual bool processArgs(const Strings& args, bool finish); - virtual void printFlags(std::ostream & out); + std::set<std::string> hiddenCategories; + + public: + class FlagMaker { + Args& args; + Flag::ptr flag; + friend class Args; + FlagMaker(Args& args) : args(args), flag(std::make_shared<Flag>()){}; + + public: + ~FlagMaker(); + FlagMaker& longName(const std::string& s) { + flag->longName = s; + return *this; + }; + FlagMaker& shortName(char s) { + flag->shortName = s; + return *this; + }; + FlagMaker& description(const std::string& s) { + flag->description = s; + return *this; + }; + FlagMaker& label(const std::string& l) { + flag->arity = 1; + flag->labels = {l}; + return *this; + }; + FlagMaker& labels(const Strings& ls) { + flag->arity = ls.size(); + flag->labels = ls; + return *this; + }; + FlagMaker& arity(size_t arity) { + flag->arity = arity; + return *this; + }; + FlagMaker& handler(std::function<void(std::vector<std::string>)> handler) { + flag->handler = handler; + return *this; + }; + FlagMaker& handler(std::function<void()> handler) { + flag->handler = [handler](std::vector<std::string>) { handler(); }; + return *this; + }; + FlagMaker& handler(std::function<void(std::string)> handler) { + flag->arity = 1; + flag->handler = [handler](std::vector<std::string> ss) { + handler(std::move(ss[0])); + }; + return *this; + }; + FlagMaker& category(const std::string& s) { + flag->category = s; + return *this; + }; - /* Positional arguments. */ - struct ExpectedArg - { - std::string label; - size_t arity; // 0 = any - bool optional; - std::function<void(std::vector<std::string>)> handler; + template <class T> + FlagMaker& dest(T* dest) { + flag->arity = 1; + flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; }; + return *this; }; - std::list<ExpectedArg> expectedArgs; - - virtual bool processArgs(const Strings & args, bool finish); - - std::set<std::string> hiddenCategories; - -public: - - class FlagMaker - { - Args & args; - Flag::ptr flag; - friend class Args; - FlagMaker(Args & args) : args(args), flag(std::make_shared<Flag>()) { }; - public: - ~FlagMaker(); - FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; }; - FlagMaker & shortName(char s) { flag->shortName = s; return *this; }; - FlagMaker & description(const std::string & s) { flag->description = s; return *this; }; - FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; }; - FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; }; - FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; }; - FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; }; - FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; }; - FlagMaker & handler(std::function<void(std::string)> handler) { - flag->arity = 1; - flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); }; - return *this; - }; - FlagMaker & category(const std::string & s) { flag->category = s; return *this; }; - - template<class T> - FlagMaker & dest(T * dest) - { - flag->arity = 1; - flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; }; - return *this; - }; - - template<class T> - FlagMaker & set(T * dest, const T & val) - { - flag->arity = 0; - flag->handler = [=](std::vector<std::string> ss) { *dest = val; }; - return *this; - }; - - FlagMaker & mkHashTypeFlag(HashType * ht); + template <class T> + FlagMaker& set(T* dest, const T& val) { + flag->arity = 0; + flag->handler = [=](std::vector<std::string> ss) { *dest = val; }; + return *this; }; - FlagMaker mkFlag(); - - /* Helper functions for constructing flags / positional - arguments. */ - - void mkFlag1(char shortName, const std::string & longName, - const std::string & label, const std::string & description, - std::function<void(std::string)> fun) - { - mkFlag() - .shortName(shortName) - .longName(longName) - .labels({label}) - .description(description) - .arity(1) - .handler([=](std::vector<std::string> ss) { fun(ss[0]); }); - } - - void mkFlag(char shortName, const std::string & name, - const std::string & description, bool * dest) - { - mkFlag(shortName, name, description, dest, true); - } - - template<class T> - void mkFlag(char shortName, const std::string & longName, const std::string & description, - T * dest, const T & value) - { - mkFlag() - .shortName(shortName) - .longName(longName) - .description(description) - .handler([=](std::vector<std::string> ss) { *dest = value; }); - } - - template<class I> - void mkIntFlag(char shortName, const std::string & longName, - const std::string & description, I * dest) - { - mkFlag<I>(shortName, longName, description, [=](I n) { - *dest = n; + FlagMaker& mkHashTypeFlag(HashType* ht); + }; + + FlagMaker mkFlag(); + + /* Helper functions for constructing flags / positional + arguments. */ + + void mkFlag1(char shortName, const std::string& longName, + const std::string& label, const std::string& description, + std::function<void(std::string)> fun) { + mkFlag() + .shortName(shortName) + .longName(longName) + .labels({label}) + .description(description) + .arity(1) + .handler([=](std::vector<std::string> ss) { fun(ss[0]); }); + } + + void mkFlag(char shortName, const std::string& name, + const std::string& description, bool* dest) { + mkFlag(shortName, name, description, dest, true); + } + + template <class T> + void mkFlag(char shortName, const std::string& longName, + const std::string& description, T* dest, const T& value) { + mkFlag() + .shortName(shortName) + .longName(longName) + .description(description) + .handler([=](std::vector<std::string> ss) { *dest = value; }); + } + + template <class I> + void mkIntFlag(char shortName, const std::string& longName, + const std::string& description, I* dest) { + mkFlag<I>(shortName, longName, description, [=](I n) { *dest = n; }); + } + + template <class I> + void mkFlag(char shortName, const std::string& longName, + const std::string& description, std::function<void(I)> fun) { + mkFlag() + .shortName(shortName) + .longName(longName) + .labels({"N"}) + .description(description) + .arity(1) + .handler([=](std::vector<std::string> ss) { + I n; + if (!string2Int(ss[0], n)) + throw UsageError("flag '--%s' requires a integer argument", + longName); + fun(n); }); - } - - template<class I> - void mkFlag(char shortName, const std::string & longName, - const std::string & description, std::function<void(I)> fun) - { - mkFlag() - .shortName(shortName) - .longName(longName) - .labels({"N"}) - .description(description) - .arity(1) - .handler([=](std::vector<std::string> ss) { - I n; - if (!string2Int(ss[0], n)) - throw UsageError("flag '--%s' requires a integer argument", longName); - fun(n); - }); - } - - /* Expect a string argument. */ - void expectArg(const std::string & label, string * dest, bool optional = false) - { - expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) { - *dest = ss[0]; - }}); - } - - /* Expect 0 or more arguments. */ - void expectArgs(const std::string & label, std::vector<std::string> * dest) - { - expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) { - *dest = std::move(ss); - }}); - } - - friend class MultiCommand; + } + + /* Expect a string argument. */ + void expectArg(const std::string& label, string* dest, + bool optional = false) { + expectedArgs.push_back( + ExpectedArg{label, 1, optional, + [=](std::vector<std::string> ss) { *dest = ss[0]; }}); + } + + /* Expect 0 or more arguments. */ + void expectArgs(const std::string& label, std::vector<std::string>* dest) { + expectedArgs.push_back(ExpectedArg{ + label, 0, false, + [=](std::vector<std::string> ss) { *dest = std::move(ss); }}); + } + + friend class MultiCommand; }; -Strings argvToStrings(int argc, char * * argv); +Strings argvToStrings(int argc, char** argv); /* Helper function for rendering argument labels. */ -std::string renderLabels(const Strings & labels); +std::string renderLabels(const Strings& labels); /* Helper function for printing 2-column tables. */ typedef std::vector<std::pair<std::string, std::string>> Table2; -void printTable(std::ostream & out, const Table2 & table); +void printTable(std::ostream& out, const Table2& table); -} +} // namespace nix diff --git a/third_party/nix/src/libutil/compression.cc b/third_party/nix/src/libutil/compression.cc index 0dd84e32034a..39bde37e0fd4 100644 --- a/third_party/nix/src/libutil/compression.cc +++ b/third_party/nix/src/libutil/compression.cc @@ -1,432 +1,379 @@ #include "compression.hh" -#include "util.hh" -#include "finally.hh" -#include "logging.hh" - -#include <lzma.h> +#include <brotli/decode.h> +#include <brotli/encode.h> #include <bzlib.h> +#include <lzma.h> #include <cstdio> #include <cstring> - -#include <brotli/decode.h> -#include <brotli/encode.h> - #include <iostream> +#include "finally.hh" +#include "logging.hh" +#include "util.hh" namespace nix { // Don't feed brotli too much at once. -struct ChunkedCompressionSink : CompressionSink -{ - uint8_t outbuf[32 * 1024]; - - void write(const unsigned char * data, size_t len) override - { - const size_t CHUNK_SIZE = sizeof(outbuf) << 2; - while (len) { - size_t n = std::min(CHUNK_SIZE, len); - writeInternal(data, n); - data += n; - len -= n; - } +struct ChunkedCompressionSink : CompressionSink { + uint8_t outbuf[32 * 1024]; + + void write(const unsigned char* data, size_t len) override { + const size_t CHUNK_SIZE = sizeof(outbuf) << 2; + while (len) { + size_t n = std::min(CHUNK_SIZE, len); + writeInternal(data, n); + data += n; + len -= n; } + } - virtual void writeInternal(const unsigned char * data, size_t len) = 0; + virtual void writeInternal(const unsigned char* data, size_t len) = 0; }; -struct NoneSink : CompressionSink -{ - Sink & nextSink; - NoneSink(Sink & nextSink) : nextSink(nextSink) { } - void finish() override { flush(); } - void write(const unsigned char * data, size_t len) override { nextSink(data, len); } +struct NoneSink : CompressionSink { + Sink& nextSink; + NoneSink(Sink& nextSink) : nextSink(nextSink) {} + void finish() override { flush(); } + void write(const unsigned char* data, size_t len) override { + nextSink(data, len); + } }; -struct XzDecompressionSink : CompressionSink -{ - Sink & nextSink; - uint8_t outbuf[BUFSIZ]; - lzma_stream strm = LZMA_STREAM_INIT; - bool finished = false; +struct XzDecompressionSink : CompressionSink { + Sink& nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; - XzDecompressionSink(Sink & nextSink) : nextSink(nextSink) - { - lzma_ret ret = lzma_stream_decoder( - &strm, UINT64_MAX, LZMA_CONCATENATED); - if (ret != LZMA_OK) - throw CompressionError("unable to initialise lzma decoder"); + XzDecompressionSink(Sink& nextSink) : nextSink(nextSink) { + lzma_ret ret = lzma_stream_decoder(&strm, UINT64_MAX, LZMA_CONCATENATED); + if (ret != LZMA_OK) + throw CompressionError("unable to initialise lzma decoder"); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } - ~XzDecompressionSink() - { - lzma_end(&strm); - } + ~XzDecompressionSink() { lzma_end(&strm); } - void finish() override - { - CompressionSink::flush(); - write(nullptr, 0); - } + void finish() override { + CompressionSink::flush(); + write(nullptr, 0); + } - void write(const unsigned char * data, size_t len) override - { - strm.next_in = data; - strm.avail_in = len; + void write(const unsigned char* data, size_t len) override { + strm.next_in = data; + strm.avail_in = len; - while (!finished && (!data || strm.avail_in)) { - checkInterrupt(); + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); - lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); - if (ret != LZMA_OK && ret != LZMA_STREAM_END) - throw CompressionError("error %d while decompressing xz file", ret); + lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw CompressionError("error %d while decompressing xz file", ret); - finished = ret == LZMA_STREAM_END; + finished = ret == LZMA_STREAM_END; - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - } + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } } + } }; -struct BzipDecompressionSink : ChunkedCompressionSink -{ - Sink & nextSink; - bz_stream strm; - bool finished = false; +struct BzipDecompressionSink : ChunkedCompressionSink { + Sink& nextSink; + bz_stream strm; + bool finished = false; - BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink) - { - memset(&strm, 0, sizeof(strm)); - int ret = BZ2_bzDecompressInit(&strm, 0, 0); - if (ret != BZ_OK) - throw CompressionError("unable to initialise bzip2 decoder"); + BzipDecompressionSink(Sink& nextSink) : nextSink(nextSink) { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) + throw CompressionError("unable to initialise bzip2 decoder"); - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } + strm.next_out = (char*)outbuf; + strm.avail_out = sizeof(outbuf); + } - ~BzipDecompressionSink() - { - BZ2_bzDecompressEnd(&strm); - } + ~BzipDecompressionSink() { BZ2_bzDecompressEnd(&strm); } - void finish() override - { - flush(); - write(nullptr, 0); - } + void finish() override { + flush(); + write(nullptr, 0); + } - void writeInternal(const unsigned char * data, size_t len) override - { - assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); + void writeInternal(const unsigned char* data, size_t len) override { + assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); - strm.next_in = (char *) data; - strm.avail_in = len; + strm.next_in = (char*)data; + strm.avail_in = len; - while (strm.avail_in) { - checkInterrupt(); + while (strm.avail_in) { + checkInterrupt(); - int ret = BZ2_bzDecompress(&strm); - if (ret != BZ_OK && ret != BZ_STREAM_END) - throw CompressionError("error while decompressing bzip2 file"); + int ret = BZ2_bzDecompress(&strm); + if (ret != BZ_OK && ret != BZ_STREAM_END) + throw CompressionError("error while decompressing bzip2 file"); - finished = ret == BZ_STREAM_END; + finished = ret == BZ_STREAM_END; - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } - } + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = (char*)outbuf; + strm.avail_out = sizeof(outbuf); + } } + } }; -struct BrotliDecompressionSink : ChunkedCompressionSink -{ - Sink & nextSink; - BrotliDecoderState * state; - bool finished = false; - - BrotliDecompressionSink(Sink & nextSink) : nextSink(nextSink) - { - state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); - if (!state) - throw CompressionError("unable to initialize brotli decoder"); - } +struct BrotliDecompressionSink : ChunkedCompressionSink { + Sink& nextSink; + BrotliDecoderState* state; + bool finished = false; - ~BrotliDecompressionSink() - { - BrotliDecoderDestroyInstance(state); - } + BrotliDecompressionSink(Sink& nextSink) : nextSink(nextSink) { + state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + if (!state) throw CompressionError("unable to initialize brotli decoder"); + } - void finish() override - { - flush(); - writeInternal(nullptr, 0); - } + ~BrotliDecompressionSink() { BrotliDecoderDestroyInstance(state); } + + void finish() override { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char* data, size_t len) override { + const uint8_t* next_in = data; + size_t avail_in = len; + uint8_t* next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (!finished && (!data || avail_in)) { + checkInterrupt(); + + if (!BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out, + &next_out, nullptr)) + throw CompressionError("error while decompressing brotli file"); + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } - void writeInternal(const unsigned char * data, size_t len) override - { - const uint8_t * next_in = data; - size_t avail_in = len; - uint8_t * next_out = outbuf; - size_t avail_out = sizeof(outbuf); - - while (!finished && (!data || avail_in)) { - checkInterrupt(); - - if (!BrotliDecoderDecompressStream(state, - &avail_in, &next_in, - &avail_out, &next_out, - nullptr)) - throw CompressionError("error while decompressing brotli file"); - - if (avail_out < sizeof(outbuf) || avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - avail_out); - next_out = outbuf; - avail_out = sizeof(outbuf); - } - - finished = BrotliDecoderIsFinished(state); - } + finished = BrotliDecoderIsFinished(state); } + } }; -ref<std::string> decompress(const std::string & method, const std::string & in) -{ - StringSink ssink; - auto sink = makeDecompressionSink(method, ssink); - (*sink)(in); - sink->finish(); - return ssink.s; +ref<std::string> decompress(const std::string& method, const std::string& in) { + StringSink ssink; + auto sink = makeDecompressionSink(method, ssink); + (*sink)(in); + sink->finish(); + return ssink.s; } -ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink) -{ - if (method == "none" || method == "") - return make_ref<NoneSink>(nextSink); - else if (method == "xz") - return make_ref<XzDecompressionSink>(nextSink); - else if (method == "bzip2") - return make_ref<BzipDecompressionSink>(nextSink); - else if (method == "br") - return make_ref<BrotliDecompressionSink>(nextSink); - else - throw UnknownCompressionMethod("unknown compression method '%s'", method); +ref<CompressionSink> makeDecompressionSink(const std::string& method, + Sink& nextSink) { + if (method == "none" || method == "") + return make_ref<NoneSink>(nextSink); + else if (method == "xz") + return make_ref<XzDecompressionSink>(nextSink); + else if (method == "bzip2") + return make_ref<BzipDecompressionSink>(nextSink); + else if (method == "br") + return make_ref<BrotliDecompressionSink>(nextSink); + else + throw UnknownCompressionMethod("unknown compression method '%s'", method); } -struct XzCompressionSink : CompressionSink -{ - Sink & nextSink; - uint8_t outbuf[BUFSIZ]; - lzma_stream strm = LZMA_STREAM_INIT; - bool finished = false; +struct XzCompressionSink : CompressionSink { + Sink& nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; - XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink) - { - lzma_ret ret; - bool done = false; + XzCompressionSink(Sink& nextSink, bool parallel) : nextSink(nextSink) { + lzma_ret ret; + bool done = false; - if (parallel) { + if (parallel) { #ifdef HAVE_LZMA_MT - lzma_mt mt_options = {}; - mt_options.flags = 0; - mt_options.timeout = 300; // Using the same setting as the xz cmd line - mt_options.preset = LZMA_PRESET_DEFAULT; - mt_options.filters = NULL; - mt_options.check = LZMA_CHECK_CRC64; - mt_options.threads = lzma_cputhreads(); - mt_options.block_size = 0; - if (mt_options.threads == 0) - mt_options.threads = 1; - // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the - // number of threads. - ret = lzma_stream_encoder_mt(&strm, &mt_options); - done = true; + lzma_mt mt_options = {}; + mt_options.flags = 0; + mt_options.timeout = 300; // Using the same setting as the xz cmd line + mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.filters = NULL; + mt_options.check = LZMA_CHECK_CRC64; + mt_options.threads = lzma_cputhreads(); + mt_options.block_size = 0; + if (mt_options.threads == 0) mt_options.threads = 1; + // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the + // number of threads. + ret = lzma_stream_encoder_mt(&strm, &mt_options); + done = true; #else - printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression"); + printMsg(lvlError, + "warning: parallel XZ compression requested but not supported, " + "falling back to single-threaded compression"); #endif - } + } - if (!done) - ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); + if (!done) ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); - if (ret != LZMA_OK) - throw CompressionError("unable to initialise lzma encoder"); + if (ret != LZMA_OK) + throw CompressionError("unable to initialise lzma encoder"); - // FIXME: apply the x86 BCJ filter? + // FIXME: apply the x86 BCJ filter? - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } - ~XzCompressionSink() - { - lzma_end(&strm); - } + ~XzCompressionSink() { lzma_end(&strm); } - void finish() override - { - CompressionSink::flush(); - write(nullptr, 0); - } + void finish() override { + CompressionSink::flush(); + write(nullptr, 0); + } - void write(const unsigned char * data, size_t len) override - { - strm.next_in = data; - strm.avail_in = len; + void write(const unsigned char* data, size_t len) override { + strm.next_in = data; + strm.avail_in = len; - while (!finished && (!data || strm.avail_in)) { - checkInterrupt(); + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); - lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); - if (ret != LZMA_OK && ret != LZMA_STREAM_END) - throw CompressionError("error %d while compressing xz file", ret); + lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw CompressionError("error %d while compressing xz file", ret); - finished = ret == LZMA_STREAM_END; + finished = ret == LZMA_STREAM_END; - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - } + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } } + } }; -struct BzipCompressionSink : ChunkedCompressionSink -{ - Sink & nextSink; - bz_stream strm; - bool finished = false; +struct BzipCompressionSink : ChunkedCompressionSink { + Sink& nextSink; + bz_stream strm; + bool finished = false; - BzipCompressionSink(Sink & nextSink) : nextSink(nextSink) - { - memset(&strm, 0, sizeof(strm)); - int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); - if (ret != BZ_OK) - throw CompressionError("unable to initialise bzip2 encoder"); + BzipCompressionSink(Sink& nextSink) : nextSink(nextSink) { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); + if (ret != BZ_OK) + throw CompressionError("unable to initialise bzip2 encoder"); - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } + strm.next_out = (char*)outbuf; + strm.avail_out = sizeof(outbuf); + } - ~BzipCompressionSink() - { - BZ2_bzCompressEnd(&strm); - } + ~BzipCompressionSink() { BZ2_bzCompressEnd(&strm); } - void finish() override - { - flush(); - writeInternal(nullptr, 0); - } + void finish() override { + flush(); + writeInternal(nullptr, 0); + } - void writeInternal(const unsigned char * data, size_t len) override - { - assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); + void writeInternal(const unsigned char* data, size_t len) override { + assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); - strm.next_in = (char *) data; - strm.avail_in = len; + strm.next_in = (char*)data; + strm.avail_in = len; - while (!finished && (!data || strm.avail_in)) { - checkInterrupt(); + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); - int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH); - if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END) - throw CompressionError("error %d while compressing bzip2 file", ret); + int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH); + if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END) + throw CompressionError("error %d while compressing bzip2 file", ret); - finished = ret == BZ_STREAM_END; + finished = ret == BZ_STREAM_END; - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } - } + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = (char*)outbuf; + strm.avail_out = sizeof(outbuf); + } } + } }; -struct BrotliCompressionSink : ChunkedCompressionSink -{ - Sink & nextSink; - uint8_t outbuf[BUFSIZ]; - BrotliEncoderState *state; - bool finished = false; - - BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink) - { - state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); - if (!state) - throw CompressionError("unable to initialise brotli encoder"); - } - - ~BrotliCompressionSink() - { - BrotliEncoderDestroyInstance(state); - } - - void finish() override - { - flush(); - writeInternal(nullptr, 0); - } - - void writeInternal(const unsigned char * data, size_t len) override - { - const uint8_t * next_in = data; - size_t avail_in = len; - uint8_t * next_out = outbuf; - size_t avail_out = sizeof(outbuf); - - while (!finished && (!data || avail_in)) { - checkInterrupt(); - - if (!BrotliEncoderCompressStream(state, - data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, - &avail_in, &next_in, - &avail_out, &next_out, - nullptr)) - throw CompressionError("error while compressing brotli compression"); - - if (avail_out < sizeof(outbuf) || avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - avail_out); - next_out = outbuf; - avail_out = sizeof(outbuf); - } - - finished = BrotliEncoderIsFinished(state); - } +struct BrotliCompressionSink : ChunkedCompressionSink { + Sink& nextSink; + uint8_t outbuf[BUFSIZ]; + BrotliEncoderState* state; + bool finished = false; + + BrotliCompressionSink(Sink& nextSink) : nextSink(nextSink) { + state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); + if (!state) throw CompressionError("unable to initialise brotli encoder"); + } + + ~BrotliCompressionSink() { BrotliEncoderDestroyInstance(state); } + + void finish() override { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char* data, size_t len) override { + const uint8_t* next_in = data; + size_t avail_in = len; + uint8_t* next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (!finished && (!data || avail_in)) { + checkInterrupt(); + + if (!BrotliEncoderCompressStream( + state, data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, + &avail_in, &next_in, &avail_out, &next_out, nullptr)) + throw CompressionError("error while compressing brotli compression"); + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + + finished = BrotliEncoderIsFinished(state); } + } }; -ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) -{ - if (method == "none") - return make_ref<NoneSink>(nextSink); - else if (method == "xz") - return make_ref<XzCompressionSink>(nextSink, parallel); - else if (method == "bzip2") - return make_ref<BzipCompressionSink>(nextSink); - else if (method == "br") - return make_ref<BrotliCompressionSink>(nextSink); - else - throw UnknownCompressionMethod(format("unknown compression method '%s'") % method); +ref<CompressionSink> makeCompressionSink(const std::string& method, + Sink& nextSink, const bool parallel) { + if (method == "none") + return make_ref<NoneSink>(nextSink); + else if (method == "xz") + return make_ref<XzCompressionSink>(nextSink, parallel); + else if (method == "bzip2") + return make_ref<BzipCompressionSink>(nextSink); + else if (method == "br") + return make_ref<BrotliCompressionSink>(nextSink); + else + throw UnknownCompressionMethod(format("unknown compression method '%s'") % + method); } -ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel) -{ - StringSink ssink; - auto sink = makeCompressionSink(method, ssink, parallel); - (*sink)(in); - sink->finish(); - return ssink.s; +ref<std::string> compress(const std::string& method, const std::string& in, + const bool parallel) { + StringSink ssink; + auto sink = makeCompressionSink(method, ssink, parallel); + (*sink)(in); + sink->finish(); + return ssink.s; } -} +} // namespace nix diff --git a/third_party/nix/src/libutil/compression.hh b/third_party/nix/src/libutil/compression.hh index dd666a4e19fd..405863f3be02 100644 --- a/third_party/nix/src/libutil/compression.hh +++ b/third_party/nix/src/libutil/compression.hh @@ -1,28 +1,30 @@ #pragma once +#include <string> #include "ref.hh" -#include "types.hh" #include "serialise.hh" - -#include <string> +#include "types.hh" namespace nix { -struct CompressionSink : BufferedSink -{ - virtual void finish() = 0; +struct CompressionSink : BufferedSink { + virtual void finish() = 0; }; -ref<std::string> decompress(const std::string & method, const std::string & in); +ref<std::string> decompress(const std::string& method, const std::string& in); -ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink); +ref<CompressionSink> makeDecompressionSink(const std::string& method, + Sink& nextSink); -ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false); +ref<std::string> compress(const std::string& method, const std::string& in, + const bool parallel = false); -ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false); +ref<CompressionSink> makeCompressionSink(const std::string& method, + Sink& nextSink, + const bool parallel = false); MakeError(UnknownCompressionMethod, Error); MakeError(CompressionError, Error); -} +} // namespace nix diff --git a/third_party/nix/src/libutil/config.cc b/third_party/nix/src/libutil/config.cc index 9023cb1bb6de..d737945924e8 100644 --- a/third_party/nix/src/libutil/config.cc +++ b/third_party/nix/src/libutil/config.cc @@ -4,268 +4,251 @@ namespace nix { -bool Config::set(const std::string & name, const std::string & value) -{ - auto i = _settings.find(name); - if (i == _settings.end()) return false; - i->second.setting->set(value); - i->second.setting->overriden = true; - return true; -} - -void Config::addSetting(AbstractSetting * setting) -{ - _settings.emplace(setting->name, Config::SettingData(false, setting)); - for (auto & alias : setting->aliases) - _settings.emplace(alias, Config::SettingData(true, setting)); - - bool set = false; - - auto i = unknownSettings.find(setting->name); +bool Config::set(const std::string& name, const std::string& value) { + auto i = _settings.find(name); + if (i == _settings.end()) return false; + i->second.setting->set(value); + i->second.setting->overriden = true; + return true; +} + +void Config::addSetting(AbstractSetting* setting) { + _settings.emplace(setting->name, Config::SettingData(false, setting)); + for (auto& alias : setting->aliases) + _settings.emplace(alias, Config::SettingData(true, setting)); + + bool set = false; + + auto i = unknownSettings.find(setting->name); + if (i != unknownSettings.end()) { + setting->set(i->second); + setting->overriden = true; + unknownSettings.erase(i); + set = true; + } + + for (auto& alias : setting->aliases) { + auto i = unknownSettings.find(alias); if (i != unknownSettings.end()) { + if (set) + warn("setting '%s' is set, but it's an alias of '%s' which is also set", + alias, setting->name); + else { setting->set(i->second); setting->overriden = true; unknownSettings.erase(i); set = true; + } } - - for (auto & alias : setting->aliases) { - auto i = unknownSettings.find(alias); - if (i != unknownSettings.end()) { - if (set) - warn("setting '%s' is set, but it's an alias of '%s' which is also set", - alias, setting->name); - else { - setting->set(i->second); - setting->overriden = true; - unknownSettings.erase(i); - set = true; - } - } - } + } } -void AbstractConfig::warnUnknownSettings() -{ - for (auto & s : unknownSettings) - warn("unknown setting '%s'", s.first); +void AbstractConfig::warnUnknownSettings() { + for (auto& s : unknownSettings) warn("unknown setting '%s'", s.first); } -void AbstractConfig::reapplyUnknownSettings() -{ - auto unknownSettings2 = std::move(unknownSettings); - for (auto & s : unknownSettings2) - set(s.first, s.second); +void AbstractConfig::reapplyUnknownSettings() { + auto unknownSettings2 = std::move(unknownSettings); + for (auto& s : unknownSettings2) set(s.first, s.second); } -void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly) -{ - for (auto & opt : _settings) - if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden)) - res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); +void Config::getSettings(std::map<std::string, SettingInfo>& res, + bool overridenOnly) { + for (auto& opt : _settings) + if (!opt.second.isAlias && + (!overridenOnly || opt.second.setting->overriden)) + res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), + opt.second.setting->description}); } -void AbstractConfig::applyConfigFile(const Path & path) -{ - try { - string contents = readFile(path); - - unsigned int pos = 0; - - while (pos < contents.size()) { - string line; - while (pos < contents.size() && contents[pos] != '\n') - line += contents[pos++]; - pos++; - - string::size_type hash = line.find('#'); - if (hash != string::npos) - line = string(line, 0, hash); - - vector<string> tokens = tokenizeString<vector<string> >(line); - if (tokens.empty()) continue; - - if (tokens.size() < 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - - auto include = false; - auto ignoreMissing = false; - if (tokens[0] == "include") - include = true; - else if (tokens[0] == "!include") { - include = true; - ignoreMissing = true; - } - - if (include) { - if (tokens.size() != 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - auto p = absPath(tokens[1], dirOf(path)); - if (pathExists(p)) { - applyConfigFile(p); - } else if (!ignoreMissing) { - throw Error("file '%1%' included from '%2%' not found", p, path); - } - continue; - } - - if (tokens[1] != "=") - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - - string name = tokens[0]; - - vector<string>::iterator i = tokens.begin(); - advance(i, 2); - - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow - }; - } catch (SysError &) { } -} +void AbstractConfig::applyConfigFile(const Path& path) { + try { + string contents = readFile(path); -void Config::resetOverriden() -{ - for (auto & s : _settings) - s.second.setting->overriden = false; -} + unsigned int pos = 0; + + while (pos < contents.size()) { + string line; + while (pos < contents.size() && contents[pos] != '\n') + line += contents[pos++]; + pos++; + + string::size_type hash = line.find('#'); + if (hash != string::npos) line = string(line, 0, hash); + + vector<string> tokens = tokenizeString<vector<string> >(line); + if (tokens.empty()) continue; + + if (tokens.size() < 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, + path); -void Config::toJSON(JSONObject & out) -{ - for (auto & s : _settings) - if (!s.second.isAlias) { - JSONObject out2(out.object(s.first)); - out2.attr("description", s.second.setting->description); - JSONPlaceholder out3(out2.placeholder("value")); - s.second.setting->toJSON(out3); + auto include = false; + auto ignoreMissing = false; + if (tokens[0] == "include") + include = true; + else if (tokens[0] == "!include") { + include = true; + ignoreMissing = true; + } + + if (include) { + if (tokens.size() != 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, + path); + auto p = absPath(tokens[1], dirOf(path)); + if (pathExists(p)) { + applyConfigFile(p); + } else if (!ignoreMissing) { + throw Error("file '%1%' included from '%2%' not found", p, path); } -} + continue; + } + + if (tokens[1] != "=") + throw UsageError("illegal configuration line '%1%' in '%2%'", line, + path); -void Config::convertToArgs(Args & args, const std::string & category) -{ - for (auto & s : _settings) - if (!s.second.isAlias) - s.second.setting->convertToArg(args, category); + string name = tokens[0]; + + vector<string>::iterator i = tokens.begin(); + advance(i, 2); + + set(name, + concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + }; + } catch (SysError&) { + } } -AbstractSetting::AbstractSetting( - const std::string & name, - const std::string & description, - const std::set<std::string> & aliases) - : name(name), description(description), aliases(aliases) -{ +void Config::resetOverriden() { + for (auto& s : _settings) s.second.setting->overriden = false; } -void AbstractSetting::toJSON(JSONPlaceholder & out) -{ - out.write(to_string()); +void Config::toJSON(JSONObject& out) { + for (auto& s : _settings) + if (!s.second.isAlias) { + JSONObject out2(out.object(s.first)); + out2.attr("description", s.second.setting->description); + JSONPlaceholder out3(out2.placeholder("value")); + s.second.setting->toJSON(out3); + } } -void AbstractSetting::convertToArg(Args & args, const std::string & category) -{ +void Config::convertToArgs(Args& args, const std::string& category) { + for (auto& s : _settings) + if (!s.second.isAlias) s.second.setting->convertToArg(args, category); } -template<typename T> -void BaseSetting<T>::toJSON(JSONPlaceholder & out) -{ - out.write(value); +AbstractSetting::AbstractSetting(const std::string& name, + const std::string& description, + const std::set<std::string>& aliases) + : name(name), description(description), aliases(aliases) {} + +void AbstractSetting::toJSON(JSONPlaceholder& out) { out.write(to_string()); } + +void AbstractSetting::convertToArg(Args& args, const std::string& category) {} + +template <typename T> +void BaseSetting<T>::toJSON(JSONPlaceholder& out) { + out.write(value); } -template<typename T> -void BaseSetting<T>::convertToArg(Args & args, const std::string & category) -{ - args.mkFlag() - .longName(name) - .description(description) - .arity(1) - .handler([=](std::vector<std::string> ss) { overriden = true; set(ss[0]); }) - .category(category); +template <typename T> +void BaseSetting<T>::convertToArg(Args& args, const std::string& category) { + args.mkFlag() + .longName(name) + .description(description) + .arity(1) + .handler([=](std::vector<std::string> ss) { + overriden = true; + set(ss[0]); + }) + .category(category); } -template<> void BaseSetting<std::string>::set(const std::string & str) -{ - value = str; +template <> +void BaseSetting<std::string>::set(const std::string& str) { + value = str; } -template<> std::string BaseSetting<std::string>::to_string() -{ - return value; +template <> +std::string BaseSetting<std::string>::to_string() { + return value; } -template<typename T> -void BaseSetting<T>::set(const std::string & str) -{ - static_assert(std::is_integral<T>::value, "Integer required."); - if (!string2Int(str, value)) - throw UsageError("setting '%s' has invalid value '%s'", name, str); +template <typename T> +void BaseSetting<T>::set(const std::string& str) { + static_assert(std::is_integral<T>::value, "Integer required."); + if (!string2Int(str, value)) + throw UsageError("setting '%s' has invalid value '%s'", name, str); } -template<typename T> -std::string BaseSetting<T>::to_string() -{ - static_assert(std::is_integral<T>::value, "Integer required."); - return std::to_string(value); +template <typename T> +std::string BaseSetting<T>::to_string() { + static_assert(std::is_integral<T>::value, "Integer required."); + return std::to_string(value); } -template<> void BaseSetting<bool>::set(const std::string & str) -{ - if (str == "true" || str == "yes" || str == "1") - value = true; - else if (str == "false" || str == "no" || str == "0") - value = false; - else - throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); +template <> +void BaseSetting<bool>::set(const std::string& str) { + if (str == "true" || str == "yes" || str == "1") + value = true; + else if (str == "false" || str == "no" || str == "0") + value = false; + else + throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); } -template<> std::string BaseSetting<bool>::to_string() -{ - return value ? "true" : "false"; +template <> +std::string BaseSetting<bool>::to_string() { + return value ? "true" : "false"; } -template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & category) -{ - args.mkFlag() - .longName(name) - .description(description) - .handler([=](std::vector<std::string> ss) { override(true); }) - .category(category); - args.mkFlag() - .longName("no-" + name) - .description(description) - .handler([=](std::vector<std::string> ss) { override(false); }) - .category(category); +template <> +void BaseSetting<bool>::convertToArg(Args& args, const std::string& category) { + args.mkFlag() + .longName(name) + .description(description) + .handler([=](std::vector<std::string> ss) { override(true); }) + .category(category); + args.mkFlag() + .longName("no-" + name) + .description(description) + .handler([=](std::vector<std::string> ss) { override(false); }) + .category(category); } -template<> void BaseSetting<Strings>::set(const std::string & str) -{ - value = tokenizeString<Strings>(str); +template <> +void BaseSetting<Strings>::set(const std::string& str) { + value = tokenizeString<Strings>(str); } -template<> std::string BaseSetting<Strings>::to_string() -{ - return concatStringsSep(" ", value); +template <> +std::string BaseSetting<Strings>::to_string() { + return concatStringsSep(" ", value); } -template<> void BaseSetting<Strings>::toJSON(JSONPlaceholder & out) -{ - JSONList list(out.list()); - for (auto & s : value) - list.elem(s); +template <> +void BaseSetting<Strings>::toJSON(JSONPlaceholder& out) { + JSONList list(out.list()); + for (auto& s : value) list.elem(s); } -template<> void BaseSetting<StringSet>::set(const std::string & str) -{ - value = tokenizeString<StringSet>(str); +template <> +void BaseSetting<StringSet>::set(const std::string& str) { + value = tokenizeString<StringSet>(str); } -template<> std::string BaseSetting<StringSet>::to_string() -{ - return concatStringsSep(" ", value); +template <> +std::string BaseSetting<StringSet>::to_string() { + return concatStringsSep(" ", value); } -template<> void BaseSetting<StringSet>::toJSON(JSONPlaceholder & out) -{ - JSONList list(out.list()); - for (auto & s : value) - list.elem(s); +template <> +void BaseSetting<StringSet>::toJSON(JSONPlaceholder& out) { + JSONList list(out.list()); + for (auto& s : value) list.elem(s); } template class BaseSetting<int>; @@ -279,60 +262,51 @@ template class BaseSetting<std::string>; template class BaseSetting<Strings>; template class BaseSetting<StringSet>; -void PathSetting::set(const std::string & str) -{ - if (str == "") { - if (allowEmpty) - value = ""; - else - throw UsageError("setting '%s' cannot be empty", name); - } else - value = canonPath(str); +void PathSetting::set(const std::string& str) { + if (str == "") { + if (allowEmpty) + value = ""; + else + throw UsageError("setting '%s' cannot be empty", name); + } else + value = canonPath(str); } -bool GlobalConfig::set(const std::string & name, const std::string & value) -{ - for (auto & config : *configRegistrations) - if (config->set(name, value)) return true; +bool GlobalConfig::set(const std::string& name, const std::string& value) { + for (auto& config : *configRegistrations) + if (config->set(name, value)) return true; - unknownSettings.emplace(name, value); + unknownSettings.emplace(name, value); - return false; + return false; } -void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly) -{ - for (auto & config : *configRegistrations) - config->getSettings(res, overridenOnly); +void GlobalConfig::getSettings(std::map<std::string, SettingInfo>& res, + bool overridenOnly) { + for (auto& config : *configRegistrations) + config->getSettings(res, overridenOnly); } -void GlobalConfig::resetOverriden() -{ - for (auto & config : *configRegistrations) - config->resetOverriden(); +void GlobalConfig::resetOverriden() { + for (auto& config : *configRegistrations) config->resetOverriden(); } -void GlobalConfig::toJSON(JSONObject & out) -{ - for (auto & config : *configRegistrations) - config->toJSON(out); +void GlobalConfig::toJSON(JSONObject& out) { + for (auto& config : *configRegistrations) config->toJSON(out); } -void GlobalConfig::convertToArgs(Args & args, const std::string & category) -{ - for (auto & config : *configRegistrations) - config->convertToArgs(args, category); +void GlobalConfig::convertToArgs(Args& args, const std::string& category) { + for (auto& config : *configRegistrations) + config->convertToArgs(args, category); } GlobalConfig globalConfig; -GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; +GlobalConfig::ConfigRegistrations* GlobalConfig::configRegistrations; -GlobalConfig::Register::Register(Config * config) -{ - if (!configRegistrations) - configRegistrations = new ConfigRegistrations; - configRegistrations->emplace_back(config); +GlobalConfig::Register::Register(Config* config) { + if (!configRegistrations) configRegistrations = new ConfigRegistrations; + configRegistrations->emplace_back(config); } -} +} // namespace nix diff --git a/third_party/nix/src/libutil/config.hh b/third_party/nix/src/libutil/config.hh index d86c65ff033a..375fefa52b7c 100644 --- a/third_party/nix/src/libutil/config.hh +++ b/third_party/nix/src/libutil/config.hh @@ -1,6 +1,5 @@ #include <map> #include <set> - #include "types.hh" #pragma once @@ -12,38 +11,34 @@ class AbstractSetting; class JSONPlaceholder; class JSONObject; -class AbstractConfig -{ -protected: - StringMap unknownSettings; - - AbstractConfig(const StringMap & initials = {}) - : unknownSettings(initials) - { } +class AbstractConfig { + protected: + StringMap unknownSettings; -public: + AbstractConfig(const StringMap& initials = {}) : unknownSettings(initials) {} - virtual bool set(const std::string & name, const std::string & value) = 0; + public: + virtual bool set(const std::string& name, const std::string& value) = 0; - struct SettingInfo - { - std::string value; - std::string description; - }; + struct SettingInfo { + std::string value; + std::string description; + }; - virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0; + virtual void getSettings(std::map<std::string, SettingInfo>& res, + bool overridenOnly = false) = 0; - void applyConfigFile(const Path & path); + void applyConfigFile(const Path& path); - virtual void resetOverriden() = 0; + virtual void resetOverriden() = 0; - virtual void toJSON(JSONObject & out) = 0; + virtual void toJSON(JSONObject& out) = 0; - virtual void convertToArgs(Args & args, const std::string & category) = 0; + virtual void convertToArgs(Args& args, const std::string& category) = 0; - void warnUnknownSettings(); + void warnUnknownSettings(); - void reapplyUnknownSettings(); + void reapplyUnknownSettings(); }; /* A class to simplify providing configuration settings. The typical @@ -61,201 +56,171 @@ public: }; */ -class Config : public AbstractConfig -{ - friend class AbstractSetting; - -public: - - struct SettingData - { - bool isAlias; - AbstractSetting * setting; - SettingData(bool isAlias, AbstractSetting * setting) - : isAlias(isAlias), setting(setting) - { } - }; +class Config : public AbstractConfig { + friend class AbstractSetting; - typedef std::map<std::string, SettingData> Settings; + public: + struct SettingData { + bool isAlias; + AbstractSetting* setting; + SettingData(bool isAlias, AbstractSetting* setting) + : isAlias(isAlias), setting(setting) {} + }; -private: + typedef std::map<std::string, SettingData> Settings; - Settings _settings; + private: + Settings _settings; -public: + public: + Config(const StringMap& initials = {}) : AbstractConfig(initials) {} - Config(const StringMap & initials = {}) - : AbstractConfig(initials) - { } + bool set(const std::string& name, const std::string& value) override; - bool set(const std::string & name, const std::string & value) override; + void addSetting(AbstractSetting* setting); - void addSetting(AbstractSetting * setting); + void getSettings(std::map<std::string, SettingInfo>& res, + bool overridenOnly = false) override; - void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override; + void resetOverriden() override; - void resetOverriden() override; + void toJSON(JSONObject& out) override; - void toJSON(JSONObject & out) override; - - void convertToArgs(Args & args, const std::string & category) override; + void convertToArgs(Args& args, const std::string& category) override; }; -class AbstractSetting -{ - friend class Config; - -public: - - const std::string name; - const std::string description; - const std::set<std::string> aliases; +class AbstractSetting { + friend class Config; - int created = 123; + public: + const std::string name; + const std::string description; + const std::set<std::string> aliases; - bool overriden = false; + int created = 123; -protected: + bool overriden = false; - AbstractSetting( - const std::string & name, - const std::string & description, - const std::set<std::string> & aliases); + protected: + AbstractSetting(const std::string& name, const std::string& description, + const std::set<std::string>& aliases); - virtual ~AbstractSetting() - { - // Check against a gcc miscompilation causing our constructor - // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). - assert(created == 123); - } + virtual ~AbstractSetting() { + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); + } - virtual void set(const std::string & value) = 0; + virtual void set(const std::string& value) = 0; - virtual std::string to_string() = 0; + virtual std::string to_string() = 0; - virtual void toJSON(JSONPlaceholder & out); + virtual void toJSON(JSONPlaceholder& out); - virtual void convertToArg(Args & args, const std::string & category); + virtual void convertToArg(Args& args, const std::string& category); - bool isOverriden() { return overriden; } + bool isOverriden() { return overriden; } }; /* A setting of type T. */ -template<typename T> -class BaseSetting : public AbstractSetting -{ -protected: +template <typename T> +class BaseSetting : public AbstractSetting { + protected: + T value; - T value; + public: + BaseSetting(const T& def, const std::string& name, + const std::string& description, + const std::set<std::string>& aliases = {}) + : AbstractSetting(name, description, aliases), value(def) {} -public: + operator const T&() const { return value; } + operator T&() { return value; } + const T& get() const { return value; } + bool operator==(const T& v2) const { return value == v2; } + bool operator!=(const T& v2) const { return value != v2; } + void operator=(const T& v) { assign(v); } + virtual void assign(const T& v) { value = v; } - BaseSetting(const T & def, - const std::string & name, - const std::string & description, - const std::set<std::string> & aliases = {}) - : AbstractSetting(name, description, aliases) - , value(def) - { } + void set(const std::string& str) override; - operator const T &() const { return value; } - operator T &() { return value; } - const T & get() const { return value; } - bool operator ==(const T & v2) const { return value == v2; } - bool operator !=(const T & v2) const { return value != v2; } - void operator =(const T & v) { assign(v); } - virtual void assign(const T & v) { value = v; } + virtual void override(const T& v) { + overriden = true; + value = v; + } - void set(const std::string & str) override; + std::string to_string() override; - virtual void override(const T & v) - { - overriden = true; - value = v; - } + void convertToArg(Args& args, const std::string& category) override; - std::string to_string() override; - - void convertToArg(Args & args, const std::string & category) override; - - void toJSON(JSONPlaceholder & out) override; + void toJSON(JSONPlaceholder& out) override; }; -template<typename T> -std::ostream & operator <<(std::ostream & str, const BaseSetting<T> & opt) -{ - str << (const T &) opt; - return str; +template <typename T> +std::ostream& operator<<(std::ostream& str, const BaseSetting<T>& opt) { + str << (const T&)opt; + return str; +} + +template <typename T> +bool operator==(const T& v1, const BaseSetting<T>& v2) { + return v1 == (const T&)v2; } -template<typename T> -bool operator ==(const T & v1, const BaseSetting<T> & v2) { return v1 == (const T &) v2; } - -template<typename T> -class Setting : public BaseSetting<T> -{ -public: - Setting(Config * options, - const T & def, - const std::string & name, - const std::string & description, - const std::set<std::string> & aliases = {}) - : BaseSetting<T>(def, name, description, aliases) - { - options->addSetting(this); - } - - void operator =(const T & v) { this->assign(v); } +template <typename T> +class Setting : public BaseSetting<T> { + public: + Setting(Config* options, const T& def, const std::string& name, + const std::string& description, + const std::set<std::string>& aliases = {}) + : BaseSetting<T>(def, name, description, aliases) { + options->addSetting(this); + } + + void operator=(const T& v) { this->assign(v); } }; /* A special setting for Paths. These are automatically canonicalised (e.g. "/foo//bar/" becomes "/foo/bar"). */ -class PathSetting : public BaseSetting<Path> -{ - bool allowEmpty; - -public: +class PathSetting : public BaseSetting<Path> { + bool allowEmpty; - PathSetting(Config * options, - bool allowEmpty, - const Path & def, - const std::string & name, - const std::string & description, - const std::set<std::string> & aliases = {}) - : BaseSetting<Path>(def, name, description, aliases) - , allowEmpty(allowEmpty) - { - options->addSetting(this); - } + public: + PathSetting(Config* options, bool allowEmpty, const Path& def, + const std::string& name, const std::string& description, + const std::set<std::string>& aliases = {}) + : BaseSetting<Path>(def, name, description, aliases), + allowEmpty(allowEmpty) { + options->addSetting(this); + } - void set(const std::string & str) override; + void set(const std::string& str) override; - Path operator +(const char * p) const { return value + p; } + Path operator+(const char* p) const { return value + p; } - void operator =(const Path & v) { this->assign(v); } + void operator=(const Path& v) { this->assign(v); } }; -struct GlobalConfig : public AbstractConfig -{ - typedef std::vector<Config*> ConfigRegistrations; - static ConfigRegistrations * configRegistrations; +struct GlobalConfig : public AbstractConfig { + typedef std::vector<Config*> ConfigRegistrations; + static ConfigRegistrations* configRegistrations; - bool set(const std::string & name, const std::string & value) override; + bool set(const std::string& name, const std::string& value) override; - void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override; + void getSettings(std::map<std::string, SettingInfo>& res, + bool overridenOnly = false) override; - void resetOverriden() override; + void resetOverriden() override; - void toJSON(JSONObject & out) override; + void toJSON(JSONObject& out) override; - void convertToArgs(Args & args, const std::string & category) override; + void convertToArgs(Args& args, const std::string& category) override; - struct Register - { - Register(Config * config); - }; + struct Register { + Register(Config* config); + }; }; extern GlobalConfig globalConfig; -} +} // namespace nix diff --git a/third_party/nix/src/libutil/finally.hh b/third_party/nix/src/libutil/finally.hh index 7760cfe9a410..8d3083b6a3ac 100644 --- a/third_party/nix/src/libutil/finally.hh +++ b/third_party/nix/src/libutil/finally.hh @@ -3,12 +3,11 @@ #include <functional> /* A trivial class to run a function at the end of a scope. */ -class Finally -{ -private: - std::function<void()> fun; +class Finally { + private: + std::function<void()> fun; -public: - Finally(std::function<void()> fun) : fun(fun) { } - ~Finally() { fun(); } + public: + Finally(std::function<void()> fun) : fun(fun) {} + ~Finally() { fun(); } }; diff --git a/third_party/nix/src/libutil/hash.cc b/third_party/nix/src/libutil/hash.cc index 1c14ebb187cc..647822213278 100644 --- a/third_party/nix/src/libutil/hash.cc +++ b/third_party/nix/src/libutil/hash.cc @@ -1,355 +1,323 @@ -#include <iostream> -#include <cstring> - +#include "hash.hh" +#include <fcntl.h> #include <openssl/md5.h> #include <openssl/sha.h> - -#include "hash.hh" +#include <sys/stat.h> +#include <sys/types.h> +#include <cstring> +#include <iostream> #include "archive.hh" -#include "util.hh" #include "istringstream_nocopy.hh" - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> +#include "util.hh" namespace nix { - -void Hash::init() -{ - if (type == htMD5) hashSize = md5HashSize; - else if (type == htSHA1) hashSize = sha1HashSize; - else if (type == htSHA256) hashSize = sha256HashSize; - else if (type == htSHA512) hashSize = sha512HashSize; - else abort(); - assert(hashSize <= maxHashSize); - memset(hash, 0, maxHashSize); -} - - -bool Hash::operator == (const Hash & h2) const -{ - if (hashSize != h2.hashSize) return false; - for (unsigned int i = 0; i < hashSize; i++) - if (hash[i] != h2.hash[i]) return false; - return true; +void Hash::init() { + if (type == htMD5) + hashSize = md5HashSize; + else if (type == htSHA1) + hashSize = sha1HashSize; + else if (type == htSHA256) + hashSize = sha256HashSize; + else if (type == htSHA512) + hashSize = sha512HashSize; + else + abort(); + assert(hashSize <= maxHashSize); + memset(hash, 0, maxHashSize); } - -bool Hash::operator != (const Hash & h2) const -{ - return !(*this == h2); +bool Hash::operator==(const Hash& h2) const { + if (hashSize != h2.hashSize) return false; + for (unsigned int i = 0; i < hashSize; i++) + if (hash[i] != h2.hash[i]) return false; + return true; } +bool Hash::operator!=(const Hash& h2) const { return !(*this == h2); } -bool Hash::operator < (const Hash & h) const -{ - if (hashSize < h.hashSize) return true; - if (hashSize > h.hashSize) return false; - for (unsigned int i = 0; i < hashSize; i++) { - if (hash[i] < h.hash[i]) return true; - if (hash[i] > h.hash[i]) return false; - } - return false; +bool Hash::operator<(const Hash& h) const { + if (hashSize < h.hashSize) return true; + if (hashSize > h.hashSize) return false; + for (unsigned int i = 0; i < hashSize; i++) { + if (hash[i] < h.hash[i]) return true; + if (hash[i] > h.hash[i]) return false; + } + return false; } - const string base16Chars = "0123456789abcdef"; - -static string printHash16(const Hash & hash) -{ - char buf[hash.hashSize * 2]; - for (unsigned int i = 0; i < hash.hashSize; i++) { - buf[i * 2] = base16Chars[hash.hash[i] >> 4]; - buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; - } - return string(buf, hash.hashSize * 2); +static string printHash16(const Hash& hash) { + char buf[hash.hashSize * 2]; + for (unsigned int i = 0; i < hash.hashSize; i++) { + buf[i * 2] = base16Chars[hash.hash[i] >> 4]; + buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; + } + return string(buf, hash.hashSize * 2); } - // omitted: E O U T const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; - -static string printHash32(const Hash & hash) -{ - assert(hash.hashSize); - size_t len = hash.base32Len(); - assert(len); - - string s; - s.reserve(len); - - for (int n = (int) len - 1; n >= 0; n--) { - unsigned int b = n * 5; - unsigned int i = b / 8; - unsigned int j = b % 8; - unsigned char c = - (hash.hash[i] >> j) - | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j)); - s.push_back(base32Chars[c & 0x1f]); - } - - return s; +static string printHash32(const Hash& hash) { + assert(hash.hashSize); + size_t len = hash.base32Len(); + assert(len); + + string s; + s.reserve(len); + + for (int n = (int)len - 1; n >= 0; n--) { + unsigned int b = n * 5; + unsigned int i = b / 8; + unsigned int j = b % 8; + unsigned char c = + (hash.hash[i] >> j) | + (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j)); + s.push_back(base32Chars[c & 0x1f]); + } + + return s; } - -string printHash16or32(const Hash & hash) -{ - return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); +string printHash16or32(const Hash& hash) { + return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); } - -std::string Hash::to_string(Base base, bool includeType) const -{ - std::string s; - if (base == SRI || includeType) { - s += printHashType(type); - s += base == SRI ? '-' : ':'; - } - switch (base) { +std::string Hash::to_string(Base base, bool includeType) const { + std::string s; + if (base == SRI || includeType) { + s += printHashType(type); + s += base == SRI ? '-' : ':'; + } + switch (base) { case Base16: - s += printHash16(*this); - break; + s += printHash16(*this); + break; case Base32: - s += printHash32(*this); - break; + s += printHash32(*this); + break; case Base64: case SRI: - s += base64Encode(std::string((const char *) hash, hashSize)); - break; - } - return s; + s += base64Encode(std::string((const char*)hash, hashSize)); + break; + } + return s; } +Hash::Hash(const std::string& s, HashType type) : type(type) { + size_t pos = 0; + bool isSRI = false; -Hash::Hash(const std::string & s, HashType type) - : type(type) -{ - size_t pos = 0; - bool isSRI = false; - - auto sep = s.find(':'); - if (sep == string::npos) { - sep = s.find('-'); - if (sep != string::npos) { - isSRI = true; - } else if (type == htUnknown) - throw BadHash("hash '%s' does not include a type", s); - } - + auto sep = s.find(':'); + if (sep == string::npos) { + sep = s.find('-'); if (sep != string::npos) { - string hts = string(s, 0, sep); - this->type = parseHashType(hts); - if (this->type == htUnknown) - throw BadHash("unknown hash type '%s'", hts); - if (type != htUnknown && type != this->type) - throw BadHash("hash '%s' should have type '%s'", s, printHashType(type)); - pos = sep + 1; - } - - init(); - - size_t size = s.size() - pos; - - if (!isSRI && size == base16Len()) { + isSRI = true; + } else if (type == htUnknown) + throw BadHash("hash '%s' does not include a type", s); + } + + if (sep != string::npos) { + string hts = string(s, 0, sep); + this->type = parseHashType(hts); + if (this->type == htUnknown) throw BadHash("unknown hash type '%s'", hts); + if (type != htUnknown && type != this->type) + throw BadHash("hash '%s' should have type '%s'", s, printHashType(type)); + pos = sep + 1; + } + + init(); + + size_t size = s.size() - pos; + + if (!isSRI && size == base16Len()) { + auto parseHexDigit = [&](char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + throw BadHash("invalid base-16 hash '%s'", s); + }; - auto parseHexDigit = [&](char c) { - if (c >= '0' && c <= '9') return c - '0'; - if (c >= 'A' && c <= 'F') return c - 'A' + 10; - if (c >= 'a' && c <= 'f') return c - 'a' + 10; - throw BadHash("invalid base-16 hash '%s'", s); - }; - - for (unsigned int i = 0; i < hashSize; i++) { - hash[i] = - parseHexDigit(s[pos + i * 2]) << 4 - | parseHexDigit(s[pos + i * 2 + 1]); - } - } - - else if (!isSRI && size == base32Len()) { - - for (unsigned int n = 0; n < size; ++n) { - char c = s[pos + size - n - 1]; - unsigned char digit; - for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ - if (base32Chars[digit] == c) break; - if (digit >= 32) - throw BadHash("invalid base-32 hash '%s'", s); - unsigned int b = n * 5; - unsigned int i = b / 8; - unsigned int j = b % 8; - hash[i] |= digit << j; - - if (i < hashSize - 1) { - hash[i + 1] |= digit >> (8 - j); - } else { - if (digit >> (8 - j)) - throw BadHash("invalid base-32 hash '%s'", s); - } - } + for (unsigned int i = 0; i < hashSize; i++) { + hash[i] = parseHexDigit(s[pos + i * 2]) << 4 | + parseHexDigit(s[pos + i * 2 + 1]); } - - else if (isSRI || size == base64Len()) { - auto d = base64Decode(std::string(s, pos)); - if (d.size() != hashSize) - throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); - assert(hashSize); - memcpy(hash, d.data(), hashSize); + } + + else if (!isSRI && size == base32Len()) { + for (unsigned int n = 0; n < size; ++n) { + char c = s[pos + size - n - 1]; + unsigned char digit; + for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ + if (base32Chars[digit] == c) break; + if (digit >= 32) throw BadHash("invalid base-32 hash '%s'", s); + unsigned int b = n * 5; + unsigned int i = b / 8; + unsigned int j = b % 8; + hash[i] |= digit << j; + + if (i < hashSize - 1) { + hash[i + 1] |= digit >> (8 - j); + } else { + if (digit >> (8 - j)) throw BadHash("invalid base-32 hash '%s'", s); + } } - - else - throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); + } + + else if (isSRI || size == base64Len()) { + auto d = base64Decode(std::string(s, pos)); + if (d.size() != hashSize) + throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); + assert(hashSize); + memcpy(hash, d.data(), hashSize); + } + + else + throw BadHash("hash '%s' has wrong length for hash type '%s'", s, + printHashType(type)); } - -union Ctx -{ - MD5_CTX md5; - SHA_CTX sha1; - SHA256_CTX sha256; - SHA512_CTX sha512; +union Ctx { + MD5_CTX md5; + SHA_CTX sha1; + SHA256_CTX sha256; + SHA512_CTX sha512; }; - -static void start(HashType ht, Ctx & ctx) -{ - if (ht == htMD5) MD5_Init(&ctx.md5); - else if (ht == htSHA1) SHA1_Init(&ctx.sha1); - else if (ht == htSHA256) SHA256_Init(&ctx.sha256); - else if (ht == htSHA512) SHA512_Init(&ctx.sha512); +static void start(HashType ht, Ctx& ctx) { + if (ht == htMD5) + MD5_Init(&ctx.md5); + else if (ht == htSHA1) + SHA1_Init(&ctx.sha1); + else if (ht == htSHA256) + SHA256_Init(&ctx.sha256); + else if (ht == htSHA512) + SHA512_Init(&ctx.sha512); } - -static void update(HashType ht, Ctx & ctx, - const unsigned char * bytes, size_t len) -{ - if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); - else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); - else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len); - else if (ht == htSHA512) SHA512_Update(&ctx.sha512, bytes, len); +static void update(HashType ht, Ctx& ctx, const unsigned char* bytes, + size_t len) { + if (ht == htMD5) + MD5_Update(&ctx.md5, bytes, len); + else if (ht == htSHA1) + SHA1_Update(&ctx.sha1, bytes, len); + else if (ht == htSHA256) + SHA256_Update(&ctx.sha256, bytes, len); + else if (ht == htSHA512) + SHA512_Update(&ctx.sha512, bytes, len); } - -static void finish(HashType ht, Ctx & ctx, unsigned char * hash) -{ - if (ht == htMD5) MD5_Final(hash, &ctx.md5); - else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); - else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); - else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512); +static void finish(HashType ht, Ctx& ctx, unsigned char* hash) { + if (ht == htMD5) + MD5_Final(hash, &ctx.md5); + else if (ht == htSHA1) + SHA1_Final(hash, &ctx.sha1); + else if (ht == htSHA256) + SHA256_Final(hash, &ctx.sha256); + else if (ht == htSHA512) + SHA512_Final(hash, &ctx.sha512); } - -Hash hashString(HashType ht, const string & s) -{ - Ctx ctx; - Hash hash(ht); - start(ht, ctx); - update(ht, ctx, (const unsigned char *) s.data(), s.length()); - finish(ht, ctx, hash.hash); - return hash; +Hash hashString(HashType ht, const string& s) { + Ctx ctx; + Hash hash(ht); + start(ht, ctx); + update(ht, ctx, (const unsigned char*)s.data(), s.length()); + finish(ht, ctx, hash.hash); + return hash; } +Hash hashFile(HashType ht, const Path& path) { + Ctx ctx; + Hash hash(ht); + start(ht, ctx); -Hash hashFile(HashType ht, const Path & path) -{ - Ctx ctx; - Hash hash(ht); - start(ht, ctx); - - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError(format("opening file '%1%'") % path); + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError(format("opening file '%1%'") % path); - std::vector<unsigned char> buf(8192); - ssize_t n; - while ((n = read(fd.get(), buf.data(), buf.size()))) { - checkInterrupt(); - if (n == -1) throw SysError(format("reading file '%1%'") % path); - update(ht, ctx, buf.data(), n); - } + std::vector<unsigned char> buf(8192); + ssize_t n; + while ((n = read(fd.get(), buf.data(), buf.size()))) { + checkInterrupt(); + if (n == -1) throw SysError(format("reading file '%1%'") % path); + update(ht, ctx, buf.data(), n); + } - finish(ht, ctx, hash.hash); - return hash; + finish(ht, ctx, hash.hash); + return hash; } - -HashSink::HashSink(HashType ht) : ht(ht) -{ - ctx = new Ctx; - bytes = 0; - start(ht, *ctx); +HashSink::HashSink(HashType ht) : ht(ht) { + ctx = new Ctx; + bytes = 0; + start(ht, *ctx); } -HashSink::~HashSink() -{ - bufPos = 0; - delete ctx; +HashSink::~HashSink() { + bufPos = 0; + delete ctx; } -void HashSink::write(const unsigned char * data, size_t len) -{ - bytes += len; - update(ht, *ctx, data, len); +void HashSink::write(const unsigned char* data, size_t len) { + bytes += len; + update(ht, *ctx, data, len); } -HashResult HashSink::finish() -{ - flush(); - Hash hash(ht); - nix::finish(ht, *ctx, hash.hash); - return HashResult(hash, bytes); +HashResult HashSink::finish() { + flush(); + Hash hash(ht); + nix::finish(ht, *ctx, hash.hash); + return HashResult(hash, bytes); } -HashResult HashSink::currentHash() -{ - flush(); - Ctx ctx2 = *ctx; - Hash hash(ht); - nix::finish(ht, ctx2, hash.hash); - return HashResult(hash, bytes); +HashResult HashSink::currentHash() { + flush(); + Ctx ctx2 = *ctx; + Hash hash(ht); + nix::finish(ht, ctx2, hash.hash); + return HashResult(hash, bytes); } - -HashResult hashPath( - HashType ht, const Path & path, PathFilter & filter) -{ - HashSink sink(ht); - dumpPath(path, sink, filter); - return sink.finish(); +HashResult hashPath(HashType ht, const Path& path, PathFilter& filter) { + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish(); } - -Hash compressHash(const Hash & hash, unsigned int newSize) -{ - Hash h; - h.hashSize = newSize; - for (unsigned int i = 0; i < hash.hashSize; ++i) - h.hash[i % newSize] ^= hash.hash[i]; - return h; +Hash compressHash(const Hash& hash, unsigned int newSize) { + Hash h; + h.hashSize = newSize; + for (unsigned int i = 0; i < hash.hashSize; ++i) + h.hash[i % newSize] ^= hash.hash[i]; + return h; } - -HashType parseHashType(const string & s) -{ - if (s == "md5") return htMD5; - else if (s == "sha1") return htSHA1; - else if (s == "sha256") return htSHA256; - else if (s == "sha512") return htSHA512; - else return htUnknown; +HashType parseHashType(const string& s) { + if (s == "md5") + return htMD5; + else if (s == "sha1") + return htSHA1; + else if (s == "sha256") + return htSHA256; + else if (s == "sha512") + return htSHA512; + else + return htUnknown; } - -string printHashType(HashType ht) -{ - if (ht == htMD5) return "md5"; - else if (ht == htSHA1) return "sha1"; - else if (ht == htSHA256) return "sha256"; - else if (ht == htSHA512) return "sha512"; - else abort(); +string printHashType(HashType ht) { + if (ht == htMD5) + return "md5"; + else if (ht == htSHA1) + return "sha1"; + else if (ht == htSHA256) + return "sha256"; + else if (ht == htSHA512) + return "sha512"; + else + abort(); } - -} +} // namespace nix diff --git a/third_party/nix/src/libutil/hash.hh b/third_party/nix/src/libutil/hash.hh index 2dbc3b630814..a9002023fa31 100644 --- a/third_party/nix/src/libutil/hash.hh +++ b/third_party/nix/src/libutil/hash.hh @@ -1,18 +1,14 @@ #pragma once -#include "types.hh" #include "serialise.hh" - +#include "types.hh" namespace nix { - MakeError(BadHash, Error); - enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 }; - const int md5HashSize = 16; const int sha1HashSize = 20; const int sha256HashSize = 32; @@ -22,110 +18,95 @@ extern const string base32Chars; enum Base : int { Base64, Base32, Base16, SRI }; +struct Hash { + static const unsigned int maxHashSize = 64; + unsigned int hashSize = 0; + unsigned char hash[maxHashSize] = {}; -struct Hash -{ - static const unsigned int maxHashSize = 64; - unsigned int hashSize = 0; - unsigned char hash[maxHashSize] = {}; - - HashType type = htUnknown; + HashType type = htUnknown; - /* Create an unset hash object. */ - Hash() { }; + /* Create an unset hash object. */ + Hash(){}; - /* Create a zero-filled hash object. */ - Hash(HashType type) : type(type) { init(); }; + /* Create a zero-filled hash object. */ + Hash(HashType type) : type(type) { init(); }; - /* Initialize the hash from a string representation, in the format - "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a - Subresource Integrity hash expression). If the 'type' argument - is htUnknown, then the hash type must be specified in the - string. */ - Hash(const std::string & s, HashType type = htUnknown); + /* Initialize the hash from a string representation, in the format + "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a + Subresource Integrity hash expression). If the 'type' argument + is htUnknown, then the hash type must be specified in the + string. */ + Hash(const std::string& s, HashType type = htUnknown); - void init(); + void init(); - /* Check whether a hash is set. */ - operator bool () const { return type != htUnknown; } + /* Check whether a hash is set. */ + operator bool() const { return type != htUnknown; } - /* Check whether two hash are equal. */ - bool operator == (const Hash & h2) const; + /* Check whether two hash are equal. */ + bool operator==(const Hash& h2) const; - /* Check whether two hash are not equal. */ - bool operator != (const Hash & h2) const; + /* Check whether two hash are not equal. */ + bool operator!=(const Hash& h2) const; - /* For sorting. */ - bool operator < (const Hash & h) const; + /* For sorting. */ + bool operator<(const Hash& h) const; - /* Returns the length of a base-16 representation of this hash. */ - size_t base16Len() const - { - return hashSize * 2; - } + /* Returns the length of a base-16 representation of this hash. */ + size_t base16Len() const { return hashSize * 2; } - /* Returns the length of a base-32 representation of this hash. */ - size_t base32Len() const - { - return (hashSize * 8 - 1) / 5 + 1; - } + /* Returns the length of a base-32 representation of this hash. */ + size_t base32Len() const { return (hashSize * 8 - 1) / 5 + 1; } - /* Returns the length of a base-64 representation of this hash. */ - size_t base64Len() const - { - return ((4 * hashSize / 3) + 3) & ~3; - } + /* Returns the length of a base-64 representation of this hash. */ + size_t base64Len() const { return ((4 * hashSize / 3) + 3) & ~3; } - /* Return a string representation of the hash, in base-16, base-32 - or base-64. By default, this is prefixed by the hash type - (e.g. "sha256:"). */ - std::string to_string(Base base = Base32, bool includeType = true) const; + /* Return a string representation of the hash, in base-16, base-32 + or base-64. By default, this is prefixed by the hash type + (e.g. "sha256:"). */ + std::string to_string(Base base = Base32, bool includeType = true) const; }; - /* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ -string printHash16or32(const Hash & hash); +string printHash16or32(const Hash& hash); /* Compute the hash of the given string. */ -Hash hashString(HashType ht, const string & s); +Hash hashString(HashType ht, const string& s); /* Compute the hash of the given file. */ -Hash hashFile(HashType ht, const Path & path); +Hash hashFile(HashType ht, const Path& path); /* Compute the hash of the given path. The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair<Hash, unsigned long long> HashResult; -HashResult hashPath(HashType ht, const Path & path, - PathFilter & filter = defaultPathFilter); +HashResult hashPath(HashType ht, const Path& path, + PathFilter& filter = defaultPathFilter); /* Compress a hash to the specified number of bytes by cyclically XORing bytes together. */ -Hash compressHash(const Hash & hash, unsigned int newSize); +Hash compressHash(const Hash& hash, unsigned int newSize); /* Parse a string representing a hash type. */ -HashType parseHashType(const string & s); +HashType parseHashType(const string& s); /* And the reverse. */ string printHashType(HashType ht); - union Ctx; -class HashSink : public BufferedSink -{ -private: - HashType ht; - Ctx * ctx; - unsigned long long bytes; - -public: - HashSink(HashType ht); - HashSink(const HashSink & h); - ~HashSink(); - void write(const unsigned char * data, size_t len); - HashResult finish(); - HashResult currentHash(); +class HashSink : public BufferedSink { + private: + HashType ht; + Ctx* ctx; + unsigned long long bytes; + + public: + HashSink(HashType ht); + HashSink(const HashSink& h); + ~HashSink(); + void write(const unsigned char* data, size_t len); + HashResult finish(); + HashResult currentHash(); }; - -} +} // namespace nix diff --git a/third_party/nix/src/libutil/istringstream_nocopy.hh b/third_party/nix/src/libutil/istringstream_nocopy.hh index f7beac578e39..997965630b6c 100644 --- a/third_party/nix/src/libutil/istringstream_nocopy.hh +++ b/third_party/nix/src/libutil/istringstream_nocopy.hh @@ -5,88 +5,78 @@ #pragma once -#include <string> #include <iostream> +#include <string> -template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>> -class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits> -{ -public: - typedef std::basic_string<CharT, Traits, Allocator> string_type; +template <class CharT, class Traits = std::char_traits<CharT>, + class Allocator = std::allocator<CharT>> +class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits> { + public: + typedef std::basic_string<CharT, Traits, Allocator> string_type; - typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type; + typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type; - typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type; + typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type; - typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type; + typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type; - typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type; + typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type; -private: - const string_type & s; + private: + const string_type& s; - off_type off; + off_type off; -public: - basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0} - { - } - -private: - pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which) - { - if (which & std::ios_base::in) { - this->off = dir == std::ios_base::beg - ? off - : (dir == std::ios_base::end - ? s.size() + off - : this->off + off); - } - return pos_type(this->off); - } + public: + basic_istringbuf_nocopy(const string_type& s) : s{s}, off{0} {} - pos_type seekpos(pos_type pos, std::ios_base::openmode which) - { - return seekoff(pos, std::ios_base::beg, which); + private: + pos_type seekoff(off_type off, std::ios_base::seekdir dir, + std::ios_base::openmode which) { + if (which & std::ios_base::in) { + this->off = + dir == std::ios_base::beg + ? off + : (dir == std::ios_base::end ? s.size() + off : this->off + off); } + return pos_type(this->off); + } - std::streamsize showmanyc() - { - return s.size() - off; - } + pos_type seekpos(pos_type pos, std::ios_base::openmode which) { + return seekoff(pos, std::ios_base::beg, which); + } - int_type underflow() - { - if (typename string_type::size_type(off) == s.size()) - return traits_type::eof(); - return traits_type::to_int_type(s[off]); - } + std::streamsize showmanyc() { return s.size() - off; } - int_type uflow() - { - if (typename string_type::size_type(off) == s.size()) - return traits_type::eof(); - return traits_type::to_int_type(s[off++]); - } + int_type underflow() { + if (typename string_type::size_type(off) == s.size()) + return traits_type::eof(); + return traits_type::to_int_type(s[off]); + } - int_type pbackfail(int_type ch) - { - if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1])) - return traits_type::eof(); + int_type uflow() { + if (typename string_type::size_type(off) == s.size()) + return traits_type::eof(); + return traits_type::to_int_type(s[off++]); + } - return traits_type::to_int_type(s[--off]); - } + int_type pbackfail(int_type ch) { + if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1])) + return traits_type::eof(); + return traits_type::to_int_type(s[--off]); + } }; -template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>> -class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits> -{ - typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type; - buf_type buf; -public: - basic_istringstream_nocopy(const typename buf_type::string_type & s) : - std::basic_iostream<CharT, Traits>(&buf), buf(s) {}; +template <class CharT, class Traits = std::char_traits<CharT>, + class Allocator = std::allocator<CharT>> +class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits> { + typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type; + buf_type buf; + + public: + basic_istringstream_nocopy(const typename buf_type::string_type& s) + : std::basic_iostream<CharT, Traits>(&buf), buf(s){}; }; typedef basic_istringstream_nocopy<char> istringstream_nocopy; diff --git a/third_party/nix/src/libutil/json.cc b/third_party/nix/src/libutil/json.cc index 0a6fb65f0605..37cae1680155 100644 --- a/third_party/nix/src/libutil/json.cc +++ b/third_party/nix/src/libutil/json.cc @@ -1,174 +1,184 @@ #include "json.hh" - -#include <iomanip> #include <cstring> +#include <iomanip> namespace nix { -void toJSON(std::ostream & str, const char * start, const char * end) -{ - str << '"'; - for (auto i = start; i != end; i++) - if (*i == '\"' || *i == '\\') str << '\\' << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else if (*i >= 0 && *i < 32) - str << "\\u" << std::setfill('0') << std::setw(4) << std::hex << (uint16_t) *i << std::dec; - else str << *i; - str << '"'; +void toJSON(std::ostream& str, const char* start, const char* end) { + str << '"'; + for (auto i = start; i != end; i++) + if (*i == '\"' || *i == '\\') + str << '\\' << *i; + else if (*i == '\n') + str << "\\n"; + else if (*i == '\r') + str << "\\r"; + else if (*i == '\t') + str << "\\t"; + else if (*i >= 0 && *i < 32) + str << "\\u" << std::setfill('0') << std::setw(4) << std::hex + << (uint16_t)*i << std::dec; + else + str << *i; + str << '"'; +} + +void toJSON(std::ostream& str, const char* s) { + if (!s) + str << "null"; + else + toJSON(str, s, s + strlen(s)); } -void toJSON(std::ostream & str, const char * s) -{ - if (!s) str << "null"; else toJSON(str, s, s + strlen(s)); +template <> +void toJSON<int>(std::ostream& str, const int& n) { + str << n; } - -template<> void toJSON<int>(std::ostream & str, const int & n) { str << n; } -template<> void toJSON<unsigned int>(std::ostream & str, const unsigned int & n) { str << n; } -template<> void toJSON<long>(std::ostream & str, const long & n) { str << n; } -template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long & n) { str << n; } -template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; } -template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; } -template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; } -template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; } - -template<> void toJSON<std::string>(std::ostream & str, const std::string & s) -{ - toJSON(str, s.c_str(), s.c_str() + s.size()); +template <> +void toJSON<unsigned int>(std::ostream& str, const unsigned int& n) { + str << n; } - -template<> void toJSON<bool>(std::ostream & str, const bool & b) -{ - str << (b ? "true" : "false"); +template <> +void toJSON<long>(std::ostream& str, const long& n) { + str << n; +} +template <> +void toJSON<unsigned long>(std::ostream& str, const unsigned long& n) { + str << n; +} +template <> +void toJSON<long long>(std::ostream& str, const long long& n) { + str << n; +} +template <> +void toJSON<unsigned long long>(std::ostream& str, + const unsigned long long& n) { + str << n; +} +template <> +void toJSON<float>(std::ostream& str, const float& n) { + str << n; +} +template <> +void toJSON<double>(std::ostream& str, const double& n) { + str << n; } -template<> void toJSON<std::nullptr_t>(std::ostream & str, const std::nullptr_t & b) -{ - str << "null"; +template <> +void toJSON<std::string>(std::ostream& str, const std::string& s) { + toJSON(str, s.c_str(), s.c_str() + s.size()); } -JSONWriter::JSONWriter(std::ostream & str, bool indent) - : state(new JSONState(str, indent)) -{ - state->stack++; +template <> +void toJSON<bool>(std::ostream& str, const bool& b) { + str << (b ? "true" : "false"); } -JSONWriter::JSONWriter(JSONState * state) - : state(state) -{ - state->stack++; +template <> +void toJSON<std::nullptr_t>(std::ostream& str, const std::nullptr_t& b) { + str << "null"; } -JSONWriter::~JSONWriter() -{ - if (state) { - assertActive(); - state->stack--; - if (state->stack == 0) delete state; - } +JSONWriter::JSONWriter(std::ostream& str, bool indent) + : state(new JSONState(str, indent)) { + state->stack++; } -void JSONWriter::comma() -{ +JSONWriter::JSONWriter(JSONState* state) : state(state) { state->stack++; } + +JSONWriter::~JSONWriter() { + if (state) { assertActive(); - if (first) { - first = false; - } else { - state->str << ','; - } - if (state->indent) indent(); + state->stack--; + if (state->stack == 0) delete state; + } } -void JSONWriter::indent() -{ - state->str << '\n' << std::string(state->depth * 2, ' '); +void JSONWriter::comma() { + assertActive(); + if (first) { + first = false; + } else { + state->str << ','; + } + if (state->indent) indent(); } -void JSONList::open() -{ - state->depth++; - state->str << '['; +void JSONWriter::indent() { + state->str << '\n' << std::string(state->depth * 2, ' '); } -JSONList::~JSONList() -{ - state->depth--; - if (state->indent && !first) indent(); - state->str << "]"; +void JSONList::open() { + state->depth++; + state->str << '['; } -JSONList JSONList::list() -{ - comma(); - return JSONList(state); +JSONList::~JSONList() { + state->depth--; + if (state->indent && !first) indent(); + state->str << "]"; } -JSONObject JSONList::object() -{ - comma(); - return JSONObject(state); +JSONList JSONList::list() { + comma(); + return JSONList(state); } -JSONPlaceholder JSONList::placeholder() -{ - comma(); - return JSONPlaceholder(state); +JSONObject JSONList::object() { + comma(); + return JSONObject(state); } -void JSONObject::open() -{ - state->depth++; - state->str << '{'; +JSONPlaceholder JSONList::placeholder() { + comma(); + return JSONPlaceholder(state); } -JSONObject::~JSONObject() -{ - if (state) { - state->depth--; - if (state->indent && !first) indent(); - state->str << "}"; - } +void JSONObject::open() { + state->depth++; + state->str << '{'; } -void JSONObject::attr(const std::string & s) -{ - comma(); - toJSON(state->str, s); - state->str << ':'; - if (state->indent) state->str << ' '; +JSONObject::~JSONObject() { + if (state) { + state->depth--; + if (state->indent && !first) indent(); + state->str << "}"; + } } -JSONList JSONObject::list(const std::string & name) -{ - attr(name); - return JSONList(state); +void JSONObject::attr(const std::string& s) { + comma(); + toJSON(state->str, s); + state->str << ':'; + if (state->indent) state->str << ' '; } -JSONObject JSONObject::object(const std::string & name) -{ - attr(name); - return JSONObject(state); +JSONList JSONObject::list(const std::string& name) { + attr(name); + return JSONList(state); } -JSONPlaceholder JSONObject::placeholder(const std::string & name) -{ - attr(name); - return JSONPlaceholder(state); +JSONObject JSONObject::object(const std::string& name) { + attr(name); + return JSONObject(state); } -JSONList JSONPlaceholder::list() -{ - assertValid(); - first = false; - return JSONList(state); +JSONPlaceholder JSONObject::placeholder(const std::string& name) { + attr(name); + return JSONPlaceholder(state); } -JSONObject JSONPlaceholder::object() -{ - assertValid(); - first = false; - return JSONObject(state); +JSONList JSONPlaceholder::list() { + assertValid(); + first = false; + return JSONList(state); } +JSONObject JSONPlaceholder::object() { + assertValid(); + first = false; + return JSONObject(state); } + +} // namespace nix diff --git a/third_party/nix/src/libutil/json.hh b/third_party/nix/src/libutil/json.hh index 02a39917fb5c..a3843a8a8a7b 100644 --- a/third_party/nix/src/libutil/json.hh +++ b/third_party/nix/src/libutil/json.hh @@ -1,189 +1,142 @@ #pragma once +#include <cassert> #include <iostream> #include <vector> -#include <cassert> namespace nix { -void toJSON(std::ostream & str, const char * start, const char * end); -void toJSON(std::ostream & str, const char * s); - -template<typename T> -void toJSON(std::ostream & str, const T & n); +void toJSON(std::ostream& str, const char* start, const char* end); +void toJSON(std::ostream& str, const char* s); -class JSONWriter -{ -protected: +template <typename T> +void toJSON(std::ostream& str, const T& n); - struct JSONState - { - std::ostream & str; - bool indent; - size_t depth = 0; - size_t stack = 0; - JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { } - ~JSONState() - { - assert(stack == 0); - } - }; +class JSONWriter { + protected: + struct JSONState { + std::ostream& str; + bool indent; + size_t depth = 0; + size_t stack = 0; + JSONState(std::ostream& str, bool indent) : str(str), indent(indent) {} + ~JSONState() { assert(stack == 0); } + }; - JSONState * state; + JSONState* state; - bool first = true; + bool first = true; - JSONWriter(std::ostream & str, bool indent); + JSONWriter(std::ostream& str, bool indent); - JSONWriter(JSONState * state); + JSONWriter(JSONState* state); - ~JSONWriter(); + ~JSONWriter(); - void assertActive() - { - assert(state->stack != 0); - } + void assertActive() { assert(state->stack != 0); } - void comma(); + void comma(); - void indent(); + void indent(); }; class JSONObject; class JSONPlaceholder; -class JSONList : JSONWriter -{ -private: - - friend class JSONObject; - friend class JSONPlaceholder; - - void open(); +class JSONList : JSONWriter { + private: + friend class JSONObject; + friend class JSONPlaceholder; - JSONList(JSONState * state) - : JSONWriter(state) - { - open(); - } + void open(); -public: + JSONList(JSONState* state) : JSONWriter(state) { open(); } - JSONList(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - open(); - } + public: + JSONList(std::ostream& str, bool indent = false) : JSONWriter(str, indent) { + open(); + } - ~JSONList(); + ~JSONList(); - template<typename T> - JSONList & elem(const T & v) - { - comma(); - toJSON(state->str, v); - return *this; - } + template <typename T> + JSONList& elem(const T& v) { + comma(); + toJSON(state->str, v); + return *this; + } - JSONList list(); + JSONList list(); - JSONObject object(); + JSONObject object(); - JSONPlaceholder placeholder(); + JSONPlaceholder placeholder(); }; -class JSONObject : JSONWriter -{ -private: +class JSONObject : JSONWriter { + private: + friend class JSONList; + friend class JSONPlaceholder; - friend class JSONList; - friend class JSONPlaceholder; + void open(); - void open(); + JSONObject(JSONState* state) : JSONWriter(state) { open(); } - JSONObject(JSONState * state) - : JSONWriter(state) - { - open(); - } + void attr(const std::string& s); - void attr(const std::string & s); + public: + JSONObject(std::ostream& str, bool indent = false) : JSONWriter(str, indent) { + open(); + } -public: + JSONObject(const JSONObject& obj) = delete; - JSONObject(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - open(); - } + JSONObject(JSONObject&& obj) : JSONWriter(obj.state) { obj.state = 0; } - JSONObject(const JSONObject & obj) = delete; + ~JSONObject(); - JSONObject(JSONObject && obj) - : JSONWriter(obj.state) - { - obj.state = 0; - } + template <typename T> + JSONObject& attr(const std::string& name, const T& v) { + attr(name); + toJSON(state->str, v); + return *this; + } - ~JSONObject(); + JSONList list(const std::string& name); - template<typename T> - JSONObject & attr(const std::string & name, const T & v) - { - attr(name); - toJSON(state->str, v); - return *this; - } + JSONObject object(const std::string& name); - JSONList list(const std::string & name); - - JSONObject object(const std::string & name); - - JSONPlaceholder placeholder(const std::string & name); + JSONPlaceholder placeholder(const std::string& name); }; -class JSONPlaceholder : JSONWriter -{ - -private: - - friend class JSONList; - friend class JSONObject; - - JSONPlaceholder(JSONState * state) - : JSONWriter(state) - { - } +class JSONPlaceholder : JSONWriter { + private: + friend class JSONList; + friend class JSONObject; - void assertValid() - { - assertActive(); - assert(first); - } + JSONPlaceholder(JSONState* state) : JSONWriter(state) {} -public: + void assertValid() { + assertActive(); + assert(first); + } - JSONPlaceholder(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - } + public: + JSONPlaceholder(std::ostream& str, bool indent = false) + : JSONWriter(str, indent) {} - ~JSONPlaceholder() - { - assert(!first || std::uncaught_exception()); - } + ~JSONPlaceholder() { assert(!first || std::uncaught_exception()); } - template<typename T> - void write(const T & v) - { - assertValid(); - first = false; - toJSON(state->str, v); - } + template <typename T> + void write(const T& v) { + assertValid(); + first = false; + toJSON(state->str, v); + } - JSONList list(); + JSONList list(); - JSONObject object(); + JSONObject object(); }; -} +} // namespace nix diff --git a/third_party/nix/src/libutil/lazy.hh b/third_party/nix/src/libutil/lazy.hh index d073e486c2eb..9807b7540620 100644 --- a/third_party/nix/src/libutil/lazy.hh +++ b/third_party/nix/src/libutil/lazy.hh @@ -12,37 +12,32 @@ namespace nix { thread-safe way) on first use, that is, when var() is first called. If the initialiser code throws an exception, then all subsequent calls to var() will rethrow that exception. */ -template<typename T> -class Lazy -{ +template <typename T> +class Lazy { + typedef std::function<T()> Init; - typedef std::function<T()> Init; + Init init; - Init init; + std::once_flag done; - std::once_flag done; + T value; - T value; + std::exception_ptr ex; - std::exception_ptr ex; + public: + Lazy(Init init) : init(init) {} -public: - - Lazy(Init init) : init(init) - { } - - const T & operator () () - { - std::call_once(done, [&]() { - try { - value = init(); - } catch (...) { - ex = std::current_exception(); - } - }); - if (ex) std::rethrow_exception(ex); - return value; - } + const T& operator()() { + std::call_once(done, [&]() { + try { + value = init(); + } catch (...) { + ex = std::current_exception(); + } + }); + if (ex) std::rethrow_exception(ex); + return value; + } }; -} +} // namespace nix diff --git a/third_party/nix/src/libutil/logging.cc b/third_party/nix/src/libutil/logging.cc index b379306f6ec0..4f88044757cd 100644 --- a/third_party/nix/src/libutil/logging.cc +++ b/third_party/nix/src/libutil/logging.cc @@ -1,242 +1,227 @@ #include "logging.hh" -#include "util.hh" - #include <atomic> #include <nlohmann/json.hpp> +#include "util.hh" namespace nix { static thread_local ActivityId curActivity = 0; -ActivityId getCurActivity() -{ - return curActivity; -} -void setCurActivity(const ActivityId activityId) -{ - curActivity = activityId; -} +ActivityId getCurActivity() { return curActivity; } +void setCurActivity(const ActivityId activityId) { curActivity = activityId; } -Logger * logger = makeDefaultLogger(); +Logger* logger = makeDefaultLogger(); -void Logger::warn(const std::string & msg) -{ - log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg); +void Logger::warn(const std::string& msg) { + log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg); } -class SimpleLogger : public Logger -{ -public: - - bool systemd, tty; - - SimpleLogger() - { - systemd = getEnv("IN_SYSTEMD") == "1"; - tty = isatty(STDERR_FILENO); +class SimpleLogger : public Logger { + public: + bool systemd, tty; + + SimpleLogger() { + systemd = getEnv("IN_SYSTEMD") == "1"; + tty = isatty(STDERR_FILENO); + } + + void log(Verbosity lvl, const FormatOrString& fs) override { + if (lvl > verbosity) return; + + std::string prefix; + + if (systemd) { + char c; + switch (lvl) { + case lvlError: + c = '3'; + break; + case lvlWarn: + c = '4'; + break; + case lvlInfo: + c = '5'; + break; + case lvlTalkative: + case lvlChatty: + c = '6'; + break; + default: + c = '7'; + } + prefix = std::string("<") + c + ">"; } - void log(Verbosity lvl, const FormatOrString & fs) override - { - if (lvl > verbosity) return; - - std::string prefix; - - if (systemd) { - char c; - switch (lvl) { - case lvlError: c = '3'; break; - case lvlWarn: c = '4'; break; - case lvlInfo: c = '5'; break; - case lvlTalkative: case lvlChatty: c = '6'; break; - default: c = '7'; - } - prefix = std::string("<") + c + ">"; - } - - writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); - } + writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); + } - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, - const std::string & s, const Fields & fields, ActivityId parent) - override - { - if (lvl <= verbosity && !s.empty()) - log(lvl, s + "..."); - } + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string& s, const Fields& fields, + ActivityId parent) override { + if (lvl <= verbosity && !s.empty()) log(lvl, s + "..."); + } }; Verbosity verbosity = lvlInfo; -void warnOnce(bool & haveWarned, const FormatOrString & fs) -{ - if (!haveWarned) { - warn(fs.s); - haveWarned = true; - } +void warnOnce(bool& haveWarned, const FormatOrString& fs) { + if (!haveWarned) { + warn(fs.s); + haveWarned = true; + } } -void writeToStderr(const string & s) -{ - try { - writeFull(STDERR_FILENO, s, false); - } catch (SysError & e) { - /* Ignore failing writes to stderr. We need to ignore write - errors to ensure that cleanup code that logs to stderr runs - to completion if the other side of stderr has been closed - unexpectedly. */ - } +void writeToStderr(const string& s) { + try { + writeFull(STDERR_FILENO, s, false); + } catch (SysError& e) { + /* Ignore failing writes to stderr. We need to ignore write + errors to ensure that cleanup code that logs to stderr runs + to completion if the other side of stderr has been closed + unexpectedly. */ + } } -Logger * makeDefaultLogger() -{ - return new SimpleLogger(); +Logger* makeDefaultLogger() { return new SimpleLogger(); } + +std::atomic<uint64_t> nextId{(uint64_t)getpid() << 32}; + +Activity::Activity(Logger& logger, Verbosity lvl, ActivityType type, + const std::string& s, const Logger::Fields& fields, + ActivityId parent) + : logger(logger), id(nextId++) { + logger.startActivity(id, lvl, type, s, fields, parent); } -std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32}; +struct JSONLogger : Logger { + Logger& prevLogger; + + JSONLogger(Logger& prevLogger) : prevLogger(prevLogger) {} + + void addFields(nlohmann::json& json, const Fields& fields) { + if (fields.empty()) return; + auto& arr = json["fields"] = nlohmann::json::array(); + for (auto& f : fields) + if (f.type == Logger::Field::tInt) + arr.push_back(f.i); + else if (f.type == Logger::Field::tString) + arr.push_back(f.s); + else + abort(); + } + + void write(const nlohmann::json& json) { + prevLogger.log(lvlError, "@nix " + json.dump()); + } + + void log(Verbosity lvl, const FormatOrString& fs) override { + nlohmann::json json; + json["action"] = "msg"; + json["level"] = lvl; + json["msg"] = fs.s; + write(json); + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string& s, const Fields& fields, + ActivityId parent) override { + nlohmann::json json; + json["action"] = "start"; + json["id"] = act; + json["level"] = lvl; + json["type"] = type; + json["text"] = s; + addFields(json, fields); + // FIXME: handle parent + write(json); + } + + void stopActivity(ActivityId act) override { + nlohmann::json json; + json["action"] = "stop"; + json["id"] = act; + write(json); + } + + void result(ActivityId act, ResultType type, const Fields& fields) override { + nlohmann::json json; + json["action"] = "result"; + json["id"] = act; + json["type"] = type; + addFields(json, fields); + write(json); + } +}; -Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, - const std::string & s, const Logger::Fields & fields, ActivityId parent) - : logger(logger), id(nextId++) -{ - logger.startActivity(id, lvl, type, s, fields, parent); +Logger* makeJSONLogger(Logger& prevLogger) { + return new JSONLogger(prevLogger); } -struct JSONLogger : Logger -{ - Logger & prevLogger; - - JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { } - - void addFields(nlohmann::json & json, const Fields & fields) - { - if (fields.empty()) return; - auto & arr = json["fields"] = nlohmann::json::array(); - for (auto & f : fields) - if (f.type == Logger::Field::tInt) - arr.push_back(f.i); - else if (f.type == Logger::Field::tString) - arr.push_back(f.s); - else - abort(); - } +static Logger::Fields getFields(nlohmann::json& json) { + Logger::Fields fields; + for (auto& f : json) { + if (f.type() == nlohmann::json::value_t::number_unsigned) + fields.emplace_back(Logger::Field(f.get<uint64_t>())); + else if (f.type() == nlohmann::json::value_t::string) + fields.emplace_back(Logger::Field(f.get<std::string>())); + else + throw Error("unsupported JSON type %d", (int)f.type()); + } + return fields; +} - void write(const nlohmann::json & json) - { - prevLogger.log(lvlError, "@nix " + json.dump()); - } +bool handleJSONLogMessage(const std::string& msg, const Activity& act, + std::map<ActivityId, Activity>& activities, + bool trusted) { + if (!hasPrefix(msg, "@nix ")) return false; - void log(Verbosity lvl, const FormatOrString & fs) override - { - nlohmann::json json; - json["action"] = "msg"; - json["level"] = lvl; - json["msg"] = fs.s; - write(json); - } + try { + auto json = nlohmann::json::parse(std::string(msg, 5)); - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, - const std::string & s, const Fields & fields, ActivityId parent) override - { - nlohmann::json json; - json["action"] = "start"; - json["id"] = act; - json["level"] = lvl; - json["type"] = type; - json["text"] = s; - addFields(json, fields); - // FIXME: handle parent - write(json); - } + std::string action = json["action"]; - void stopActivity(ActivityId act) override - { - nlohmann::json json; - json["action"] = "stop"; - json["id"] = act; - write(json); + if (action == "start") { + auto type = (ActivityType)json["type"]; + if (trusted || type == actDownload) + activities.emplace( + std::piecewise_construct, std::forward_as_tuple(json["id"]), + std::forward_as_tuple(*logger, (Verbosity)json["level"], type, + json["text"], getFields(json["fields"]), + act.id)); } - void result(ActivityId act, ResultType type, const Fields & fields) override - { - nlohmann::json json; - json["action"] = "result"; - json["id"] = act; - json["type"] = type; - addFields(json, fields); - write(json); - } -}; + else if (action == "stop") + activities.erase((ActivityId)json["id"]); -Logger * makeJSONLogger(Logger & prevLogger) -{ - return new JSONLogger(prevLogger); -} + else if (action == "result") { + auto i = activities.find((ActivityId)json["id"]); + if (i != activities.end()) + i->second.result((ResultType)json["type"], getFields(json["fields"])); + } -static Logger::Fields getFields(nlohmann::json & json) -{ - Logger::Fields fields; - for (auto & f : json) { - if (f.type() == nlohmann::json::value_t::number_unsigned) - fields.emplace_back(Logger::Field(f.get<uint64_t>())); - else if (f.type() == nlohmann::json::value_t::string) - fields.emplace_back(Logger::Field(f.get<std::string>())); - else throw Error("unsupported JSON type %d", (int) f.type()); + else if (action == "setPhase") { + std::string phase = json["phase"]; + act.result(resSetPhase, phase); } - return fields; -} -bool handleJSONLogMessage(const std::string & msg, - const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted) -{ - if (!hasPrefix(msg, "@nix ")) return false; - - try { - auto json = nlohmann::json::parse(std::string(msg, 5)); - - std::string action = json["action"]; - - if (action == "start") { - auto type = (ActivityType) json["type"]; - if (trusted || type == actDownload) - activities.emplace(std::piecewise_construct, - std::forward_as_tuple(json["id"]), - std::forward_as_tuple(*logger, (Verbosity) json["level"], type, - json["text"], getFields(json["fields"]), act.id)); - } - - else if (action == "stop") - activities.erase((ActivityId) json["id"]); - - else if (action == "result") { - auto i = activities.find((ActivityId) json["id"]); - if (i != activities.end()) - i->second.result((ResultType) json["type"], getFields(json["fields"])); - } - - else if (action == "setPhase") { - std::string phase = json["phase"]; - act.result(resSetPhase, phase); - } - - else if (action == "msg") { - std::string msg = json["msg"]; - logger->log((Verbosity) json["level"], msg); - } - - } catch (std::exception & e) { - printError("bad log message from builder: %s", e.what()); + else if (action == "msg") { + std::string msg = json["msg"]; + logger->log((Verbosity)json["level"], msg); } - return true; + } catch (std::exception& e) { + printError("bad log message from builder: %s", e.what()); + } + + return true; } Activity::~Activity() { - try { - logger.stopActivity(id); - } catch (...) { - ignoreException(); - } + try { + logger.stopActivity(id); + } catch (...) { + ignoreException(); + } } -} +} // namespace nix diff --git a/third_party/nix/src/libutil/logging.hh b/third_party/nix/src/libutil/logging.hh index 5df03da74e00..8b04dfd17ee3 100644 --- a/third_party/nix/src/libutil/logging.hh +++ b/third_party/nix/src/libutil/logging.hh @@ -5,151 +5,148 @@ namespace nix { typedef enum { - lvlError = 0, - lvlWarn, - lvlInfo, - lvlTalkative, - lvlChatty, - lvlDebug, - lvlVomit + lvlError = 0, + lvlWarn, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit } Verbosity; typedef enum { - actUnknown = 0, - actCopyPath = 100, - actDownload = 101, - actRealise = 102, - actCopyPaths = 103, - actBuilds = 104, - actBuild = 105, - actOptimiseStore = 106, - actVerifyPaths = 107, - actSubstitute = 108, - actQueryPathInfo = 109, - actPostBuildHook = 110, + actUnknown = 0, + actCopyPath = 100, + actDownload = 101, + actRealise = 102, + actCopyPaths = 103, + actBuilds = 104, + actBuild = 105, + actOptimiseStore = 106, + actVerifyPaths = 107, + actSubstitute = 108, + actQueryPathInfo = 109, + actPostBuildHook = 110, } ActivityType; typedef enum { - resFileLinked = 100, - resBuildLogLine = 101, - resUntrustedPath = 102, - resCorruptedPath = 103, - resSetPhase = 104, - resProgress = 105, - resSetExpected = 106, - resPostBuildLogLine = 107, + resFileLinked = 100, + resBuildLogLine = 101, + resUntrustedPath = 102, + resCorruptedPath = 103, + resSetPhase = 104, + resProgress = 105, + resSetExpected = 106, + resPostBuildLogLine = 107, } ResultType; typedef uint64_t ActivityId; -class Logger -{ - friend struct Activity; +class Logger { + friend struct Activity; -public: + public: + struct Field { + // FIXME: use std::variant. + enum { tInt = 0, tString = 1 } type; + uint64_t i = 0; + std::string s; + Field(const std::string& s) : type(tString), s(s) {} + Field(const char* s) : type(tString), s(s) {} + Field(const uint64_t& i) : type(tInt), i(i) {} + }; - struct Field - { - // FIXME: use std::variant. - enum { tInt = 0, tString = 1 } type; - uint64_t i = 0; - std::string s; - Field(const std::string & s) : type(tString), s(s) { } - Field(const char * s) : type(tString), s(s) { } - Field(const uint64_t & i) : type(tInt), i(i) { } - }; + typedef std::vector<Field> Fields; - typedef std::vector<Field> Fields; + virtual ~Logger() {} - virtual ~Logger() { } + virtual void log(Verbosity lvl, const FormatOrString& fs) = 0; - virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + void log(const FormatOrString& fs) { log(lvlInfo, fs); } - void log(const FormatOrString & fs) - { - log(lvlInfo, fs); - } + virtual void warn(const std::string& msg); - virtual void warn(const std::string & msg); + virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string& s, const Fields& fields, + ActivityId parent){}; - virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type, - const std::string & s, const Fields & fields, ActivityId parent) { }; + virtual void stopActivity(ActivityId act){}; - virtual void stopActivity(ActivityId act) { }; - - virtual void result(ActivityId act, ResultType type, const Fields & fields) { }; + virtual void result(ActivityId act, ResultType type, const Fields& fields){}; }; ActivityId getCurActivity(); void setCurActivity(const ActivityId activityId); -struct Activity -{ - Logger & logger; +struct Activity { + Logger& logger; - const ActivityId id; + const ActivityId id; - Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s = "", - const Logger::Fields & fields = {}, ActivityId parent = getCurActivity()); + Activity(Logger& logger, Verbosity lvl, ActivityType type, + const std::string& s = "", const Logger::Fields& fields = {}, + ActivityId parent = getCurActivity()); - Activity(Logger & logger, ActivityType type, - const Logger::Fields & fields = {}, ActivityId parent = getCurActivity()) - : Activity(logger, lvlError, type, "", fields, parent) { }; + Activity(Logger& logger, ActivityType type, const Logger::Fields& fields = {}, + ActivityId parent = getCurActivity()) + : Activity(logger, lvlError, type, "", fields, parent){}; - Activity(const Activity & act) = delete; + Activity(const Activity& act) = delete; - ~Activity(); + ~Activity(); - void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const - { result(resProgress, done, expected, running, failed); } + void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, + uint64_t failed = 0) const { + result(resProgress, done, expected, running, failed); + } - void setExpected(ActivityType type2, uint64_t expected) const - { result(resSetExpected, type2, expected); } + void setExpected(ActivityType type2, uint64_t expected) const { + result(resSetExpected, type2, expected); + } - template<typename... Args> - void result(ResultType type, const Args & ... args) const - { - Logger::Fields fields; - nop{(fields.emplace_back(Logger::Field(args)), 1)...}; - result(type, fields); - } + template <typename... Args> + void result(ResultType type, const Args&... args) const { + Logger::Fields fields; + nop{(fields.emplace_back(Logger::Field(args)), 1)...}; + result(type, fields); + } - void result(ResultType type, const Logger::Fields & fields) const - { - logger.result(id, type, fields); - } + void result(ResultType type, const Logger::Fields& fields) const { + logger.result(id, type, fields); + } - friend class Logger; + friend class Logger; }; -struct PushActivity -{ - const ActivityId prevAct; - PushActivity(ActivityId act) : prevAct(getCurActivity()) { setCurActivity(act); } - ~PushActivity() { setCurActivity(prevAct); } +struct PushActivity { + const ActivityId prevAct; + PushActivity(ActivityId act) : prevAct(getCurActivity()) { + setCurActivity(act); + } + ~PushActivity() { setCurActivity(prevAct); } }; -extern Logger * logger; +extern Logger* logger; -Logger * makeDefaultLogger(); +Logger* makeDefaultLogger(); -Logger * makeJSONLogger(Logger & prevLogger); +Logger* makeJSONLogger(Logger& prevLogger); -bool handleJSONLogMessage(const std::string & msg, - const Activity & act, std::map<ActivityId, Activity> & activities, - bool trusted); +bool handleJSONLogMessage(const std::string& msg, const Activity& act, + std::map<ActivityId, Activity>& activities, + bool trusted); extern Verbosity verbosity; /* suppress msgs > this */ /* Print a message if the current log level is at least the specified level. Note that this has to be implemented as a macro to ensure that the arguments are evaluated lazily. */ -#define printMsg(level, args...) \ - do { \ - if (level <= nix::verbosity) { \ - logger->log(level, fmt(args)); \ - } \ - } while (0) +#define printMsg(level, args...) \ + do { \ + if (level <= nix::verbosity) { \ + logger->log(level, fmt(args)); \ + } \ + } while (0) #define printError(args...) printMsg(lvlError, args) #define printInfo(args...) printMsg(lvlInfo, args) @@ -157,16 +154,15 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) -template<typename... Args> -inline void warn(const std::string & fs, Args... args) -{ - boost::format f(fs); - nop{boost::io::detail::feed(f, args)...}; - logger->warn(f.str()); +template <typename... Args> +inline void warn(const std::string& fs, Args... args) { + boost::format f(fs); + nop{boost::io::detail::feed(f, args)...}; + logger->warn(f.str()); } -void warnOnce(bool & haveWarned, const FormatOrString & fs); +void warnOnce(bool& haveWarned, const FormatOrString& fs); -void writeToStderr(const string & s); +void writeToStderr(const string& s); -} +} // namespace nix diff --git a/third_party/nix/src/libutil/lru-cache.hh b/third_party/nix/src/libutil/lru-cache.hh index 8b83f842c324..bf7263243c31 100644 --- a/third_party/nix/src/libutil/lru-cache.hh +++ b/third_party/nix/src/libutil/lru-cache.hh @@ -1,92 +1,84 @@ #pragma once -#include <map> #include <list> +#include <map> #include <optional> namespace nix { /* A simple least-recently used cache. Not thread-safe. */ -template<typename Key, typename Value> -class LRUCache -{ -private: - - size_t capacity; +template <typename Key, typename Value> +class LRUCache { + private: + size_t capacity; - // Stupid wrapper to get around circular dependency between Data - // and LRU. - struct LRUIterator; + // Stupid wrapper to get around circular dependency between Data + // and LRU. + struct LRUIterator; - using Data = std::map<Key, std::pair<LRUIterator, Value>>; - using LRU = std::list<typename Data::iterator>; + using Data = std::map<Key, std::pair<LRUIterator, Value>>; + using LRU = std::list<typename Data::iterator>; - struct LRUIterator { typename LRU::iterator it; }; + struct LRUIterator { + typename LRU::iterator it; + }; - Data data; - LRU lru; + Data data; + LRU lru; -public: + public: + LRUCache(size_t capacity) : capacity(capacity) {} - LRUCache(size_t capacity) : capacity(capacity) { } + /* Insert or upsert an item in the cache. */ + void upsert(const Key& key, const Value& value) { + if (capacity == 0) return; - /* Insert or upsert an item in the cache. */ - void upsert(const Key & key, const Value & value) - { - if (capacity == 0) return; + erase(key); - erase(key); - - if (data.size() >= capacity) { - /* Retire the oldest item. */ - auto oldest = lru.begin(); - data.erase(*oldest); - lru.erase(oldest); - } + if (data.size() >= capacity) { + /* Retire the oldest item. */ + auto oldest = lru.begin(); + data.erase(*oldest); + lru.erase(oldest); + } - auto res = data.emplace(key, std::make_pair(LRUIterator(), value)); - assert(res.second); - auto & i(res.first); + auto res = data.emplace(key, std::make_pair(LRUIterator(), value)); + assert(res.second); + auto& i(res.first); - auto j = lru.insert(lru.end(), i); + auto j = lru.insert(lru.end(), i); - i->second.first.it = j; - } + i->second.first.it = j; + } - bool erase(const Key & key) - { - auto i = data.find(key); - if (i == data.end()) return false; - lru.erase(i->second.first.it); - data.erase(i); - return true; - } + bool erase(const Key& key) { + auto i = data.find(key); + if (i == data.end()) return false; + lru.erase(i->second.first.it); + data.erase(i); + return true; + } - /* Look up an item in the cache. If it exists, it becomes the most - recently used item. */ - std::optional<Value> get(const Key & key) - { - auto i = data.find(key); - if (i == data.end()) return {}; + /* Look up an item in the cache. If it exists, it becomes the most + recently used item. */ + std::optional<Value> get(const Key& key) { + auto i = data.find(key); + if (i == data.end()) return {}; - /* Move this item to the back of the LRU list. */ - lru.erase(i->second.first.it); - auto j = lru.insert(lru.end(), i); - i->second.first.it = j; + /* Move this item to the back of the LRU list. */ + lru.erase(i->second.first.it); + auto j = lru.insert(lru.end(), i); + i->second.first.it = j; - return i->second.second; - } + return i->second.second; + } - size_t size() - { - return data.size(); - } + size_t size() { return data.size(); } - void clear() - { - data.clear(); - lru.clear(); - } + void clear() { + data.clear(); + lru.clear(); + } }; -} +} // namespace nix diff --git a/third_party/nix/src/libutil/monitor-fd.hh b/third_party/nix/src/libutil/monitor-fd.hh index 5ee0b88ef50f..faf79e6381d2 100644 --- a/third_party/nix/src/libutil/monitor-fd.hh +++ b/third_party/nix/src/libutil/monitor-fd.hh @@ -1,58 +1,52 @@ #pragma once -#include <thread> -#include <atomic> - -#include <cstdlib> #include <poll.h> +#include <signal.h> #include <sys/types.h> #include <unistd.h> -#include <signal.h> +#include <atomic> +#include <cstdlib> +#include <thread> namespace nix { - -class MonitorFdHup -{ -private: - std::thread thread; - -public: - MonitorFdHup(int fd) - { - thread = std::thread([fd]() { - while (true) { - /* Wait indefinitely until a POLLHUP occurs. */ - struct pollfd fds[1]; - fds[0].fd = fd; - /* This shouldn't be necessary, but macOS doesn't seem to - like a zeroed out events field. - See rdar://37537852. - */ - fds[0].events = POLLHUP; - auto count = poll(fds, 1, -1); - if (count == -1) abort(); // can't happen - /* This shouldn't happen, but can on macOS due to a bug. - See rdar://37550628. - - This may eventually need a delay or further - coordination with the main thread if spinning proves - too harmful. - */ - if (count == 0) continue; - assert(fds[0].revents & POLLHUP); - triggerInterrupt(); - break; - } - }); - }; - - ~MonitorFdHup() - { - pthread_cancel(thread.native_handle()); - thread.join(); - } +class MonitorFdHup { + private: + std::thread thread; + + public: + MonitorFdHup(int fd) { + thread = std::thread([fd]() { + while (true) { + /* Wait indefinitely until a POLLHUP occurs. */ + struct pollfd fds[1]; + fds[0].fd = fd; + /* This shouldn't be necessary, but macOS doesn't seem to + like a zeroed out events field. + See rdar://37537852. + */ + fds[0].events = POLLHUP; + auto count = poll(fds, 1, -1); + if (count == -1) abort(); // can't happen + /* This shouldn't happen, but can on macOS due to a bug. + See rdar://37550628. + + This may eventually need a delay or further + coordination with the main thread if spinning proves + too harmful. + */ + if (count == 0) continue; + assert(fds[0].revents & POLLHUP); + triggerInterrupt(); + break; + } + }); + }; + + ~MonitorFdHup() { + pthread_cancel(thread.native_handle()); + thread.join(); + } }; - -} +} // namespace nix diff --git a/third_party/nix/src/libutil/pool.hh b/third_party/nix/src/libutil/pool.hh index d49067bb95dc..592bd50897cc 100644 --- a/third_party/nix/src/libutil/pool.hh +++ b/third_party/nix/src/libutil/pool.hh @@ -1,13 +1,12 @@ #pragma once +#include <cassert> #include <functional> #include <limits> #include <list> #include <memory> -#include <cassert> - -#include "sync.hh" #include "ref.hh" +#include "sync.hh" namespace nix { @@ -29,159 +28,140 @@ namespace nix { */ template <class R> -class Pool -{ -public: - - /* A function that produces new instances of R on demand. */ - typedef std::function<ref<R>()> Factory; - - /* A function that checks whether an instance of R is still - usable. Unusable instances are removed from the pool. */ - typedef std::function<bool(const ref<R> &)> Validator; - -private: - - Factory factory; - Validator validator; - - struct State - { - size_t inUse = 0; - size_t max; - std::vector<ref<R>> idle; - }; - - Sync<State> state; - - std::condition_variable wakeup; - -public: - - Pool(size_t max = std::numeric_limits<size_t>::max(), - const Factory & factory = []() { return make_ref<R>(); }, - const Validator & validator = [](ref<R> r) { return true; }) - : factory(factory) - , validator(validator) - { - auto state_(state.lock()); - state_->max = max; - } - - void incCapacity() - { - auto state_(state.lock()); - state_->max++; - /* we could wakeup here, but this is only used when we're - * about to nest Pool usages, and we want to save the slot for - * the nested use if we can - */ +class Pool { + public: + /* A function that produces new instances of R on demand. */ + typedef std::function<ref<R>()> Factory; + + /* A function that checks whether an instance of R is still + usable. Unusable instances are removed from the pool. */ + typedef std::function<bool(const ref<R>&)> Validator; + + private: + Factory factory; + Validator validator; + + struct State { + size_t inUse = 0; + size_t max; + std::vector<ref<R>> idle; + }; + + Sync<State> state; + + std::condition_variable wakeup; + + public: + Pool( + size_t max = std::numeric_limits<size_t>::max(), + const Factory& factory = []() { return make_ref<R>(); }, + const Validator& validator = [](ref<R> r) { return true; }) + : factory(factory), validator(validator) { + auto state_(state.lock()); + state_->max = max; + } + + void incCapacity() { + auto state_(state.lock()); + state_->max++; + /* we could wakeup here, but this is only used when we're + * about to nest Pool usages, and we want to save the slot for + * the nested use if we can + */ + } + + void decCapacity() { + auto state_(state.lock()); + state_->max--; + } + + ~Pool() { + auto state_(state.lock()); + assert(!state_->inUse); + state_->max = 0; + state_->idle.clear(); + } + + class Handle { + private: + Pool& pool; + std::shared_ptr<R> r; + bool bad = false; + + friend Pool; + + Handle(Pool& pool, std::shared_ptr<R> r) : pool(pool), r(r) {} + + public: + Handle(Handle&& h) : pool(h.pool), r(h.r) { h.r.reset(); } + + Handle(const Handle& l) = delete; + + ~Handle() { + if (!r) return; + { + auto state_(pool.state.lock()); + if (!bad) state_->idle.push_back(ref<R>(r)); + assert(state_->inUse); + state_->inUse--; + } + pool.wakeup.notify_one(); } - void decCapacity() - { - auto state_(state.lock()); - state_->max--; - } - - ~Pool() - { - auto state_(state.lock()); - assert(!state_->inUse); - state_->max = 0; - state_->idle.clear(); - } - - class Handle - { - private: - Pool & pool; - std::shared_ptr<R> r; - bool bad = false; - - friend Pool; - - Handle(Pool & pool, std::shared_ptr<R> r) : pool(pool), r(r) { } - - public: - Handle(Handle && h) : pool(h.pool), r(h.r) { h.r.reset(); } - - Handle(const Handle & l) = delete; - - ~Handle() - { - if (!r) return; - { - auto state_(pool.state.lock()); - if (!bad) - state_->idle.push_back(ref<R>(r)); - assert(state_->inUse); - state_->inUse--; - } - pool.wakeup.notify_one(); - } - - R * operator -> () { return &*r; } - R & operator * () { return *r; } + R* operator->() { return &*r; } + R& operator*() { return *r; } - void markBad() { bad = true; } - }; + void markBad() { bad = true; } + }; - Handle get() + Handle get() { { - { - auto state_(state.lock()); - - /* If we're over the maximum number of instance, we need - to wait until a slot becomes available. */ - while (state_->idle.empty() && state_->inUse >= state_->max) - state_.wait(wakeup); - - while (!state_->idle.empty()) { - auto p = state_->idle.back(); - state_->idle.pop_back(); - if (validator(p)) { - state_->inUse++; - return Handle(*this, p); - } - } - - state_->inUse++; + auto state_(state.lock()); + + /* If we're over the maximum number of instance, we need + to wait until a slot becomes available. */ + while (state_->idle.empty() && state_->inUse >= state_->max) + state_.wait(wakeup); + + while (!state_->idle.empty()) { + auto p = state_->idle.back(); + state_->idle.pop_back(); + if (validator(p)) { + state_->inUse++; + return Handle(*this, p); } + } - /* We need to create a new instance. Because that might take a - while, we don't hold the lock in the meantime. */ - try { - Handle h(*this, factory()); - return h; - } catch (...) { - auto state_(state.lock()); - state_->inUse--; - wakeup.notify_one(); - throw; - } - } - - size_t count() - { - auto state_(state.lock()); - return state_->idle.size() + state_->inUse; + state_->inUse++; } - size_t capacity() - { - return state.lock()->max; - } - - void flushBad() - { - auto state_(state.lock()); - std::vector<ref<R>> left; - for (auto & p : state_->idle) - if (validator(p)) - left.push_back(p); - std::swap(state_->idle, left); + /* We need to create a new instance. Because that might take a + while, we don't hold the lock in the meantime. */ + try { + Handle h(*this, factory()); + return h; + } catch (...) { + auto state_(state.lock()); + state_->inUse--; + wakeup.notify_one(); + throw; } + } + + size_t count() { + auto state_(state.lock()); + return state_->idle.size() + state_->inUse; + } + + size_t capacity() { return state.lock()->max; } + + void flushBad() { + auto state_(state.lock()); + std::vector<ref<R>> left; + for (auto& p : state_->idle) + if (validator(p)) left.push_back(p); + std::swap(state_->idle, left); + } }; -} +} // namespace nix diff --git a/third_party/nix/src/libutil/ref.hh b/third_party/nix/src/libutil/ref.hh index 0be2a7e74ae0..5f0d7db4b6a1 100644 --- a/third_party/nix/src/libutil/ref.hh +++ b/third_party/nix/src/libutil/ref.hh @@ -1,92 +1,61 @@ #pragma once -#include <memory> #include <exception> +#include <memory> #include <stdexcept> namespace nix { /* A simple non-nullable reference-counted pointer. Actually a wrapper around std::shared_ptr that prevents non-null constructions. */ -template<typename T> -class ref -{ -private: - - std::shared_ptr<T> p; - -public: - - ref<T>(const ref<T> & r) - : p(r.p) - { } - - explicit ref<T>(const std::shared_ptr<T> & p) - : p(p) - { - if (!p) - throw std::invalid_argument("null pointer cast to ref"); - } - - explicit ref<T>(T * p) - : p(p) - { - if (!p) - throw std::invalid_argument("null pointer cast to ref"); - } - - T* operator ->() const - { - return &*p; - } - - T& operator *() const - { - return *p; - } - - operator std::shared_ptr<T> () const - { - return p; - } - - std::shared_ptr<T> get_ptr() const - { - return p; - } - - template<typename T2> - ref<T2> cast() const - { - return ref<T2>(std::dynamic_pointer_cast<T2>(p)); - } - - template<typename T2> - std::shared_ptr<T2> dynamic_pointer_cast() const - { - return std::dynamic_pointer_cast<T2>(p); - } - - template<typename T2> - operator ref<T2> () const - { - return ref<T2>((std::shared_ptr<T2>) p); - } - -private: - - template<typename T2, typename... Args> - friend ref<T2> - make_ref(Args&&... args); +template <typename T> +class ref { + private: + std::shared_ptr<T> p; + + public: + ref<T>(const ref<T>& r) : p(r.p) {} + + explicit ref<T>(const std::shared_ptr<T>& p) : p(p) { + if (!p) throw std::invalid_argument("null pointer cast to ref"); + } + + explicit ref<T>(T* p) : p(p) { + if (!p) throw std::invalid_argument("null pointer cast to ref"); + } + + T* operator->() const { return &*p; } + + T& operator*() const { return *p; } + operator std::shared_ptr<T>() const { return p; } + + std::shared_ptr<T> get_ptr() const { return p; } + + template <typename T2> + ref<T2> cast() const { + return ref<T2>(std::dynamic_pointer_cast<T2>(p)); + } + + template <typename T2> + std::shared_ptr<T2> dynamic_pointer_cast() const { + return std::dynamic_pointer_cast<T2>(p); + } + + template <typename T2> + operator ref<T2>() const { + return ref<T2>((std::shared_ptr<T2>)p); + } + + private: + template <typename T2, typename... Args> + friend ref<T2> make_ref(Args&&... args); }; -template<typename T, typename... Args> -inline ref<T> -make_ref(Args&&... args) -{ - auto p = std::make_shared<T>(std::forward<Args>(args)...); - return ref<T>(p); +template <typename T, typename... Args> +inline ref<T> make_ref(Args&&... args) { + auto p = std::make_shared<T>(std::forward<Args>(args)...); + return ref<T>(p); } -} +} // namespace nix diff --git a/third_party/nix/src/libutil/serialise.cc b/third_party/nix/src/libutil/serialise.cc index 8201549fd7d0..6d7cb1d8d156 100644 --- a/third_party/nix/src/libutil/serialise.cc +++ b/third_party/nix/src/libutil/serialise.cc @@ -1,323 +1,277 @@ #include "serialise.hh" -#include "util.hh" - -#include <cstring> +#include <boost/coroutine2/coroutine.hpp> #include <cerrno> +#include <cstring> #include <memory> - -#include <boost/coroutine2/coroutine.hpp> - +#include "util.hh" namespace nix { +void BufferedSink::operator()(const unsigned char* data, size_t len) { + if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); -void BufferedSink::operator () (const unsigned char * data, size_t len) -{ - if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); - - while (len) { - /* Optimisation: bypass the buffer if the data exceeds the - buffer size. */ - if (bufPos + len >= bufSize) { - flush(); - write(data, len); - break; - } - /* Otherwise, copy the bytes to the buffer. Flush the buffer - when it's full. */ - size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; - memcpy(buffer.get() + bufPos, data, n); - data += n; bufPos += n; len -= n; - if (bufPos == bufSize) flush(); + while (len) { + /* Optimisation: bypass the buffer if the data exceeds the + buffer size. */ + if (bufPos + len >= bufSize) { + flush(); + write(data, len); + break; } + /* Otherwise, copy the bytes to the buffer. Flush the buffer + when it's full. */ + size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; + memcpy(buffer.get() + bufPos, data, n); + data += n; + bufPos += n; + len -= n; + if (bufPos == bufSize) flush(); + } } - -void BufferedSink::flush() -{ - if (bufPos == 0) return; - size_t n = bufPos; - bufPos = 0; // don't trigger the assert() in ~BufferedSink() - write(buffer.get(), n); +void BufferedSink::flush() { + if (bufPos == 0) return; + size_t n = bufPos; + bufPos = 0; // don't trigger the assert() in ~BufferedSink() + write(buffer.get(), n); } - -FdSink::~FdSink() -{ - try { flush(); } catch (...) { ignoreException(); } +FdSink::~FdSink() { + try { + flush(); + } catch (...) { + ignoreException(); + } } - size_t threshold = 256 * 1024 * 1024; -static void warnLargeDump() -{ - printError("warning: dumping very large path (> 256 MiB); this may run out of memory"); +static void warnLargeDump() { + printError( + "warning: dumping very large path (> 256 MiB); this may run out of " + "memory"); } - -void FdSink::write(const unsigned char * data, size_t len) -{ - written += len; - static bool warned = false; - if (warn && !warned) { - if (written > threshold) { - warnLargeDump(); - warned = true; - } - } - try { - writeFull(fd, data, len); - } catch (SysError & e) { - _good = false; - throw; +void FdSink::write(const unsigned char* data, size_t len) { + written += len; + static bool warned = false; + if (warn && !warned) { + if (written > threshold) { + warnLargeDump(); + warned = true; } + } + try { + writeFull(fd, data, len); + } catch (SysError& e) { + _good = false; + throw; + } } +bool FdSink::good() { return _good; } -bool FdSink::good() -{ - return _good; +void Source::operator()(unsigned char* data, size_t len) { + while (len) { + size_t n = read(data, len); + data += n; + len -= n; + } } - -void Source::operator () (unsigned char * data, size_t len) -{ - while (len) { - size_t n = read(data, len); - data += n; len -= n; - } -} - - -std::string Source::drain() -{ - std::string s; - std::vector<unsigned char> buf(8192); - while (true) { - size_t n; - try { - n = read(buf.data(), buf.size()); - s.append((char *) buf.data(), n); - } catch (EndOfFile &) { - break; - } +std::string Source::drain() { + std::string s; + std::vector<unsigned char> buf(8192); + while (true) { + size_t n; + try { + n = read(buf.data(), buf.size()); + s.append((char*)buf.data(), n); + } catch (EndOfFile&) { + break; } - return s; -} - - -size_t BufferedSource::read(unsigned char * data, size_t len) -{ - if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); - - if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize); - - /* Copy out the data in the buffer. */ - size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; - memcpy(data, buffer.get() + bufPosOut, n); - bufPosOut += n; - if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; - return n; + } + return s; } +size_t BufferedSource::read(unsigned char* data, size_t len) { + if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); -bool BufferedSource::hasData() -{ - return bufPosOut < bufPosIn; -} - + if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize); -size_t FdSource::readUnbuffered(unsigned char * data, size_t len) -{ - ssize_t n; - do { - checkInterrupt(); - n = ::read(fd, (char *) data, len); - } while (n == -1 && errno == EINTR); - if (n == -1) { _good = false; throw SysError("reading from file"); } - if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } - read += n; - return n; + /* Copy out the data in the buffer. */ + size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + memcpy(data, buffer.get() + bufPosOut, n); + bufPosOut += n; + if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; + return n; } - -bool FdSource::good() -{ - return _good; +bool BufferedSource::hasData() { return bufPosOut < bufPosIn; } + +size_t FdSource::readUnbuffered(unsigned char* data, size_t len) { + ssize_t n; + do { + checkInterrupt(); + n = ::read(fd, (char*)data, len); + } while (n == -1 && errno == EINTR); + if (n == -1) { + _good = false; + throw SysError("reading from file"); + } + if (n == 0) { + _good = false; + throw EndOfFile("unexpected end-of-file"); + } + read += n; + return n; } +bool FdSource::good() { return _good; } -size_t StringSource::read(unsigned char * data, size_t len) -{ - if (pos == s.size()) throw EndOfFile("end of string reached"); - size_t n = s.copy((char *) data, len, pos); - pos += n; - return n; +size_t StringSource::read(unsigned char* data, size_t len) { + if (pos == s.size()) throw EndOfFile("end of string reached"); + size_t n = s.copy((char*)data, len, pos); + pos += n; + return n; } - #if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600 #error Coroutines are broken in this version of Boost! #endif -std::unique_ptr<Source> sinkToSource( - std::function<void(Sink &)> fun, - std::function<void()> eof) -{ - struct SinkToSource : Source - { - typedef boost::coroutines2::coroutine<std::string> coro_t; - - std::function<void(Sink &)> fun; - std::function<void()> eof; - std::optional<coro_t::pull_type> coro; - bool started = false; - - SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof) - : fun(fun), eof(eof) - { - } - - std::string cur; - size_t pos = 0; - - size_t read(unsigned char * data, size_t len) override - { - if (!coro) - coro = coro_t::pull_type([&](coro_t::push_type & yield) { - LambdaSink sink([&](const unsigned char * data, size_t len) { - if (len) yield(std::string((const char *) data, len)); - }); - fun(sink); - }); - - if (!*coro) { eof(); abort(); } - - if (pos == cur.size()) { - if (!cur.empty()) (*coro)(); - cur = coro->get(); - pos = 0; - } - - auto n = std::min(cur.size() - pos, len); - memcpy(data, (unsigned char *) cur.data() + pos, n); - pos += n; - - return n; - } - }; - - return std::make_unique<SinkToSource>(fun, eof); -} - - -void writePadding(size_t len, Sink & sink) -{ - if (len % 8) { - unsigned char zero[8]; - memset(zero, 0, sizeof(zero)); - sink(zero, 8 - (len % 8)); +std::unique_ptr<Source> sinkToSource(std::function<void(Sink&)> fun, + std::function<void()> eof) { + struct SinkToSource : Source { + typedef boost::coroutines2::coroutine<std::string> coro_t; + + std::function<void(Sink&)> fun; + std::function<void()> eof; + std::optional<coro_t::pull_type> coro; + bool started = false; + + SinkToSource(std::function<void(Sink&)> fun, std::function<void()> eof) + : fun(fun), eof(eof) {} + + std::string cur; + size_t pos = 0; + + size_t read(unsigned char* data, size_t len) override { + if (!coro) + coro = coro_t::pull_type([&](coro_t::push_type& yield) { + LambdaSink sink([&](const unsigned char* data, size_t len) { + if (len) yield(std::string((const char*)data, len)); + }); + fun(sink); + }); + + if (!*coro) { + eof(); + abort(); + } + + if (pos == cur.size()) { + if (!cur.empty()) (*coro)(); + cur = coro->get(); + pos = 0; + } + + auto n = std::min(cur.size() - pos, len); + memcpy(data, (unsigned char*)cur.data() + pos, n); + pos += n; + + return n; } -} - + }; -void writeString(const unsigned char * buf, size_t len, Sink & sink) -{ - sink << len; - sink(buf, len); - writePadding(len, sink); + return std::make_unique<SinkToSource>(fun, eof); } - -Sink & operator << (Sink & sink, const string & s) -{ - writeString((const unsigned char *) s.data(), s.size(), sink); - return sink; +void writePadding(size_t len, Sink& sink) { + if (len % 8) { + unsigned char zero[8]; + memset(zero, 0, sizeof(zero)); + sink(zero, 8 - (len % 8)); + } } - -template<class T> void writeStrings(const T & ss, Sink & sink) -{ - sink << ss.size(); - for (auto & i : ss) - sink << i; +void writeString(const unsigned char* buf, size_t len, Sink& sink) { + sink << len; + sink(buf, len); + writePadding(len, sink); } -Sink & operator << (Sink & sink, const Strings & s) -{ - writeStrings(s, sink); - return sink; +Sink& operator<<(Sink& sink, const string& s) { + writeString((const unsigned char*)s.data(), s.size(), sink); + return sink; } -Sink & operator << (Sink & sink, const StringSet & s) -{ - writeStrings(s, sink); - return sink; +template <class T> +void writeStrings(const T& ss, Sink& sink) { + sink << ss.size(); + for (auto& i : ss) sink << i; } - -void readPadding(size_t len, Source & source) -{ - if (len % 8) { - unsigned char zero[8]; - size_t n = 8 - (len % 8); - source(zero, n); - for (unsigned int i = 0; i < n; i++) - if (zero[i]) throw SerialisationError("non-zero padding"); - } +Sink& operator<<(Sink& sink, const Strings& s) { + writeStrings(s, sink); + return sink; } - -size_t readString(unsigned char * buf, size_t max, Source & source) -{ - auto len = readNum<size_t>(source); - if (len > max) throw SerialisationError("string is too long"); - source(buf, len); - readPadding(len, source); - return len; +Sink& operator<<(Sink& sink, const StringSet& s) { + writeStrings(s, sink); + return sink; } - -string readString(Source & source, size_t max) -{ - auto len = readNum<size_t>(source); - if (len > max) throw SerialisationError("string is too long"); - std::string res(len, 0); - source((unsigned char*) res.data(), len); - readPadding(len, source); - return res; +void readPadding(size_t len, Source& source) { + if (len % 8) { + unsigned char zero[8]; + size_t n = 8 - (len % 8); + source(zero, n); + for (unsigned int i = 0; i < n; i++) + if (zero[i]) throw SerialisationError("non-zero padding"); + } } -Source & operator >> (Source & in, string & s) -{ - s = readString(in); - return in; +size_t readString(unsigned char* buf, size_t max, Source& source) { + auto len = readNum<size_t>(source); + if (len > max) throw SerialisationError("string is too long"); + source(buf, len); + readPadding(len, source); + return len; } - -template<class T> T readStrings(Source & source) -{ - auto count = readNum<size_t>(source); - T ss; - while (count--) - ss.insert(ss.end(), readString(source)); - return ss; +string readString(Source& source, size_t max) { + auto len = readNum<size_t>(source); + if (len > max) throw SerialisationError("string is too long"); + std::string res(len, 0); + source((unsigned char*)res.data(), len); + readPadding(len, source); + return res; } -template Paths readStrings(Source & source); -template PathSet readStrings(Source & source); - +Source& operator>>(Source& in, string& s) { + s = readString(in); + return in; +} -void StringSink::operator () (const unsigned char * data, size_t len) -{ - static bool warned = false; - if (!warned && s->size() > threshold) { - warnLargeDump(); - warned = true; - } - s->append((const char *) data, len); +template <class T> +T readStrings(Source& source) { + auto count = readNum<size_t>(source); + T ss; + while (count--) ss.insert(ss.end(), readString(source)); + return ss; } +template Paths readStrings(Source& source); +template PathSet readStrings(Source& source); +void StringSink::operator()(const unsigned char* data, size_t len) { + static bool warned = false; + if (!warned && s->size() > threshold) { + warnLargeDump(); + warned = true; + } + s->append((const char*)data, len); } + +} // namespace nix diff --git a/third_party/nix/src/libutil/serialise.hh b/third_party/nix/src/libutil/serialise.hh index a344a5ac7520..2ab4bdff6f46 100644 --- a/third_party/nix/src/libutil/serialise.hh +++ b/third_party/nix/src/libutil/serialise.hh @@ -1,337 +1,285 @@ #pragma once #include <memory> - #include "types.hh" #include "util.hh" - namespace nix { - /* Abstract destination of binary data. */ -struct Sink -{ - virtual ~Sink() { } - virtual void operator () (const unsigned char * data, size_t len) = 0; - virtual bool good() { return true; } - - void operator () (const std::string & s) - { - (*this)((const unsigned char *) s.data(), s.size()); - } +struct Sink { + virtual ~Sink() {} + virtual void operator()(const unsigned char* data, size_t len) = 0; + virtual bool good() { return true; } + + void operator()(const std::string& s) { + (*this)((const unsigned char*)s.data(), s.size()); + } }; - /* A buffered abstract sink. */ -struct BufferedSink : Sink -{ - size_t bufSize, bufPos; - std::unique_ptr<unsigned char[]> buffer; +struct BufferedSink : Sink { + size_t bufSize, bufPos; + std::unique_ptr<unsigned char[]> buffer; - BufferedSink(size_t bufSize = 32 * 1024) - : bufSize(bufSize), bufPos(0), buffer(nullptr) { } + BufferedSink(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPos(0), buffer(nullptr) {} - void operator () (const unsigned char * data, size_t len) override; + void operator()(const unsigned char* data, size_t len) override; - void operator () (const std::string & s) - { - Sink::operator()(s); - } + void operator()(const std::string& s) { Sink::operator()(s); } - void flush(); + void flush(); - virtual void write(const unsigned char * data, size_t len) = 0; + virtual void write(const unsigned char* data, size_t len) = 0; }; - /* Abstract source of binary data. */ -struct Source -{ - virtual ~Source() { } +struct Source { + virtual ~Source() {} - /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. - It blocks until all the requested data is available, or throws - an error if it is not going to be available. */ - void operator () (unsigned char * data, size_t len); + /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + It blocks until all the requested data is available, or throws + an error if it is not going to be available. */ + void operator()(unsigned char* data, size_t len); - /* Store up to ‘len’ in the buffer pointed to by ‘data’, and - return the number of bytes stored. It blocks until at least - one byte is available. */ - virtual size_t read(unsigned char * data, size_t len) = 0; + /* Store up to ‘len’ in the buffer pointed to by ‘data’, and + return the number of bytes stored. It blocks until at least + one byte is available. */ + virtual size_t read(unsigned char* data, size_t len) = 0; - virtual bool good() { return true; } + virtual bool good() { return true; } - std::string drain(); + std::string drain(); }; - /* A buffered abstract source. */ -struct BufferedSource : Source -{ - size_t bufSize, bufPosIn, bufPosOut; - std::unique_ptr<unsigned char[]> buffer; - - BufferedSource(size_t bufSize = 32 * 1024) - : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) { } +struct BufferedSource : Source { + size_t bufSize, bufPosIn, bufPosOut; + std::unique_ptr<unsigned char[]> buffer; - size_t read(unsigned char * data, size_t len) override; + BufferedSource(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) {} + size_t read(unsigned char* data, size_t len) override; - bool hasData(); + bool hasData(); -protected: - /* Underlying read call, to be overridden. */ - virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; + protected: + /* Underlying read call, to be overridden. */ + virtual size_t readUnbuffered(unsigned char* data, size_t len) = 0; }; - /* A sink that writes data to a file descriptor. */ -struct FdSink : BufferedSink -{ - int fd; - bool warn = false; - size_t written = 0; - - FdSink() : fd(-1) { } - FdSink(int fd) : fd(fd) { } - FdSink(FdSink&&) = default; - - FdSink& operator=(FdSink && s) - { - flush(); - fd = s.fd; - s.fd = -1; - warn = s.warn; - written = s.written; - return *this; - } +struct FdSink : BufferedSink { + int fd; + bool warn = false; + size_t written = 0; - ~FdSink(); + FdSink() : fd(-1) {} + FdSink(int fd) : fd(fd) {} + FdSink(FdSink&&) = default; - void write(const unsigned char * data, size_t len) override; + FdSink& operator=(FdSink&& s) { + flush(); + fd = s.fd; + s.fd = -1; + warn = s.warn; + written = s.written; + return *this; + } - bool good() override; + ~FdSink(); -private: - bool _good = true; -}; + void write(const unsigned char* data, size_t len) override; + bool good() override; + + private: + bool _good = true; +}; /* A source that reads data from a file descriptor. */ -struct FdSource : BufferedSource -{ - int fd; - size_t read = 0; - - FdSource() : fd(-1) { } - FdSource(int fd) : fd(fd) { } - FdSource(FdSource&&) = default; - - FdSource& operator=(FdSource && s) - { - fd = s.fd; - s.fd = -1; - read = s.read; - return *this; - } +struct FdSource : BufferedSource { + int fd; + size_t read = 0; - bool good() override; -protected: - size_t readUnbuffered(unsigned char * data, size_t len) override; -private: - bool _good = true; -}; + FdSource() : fd(-1) {} + FdSource(int fd) : fd(fd) {} + FdSource(FdSource&&) = default; + FdSource& operator=(FdSource&& s) { + fd = s.fd; + s.fd = -1; + read = s.read; + return *this; + } -/* A sink that writes data to a string. */ -struct StringSink : Sink -{ - ref<std::string> s; - StringSink() : s(make_ref<std::string>()) { }; - StringSink(ref<std::string> s) : s(s) { }; - void operator () (const unsigned char * data, size_t len) override; + bool good() override; + + protected: + size_t readUnbuffered(unsigned char* data, size_t len) override; + + private: + bool _good = true; }; +/* A sink that writes data to a string. */ +struct StringSink : Sink { + ref<std::string> s; + StringSink() : s(make_ref<std::string>()){}; + StringSink(ref<std::string> s) : s(s){}; + void operator()(const unsigned char* data, size_t len) override; +}; /* A source that reads data from a string. */ -struct StringSource : Source -{ - const string & s; - size_t pos; - StringSource(const string & _s) : s(_s), pos(0) { } - size_t read(unsigned char * data, size_t len) override; +struct StringSource : Source { + const string& s; + size_t pos; + StringSource(const string& _s) : s(_s), pos(0) {} + size_t read(unsigned char* data, size_t len) override; }; - /* Adapter class of a Source that saves all data read to `s'. */ -struct TeeSource : Source -{ - Source & orig; - ref<std::string> data; - TeeSource(Source & orig) - : orig(orig), data(make_ref<std::string>()) { } - size_t read(unsigned char * data, size_t len) - { - size_t n = orig.read(data, len); - this->data->append((const char *) data, n); - return n; - } +struct TeeSource : Source { + Source& orig; + ref<std::string> data; + TeeSource(Source& orig) : orig(orig), data(make_ref<std::string>()) {} + size_t read(unsigned char* data, size_t len) { + size_t n = orig.read(data, len); + this->data->append((const char*)data, n); + return n; + } }; /* A reader that consumes the original Source until 'size'. */ -struct SizedSource : Source -{ - Source & orig; - size_t remain; - SizedSource(Source & orig, size_t size) - : orig(orig), remain(size) { } - size_t read(unsigned char * data, size_t len) - { - if (this->remain <= 0) { - throw EndOfFile("sized: unexpected end-of-file"); - } - len = std::min(len, this->remain); - size_t n = this->orig.read(data, len); - this->remain -= n; - return n; +struct SizedSource : Source { + Source& orig; + size_t remain; + SizedSource(Source& orig, size_t size) : orig(orig), remain(size) {} + size_t read(unsigned char* data, size_t len) { + if (this->remain <= 0) { + throw EndOfFile("sized: unexpected end-of-file"); } - - /* Consume the original source until no remain data is left to consume. */ - size_t drainAll() - { - std::vector<unsigned char> buf(8192); - size_t sum = 0; - while (this->remain > 0) { - size_t n = read(buf.data(), buf.size()); - sum += n; - } - return sum; + len = std::min(len, this->remain); + size_t n = this->orig.read(data, len); + this->remain -= n; + return n; + } + + /* Consume the original source until no remain data is left to consume. */ + size_t drainAll() { + std::vector<unsigned char> buf(8192); + size_t sum = 0; + while (this->remain > 0) { + size_t n = read(buf.data(), buf.size()); + sum += n; } + return sum; + } }; /* Convert a function into a sink. */ -struct LambdaSink : Sink -{ - typedef std::function<void(const unsigned char *, size_t)> lambda_t; +struct LambdaSink : Sink { + typedef std::function<void(const unsigned char*, size_t)> lambda_t; - lambda_t lambda; + lambda_t lambda; - LambdaSink(const lambda_t & lambda) : lambda(lambda) { } + LambdaSink(const lambda_t& lambda) : lambda(lambda) {} - virtual void operator () (const unsigned char * data, size_t len) - { - lambda(data, len); - } + virtual void operator()(const unsigned char* data, size_t len) { + lambda(data, len); + } }; - /* Convert a function into a source. */ -struct LambdaSource : Source -{ - typedef std::function<size_t(unsigned char *, size_t)> lambda_t; +struct LambdaSource : Source { + typedef std::function<size_t(unsigned char*, size_t)> lambda_t; - lambda_t lambda; + lambda_t lambda; - LambdaSource(const lambda_t & lambda) : lambda(lambda) { } + LambdaSource(const lambda_t& lambda) : lambda(lambda) {} - size_t read(unsigned char * data, size_t len) override - { - return lambda(data, len); - } + size_t read(unsigned char* data, size_t len) override { + return lambda(data, len); + } }; - /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ std::unique_ptr<Source> sinkToSource( - std::function<void(Sink &)> fun, - std::function<void()> eof = []() { - throw EndOfFile("coroutine has finished"); + std::function<void(Sink&)> fun, std::function<void()> eof = []() { + throw EndOfFile("coroutine has finished"); }); - -void writePadding(size_t len, Sink & sink); -void writeString(const unsigned char * buf, size_t len, Sink & sink); - -inline Sink & operator << (Sink & sink, uint64_t n) -{ - unsigned char buf[8]; - buf[0] = n & 0xff; - buf[1] = (n >> 8) & 0xff; - buf[2] = (n >> 16) & 0xff; - buf[3] = (n >> 24) & 0xff; - buf[4] = (n >> 32) & 0xff; - buf[5] = (n >> 40) & 0xff; - buf[6] = (n >> 48) & 0xff; - buf[7] = (unsigned char) (n >> 56) & 0xff; - sink(buf, sizeof(buf)); - return sink; +void writePadding(size_t len, Sink& sink); +void writeString(const unsigned char* buf, size_t len, Sink& sink); + +inline Sink& operator<<(Sink& sink, uint64_t n) { + unsigned char buf[8]; + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + buf[4] = (n >> 32) & 0xff; + buf[5] = (n >> 40) & 0xff; + buf[6] = (n >> 48) & 0xff; + buf[7] = (unsigned char)(n >> 56) & 0xff; + sink(buf, sizeof(buf)); + return sink; } -Sink & operator << (Sink & sink, const string & s); -Sink & operator << (Sink & sink, const Strings & s); -Sink & operator << (Sink & sink, const StringSet & s); - +Sink& operator<<(Sink& sink, const string& s); +Sink& operator<<(Sink& sink, const Strings& s); +Sink& operator<<(Sink& sink, const StringSet& s); MakeError(SerialisationError, Error) + template <typename T> + T readNum(Source& source) { + unsigned char buf[8]; + source(buf, sizeof(buf)); -template<typename T> -T readNum(Source & source) -{ - unsigned char buf[8]; - source(buf, sizeof(buf)); + uint64_t n = + ((unsigned long long)buf[0]) | ((unsigned long long)buf[1] << 8) | + ((unsigned long long)buf[2] << 16) | ((unsigned long long)buf[3] << 24) | + ((unsigned long long)buf[4] << 32) | ((unsigned long long)buf[5] << 40) | + ((unsigned long long)buf[6] << 48) | ((unsigned long long)buf[7] << 56); - uint64_t n = - ((unsigned long long) buf[0]) | - ((unsigned long long) buf[1] << 8) | - ((unsigned long long) buf[2] << 16) | - ((unsigned long long) buf[3] << 24) | - ((unsigned long long) buf[4] << 32) | - ((unsigned long long) buf[5] << 40) | - ((unsigned long long) buf[6] << 48) | - ((unsigned long long) buf[7] << 56); + if (n > std::numeric_limits<T>::max()) + throw SerialisationError("serialised integer %d is too large for type '%s'", + n, typeid(T).name()); - if (n > std::numeric_limits<T>::max()) - throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); - - return (T) n; + return (T)n; } - -inline unsigned int readInt(Source & source) -{ - return readNum<unsigned int>(source); +inline unsigned int readInt(Source& source) { + return readNum<unsigned int>(source); } - -inline uint64_t readLongLong(Source & source) -{ - return readNum<uint64_t>(source); +inline uint64_t readLongLong(Source& source) { + return readNum<uint64_t>(source); } +void readPadding(size_t len, Source& source); +size_t readString(unsigned char* buf, size_t max, Source& source); +string readString(Source& source, + size_t max = std::numeric_limits<size_t>::max()); +template <class T> +T readStrings(Source& source); -void readPadding(size_t len, Source & source); -size_t readString(unsigned char * buf, size_t max, Source & source); -string readString(Source & source, size_t max = std::numeric_limits<size_t>::max()); -template<class T> T readStrings(Source & source); - -Source & operator >> (Source & in, string & s); +Source& operator>>(Source& in, string& s); -template<typename T> -Source & operator >> (Source & in, T & n) -{ - n = readNum<T>(in); - return in; +template <typename T> +Source& operator>>(Source& in, T& n) { + n = readNum<T>(in); + return in; } -template<typename T> -Source & operator >> (Source & in, bool & b) -{ - b = readNum<uint64_t>(in); - return in; +template <typename T> +Source& operator>>(Source& in, bool& b) { + b = readNum<uint64_t>(in); + return in; } - -} +} // namespace nix diff --git a/third_party/nix/src/libutil/sync.hh b/third_party/nix/src/libutil/sync.hh index e1d591d77a84..b79d1176b9f3 100644 --- a/third_party/nix/src/libutil/sync.hh +++ b/third_party/nix/src/libutil/sync.hh @@ -1,9 +1,9 @@ #pragma once +#include <cassert> +#include <condition_variable> #include <cstdlib> #include <mutex> -#include <condition_variable> -#include <cassert> namespace nix { @@ -23,66 +23,62 @@ namespace nix { scope. */ -template<class T, class M = std::mutex> -class Sync -{ -private: - M mutex; - T data; - -public: - - Sync() { } - Sync(const T & data) : data(data) { } - Sync(T && data) noexcept : data(std::move(data)) { } - - class Lock - { - private: - Sync * s; - std::unique_lock<M> lk; - friend Sync; - Lock(Sync * s) : s(s), lk(s->mutex) { } - public: - Lock(Lock && l) : s(l.s) { abort(); } - Lock(const Lock & l) = delete; - ~Lock() { } - T * operator -> () { return &s->data; } - T & operator * () { return s->data; } - - void wait(std::condition_variable & cv) - { - assert(s); - cv.wait(lk); - } - - template<class Rep, class Period> - std::cv_status wait_for(std::condition_variable & cv, - const std::chrono::duration<Rep, Period> & duration) - { - assert(s); - return cv.wait_for(lk, duration); - } - - template<class Rep, class Period, class Predicate> - bool wait_for(std::condition_variable & cv, - const std::chrono::duration<Rep, Period> & duration, - Predicate pred) - { - assert(s); - return cv.wait_for(lk, duration, pred); - } - - template<class Clock, class Duration> - std::cv_status wait_until(std::condition_variable & cv, - const std::chrono::time_point<Clock, Duration> & duration) - { - assert(s); - return cv.wait_until(lk, duration); - } - }; - - Lock lock() { return Lock(this); } +template <class T, class M = std::mutex> +class Sync { + private: + M mutex; + T data; + + public: + Sync() {} + Sync(const T& data) : data(data) {} + Sync(T&& data) noexcept : data(std::move(data)) {} + + class Lock { + private: + Sync* s; + std::unique_lock<M> lk; + friend Sync; + Lock(Sync* s) : s(s), lk(s->mutex) {} + + public: + Lock(Lock&& l) : s(l.s) { abort(); } + Lock(const Lock& l) = delete; + ~Lock() {} + T* operator->() { return &s->data; } + T& operator*() { return s->data; } + + void wait(std::condition_variable& cv) { + assert(s); + cv.wait(lk); + } + + template <class Rep, class Period> + std::cv_status wait_for( + std::condition_variable& cv, + const std::chrono::duration<Rep, Period>& duration) { + assert(s); + return cv.wait_for(lk, duration); + } + + template <class Rep, class Period, class Predicate> + bool wait_for(std::condition_variable& cv, + const std::chrono::duration<Rep, Period>& duration, + Predicate pred) { + assert(s); + return cv.wait_for(lk, duration, pred); + } + + template <class Clock, class Duration> + std::cv_status wait_until( + std::condition_variable& cv, + const std::chrono::time_point<Clock, Duration>& duration) { + assert(s); + return cv.wait_until(lk, duration); + } + }; + + Lock lock() { return Lock(this); } }; -} +} // namespace nix diff --git a/third_party/nix/src/libutil/thread-pool.cc b/third_party/nix/src/libutil/thread-pool.cc index 857ee91f87d0..5b5be926539d 100644 --- a/third_party/nix/src/libutil/thread-pool.cc +++ b/third_party/nix/src/libutil/thread-pool.cc @@ -3,154 +3,141 @@ namespace nix { -ThreadPool::ThreadPool(size_t _maxThreads) - : maxThreads(_maxThreads) -{ - restoreAffinity(); // FIXME - - if (!maxThreads) { - maxThreads = std::thread::hardware_concurrency(); - if (!maxThreads) maxThreads = 1; - } +ThreadPool::ThreadPool(size_t _maxThreads) : maxThreads(_maxThreads) { + restoreAffinity(); // FIXME - debug("starting pool of %d threads", maxThreads - 1); -} + if (!maxThreads) { + maxThreads = std::thread::hardware_concurrency(); + if (!maxThreads) maxThreads = 1; + } -ThreadPool::~ThreadPool() -{ - shutdown(); + debug("starting pool of %d threads", maxThreads - 1); } -void ThreadPool::shutdown() -{ - std::vector<std::thread> workers; - { - auto state(state_.lock()); - quit = true; - std::swap(workers, state->workers); - } +ThreadPool::~ThreadPool() { shutdown(); } + +void ThreadPool::shutdown() { + std::vector<std::thread> workers; + { + auto state(state_.lock()); + quit = true; + std::swap(workers, state->workers); + } - if (workers.empty()) return; + if (workers.empty()) return; - debug("reaping %d worker threads", workers.size()); + debug("reaping %d worker threads", workers.size()); - work.notify_all(); + work.notify_all(); - for (auto & thr : workers) - thr.join(); + for (auto& thr : workers) thr.join(); } -void ThreadPool::enqueue(const work_t & t) -{ - auto state(state_.lock()); - if (quit) - throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down"); - state->pending.push(t); - /* Note: process() also executes items, so count it as a worker. */ - if (state->pending.size() > state->workers.size() + 1 && state->workers.size() + 1 < maxThreads) - state->workers.emplace_back(&ThreadPool::doWork, this, false); - work.notify_one(); +void ThreadPool::enqueue(const work_t& t) { + auto state(state_.lock()); + if (quit) + throw ThreadPoolShutDown( + "cannot enqueue a work item while the thread pool is shutting down"); + state->pending.push(t); + /* Note: process() also executes items, so count it as a worker. */ + if (state->pending.size() > state->workers.size() + 1 && + state->workers.size() + 1 < maxThreads) + state->workers.emplace_back(&ThreadPool::doWork, this, false); + work.notify_one(); } -void ThreadPool::process() -{ - state_.lock()->draining = true; +void ThreadPool::process() { + state_.lock()->draining = true; - /* Do work until no more work is pending or active. */ - try { - doWork(true); + /* Do work until no more work is pending or active. */ + try { + doWork(true); - auto state(state_.lock()); + auto state(state_.lock()); - assert(quit); + assert(quit); - if (state->exception) - std::rethrow_exception(state->exception); + if (state->exception) std::rethrow_exception(state->exception); - } catch (...) { - /* In the exceptional case, some workers may still be - active. They may be referencing the stack frame of the - caller. So wait for them to finish. (~ThreadPool also does - this, but it might be destroyed after objects referenced by - the work item lambdas.) */ - shutdown(); - throw; - } + } catch (...) { + /* In the exceptional case, some workers may still be + active. They may be referencing the stack frame of the + caller. So wait for them to finish. (~ThreadPool also does + this, but it might be destroyed after objects referenced by + the work item lambdas.) */ + shutdown(); + throw; + } } -void ThreadPool::doWork(bool mainThread) -{ - if (!mainThread) - interruptCheck = [&]() { return (bool) quit; }; - - bool didWork = false; - std::exception_ptr exc; - - while (true) { - work_t w; - { - auto state(state_.lock()); - - if (didWork) { - assert(state->active); - state->active--; - - if (exc) { - - if (!state->exception) { - state->exception = exc; - // Tell the other workers to quit. - quit = true; - work.notify_all(); - } else { - /* Print the exception, since we can't - propagate it. */ - try { - std::rethrow_exception(exc); - } catch (std::exception & e) { - if (!dynamic_cast<Interrupted*>(&e) && - !dynamic_cast<ThreadPoolShutDown*>(&e)) - ignoreException(); - } catch (...) { - } - } - } - } - - /* Wait until a work item is available or we're asked to - quit. */ - while (true) { - if (quit) return; +void ThreadPool::doWork(bool mainThread) { + if (!mainThread) interruptCheck = [&]() { return (bool)quit; }; - if (!state->pending.empty()) break; + bool didWork = false; + std::exception_ptr exc; - /* If there are no active or pending items, and the - main thread is running process(), then no new items - can be added. So exit. */ - if (!state->active && state->draining) { - quit = true; - work.notify_all(); - return; - } - - state.wait(work); + while (true) { + work_t w; + { + auto state(state_.lock()); + + if (didWork) { + assert(state->active); + state->active--; + + if (exc) { + if (!state->exception) { + state->exception = exc; + // Tell the other workers to quit. + quit = true; + work.notify_all(); + } else { + /* Print the exception, since we can't + propagate it. */ + try { + std::rethrow_exception(exc); + } catch (std::exception& e) { + if (!dynamic_cast<Interrupted*>(&e) && + !dynamic_cast<ThreadPoolShutDown*>(&e)) + ignoreException(); + } catch (...) { } - - w = std::move(state->pending.front()); - state->pending.pop(); - state->active++; + } } - - try { - w(); - } catch (...) { - exc = std::current_exception(); + } + + /* Wait until a work item is available or we're asked to + quit. */ + while (true) { + if (quit) return; + + if (!state->pending.empty()) break; + + /* If there are no active or pending items, and the + main thread is running process(), then no new items + can be added. So exit. */ + if (!state->active && state->draining) { + quit = true; + work.notify_all(); + return; } - didWork = true; + state.wait(work); + } + + w = std::move(state->pending.front()); + state->pending.pop(); + state->active++; } -} -} + try { + w(); + } catch (...) { + exc = std::current_exception(); + } + didWork = true; + } +} +} // namespace nix diff --git a/third_party/nix/src/libutil/thread-pool.hh b/third_party/nix/src/libutil/thread-pool.hh index bb16b639a591..18fd208b1bb4 100644 --- a/third_party/nix/src/libutil/thread-pool.hh +++ b/third_party/nix/src/libutil/thread-pool.hh @@ -1,143 +1,129 @@ #pragma once -#include "sync.hh" -#include "util.hh" - -#include <queue> +#include <atomic> #include <functional> -#include <thread> #include <map> -#include <atomic> +#include <queue> +#include <thread> +#include "sync.hh" +#include "util.hh" namespace nix { MakeError(ThreadPoolShutDown, Error) -/* A simple thread pool that executes a queue of work items - (lambdas). */ -class ThreadPool -{ -public: + /* A simple thread pool that executes a queue of work items + (lambdas). */ + class ThreadPool { + public: + ThreadPool(size_t maxThreads = 0); - ThreadPool(size_t maxThreads = 0); + ~ThreadPool(); - ~ThreadPool(); + // FIXME: use std::packaged_task? + typedef std::function<void()> work_t; - // FIXME: use std::packaged_task? - typedef std::function<void()> work_t; + /* Enqueue a function to be executed by the thread pool. */ + void enqueue(const work_t& t); - /* Enqueue a function to be executed by the thread pool. */ - void enqueue(const work_t & t); + /* Execute work items until the queue is empty. Note that work + items are allowed to add new items to the queue; this is + handled correctly. Queue processing stops prematurely if any + work item throws an exception. This exception is propagated to + the calling thread. If multiple work items throw an exception + concurrently, only one item is propagated; the others are + printed on stderr and otherwise ignored. */ + void process(); - /* Execute work items until the queue is empty. Note that work - items are allowed to add new items to the queue; this is - handled correctly. Queue processing stops prematurely if any - work item throws an exception. This exception is propagated to - the calling thread. If multiple work items throw an exception - concurrently, only one item is propagated; the others are - printed on stderr and otherwise ignored. */ - void process(); + private: + size_t maxThreads; -private: + struct State { + std::queue<work_t> pending; + size_t active = 0; + std::exception_ptr exception; + std::vector<std::thread> workers; + bool draining = false; + }; - size_t maxThreads; + std::atomic_bool quit{false}; - struct State - { - std::queue<work_t> pending; - size_t active = 0; - std::exception_ptr exception; - std::vector<std::thread> workers; - bool draining = false; - }; + Sync<State> state_; - std::atomic_bool quit{false}; + std::condition_variable work; - Sync<State> state_; + void doWork(bool mainThread); - std::condition_variable work; - - void doWork(bool mainThread); - - void shutdown(); + void shutdown(); }; /* Process in parallel a set of items of type T that have a partial ordering between them. Thus, any item is only processed after all its dependencies have been processed. */ -template<typename T> -void processGraph( - ThreadPool & pool, - const std::set<T> & nodes, - std::function<std::set<T>(const T &)> getEdges, - std::function<void(const T &)> processNode) -{ - struct Graph { - std::set<T> left; - std::map<T, std::set<T>> refs, rrefs; - }; - - Sync<Graph> graph_(Graph{nodes, {}, {}}); - - std::function<void(const T &)> worker; - - worker = [&](const T & node) { - - { - auto graph(graph_.lock()); - auto i = graph->refs.find(node); - if (i == graph->refs.end()) - goto getRefs; - goto doWork; - } +template <typename T> +void processGraph(ThreadPool& pool, const std::set<T>& nodes, + std::function<std::set<T>(const T&)> getEdges, + std::function<void(const T&)> processNode) { + struct Graph { + std::set<T> left; + std::map<T, std::set<T>> refs, rrefs; + }; - getRefs: - { - auto refs = getEdges(node); - refs.erase(node); - - { - auto graph(graph_.lock()); - for (auto & ref : refs) - if (graph->left.count(ref)) { - graph->refs[node].insert(ref); - graph->rrefs[ref].insert(node); - } - if (graph->refs[node].empty()) - goto doWork; - } - } + Sync<Graph> graph_(Graph{nodes, {}, {}}); + + std::function<void(const T&)> worker; - return; - - doWork: - processNode(node); - - /* Enqueue work for all nodes that were waiting on this one - and have no unprocessed dependencies. */ - { - auto graph(graph_.lock()); - for (auto & rref : graph->rrefs[node]) { - auto & refs(graph->refs[rref]); - auto i = refs.find(node); - assert(i != refs.end()); - refs.erase(i); - if (refs.empty()) - pool.enqueue(std::bind(worker, rref)); - } - graph->left.erase(node); - graph->refs.erase(node); - graph->rrefs.erase(node); + worker = [&](const T& node) { + { + auto graph(graph_.lock()); + auto i = graph->refs.find(node); + if (i == graph->refs.end()) goto getRefs; + goto doWork; + } + + getRefs : { + auto refs = getEdges(node); + refs.erase(node); + + { + auto graph(graph_.lock()); + for (auto& ref : refs) + if (graph->left.count(ref)) { + graph->refs[node].insert(ref); + graph->rrefs[ref].insert(node); } - }; + if (graph->refs[node].empty()) goto doWork; + } + } - for (auto & node : nodes) - pool.enqueue(std::bind(worker, std::ref(node))); + return; - pool.process(); + doWork: + processNode(node); - if (!graph_.lock()->left.empty()) - throw Error("graph processing incomplete (cyclic reference?)"); + /* Enqueue work for all nodes that were waiting on this one + and have no unprocessed dependencies. */ + { + auto graph(graph_.lock()); + for (auto& rref : graph->rrefs[node]) { + auto& refs(graph->refs[rref]); + auto i = refs.find(node); + assert(i != refs.end()); + refs.erase(i); + if (refs.empty()) pool.enqueue(std::bind(worker, rref)); + } + graph->left.erase(node); + graph->refs.erase(node); + graph->rrefs.erase(node); + } + }; + + for (auto& node : nodes) pool.enqueue(std::bind(worker, std::ref(node))); + + pool.process(); + + if (!graph_.lock()->left.empty()) + throw Error("graph processing incomplete (cyclic reference?)"); } -} +} // namespace nix diff --git a/third_party/nix/src/libutil/types.hh b/third_party/nix/src/libutil/types.hh index 92bf469b5c6f..ab2fee392286 100644 --- a/third_party/nix/src/libutil/types.hh +++ b/third_party/nix/src/libutil/types.hh @@ -1,15 +1,12 @@ #pragma once - -#include "ref.hh" - -#include <string> +#include <boost/format.hpp> #include <list> -#include <set> -#include <memory> #include <map> - -#include <boost/format.hpp> +#include <memory> +#include <set> +#include <string> +#include "ref.hh" /* Before 4.7, gcc's std::exception uses empty throw() specifiers for * its (virtual) destructor and what() in c++11 mode, in violation of spec @@ -20,131 +17,103 @@ #endif #endif - namespace nix { - /* Inherit some names from other namespaces for convenience. */ -using std::string; +using boost::format; using std::list; using std::set; +using std::string; using std::vector; -using boost::format; - /* A variadic template that does nothing. Useful to call a function for all variadic arguments but ignoring the result. */ -struct nop { template<typename... T> nop(T...) {} }; - - -struct FormatOrString -{ - string s; - FormatOrString(const string & s) : s(s) { }; - FormatOrString(const format & f) : s(f.str()) { }; - FormatOrString(const char * s) : s(s) { }; +struct nop { + template <typename... T> + nop(T...) {} }; +struct FormatOrString { + string s; + FormatOrString(const string& s) : s(s){}; + FormatOrString(const format& f) : s(f.str()){}; + FormatOrString(const char* s) : s(s){}; +}; /* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is equivalent to ‘boost::format(format) % a_0 % ... % ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion takes place). */ -inline std::string fmt(const std::string & s) -{ - return s; -} +inline std::string fmt(const std::string& s) { return s; } -inline std::string fmt(const char * s) -{ - return s; -} +inline std::string fmt(const char* s) { return s; } -inline std::string fmt(const FormatOrString & fs) -{ - return fs.s; -} +inline std::string fmt(const FormatOrString& fs) { return fs.s; } -template<typename... Args> -inline std::string fmt(const std::string & fs, Args... args) -{ - boost::format f(fs); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - nop{boost::io::detail::feed(f, args)...}; - return f.str(); +template <typename... Args> +inline std::string fmt(const std::string& fs, Args... args) { + boost::format f(fs); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + nop{boost::io::detail::feed(f, args)...}; + return f.str(); } - /* BaseError should generally not be caught, as it has Interrupted as a subclass. Catch Error instead. */ -class BaseError : public std::exception -{ -protected: - string prefix_; // used for location traces etc. - string err; -public: - unsigned int status = 1; // exit status - - template<typename... Args> - BaseError(unsigned int status, Args... args) - : err(fmt(args...)) - , status(status) - { - } - - template<typename... Args> - BaseError(Args... args) - : err(fmt(args...)) - { - } +class BaseError : public std::exception { + protected: + string prefix_; // used for location traces etc. + string err; + + public: + unsigned int status = 1; // exit status + + template <typename... Args> + BaseError(unsigned int status, Args... args) + : err(fmt(args...)), status(status) {} + + template <typename... Args> + BaseError(Args... args) : err(fmt(args...)) {} #ifdef EXCEPTION_NEEDS_THROW_SPEC - ~BaseError() throw () { }; - const char * what() const throw () { return err.c_str(); } + ~BaseError() throw(){}; + const char* what() const throw() { return err.c_str(); } #else - const char * what() const noexcept { return err.c_str(); } + const char* what() const noexcept { return err.c_str(); } #endif - const string & msg() const { return err; } - const string & prefix() const { return prefix_; } - BaseError & addPrefix(const FormatOrString & fs); + const string& msg() const { return err; } + const string& prefix() const { return prefix_; } + BaseError& addPrefix(const FormatOrString& fs); }; #define MakeError(newClass, superClass) \ - class newClass : public superClass \ - { \ - public: \ - using superClass::superClass; \ - }; + class newClass : public superClass { \ + public: \ + using superClass::superClass; \ + }; MakeError(Error, BaseError) -class SysError : public Error -{ -public: - int errNo; - - template<typename... Args> - SysError(Args... args) - : Error(addErrno(fmt(args...))) - { } + class SysError : public Error { + public: + int errNo; -private: + template <typename... Args> + SysError(Args... args) : Error(addErrno(fmt(args...))) {} - std::string addErrno(const std::string & s); + private: + std::string addErrno(const std::string& s); }; - typedef list<string> Strings; typedef set<string> StringSet; typedef std::map<std::string, std::string> StringMap; - /* Paths are just strings. */ typedef string Path; typedef list<Path> Paths; typedef set<Path> PathSet; - -} +} // namespace nix diff --git a/third_party/nix/src/libutil/util.cc b/third_party/nix/src/libutil/util.cc index 05527473210d..1f0bc74cc72d 100644 --- a/third_party/nix/src/libutil/util.cc +++ b/third_party/nix/src/libutil/util.cc @@ -1,20 +1,4 @@ -#include "lazy.hh" #include "util.hh" -#include "affinity.hh" -#include "sync.hh" -#include "finally.hh" -#include "serialise.hh" - -#include <cctype> -#include <cerrno> -#include <cstdio> -#include <cstdlib> -#include <cstring> -#include <iostream> -#include <sstream> -#include <thread> -#include <future> - #include <fcntl.h> #include <grp.h> #include <limits.h> @@ -23,6 +7,20 @@ #include <sys/types.h> #include <sys/wait.h> #include <unistd.h> +#include <cctype> +#include <cerrno> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <future> +#include <iostream> +#include <sstream> +#include <thread> +#include "affinity.hh" +#include "finally.hh" +#include "lazy.hh" +#include "serialise.hh" +#include "sync.hh" #ifdef __APPLE__ #include <sys/syscall.h> @@ -32,1040 +30,864 @@ #include <sys/prctl.h> #endif - -extern char * * environ __attribute__((weak)); - +extern char** environ __attribute__((weak)); namespace nix { - const std::string nativeSystem = SYSTEM; - -BaseError & BaseError::addPrefix(const FormatOrString & fs) -{ - prefix_ = fs.s + prefix_; - return *this; +BaseError& BaseError::addPrefix(const FormatOrString& fs) { + prefix_ = fs.s + prefix_; + return *this; } - -std::string SysError::addErrno(const std::string & s) -{ - errNo = errno; - return s + ": " + strerror(errNo); +std::string SysError::addErrno(const std::string& s) { + errNo = errno; + return s + ": " + strerror(errNo); } - -string getEnv(const string & key, const string & def) -{ - char * value = getenv(key.c_str()); - return value ? string(value) : def; +string getEnv(const string& key, const string& def) { + char* value = getenv(key.c_str()); + return value ? string(value) : def; } - -std::map<std::string, std::string> getEnv() -{ - std::map<std::string, std::string> env; - for (size_t i = 0; environ[i]; ++i) { - auto s = environ[i]; - auto eq = strchr(s, '='); - if (!eq) - // invalid env, just keep going - continue; - env.emplace(std::string(s, eq), std::string(eq + 1)); - } - return env; +std::map<std::string, std::string> getEnv() { + std::map<std::string, std::string> env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; } - -void clearEnv() -{ - for (auto & name : getEnv()) - unsetenv(name.first.c_str()); +void clearEnv() { + for (auto& name : getEnv()) unsetenv(name.first.c_str()); } -void replaceEnv(std::map<std::string, std::string> newEnv) -{ - clearEnv(); - for (auto newEnvVar : newEnv) - { - setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); - } +void replaceEnv(std::map<std::string, std::string> newEnv) { + clearEnv(); + for (auto newEnvVar : newEnv) { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } } - -Path absPath(Path path, Path dir) -{ - if (path[0] != '/') { - if (dir == "") { +Path absPath(Path path, Path dir) { + if (path[0] != '/') { + if (dir == "") { #ifdef __GNU__ - /* GNU (aka. GNU/Hurd) doesn't have any limitation on path - lengths and doesn't define `PATH_MAX'. */ - char *buf = getcwd(NULL, 0); - if (buf == NULL) + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char* buf = getcwd(NULL, 0); + if (buf == NULL) #else - char buf[PATH_MAX]; - if (!getcwd(buf, sizeof(buf))) + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) #endif - throw SysError("cannot get cwd"); - dir = buf; + throw SysError("cannot get cwd"); + dir = buf; #ifdef __GNU__ - free(buf); + free(buf); #endif - } - path = dir + "/" + path; } - return canonPath(path); + path = dir + "/" + path; + } + return canonPath(path); } +Path canonPath(const Path& path, bool resolveSymlinks) { + assert(path != ""); -Path canonPath(const Path & path, bool resolveSymlinks) -{ - assert(path != ""); - - string s; + string s; - if (path[0] != '/') - throw Error(format("not an absolute path: '%1%'") % path); + if (path[0] != '/') throw Error(format("not an absolute path: '%1%'") % path); - string::const_iterator i = path.begin(), end = path.end(); - string temp; + string::const_iterator i = path.begin(), end = path.end(); + string temp; - /* Count the number of times we follow a symlink and stop at some - arbitrary (but high) limit to prevent infinite loops. */ - unsigned int followCount = 0, maxFollow = 1024; + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; - while (1) { + while (1) { + /* Skip slashes. */ + while (i != end && *i == '/') i++; + if (i == end) break; - /* Skip slashes. */ - while (i != end && *i == '/') i++; - if (i == end) break; + /* Ignore `.'. */ + if (*i == '.' && (i + 1 == end || i[1] == '/')) i++; - /* Ignore `.'. */ - if (*i == '.' && (i + 1 == end || i[1] == '/')) - i++; - - /* If `..', delete the last component. */ - else if (*i == '.' && i + 1 < end && i[1] == '.' && - (i + 2 == end || i[2] == '/')) - { - if (!s.empty()) s.erase(s.rfind('/')); - i += 2; - } + /* If `..', delete the last component. */ + else if (*i == '.' && i + 1 < end && i[1] == '.' && + (i + 2 == end || i[2] == '/')) { + if (!s.empty()) s.erase(s.rfind('/')); + i += 2; + } - /* Normal component; copy it. */ - else { - s += '/'; - while (i != end && *i != '/') s += *i++; - - /* If s points to a symlink, resolve it and restart (since - the symlink target might contain new symlinks). */ - if (resolveSymlinks && isLink(s)) { - if (++followCount >= maxFollow) - throw Error(format("infinite symlink recursion in path '%1%'") % path); - temp = absPath(readLink(s), dirOf(s)) - + string(i, end); - i = temp.begin(); /* restart */ - end = temp.end(); - s = ""; - } - } + /* Normal component; copy it. */ + else { + s += '/'; + while (i != end && *i != '/') s += *i++; + + /* If s points to a symlink, resolve it and restart (since + the symlink target might contain new symlinks). */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error(format("infinite symlink recursion in path '%1%'") % + path); + temp = absPath(readLink(s), dirOf(s)) + string(i, end); + i = temp.begin(); /* restart */ + end = temp.end(); + s = ""; + } } + } - return s.empty() ? "/" : s; + return s.empty() ? "/" : s; } - -Path dirOf(const Path & path) -{ - Path::size_type pos = path.rfind('/'); - if (pos == string::npos) - return "."; - return pos == 0 ? "/" : Path(path, 0, pos); +Path dirOf(const Path& path) { + Path::size_type pos = path.rfind('/'); + if (pos == string::npos) return "."; + return pos == 0 ? "/" : Path(path, 0, pos); } +string baseNameOf(const Path& path) { + if (path.empty()) return ""; -string baseNameOf(const Path & path) -{ - if (path.empty()) - return ""; - - Path::size_type last = path.length() - 1; - if (path[last] == '/' && last > 0) - last -= 1; + Path::size_type last = path.length() - 1; + if (path[last] == '/' && last > 0) last -= 1; - Path::size_type pos = path.rfind('/', last); - if (pos == string::npos) - pos = 0; - else - pos += 1; + Path::size_type pos = path.rfind('/', last); + if (pos == string::npos) + pos = 0; + else + pos += 1; - return string(path, pos, last - pos + 1); + return string(path, pos, last - pos + 1); } - -bool isInDir(const Path & path, const Path & dir) -{ - return path[0] == '/' - && string(path, 0, dir.size()) == dir - && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; +bool isInDir(const Path& path, const Path& dir) { + return path[0] == '/' && string(path, 0, dir.size()) == dir && + path.size() >= dir.size() + 2 && path[dir.size()] == '/'; } - -bool isDirOrInDir(const Path & path, const Path & dir) -{ - return path == dir || isInDir(path, dir); +bool isDirOrInDir(const Path& path, const Path& dir) { + return path == dir || isInDir(path, dir); } - -struct stat lstat(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of '%1%'") % path); - return st; +struct stat lstat(const Path& path) { + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of '%1%'") % path); + return st; } - -bool pathExists(const Path & path) -{ - int res; - struct stat st; - res = lstat(path.c_str(), &st); - if (!res) return true; - if (errno != ENOENT && errno != ENOTDIR) - throw SysError(format("getting status of %1%") % path); - return false; +bool pathExists(const Path& path) { + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (!res) return true; + if (errno != ENOENT && errno != ENOTDIR) + throw SysError(format("getting status of %1%") % path); + return false; } - -Path readLink(const Path & path) -{ - checkInterrupt(); - std::vector<char> buf; - for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { - buf.resize(bufSize); - ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); - if (rlSize == -1) - if (errno == EINVAL) - throw Error("'%1%' is not a symlink", path); - else - throw SysError("reading symbolic link '%1%'", path); - else if (rlSize < bufSize) - return string(buf.data(), rlSize); - } +Path readLink(const Path& path) { + checkInterrupt(); + std::vector<char> buf; + for (ssize_t bufSize = PATH_MAX / 4; true; bufSize += bufSize / 2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) + if (errno == EINVAL) + throw Error("'%1%' is not a symlink", path); + else + throw SysError("reading symbolic link '%1%'", path); + else if (rlSize < bufSize) + return string(buf.data(), rlSize); + } } - -bool isLink(const Path & path) -{ - struct stat st = lstat(path); - return S_ISLNK(st.st_mode); +bool isLink(const Path& path) { + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); } +DirEntries readDirectory(const Path& path) { + DirEntries entries; + entries.reserve(64); -DirEntries readDirectory(const Path & path) -{ - DirEntries entries; - entries.reserve(64); + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % path); - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { /* sic */ - checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - entries.emplace_back(name, dirent->d_ino, + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + entries.emplace_back(name, dirent->d_ino, #ifdef HAVE_STRUCT_DIRENT_D_TYPE - dirent->d_type + dirent->d_type #else - DT_UNKNOWN + DT_UNKNOWN #endif - ); - } - if (errno) throw SysError(format("reading directory '%1%'") % path); + ); + } + if (errno) throw SysError(format("reading directory '%1%'") % path); - return entries; + return entries; } - -unsigned char getFileType(const Path & path) -{ - struct stat st = lstat(path); - if (S_ISDIR(st.st_mode)) return DT_DIR; - if (S_ISLNK(st.st_mode)) return DT_LNK; - if (S_ISREG(st.st_mode)) return DT_REG; - return DT_UNKNOWN; +unsigned char getFileType(const Path& path) { + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) return DT_DIR; + if (S_ISLNK(st.st_mode)) return DT_LNK; + if (S_ISREG(st.st_mode)) return DT_REG; + return DT_UNKNOWN; } +string readFile(int fd) { + struct stat st; + if (fstat(fd, &st) == -1) throw SysError("statting file"); -string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); + std::vector<unsigned char> buf(st.st_size); + readFull(fd, buf.data(), st.st_size); - std::vector<unsigned char> buf(st.st_size); - readFull(fd, buf.data(), st.st_size); - - return string((char *) buf.data(), st.st_size); + return string((char*)buf.data(), st.st_size); } - -string readFile(const Path & path, bool drain) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError(format("opening file '%1%'") % path); - return drain ? drainFD(fd.get()) : readFile(fd.get()); +string readFile(const Path& path, bool drain) { + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError(format("opening file '%1%'") % path); + return drain ? drainFD(fd.get()) : readFile(fd.get()); } - -void readFile(const Path & path, Sink & sink) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError("opening file '%s'", path); - drainFD(fd.get(), sink); +void readFile(const Path& path, Sink& sink) { + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); } - -void writeFile(const Path & path, const string & s, mode_t mode) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError(format("opening file '%1%'") % path); - writeFull(fd.get(), s); +void writeFile(const Path& path, const string& s, mode_t mode) { + AutoCloseFD fd = + open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) throw SysError(format("opening file '%1%'") % path); + writeFull(fd.get(), s); } +void writeFile(const Path& path, Source& source, mode_t mode) { + AutoCloseFD fd = + open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) throw SysError(format("opening file '%1%'") % path); -void writeFile(const Path & path, Source & source, mode_t mode) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError(format("opening file '%1%'") % path); + std::vector<unsigned char> buf(64 * 1024); - std::vector<unsigned char> buf(64 * 1024); - - while (true) { - try { - auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), (unsigned char *) buf.data(), n); - } catch (EndOfFile &) { break; } + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), (unsigned char*)buf.data(), n); + } catch (EndOfFile&) { + break; } + } } - -string readLine(int fd) -{ - string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } +string readLine(int fd) { + string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; } + } } - -void writeLine(int fd, string s) -{ - s += '\n'; - writeFull(fd, s); +void writeLine(int fd, string s) { + s += '\n'; + writeFull(fd, s); } +static void _deletePath(const Path& path, unsigned long long& bytesFreed) { + checkInterrupt(); -static void _deletePath(const Path & path, unsigned long long & bytesFreed) -{ - checkInterrupt(); + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + if (errno == ENOENT) return; + throw SysError(format("getting status of '%1%'") % path); + } - struct stat st; - if (lstat(path.c_str(), &st) == -1) { - if (errno == ENOENT) return; - throw SysError(format("getting status of '%1%'") % path); - } - - if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) - bytesFreed += st.st_size; - - if (S_ISDIR(st.st_mode)) { - /* Make the directory accessible. */ - const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; - if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1) - throw SysError(format("chmod '%1%'") % path); - } + if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) bytesFreed += st.st_size; - for (auto & i : readDirectory(path)) - _deletePath(path + "/" + i.name, bytesFreed); + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1) + throw SysError(format("chmod '%1%'") % path); } - if (remove(path.c_str()) == -1) { - if (errno == ENOENT) return; - throw SysError(format("cannot unlink '%1%'") % path); - } -} + for (auto& i : readDirectory(path)) + _deletePath(path + "/" + i.name, bytesFreed); + } - -void deletePath(const Path & path) -{ - unsigned long long dummy; - deletePath(path, dummy); + if (remove(path.c_str()) == -1) { + if (errno == ENOENT) return; + throw SysError(format("cannot unlink '%1%'") % path); + } } - -void deletePath(const Path & path, unsigned long long & bytesFreed) -{ - //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path); - bytesFreed = 0; - _deletePath(path, bytesFreed); +void deletePath(const Path& path) { + unsigned long long dummy; + deletePath(path, dummy); } - -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - int & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true); - if (includePid) - return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); - else - return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +void deletePath(const Path& path, unsigned long long& bytesFreed) { + // Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % + // path); + bytesFreed = 0; + _deletePath(path, bytesFreed); } +static Path tempName(Path tmpRoot, const Path& prefix, bool includePid, + int& counter) { + tmpRoot = + canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true); + if (includePid) + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++) + .str(); + else + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static int globalCounter = 0; - int localCounter = 0; - int & counter(useGlobalCounter ? globalCounter : localCounter); +Path createTempDir(const Path& tmpRoot, const Path& prefix, bool includePid, + bool useGlobalCounter, mode_t mode) { + static int globalCounter = 0; + int localCounter = 0; + int& counter(useGlobalCounter ? globalCounter : localCounter); - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { #if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError(format("setting group of directory '%1%'") % tmpDir); + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t)-1, getegid()) != 0) + throw SysError(format("setting group of directory '%1%'") % tmpDir); #endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError(format("creating directory '%1%'") % tmpDir); + return tmpDir; } + if (errno != EEXIST) + throw SysError(format("creating directory '%1%'") % tmpDir); + } } - -std::string getUserName() -{ - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER", ""); - if (name.empty()) - throw Error("cannot figure out user name"); - return name; +std::string getUserName() { + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER", ""); + if (name.empty()) throw Error("cannot figure out user name"); + return name; } - static Lazy<Path> getHome2([]() { - Path homeDir = getEnv("HOME"); - if (homeDir.empty()) { - std::vector<char> buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - homeDir = pw->pw_dir; - } - return homeDir; + Path homeDir = getEnv("HOME"); + if (homeDir.empty()) { + std::vector<char> buf(16384); + struct passwd pwbuf; + struct passwd* pw; + if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 || + !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + homeDir = pw->pw_dir; + } + return homeDir; }); Path getHome() { return getHome2(); } - -Path getCacheDir() -{ - Path cacheDir = getEnv("XDG_CACHE_HOME"); - if (cacheDir.empty()) - cacheDir = getHome() + "/.cache"; - return cacheDir; +Path getCacheDir() { + Path cacheDir = getEnv("XDG_CACHE_HOME"); + if (cacheDir.empty()) cacheDir = getHome() + "/.cache"; + return cacheDir; } - -Path getConfigDir() -{ - Path configDir = getEnv("XDG_CONFIG_HOME"); - if (configDir.empty()) - configDir = getHome() + "/.config"; - return configDir; +Path getConfigDir() { + Path configDir = getEnv("XDG_CONFIG_HOME"); + if (configDir.empty()) configDir = getHome() + "/.config"; + return configDir; } -std::vector<Path> getConfigDirs() -{ - Path configHome = getConfigDir(); - string configDirs = getEnv("XDG_CONFIG_DIRS"); - std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":"); - result.insert(result.begin(), configHome); - return result; +std::vector<Path> getConfigDirs() { + Path configHome = getConfigDir(); + string configDirs = getEnv("XDG_CONFIG_DIRS"); + std::vector<Path> result = + tokenizeString<std::vector<string>>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; } - -Path getDataDir() -{ - Path dataDir = getEnv("XDG_DATA_HOME"); - if (dataDir.empty()) - dataDir = getHome() + "/.local/share"; - return dataDir; +Path getDataDir() { + Path dataDir = getEnv("XDG_DATA_HOME"); + if (dataDir.empty()) dataDir = getHome() + "/.local/share"; + return dataDir; } +Paths createDirs(const Path& path) { + Paths created; + if (path == "/") return created; -Paths createDirs(const Path & path) -{ - Paths created; - if (path == "/") return created; - - struct stat st; - if (lstat(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError(format("creating directory '%1%'") % path); - st = lstat(path); - created.push_back(path); - } + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError(format("creating directory '%1%'") % path); + st = lstat(path); + created.push_back(path); + } - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError(format("statting symlink '%1%'") % path); + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) + throw SysError(format("statting symlink '%1%'") % path); - if (!S_ISDIR(st.st_mode)) throw Error(format("'%1%' is not a directory") % path); + if (!S_ISDIR(st.st_mode)) + throw Error(format("'%1%' is not a directory") % path); - return created; + return created; } - -void createSymlink(const Path & target, const Path & link) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target); +void createSymlink(const Path& target, const Path& link) { + if (symlink(target.c_str(), link.c_str())) + throw SysError(format("creating symlink from '%1%' to '%2%'") % link % + target); } +void replaceSymlink(const Path& target, const Path& link) { + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); -void replaceSymlink(const Path & target, const Path & link) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError(format("renaming '%1%' to '%2%'") % tmp % link); - - break; + try { + createSymlink(target, tmp); + } catch (SysError& e) { + if (e.errNo == EEXIST) continue; + throw; } -} + if (rename(tmp.c_str(), link.c_str()) != 0) + throw SysError(format("renaming '%1%' to '%2%'") % tmp % link); -void readFull(int fd, unsigned char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, (char *) buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } + break; + } } - -void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts) -{ - while (count) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, (char *) buf, count); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) { - count -= res; - buf += res; - } +void readFull(int fd, unsigned char* buf, size_t count) { + while (count) { + checkInterrupt(); + ssize_t res = read(fd, (char*)buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + +void writeFull(int fd, const unsigned char* buf, size_t count, + bool allowInterrupts) { + while (count) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, (char*)buf, count); + if (res == -1 && errno != EINTR) throw SysError("writing to file"); + if (res > 0) { + count -= res; + buf += res; } + } } - -void writeFull(int fd, const string & s, bool allowInterrupts) -{ - writeFull(fd, (const unsigned char *) s.data(), s.size(), allowInterrupts); +void writeFull(int fd, const string& s, bool allowInterrupts) { + writeFull(fd, (const unsigned char*)s.data(), s.size(), allowInterrupts); } - -string drainFD(int fd, bool block) -{ - StringSink sink; - drainFD(fd, sink, block); - return std::move(*sink.s); +string drainFD(int fd, bool block) { + StringSink sink; + drainFD(fd, sink, block); + return std::move(*sink.s); } +void drainFD(int fd, Sink& sink, bool block) { + int saved; -void drainFD(int fd, Sink & sink, bool block) -{ - int saved; - - Finally finally([&]() { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - + Finally finally([&]() { if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - std::vector<unsigned char> buf(64 * 1024); - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else sink(buf.data(), rd); + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); } -} + }); + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + std::vector<unsigned char> buf(64 * 1024); + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) break; + if (errno != EINTR) throw SysError("reading from file"); + } else if (rd == 0) + break; + else + sink(buf.data(), rd); + } +} ////////////////////////////////////////////////////////////////////// - AutoDelete::AutoDelete() : del{false} {} -AutoDelete::AutoDelete(const string & p, bool recursive) : path(p) -{ - del = true; - this->recursive = recursive; +AutoDelete::AutoDelete(const string& p, bool recursive) : path(p) { + del = true; + this->recursive = recursive; } -AutoDelete::~AutoDelete() -{ - try { - if (del) { - if (recursive) - deletePath(path); - else { - if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink '%1%'") % path); - } - } - } catch (...) { - ignoreException(); +AutoDelete::~AutoDelete() { + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink '%1%'") % path); + } } + } catch (...) { + ignoreException(); + } } -void AutoDelete::cancel() -{ - del = false; -} +void AutoDelete::cancel() { del = false; } -void AutoDelete::reset(const Path & p, bool recursive) { - path = p; - this->recursive = recursive; - del = true; +void AutoDelete::reset(const Path& p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; } - - ////////////////////////////////////////////////////////////////////// - AutoCloseFD::AutoCloseFD() : fd{-1} {} - AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} +AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd} { that.fd = -1; } -AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd} -{ - that.fd = -1; +AutoCloseFD& AutoCloseFD::operator=(AutoCloseFD&& that) { + close(); + fd = that.fd; + that.fd = -1; + return *this; } - -AutoCloseFD& AutoCloseFD::operator =(AutoCloseFD&& that) -{ +AutoCloseFD::~AutoCloseFD() { + try { close(); - fd = that.fd; - that.fd = -1; - return *this; + } catch (...) { + ignoreException(); + } } +int AutoCloseFD::get() const { return fd; } -AutoCloseFD::~AutoCloseFD() -{ - try { - close(); - } catch (...) { - ignoreException(); - } +void AutoCloseFD::close() { + if (fd != -1) { + if (::close(fd) == -1) /* This should never happen. */ + throw SysError(format("closing file descriptor %1%") % fd); + } } +AutoCloseFD::operator bool() const { return fd != -1; } -int AutoCloseFD::get() const -{ - return fd; +int AutoCloseFD::release() { + int oldFD = fd; + fd = -1; + return oldFD; } - -void AutoCloseFD::close() -{ - if (fd != -1) { - if (::close(fd) == -1) - /* This should never happen. */ - throw SysError(format("closing file descriptor %1%") % fd); - } -} - - -AutoCloseFD::operator bool() const -{ - return fd != -1; -} - - -int AutoCloseFD::release() -{ - int oldFD = fd; - fd = -1; - return oldFD; -} - - -void Pipe::create() -{ - int fds[2]; +void Pipe::create() { + int fds[2]; #if HAVE_PIPE2 - if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); #else - if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); #endif - readSide = fds[0]; - writeSide = fds[1]; + readSide = fds[0]; + writeSide = fds[1]; } - - ////////////////////////////////////////////////////////////////////// +Pid::Pid() {} -Pid::Pid() -{ -} - - -Pid::Pid(pid_t pid) - : pid(pid) -{ -} - - -Pid::~Pid() -{ - if (pid != -1) kill(); -} - +Pid::Pid(pid_t pid) : pid(pid) {} -void Pid::operator =(pid_t pid) -{ - if (this->pid != -1 && this->pid != pid) kill(); - this->pid = pid; - killSignal = SIGKILL; // reset signal to default +Pid::~Pid() { + if (pid != -1) kill(); } - -Pid::operator pid_t() -{ - return pid; +void Pid::operator=(pid_t pid) { + if (this->pid != -1 && this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default } +Pid::operator pid_t() { return pid; } -int Pid::kill() -{ - assert(pid != -1); +int Pid::kill() { + assert(pid != -1); - debug(format("killing process %1%") % pid); + debug(format("killing process %1%") % pid); - /* Send the requested signal to the child. If it has its own - process group, send the signal to every process in the child - process group (which hopefully includes *all* its children). */ - if (::kill(separatePG ? -pid : pid, killSignal) != 0) { - /* On BSDs, killing a process group will return EPERM if all - processes in the group are zombies (or something like - that). So try to detect and ignore that situation. */ + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ #if __FreeBSD__ || __APPLE__ - if (errno != EPERM || ::kill(pid, 0) != 0) + if (errno != EPERM || ::kill(pid, 0) != 0) #endif - printError((SysError("killing process %d", pid).msg())); - } + printError((SysError("killing process %d", pid).msg())); + } - return wait(); + return wait(); } - -int Pid::wait() -{ - assert(pid != -1); - while (1) { - int status; - int res = waitpid(pid, &status, 0); - if (res == pid) { - pid = -1; - return status; - } - if (errno != EINTR) - throw SysError("cannot get child exit status"); - checkInterrupt(); +int Pid::wait() { + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; } + if (errno != EINTR) throw SysError("cannot get child exit status"); + checkInterrupt(); + } } +void Pid::setSeparatePG(bool separatePG) { this->separatePG = separatePG; } -void Pid::setSeparatePG(bool separatePG) -{ - this->separatePG = separatePG; -} - - -void Pid::setKillSignal(int signal) -{ - this->killSignal = signal; -} - +void Pid::setKillSignal(int signal) { this->killSignal = signal; } -pid_t Pid::release() -{ - pid_t p = pid; - pid = -1; - return p; +pid_t Pid::release() { + pid_t p = pid; + pid = -1; + return p; } +void killUser(uid_t uid) { + debug(format("killing all processes running under uid '%1%'") % uid); -void killUser(uid_t uid) -{ - debug(format("killing all processes running under uid '%1%'") % uid); + assert(uid != 0); /* just to be safe... */ - assert(uid != 0); /* just to be safe... */ + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ - /* The system call kill(-1, sig) sends the signal `sig' to all - users to which the current process can send signals. So we - fork a process, switch to uid, and send a mass kill. */ + ProcessOptions options; + options.allowVfork = false; - ProcessOptions options; - options.allowVfork = false; - - Pid pid = startProcess([&]() { - - if (setuid(uid) == -1) - throw SysError("setting uid"); + Pid pid = startProcess( + [&]() { + if (setuid(uid) == -1) throw SysError("setting uid"); while (true) { #ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among - other things, determines if kill(-1, signo) affects the - calling process. In the OSX libc, it's set to true, - which means "follow POSIX", which we don't want here - */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; #else - if (kill(-1, SIGKILL) == 0) break; + if (kill(-1, SIGKILL) == 0) break; #endif - if (errno == ESRCH) break; /* no more processes */ - if (errno != EINTR) - throw SysError(format("cannot kill processes for uid '%1%'") % uid); + if (errno == ESRCH) break; /* no more processes */ + if (errno != EINTR) + throw SysError(format("cannot kill processes for uid '%1%'") % uid); } _exit(0); - }, options); + }, + options); - int status = pid.wait(); - if (status != 0) - throw Error(format("cannot kill processes for uid '%1%': %2%") % uid % statusToString(status)); + int status = pid.wait(); + if (status != 0) + throw Error(format("cannot kill processes for uid '%1%': %2%") % uid % + statusToString(status)); - /* !!! We should really do some check to make sure that there are - no processes left running under `uid', but there is no portable - way to do so (I think). The most reliable way may be `ps -eo - uid | grep -q $uid'. */ + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ } - ////////////////////////////////////////////////////////////////////// - /* Wrapper around vfork to prevent the child process from clobbering the caller's stack frame in the parent. */ -static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline)); static pid_t doFork(bool allowVfork, std::function<void()> fun) -{ + __attribute__((noinline)); +static pid_t doFork(bool allowVfork, std::function<void()> fun) { #ifdef __linux__ - pid_t pid = allowVfork ? vfork() : fork(); + pid_t pid = allowVfork ? vfork() : fork(); #else - pid_t pid = fork(); + pid_t pid = fork(); #endif - if (pid != 0) return pid; - fun(); - abort(); + if (pid != 0) return pid; + fun(); + abort(); } - -pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) -{ - auto wrapper = [&]() { - if (!options.allowVfork) - logger = makeDefaultLogger(); - try { +pid_t startProcess(std::function<void()> fun, const ProcessOptions& options) { + auto wrapper = [&]() { + if (!options.allowVfork) logger = makeDefaultLogger(); + try { #if __linux__ - if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) - throw SysError("setting death signal"); + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + throw SysError("setting death signal"); #endif - restoreAffinity(); - fun(); - } catch (std::exception & e) { - try { - std::cerr << options.errorPrefix << e.what() << "\n"; - } catch (...) { } - } catch (...) { } - if (options.runExitHandlers) - exit(1); - else - _exit(1); - }; + restoreAffinity(); + fun(); + } catch (std::exception& e) { + try { + std::cerr << options.errorPrefix << e.what() << "\n"; + } catch (...) { + } + } catch (...) { + } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; - pid_t pid = doFork(options.allowVfork, wrapper); - if (pid == -1) throw SysError("unable to fork"); + pid_t pid = doFork(options.allowVfork, wrapper); + if (pid == -1) throw SysError("unable to fork"); - return pid; + return pid; } - -std::vector<char *> stringsToCharPtrs(const Strings & ss) -{ - std::vector<char *> res; - for (auto & s : ss) res.push_back((char *) s.c_str()); - res.push_back(0); - return res; +std::vector<char*> stringsToCharPtrs(const Strings& ss) { + std::vector<char*> res; + for (auto& s : ss) res.push_back((char*)s.c_str()); + res.push_back(0); + return res; } +string runProgram(Path program, bool searchPath, const Strings& args, + const std::optional<std::string>& input) { + RunOptions opts(program, args); + opts.searchPath = searchPath; + opts.input = input; -string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional<std::string> & input) -{ - RunOptions opts(program, args); - opts.searchPath = searchPath; - opts.input = input; - - auto res = runProgram(opts); + auto res = runProgram(opts); - if (!statusOk(res.first)) - throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + if (!statusOk(res.first)) + throw ExecError(res.first, fmt("program '%1%' %2%", program, + statusToString(res.first))); - return res.second; + return res.second; } -std::pair<int, std::string> runProgram(const RunOptions & options_) -{ - RunOptions options(options_); - StringSink sink; - options.standardOut = &sink; +std::pair<int, std::string> runProgram(const RunOptions& options_) { + RunOptions options(options_); + StringSink sink; + options.standardOut = &sink; - int status = 0; + int status = 0; - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } + try { + runProgram2(options); + } catch (ExecError& e) { + status = e.status; + } - return {status, std::move(*sink.s)}; + return {status, std::move(*sink.s)}; } -void runProgram2(const RunOptions & options) -{ - checkInterrupt(); +void runProgram2(const RunOptions& options) { + checkInterrupt(); - assert(!(options.standardIn && options.input)); + assert(!(options.standardIn && options.input)); - std::unique_ptr<Source> source_; - Source * source = options.standardIn; + std::unique_ptr<Source> source_; + Source* source = options.standardIn; - if (options.input) { - source_ = std::make_unique<StringSource>(*options.input); - source = source_.get(); - } + if (options.input) { + source_ = std::make_unique<StringSource>(*options.input); + source = source_.get(); + } - /* Create a pipe. */ - Pipe out, in; - if (options.standardOut) out.create(); - if (source) in.create(); - - ProcessOptions processOptions; - // vfork implies that the environment of the main process and the fork will - // be shared (technically this is undefined, but in practice that's the - // case), so we can't use it if we alter the environment - if (options.environment) - processOptions.allowVfork = false; - - /* Fork. */ - Pid pid = startProcess([&]() { - if (options.environment) - replaceEnv(*options.environment); - if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); + /* Create a pipe. */ + Pipe out, in; + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + if (options.environment) processOptions.allowVfork = false; + + /* Fork. */ + Pid pid = startProcess( + [&]() { + if (options.environment) replaceEnv(*options.environment); + if (options.standardOut && + dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); if (options.mergeStderrToStdout) - if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) - throw SysError("cannot dup stdout into stderr"); + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); + throw SysError("dupping stdin"); if (options.chdir && chdir((*options.chdir).c_str()) == -1) - throw SysError("chdir failed"); + throw SysError("chdir failed"); if (options.gid && setgid(*options.gid) == -1) - throw SysError("setgid failed"); + throw SysError("setgid failed"); /* Drop all other groups if we're setgid. */ if (options.gid && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); + throw SysError("setgroups failed"); if (options.uid && setuid(*options.uid) == -1) - throw SysError("setuid failed"); + throw SysError("setuid failed"); Strings args_(options.args); args_.push_front(options.program); @@ -1073,480 +895,427 @@ void runProgram2(const RunOptions & options) restoreSignals(); if (options.searchPath) - execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); else - execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); throw SysError("executing '%1%'", options.program); - }, processOptions); + }, + processOptions); - out.writeSide = -1; + out.writeSide = -1; - std::thread writerThread; + std::thread writerThread; - std::promise<void> promise; - - Finally doJoin([&]() { - if (writerThread.joinable()) - writerThread.join(); - }); + std::promise<void> promise; + Finally doJoin([&]() { + if (writerThread.joinable()) writerThread.join(); + }); - if (source) { - in.readSide = -1; - writerThread = std::thread([&]() { - try { - std::vector<unsigned char> buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), buf.data(), n); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide = -1; - }); - } + if (source) { + in.readSide = -1; + writerThread = std::thread([&]() { + try { + std::vector<unsigned char> buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile&) { + break; + } + writeFull(in.writeSide.get(), buf.data(), n); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide = -1; + }); + } - if (options.standardOut) - drainFD(out.readSide.get(), *options.standardOut); + if (options.standardOut) drainFD(out.readSide.get(), *options.standardOut); - /* Wait for the child to finish. */ - int status = pid.wait(); + /* Wait for the child to finish. */ + int status = pid.wait(); - /* Wait for the writer thread to finish. */ - if (source) promise.get_future().get(); + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); - if (status) - throw ExecError(status, fmt("program '%1%' %2%", options.program, statusToString(status))); + if (status) + throw ExecError(status, fmt("program '%1%' %2%", options.program, + statusToString(status))); } - -void closeMostFDs(const set<int> & exceptions) -{ +void closeMostFDs(const set<int>& exceptions) { #if __linux__ - try { - for (auto & s : readDirectory("/proc/self/fd")) { - auto fd = std::stoi(s.name); - if (!exceptions.count(fd)) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (SysError &) { + try { + for (auto& s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } } + return; + } catch (SysError&) { + } #endif - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) close(fd); /* ignore result */ } - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); +void closeOnExec(int fd) { + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); } - ////////////////////////////////////////////////////////////////////// - bool _isInterrupted = false; static thread_local bool interruptThrown = false; thread_local std::function<bool()> interruptCheck; -void setInterruptThrown() -{ - interruptThrown = true; -} +void setInterruptThrown() { interruptThrown = true; } -void _interrupted() -{ - /* Block user interrupts while an exception is being handled. - Throwing an exception while another exception is being handled - kills the program! */ - if (!interruptThrown && !std::uncaught_exceptions()) { - interruptThrown = true; - throw Interrupted("interrupted by the user"); - } +void _interrupted() { + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!interruptThrown && !std::uncaught_exceptions()) { + interruptThrown = true; + throw Interrupted("interrupted by the user"); + } } - ////////////////////////////////////////////////////////////////////// - -template<class C> C tokenizeString(const string & s, const string & separators) -{ - C result; - string::size_type pos = s.find_first_not_of(separators, 0); - while (pos != string::npos) { - string::size_type end = s.find_first_of(separators, pos + 1); - if (end == string::npos) end = s.size(); - string token(s, pos, end - pos); - result.insert(result.end(), token); - pos = s.find_first_not_of(separators, end); - } - return result; -} - -template Strings tokenizeString(const string & s, const string & separators); -template StringSet tokenizeString(const string & s, const string & separators); -template vector<string> tokenizeString(const string & s, const string & separators); - - -string concatStringsSep(const string & sep, const Strings & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - -string concatStringsSep(const string & sep, const StringSet & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - -string chomp(const string & s) -{ - size_t i = s.find_last_not_of(" \n\r\t"); - return i == string::npos ? "" : string(s, 0, i + 1); -} - - -string trim(const string & s, const string & whitespace) -{ - auto i = s.find_first_not_of(whitespace); - if (i == string::npos) return ""; - auto j = s.find_last_not_of(whitespace); - return string(s, i, j == string::npos ? j : j - i + 1); -} - - -string replaceStrings(const std::string & s, - const std::string & from, const std::string & to) -{ - if (from.empty()) return s; - string res = s; - size_t pos = 0; - while ((pos = res.find(from, pos)) != std::string::npos) { - res.replace(pos, from.size(), to); - pos += to.size(); - } - return res; -} - - -string statusToString(int status) -{ - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - if (WIFEXITED(status)) - return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); - else if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); +template <class C> +C tokenizeString(const string& s, const string& separators) { + C result; + string::size_type pos = s.find_first_not_of(separators, 0); + while (pos != string::npos) { + string::size_type end = s.find_first_of(separators, pos + 1); + if (end == string::npos) end = s.size(); + string token(s, pos, end - pos); + result.insert(result.end(), token); + pos = s.find_first_not_of(separators, end); + } + return result; +} + +template Strings tokenizeString(const string& s, const string& separators); +template StringSet tokenizeString(const string& s, const string& separators); +template vector<string> tokenizeString(const string& s, + const string& separators); + +string concatStringsSep(const string& sep, const Strings& ss) { + string s; + for (auto& i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + +string concatStringsSep(const string& sep, const StringSet& ss) { + string s; + for (auto& i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + +string chomp(const string& s) { + size_t i = s.find_last_not_of(" \n\r\t"); + return i == string::npos ? "" : string(s, 0, i + 1); +} + +string trim(const string& s, const string& whitespace) { + auto i = s.find_first_not_of(whitespace); + if (i == string::npos) return ""; + auto j = s.find_last_not_of(whitespace); + return string(s, i, j == string::npos ? j : j - i + 1); +} + +string replaceStrings(const std::string& s, const std::string& from, + const std::string& to) { + if (from.empty()) return s; + string res = s; + size_t pos = 0; + while ((pos = res.find(from, pos)) != std::string::npos) { + res.replace(pos, from.size(), to); + pos += to.size(); + } + return res; +} + +string statusToString(int status) { + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); #if HAVE_STRSIGNAL - const char * description = strsignal(sig); - return (format("failed due to signal %1% (%2%)") % sig % description).str(); + const char* description = strsignal(sig); + return (format("failed due to signal %1% (%2%)") % sig % description) + .str(); #else - return (format("failed due to signal %1%") % sig).str(); + return (format("failed due to signal %1%") % sig).str(); #endif - } - else - return "died abnormally"; - } else return "succeeded"; + } else + return "died abnormally"; + } else + return "succeeded"; } - -bool statusOk(int status) -{ - return WIFEXITED(status) && WEXITSTATUS(status) == 0; +bool statusOk(int status) { + return WIFEXITED(status) && WEXITSTATUS(status) == 0; } - -bool hasPrefix(const string & s, const string & prefix) -{ - return s.compare(0, prefix.size(), prefix) == 0; +bool hasPrefix(const string& s, const string& prefix) { + return s.compare(0, prefix.size(), prefix) == 0; } - -bool hasSuffix(const string & s, const string & suffix) -{ - return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; -} - - -std::string toLower(const std::string & s) -{ - std::string r(s); - for (auto & c : r) - c = std::tolower(c); - return r; +bool hasSuffix(const string& s, const string& suffix) { + return s.size() >= suffix.size() && + string(s, s.size() - suffix.size()) == suffix; } - -std::string shellEscape(const std::string & s) -{ - std::string r = "'"; - for (auto & i : s) - if (i == '\'') r += "'\\''"; else r += i; - r += '\''; - return r; +std::string toLower(const std::string& s) { + std::string r(s); + for (auto& c : r) c = std::tolower(c); + return r; } - -void ignoreException() -{ - try { - throw; - } catch (std::exception & e) { - printError(format("error (ignored): %1%") % e.what()); +std::string shellEscape(const std::string& s) { + std::string r = "'"; + for (auto& i : s) + if (i == '\'') + r += "'\\''"; + else + r += i; + r += '\''; + return r; +} + +void ignoreException() { + try { + throw; + } catch (std::exception& e) { + printError(format("error (ignored): %1%") % e.what()); + } +} + +std::string filterANSIEscapes(const std::string& s, bool filterAll, + unsigned int width) { + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t)width && i != s.end()) { + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; + } + + if (!filterAll && last == 'm') t += e; } -} - - -std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) -{ - std::string t, e; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - - if (*i == '\e') { - std::string e; - e += *i++; - char last = 0; - if (i != s.end() && *i == '[') { - e += *i++; - // eat parameter bytes - while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; - // eat intermediate bytes - while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; - // eat final byte - if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; - } else { - if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; - } - - if (!filterAll && last == 'm') - t += e; - } - - else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } - } + else if (*i == '\t') { + i++; + t += ' '; + w++; + while (w < (size_t)width && w % 8) { + t += ' '; + w++; + } + } - else if (*i == '\r') - // do nothing for now - i++; + else if (*i == '\r') + // do nothing for now + i++; - else { - t += *i++; w++; - } + else { + t += *i++; + w++; } + } - return t; + return t; } +static char base64Chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - - -string base64Encode(const string & s) -{ - string res; - int data = 0, nbits = 0; +string base64Encode(const string& s) { + string res; + int data = 0, nbits = 0; - for (char c : s) { - data = data << 8 | (unsigned char) c; - nbits += 8; - while (nbits >= 6) { - nbits -= 6; - res.push_back(base64Chars[data >> nbits & 0x3f]); - } + for (char c : s) { + data = data << 8 | (unsigned char)c; + nbits += 8; + while (nbits >= 6) { + nbits -= 6; + res.push_back(base64Chars[data >> nbits & 0x3f]); } + } - if (nbits) res.push_back(base64Chars[data << (6 - nbits) & 0x3f]); - while (res.size() % 4) res.push_back('='); + if (nbits) res.push_back(base64Chars[data << (6 - nbits) & 0x3f]); + while (res.size() % 4) res.push_back('='); - return res; + return res; } +string base64Decode(const string& s) { + bool init = false; + char decode[256]; + if (!init) { + // FIXME: not thread-safe. + memset(decode, -1, sizeof(decode)); + for (int i = 0; i < 64; i++) decode[(int)base64Chars[i]] = i; + init = true; + } -string base64Decode(const string & s) -{ - bool init = false; - char decode[256]; - if (!init) { - // FIXME: not thread-safe. - memset(decode, -1, sizeof(decode)); - for (int i = 0; i < 64; i++) - decode[(int) base64Chars[i]] = i; - init = true; - } - - string res; - unsigned int d = 0, bits = 0; + string res; + unsigned int d = 0, bits = 0; - for (char c : s) { - if (c == '=') break; - if (c == '\n') continue; + for (char c : s) { + if (c == '=') break; + if (c == '\n') continue; - char digit = decode[(unsigned char) c]; - if (digit == -1) - throw Error("invalid character in Base64 string"); + char digit = decode[(unsigned char)c]; + if (digit == -1) throw Error("invalid character in Base64 string"); - bits += 6; - d = d << 6 | digit; - if (bits >= 8) { - res.push_back(d >> (bits - 8) & 0xff); - bits -= 8; - } + bits += 6; + d = d << 6 | digit; + if (bits >= 8) { + res.push_back(d >> (bits - 8) & 0xff); + bits -= 8; } + } - return res; + return res; } - -void callFailure(const std::function<void(std::exception_ptr exc)> & failure, std::exception_ptr exc) -{ - try { - failure(exc); - } catch (std::exception & e) { - printError(format("uncaught exception: %s") % e.what()); - abort(); - } +void callFailure(const std::function<void(std::exception_ptr exc)>& failure, + std::exception_ptr exc) { + try { + failure(exc); + } catch (std::exception& e) { + printError(format("uncaught exception: %s") % e.what()); + abort(); + } } - static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}}; - -static void updateWindowSize() -{ - struct winsize ws; - if (ioctl(2, TIOCGWINSZ, &ws) == 0) { - auto windowSize_(windowSize.lock()); - windowSize_->first = ws.ws_row; - windowSize_->second = ws.ws_col; - } +static void updateWindowSize() { + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } } - -std::pair<unsigned short, unsigned short> getWindowSize() -{ - return *windowSize.lock(); +std::pair<unsigned short, unsigned short> getWindowSize() { + return *windowSize.lock(); } - static Sync<std::list<std::function<void()>>> _interruptCallbacks; -static void signalHandlerThread(sigset_t set) -{ - while (true) { - int signal = 0; - sigwait(&set, &signal); +static void signalHandlerThread(sigset_t set) { + while (true) { + int signal = 0; + sigwait(&set, &signal); - if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) - triggerInterrupt(); + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) + triggerInterrupt(); - else if (signal == SIGWINCH) { - updateWindowSize(); - } + else if (signal == SIGWINCH) { + updateWindowSize(); } + } } -void triggerInterrupt() -{ - _isInterrupted = true; +void triggerInterrupt() { + _isInterrupted = true; - { - auto interruptCallbacks(_interruptCallbacks.lock()); - for (auto & callback : *interruptCallbacks) { - try { - callback(); - } catch (...) { - ignoreException(); - } - } + { + auto interruptCallbacks(_interruptCallbacks.lock()); + for (auto& callback : *interruptCallbacks) { + try { + callback(); + } catch (...) { + ignoreException(); + } } + } } static sigset_t savedSignalMask; -void startSignalHandlerThread() -{ - updateWindowSize(); +void startSignalHandlerThread() { + updateWindowSize(); - if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("quering signal mask"); + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("quering signal mask"); - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGINT); - sigaddset(&set, SIGTERM); - sigaddset(&set, SIGHUP); - sigaddset(&set, SIGPIPE); - sigaddset(&set, SIGWINCH); - if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) - throw SysError("blocking signals"); + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGWINCH); + if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) + throw SysError("blocking signals"); - std::thread(signalHandlerThread, set).detach(); + std::thread(signalHandlerThread, set).detach(); } -void restoreSignals() -{ - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); +void restoreSignals() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); } /* RAII helper to automatically deregister a callback. */ -struct InterruptCallbackImpl : InterruptCallback -{ - std::list<std::function<void()>>::iterator it; - ~InterruptCallbackImpl() override - { - _interruptCallbacks.lock()->erase(it); - } +struct InterruptCallbackImpl : InterruptCallback { + std::list<std::function<void()>>::iterator it; + ~InterruptCallbackImpl() override { _interruptCallbacks.lock()->erase(it); } }; -std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> callback) -{ - auto interruptCallbacks(_interruptCallbacks.lock()); - interruptCallbacks->push_back(callback); +std::unique_ptr<InterruptCallback> createInterruptCallback( + std::function<void()> callback) { + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->push_back(callback); - auto res = std::make_unique<InterruptCallbackImpl>(); - res->it = interruptCallbacks->end(); - res->it--; + auto res = std::make_unique<InterruptCallbackImpl>(); + res->it = interruptCallbacks->end(); + res->it--; - return std::unique_ptr<InterruptCallback>(res.release()); + return std::unique_ptr<InterruptCallback>(res.release()); } -} +} // namespace nix diff --git a/third_party/nix/src/libutil/util.hh b/third_party/nix/src/libutil/util.hh index f057fdb2c041..8a23090def97 100644 --- a/third_party/nix/src/libutil/util.hh +++ b/third_party/nix/src/libutil/util.hh @@ -1,21 +1,19 @@ #pragma once -#include "types.hh" -#include "logging.hh" - -#include <sys/types.h> -#include <sys/stat.h> #include <dirent.h> -#include <unistd.h> #include <signal.h> - +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <cstdio> #include <functional> +#include <future> #include <limits> -#include <cstdio> #include <map> -#include <sstream> #include <optional> -#include <future> +#include <sstream> +#include "logging.hh" +#include "types.hh" #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 @@ -29,13 +27,11 @@ namespace nix { struct Sink; struct Source; - /* The system for which Nix is compiled. */ extern const std::string nativeSystem; - /* Return an environment variable. */ -string getEnv(const string & key, const string & def = ""); +string getEnv(const string& key, const string& def = ""); /* Get the entire environment. */ std::map<std::string, std::string> getEnv(); @@ -52,62 +48,61 @@ Path absPath(Path path, Path dir = ""); double or trailing slashes. Optionally resolves all symlink components such that each component of the resulting path is *not* a symbolic link. */ -Path canonPath(const Path & path, bool resolveSymlinks = false); +Path canonPath(const Path& path, bool resolveSymlinks = false); /* Return the directory part of the given canonical path, i.e., everything before the final `/'. If the path is the root or an immediate child thereof (e.g., `/foo'), this means an empty string is returned. */ -Path dirOf(const Path & path); +Path dirOf(const Path& path); /* Return the base name of the given canonical path, i.e., everything following the final `/'. */ -string baseNameOf(const Path & path); +string baseNameOf(const Path& path); /* Check whether 'path' is a descendant of 'dir'. */ -bool isInDir(const Path & path, const Path & dir); +bool isInDir(const Path& path, const Path& dir); /* Check whether 'path' is equal to 'dir' or a descendant of 'dir'. */ -bool isDirOrInDir(const Path & path, const Path & dir); +bool isDirOrInDir(const Path& path, const Path& dir); /* Get status of `path'. */ -struct stat lstat(const Path & path); +struct stat lstat(const Path& path); /* Return true iff the given path exists. */ -bool pathExists(const Path & path); +bool pathExists(const Path& path); /* Read the contents (target) of a symbolic link. The result is not in any way canonicalised. */ -Path readLink(const Path & path); +Path readLink(const Path& path); -bool isLink(const Path & path); +bool isLink(const Path& path); /* Read the contents of a directory. The entries `.' and `..' are removed. */ -struct DirEntry -{ - string name; - ino_t ino; - unsigned char type; // one of DT_* - DirEntry(const string & name, ino_t ino, unsigned char type) - : name(name), ino(ino), type(type) { } +struct DirEntry { + string name; + ino_t ino; + unsigned char type; // one of DT_* + DirEntry(const string& name, ino_t ino, unsigned char type) + : name(name), ino(ino), type(type) {} }; typedef vector<DirEntry> DirEntries; -DirEntries readDirectory(const Path & path); +DirEntries readDirectory(const Path& path); -unsigned char getFileType(const Path & path); +unsigned char getFileType(const Path& path); /* Read the contents of a file into a string. */ string readFile(int fd); -string readFile(const Path & path, bool drain = false); -void readFile(const Path & path, Sink & sink); +string readFile(const Path& path, bool drain = false); +void readFile(const Path& path, Sink& sink); /* Write a string to a file. */ -void writeFile(const Path & path, const string & s, mode_t mode = 0666); +void writeFile(const Path& path, const string& s, mode_t mode = 0666); -void writeFile(const Path & path, Source & source, mode_t mode = 0666); +void writeFile(const Path& path, Source& source, mode_t mode = 0666); /* Read a line from a file descriptor. */ string readLine(int fd); @@ -118,13 +113,14 @@ void writeLine(int fd, string s); /* Delete a path; i.e., in the case of a directory, it is deleted recursively. It's not an error if the path does not exist. The second variant returns the number of bytes and blocks freed. */ -void deletePath(const Path & path); +void deletePath(const Path& path); -void deletePath(const Path & path, unsigned long long & bytesFreed); +void deletePath(const Path& path, unsigned long long& bytesFreed); /* Create a temporary directory. */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); +Path createTempDir(const Path& tmpRoot = "", const Path& prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, + mode_t mode = 0755); std::string getUserName(); @@ -145,178 +141,161 @@ Path getDataDir(); /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ -Paths createDirs(const Path & path); +Paths createDirs(const Path& path); /* Create a symlink. */ -void createSymlink(const Path & target, const Path & link); +void createSymlink(const Path& target, const Path& link); /* Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link); - +void replaceSymlink(const Path& target, const Path& link); /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ -void readFull(int fd, unsigned char * buf, size_t count); -void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true); -void writeFull(int fd, const string & s, bool allowInterrupts = true); +void readFull(int fd, unsigned char* buf, size_t count); +void writeFull(int fd, const unsigned char* buf, size_t count, + bool allowInterrupts = true); +void writeFull(int fd, const string& s, bool allowInterrupts = true); MakeError(EndOfFile, Error) + /* Read a file descriptor until EOF occurs. */ + string drainFD(int fd, bool block = true); -/* Read a file descriptor until EOF occurs. */ -string drainFD(int fd, bool block = true); - -void drainFD(int fd, Sink & sink, bool block = true); - +void drainFD(int fd, Sink& sink, bool block = true); /* Automatic cleanup of resources. */ - -class AutoDelete -{ - Path path; - bool del; - bool recursive; -public: - AutoDelete(); - AutoDelete(const Path & p, bool recursive = true); - ~AutoDelete(); - void cancel(); - void reset(const Path & p, bool recursive = true); - operator Path() const { return path; } +class AutoDelete { + Path path; + bool del; + bool recursive; + + public: + AutoDelete(); + AutoDelete(const Path& p, bool recursive = true); + ~AutoDelete(); + void cancel(); + void reset(const Path& p, bool recursive = true); + operator Path() const { return path; } }; - -class AutoCloseFD -{ - int fd; - void close(); -public: - AutoCloseFD(); - AutoCloseFD(int fd); - AutoCloseFD(const AutoCloseFD & fd) = delete; - AutoCloseFD(AutoCloseFD&& fd); - ~AutoCloseFD(); - AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; - AutoCloseFD& operator =(AutoCloseFD&& fd); - int get() const; - explicit operator bool() const; - int release(); +class AutoCloseFD { + int fd; + void close(); + + public: + AutoCloseFD(); + AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD& fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator=(const AutoCloseFD& fd) = delete; + AutoCloseFD& operator=(AutoCloseFD&& fd); + int get() const; + explicit operator bool() const; + int release(); }; - -class Pipe -{ -public: - AutoCloseFD readSide, writeSide; - void create(); +class Pipe { + public: + AutoCloseFD readSide, writeSide; + void create(); }; - -struct DIRDeleter -{ - void operator()(DIR * dir) const { - closedir(dir); - } +struct DIRDeleter { + void operator()(DIR* dir) const { closedir(dir); } }; typedef std::unique_ptr<DIR, DIRDeleter> AutoCloseDir; - -class Pid -{ - pid_t pid = -1; - bool separatePG = false; - int killSignal = SIGKILL; -public: - Pid(); - Pid(pid_t pid); - ~Pid(); - void operator =(pid_t pid); - operator pid_t(); - int kill(); - int wait(); - - void setSeparatePG(bool separatePG); - void setKillSignal(int signal); - pid_t release(); +class Pid { + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; + + public: + Pid(); + Pid(pid_t pid); + ~Pid(); + void operator=(pid_t pid); + operator pid_t(); + int kill(); + int wait(); + + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); }; - /* Kill all processes running under the specified uid by sending them a SIGKILL. */ void killUser(uid_t uid); - /* Fork a process that runs the given function, and return the child pid to the caller. */ -struct ProcessOptions -{ - string errorPrefix = "error: "; - bool dieWithParent = true; - bool runExitHandlers = false; - bool allowVfork = true; +struct ProcessOptions { + string errorPrefix = "error: "; + bool dieWithParent = true; + bool runExitHandlers = false; + bool allowVfork = true; }; -pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); - +pid_t startProcess(std::function<void()> fun, + const ProcessOptions& options = ProcessOptions()); /* Run a program and return its stdout in a string (i.e., like the shell backtick operator). */ string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings(), - const std::optional<std::string> & input = {}); - -struct RunOptions -{ - std::optional<uid_t> uid; - std::optional<uid_t> gid; - std::optional<Path> chdir; - std::optional<std::map<std::string, std::string>> environment; - Path program; - bool searchPath = true; - Strings args; - std::optional<std::string> input; - Source * standardIn = nullptr; - Sink * standardOut = nullptr; - bool mergeStderrToStdout = false; - bool _killStderr = false; - - RunOptions(const Path & program, const Strings & args) - : program(program), args(args) { }; - - RunOptions & killStderr(bool v) { _killStderr = true; return *this; } + const Strings& args = Strings(), + const std::optional<std::string>& input = {}); + +struct RunOptions { + std::optional<uid_t> uid; + std::optional<uid_t> gid; + std::optional<Path> chdir; + std::optional<std::map<std::string, std::string>> environment; + Path program; + bool searchPath = true; + Strings args; + std::optional<std::string> input; + Source* standardIn = nullptr; + Sink* standardOut = nullptr; + bool mergeStderrToStdout = false; + bool _killStderr = false; + + RunOptions(const Path& program, const Strings& args) + : program(program), args(args){}; + + RunOptions& killStderr(bool v) { + _killStderr = true; + return *this; + } }; -std::pair<int, std::string> runProgram(const RunOptions & options); +std::pair<int, std::string> runProgram(const RunOptions& options); -void runProgram2(const RunOptions & options); +void runProgram2(const RunOptions& options); +class ExecError : public Error { + public: + int status; -class ExecError : public Error -{ -public: - int status; - - template<typename... Args> - ExecError(int status, Args... args) - : Error(args...), status(status) - { } + template <typename... Args> + ExecError(int status, Args... args) : Error(args...), status(status) {} }; /* Convert a list of strings to a null-terminated vector of char *'s. The result must not be accessed beyond the lifetime of the list of strings. */ -std::vector<char *> stringsToCharPtrs(const Strings & ss); +std::vector<char*> stringsToCharPtrs(const Strings& ss); /* Close all file descriptors except those listed in the given set. Good practice in child processes. */ -void closeMostFDs(const set<int> & exceptions); +void closeMostFDs(const set<int>& exceptions); /* Set the close-on-exec flag for the given file descriptor. */ void closeOnExec(int fd); - /* User interruption. */ extern bool _isInterrupted; @@ -327,40 +306,32 @@ void setInterruptThrown(); void _interrupted(); -void inline checkInterrupt() -{ - if (_isInterrupted || (interruptCheck && interruptCheck())) - _interrupted(); +void inline checkInterrupt() { + if (_isInterrupted || (interruptCheck && interruptCheck())) _interrupted(); } MakeError(Interrupted, BaseError) + MakeError(FormatError, Error) -MakeError(FormatError, Error) - - -/* String tokenizer. */ -template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r"); - + /* String tokenizer. */ + template <class C> + C tokenizeString(const string& s, const string& separators = " \t\n\r"); /* Concatenate the given strings with a separator between the elements. */ -string concatStringsSep(const string & sep, const Strings & ss); -string concatStringsSep(const string & sep, const StringSet & ss); - +string concatStringsSep(const string& sep, const Strings& ss); +string concatStringsSep(const string& sep, const StringSet& ss); /* Remove trailing whitespace from a string. */ -string chomp(const string & s); - +string chomp(const string& s); /* Remove whitespace from the start and end of a string. */ -string trim(const string & s, const string & whitespace = " \n\r\t"); - +string trim(const string& s, const string& whitespace = " \n\r\t"); /* Replace all occurrences of a string inside another string. */ -string replaceStrings(const std::string & s, - const std::string & from, const std::string & to); - +string replaceStrings(const std::string& s, const std::string& from, + const std::string& to); /* Convert the exit status of a child as returned by wait() into an error string. */ @@ -368,47 +339,40 @@ string statusToString(int status); bool statusOk(int status); - /* Parse a string into an integer. */ -template<class N> bool string2Int(const string & s, N & n) -{ - if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed) - return false; - std::istringstream str(s); - str >> n; - return str && str.get() == EOF; +template <class N> +bool string2Int(const string& s, N& n) { + if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed) + return false; + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; } /* Parse a string into a float. */ -template<class N> bool string2Float(const string & s, N & n) -{ - std::istringstream str(s); - str >> n; - return str && str.get() == EOF; +template <class N> +bool string2Float(const string& s, N& n) { + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; } - /* Return true iff `s' starts with `prefix'. */ -bool hasPrefix(const string & s, const string & prefix); - +bool hasPrefix(const string& s, const string& prefix); /* Return true iff `s' ends in `suffix'. */ -bool hasSuffix(const string & s, const string & suffix); - +bool hasSuffix(const string& s, const string& suffix); /* Convert a string to lower case. */ -std::string toLower(const std::string & s); - +std::string toLower(const std::string& s); /* Escape a string as a shell word. */ -std::string shellEscape(const std::string & s); - +std::string shellEscape(const std::string& s); /* Exception handling in destructors: print an error message, then ignore the exception. */ void ignoreException(); - /* Some ANSI escape sequences. */ #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" @@ -417,71 +381,61 @@ void ignoreException(); #define ANSI_GREEN "\e[32;1m" #define ANSI_BLUE "\e[34;1m" - /* Truncate a string to 'width' printable characters. If 'filterAll' is true, all ANSI escape sequences are filtered out. Otherwise, some escape sequences (such as colour setting) are copied but not included in the character count. Also, tabs are expanded to spaces. */ -std::string filterANSIEscapes(const std::string & s, - bool filterAll = false, +std::string filterANSIEscapes( + const std::string& s, bool filterAll = false, unsigned int width = std::numeric_limits<unsigned int>::max()); - /* Base64 encoding/decoding. */ -string base64Encode(const string & s); -string base64Decode(const string & s); - +string base64Encode(const string& s); +string base64Decode(const string& s); /* Get a value for the specified key from an associate container, or a default value if the key doesn't exist. */ template <class T> -string get(const T & map, const string & key, const string & def = "") -{ - auto i = map.find(key); - return i == map.end() ? def : i->second; +string get(const T& map, const string& key, const string& def = "") { + auto i = map.find(key); + return i == map.end() ? def : i->second; } - /* A callback is a wrapper around a lambda that accepts a valid of type T or an exception. (We abuse std::future<T> to pass the value or exception.) */ -template<typename T> -class Callback -{ - std::function<void(std::future<T>)> fun; - std::atomic_flag done = ATOMIC_FLAG_INIT; - -public: - - Callback(std::function<void(std::future<T>)> fun) : fun(fun) { } - - Callback(Callback && callback) : fun(std::move(callback.fun)) - { - auto prev = callback.done.test_and_set(); - if (prev) done.test_and_set(); - } - - void operator()(T && t) noexcept - { - auto prev = done.test_and_set(); - assert(!prev); - std::promise<T> promise; - promise.set_value(std::move(t)); - fun(promise.get_future()); - } - - void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept - { - auto prev = done.test_and_set(); - assert(!prev); - std::promise<T> promise; - promise.set_exception(exc); - fun(promise.get_future()); - } +template <typename T> +class Callback { + std::function<void(std::future<T>)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + + public: + Callback(std::function<void(std::future<T>)> fun) : fun(fun) {} + + Callback(Callback&& callback) : fun(std::move(callback.fun)) { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T&& t) noexcept { + auto prev = done.test_and_set(); + assert(!prev); + std::promise<T> promise; + promise.set_value(std::move(t)); + fun(promise.get_future()); + } + + void rethrow( + const std::exception_ptr& exc = std::current_exception()) noexcept { + auto prev = done.test_and_set(); + assert(!prev); + std::promise<T> promise; + promise.set_exception(exc); + fun(promise.get_future()); + } }; - /* Start a thread that handles various signals. Also block those signals on the current thread (and thus any threads created by it). */ void startSignalHandlerThread(); @@ -489,9 +443,8 @@ void startSignalHandlerThread(); /* Restore default signal handling. */ void restoreSignals(); -struct InterruptCallback -{ - virtual ~InterruptCallback() { }; +struct InterruptCallback { + virtual ~InterruptCallback(){}; }; /* Register a function that gets called on SIGINT (in a non-signal @@ -504,39 +457,34 @@ void triggerInterrupt(); /* A RAII class that causes the current thread to receive SIGUSR1 when the signal handler thread receives SIGINT. That is, this allows SIGINT to be multiplexed to multiple threads. */ -struct ReceiveInterrupts -{ - pthread_t target; - std::unique_ptr<InterruptCallback> callback; - - ReceiveInterrupts() - : target(pthread_self()) - , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) - { } +struct ReceiveInterrupts { + pthread_t target; + std::unique_ptr<InterruptCallback> callback; + + ReceiveInterrupts() + : target(pthread_self()), callback(createInterruptCallback([&]() { + pthread_kill(target, SIGUSR1); + })) {} }; - - /* A RAII helper that increments a counter on construction and decrements it on destruction. */ -template<typename T> -struct MaintainCount -{ - T & counter; - long delta; - MaintainCount(T & counter, long delta = 1) : counter(counter), delta(delta) { counter += delta; } - ~MaintainCount() { counter -= delta; } +template <typename T> +struct MaintainCount { + T& counter; + long delta; + MaintainCount(T& counter, long delta = 1) : counter(counter), delta(delta) { + counter += delta; + } + ~MaintainCount() { counter -= delta; } }; - /* Return the number of rows and columns of the terminal. */ std::pair<unsigned short, unsigned short> getWindowSize(); - /* Used in various places. */ -typedef std::function<bool(const Path & path)> PathFilter; +typedef std::function<bool(const Path& path)> PathFilter; extern PathFilter defaultPathFilter; - -} +} // namespace nix diff --git a/third_party/nix/src/libutil/xml-writer.cc b/third_party/nix/src/libutil/xml-writer.cc index e5cc2e9fc719..cabf7812128e 100644 --- a/third_party/nix/src/libutil/xml-writer.cc +++ b/third_party/nix/src/libutil/xml-writer.cc @@ -1,94 +1,77 @@ -#include <assert.h> - #include "xml-writer.hh" - +#include <assert.h> namespace nix { - - -XMLWriter::XMLWriter(bool indent, std::ostream & output) - : output(output), indent(indent) -{ - output << "<?xml version='1.0' encoding='utf-8'?>" << std::endl; - closed = false; -} - -XMLWriter::~XMLWriter() -{ - close(); +XMLWriter::XMLWriter(bool indent, std::ostream& output) + : output(output), indent(indent) { + output << "<?xml version='1.0' encoding='utf-8'?>" << std::endl; + closed = false; } +XMLWriter::~XMLWriter() { close(); } -void XMLWriter::close() -{ - if (closed) return; - while (!pendingElems.empty()) closeElement(); - closed = true; +void XMLWriter::close() { + if (closed) return; + while (!pendingElems.empty()) closeElement(); + closed = true; } - -void XMLWriter::indent_(size_t depth) -{ - if (!indent) return; - output << string(depth * 2, ' '); +void XMLWriter::indent_(size_t depth) { + if (!indent) return; + output << string(depth * 2, ' '); } - -void XMLWriter::openElement(const string & name, - const XMLAttrs & attrs) -{ - assert(!closed); - indent_(pendingElems.size()); - output << "<" << name; - writeAttrs(attrs); - output << ">"; - if (indent) output << std::endl; - pendingElems.push_back(name); +void XMLWriter::openElement(const string& name, const XMLAttrs& attrs) { + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << ">"; + if (indent) output << std::endl; + pendingElems.push_back(name); } - -void XMLWriter::closeElement() -{ - assert(!pendingElems.empty()); - indent_(pendingElems.size() - 1); - output << "</" << pendingElems.back() << ">"; - if (indent) output << std::endl; - pendingElems.pop_back(); - if (pendingElems.empty()) closed = true; +void XMLWriter::closeElement() { + assert(!pendingElems.empty()); + indent_(pendingElems.size() - 1); + output << "</" << pendingElems.back() << ">"; + if (indent) output << std::endl; + pendingElems.pop_back(); + if (pendingElems.empty()) closed = true; } - -void XMLWriter::writeEmptyElement(const string & name, - const XMLAttrs & attrs) -{ - assert(!closed); - indent_(pendingElems.size()); - output << "<" << name; - writeAttrs(attrs); - output << " />"; - if (indent) output << std::endl; +void XMLWriter::writeEmptyElement(const string& name, const XMLAttrs& attrs) { + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << " />"; + if (indent) output << std::endl; } - -void XMLWriter::writeAttrs(const XMLAttrs & attrs) -{ - for (auto & i : attrs) { - output << " " << i.first << "=\""; - for (size_t j = 0; j < i.second.size(); ++j) { - char c = i.second[j]; - if (c == '"') output << """; - else if (c == '<') output << "<"; - else if (c == '>') output << ">"; - else if (c == '&') output << "&"; - /* Escape newlines to prevent attribute normalisation (see - XML spec, section 3.3.3. */ - else if (c == '\n') output << "
"; - else output << c; - } - output << "\""; +void XMLWriter::writeAttrs(const XMLAttrs& attrs) { + for (auto& i : attrs) { + output << " " << i.first << "=\""; + for (size_t j = 0; j < i.second.size(); ++j) { + char c = i.second[j]; + if (c == '"') + output << """; + else if (c == '<') + output << "<"; + else if (c == '>') + output << ">"; + else if (c == '&') + output << "&"; + /* Escape newlines to prevent attribute normalisation (see + XML spec, section 3.3.3. */ + else if (c == '\n') + output << "
"; + else + output << c; } + output << "\""; + } } - -} +} // namespace nix diff --git a/third_party/nix/src/libutil/xml-writer.hh b/third_party/nix/src/libutil/xml-writer.hh index b98b445265a2..3a2d9a66d8e1 100644 --- a/third_party/nix/src/libutil/xml-writer.hh +++ b/third_party/nix/src/libutil/xml-writer.hh @@ -1,69 +1,56 @@ #pragma once #include <iostream> -#include <string> #include <list> #include <map> - +#include <string> namespace nix { -using std::string; -using std::map; using std::list; - +using std::map; +using std::string; typedef map<string, string> XMLAttrs; +class XMLWriter { + private: + std::ostream& output; -class XMLWriter -{ -private: - - std::ostream & output; - - bool indent; - bool closed; + bool indent; + bool closed; - list<string> pendingElems; + list<string> pendingElems; -public: + public: + XMLWriter(bool indent, std::ostream& output); + ~XMLWriter(); - XMLWriter(bool indent, std::ostream & output); - ~XMLWriter(); + void close(); - void close(); + void openElement(const string& name, const XMLAttrs& attrs = XMLAttrs()); + void closeElement(); - void openElement(const string & name, - const XMLAttrs & attrs = XMLAttrs()); - void closeElement(); + void writeEmptyElement(const string& name, + const XMLAttrs& attrs = XMLAttrs()); - void writeEmptyElement(const string & name, - const XMLAttrs & attrs = XMLAttrs()); + private: + void writeAttrs(const XMLAttrs& attrs); -private: - void writeAttrs(const XMLAttrs & attrs); - - void indent_(size_t depth); + void indent_(size_t depth); }; - -class XMLOpenElement -{ -private: - XMLWriter & writer; -public: - XMLOpenElement(XMLWriter & writer, const string & name, - const XMLAttrs & attrs = XMLAttrs()) - : writer(writer) - { - writer.openElement(name, attrs); - } - ~XMLOpenElement() - { - writer.closeElement(); - } +class XMLOpenElement { + private: + XMLWriter& writer; + + public: + XMLOpenElement(XMLWriter& writer, const string& name, + const XMLAttrs& attrs = XMLAttrs()) + : writer(writer) { + writer.openElement(name, attrs); + } + ~XMLOpenElement() { writer.closeElement(); } }; - -} +} // namespace nix diff --git a/third_party/nix/src/nix-build/nix-build.cc b/third_party/nix/src/nix-build/nix-build.cc index 90b63d0ba5e4..3cd78d940919 100755..100644 --- a/third_party/nix/src/nix-build/nix-build.cc +++ b/third_party/nix/src/nix-build/nix-build.cc @@ -4,414 +4,425 @@ #include <regex> #include <sstream> #include <vector> - -#include "store-api.hh" -#include "globals.hh" -#include "derivations.hh" #include "affinity.hh" -#include "util.hh" -#include "shared.hh" -#include "eval.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" #include "eval-inline.hh" +#include "eval.hh" #include "get-drvs.hh" -#include "common-eval-args.hh" -#include "attr-path.hh" +#include "globals.hh" #include "legacy.hh" +#include "shared.hh" +#include "store-api.hh" +#include "util.hh" using namespace nix; using namespace std::string_literals; -extern char * * environ __attribute__((weak)); +extern char** environ __attribute__((weak)); /* Recreate the effect of the perl shellwords function, breaking up a * string into arguments like a shell word, including escapes */ -std::vector<string> shellwords(const string & s) -{ - std::regex whitespace("^(\\s+).*"); - auto begin = s.cbegin(); - std::vector<string> res; - std::string cur; - enum state { - sBegin, - sQuote - }; - state st = sBegin; - auto it = begin; - for (; it != s.cend(); ++it) { - if (st == sBegin) { - std::smatch match; - if (regex_search(it, s.cend(), match, whitespace)) { - cur.append(begin, it); - res.push_back(cur); - cur.clear(); - it = match[1].second; - begin = it; - } - } - switch (*it) { - case '"': - cur.append(begin, it); - begin = it + 1; - st = st == sBegin ? sQuote : sBegin; - break; - case '\\': - /* perl shellwords mostly just treats the next char as part of the string with no special processing */ - cur.append(begin, it); - begin = ++it; - break; - } +std::vector<string> shellwords(const string& s) { + std::regex whitespace("^(\\s+).*"); + auto begin = s.cbegin(); + std::vector<string> res; + std::string cur; + enum state { sBegin, sQuote }; + state st = sBegin; + auto it = begin; + for (; it != s.cend(); ++it) { + if (st == sBegin) { + std::smatch match; + if (regex_search(it, s.cend(), match, whitespace)) { + cur.append(begin, it); + res.push_back(cur); + cur.clear(); + it = match[1].second; + begin = it; + } + } + switch (*it) { + case '"': + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sQuote : sBegin; + break; + case '\\': + /* perl shellwords mostly just treats the next char as part of the + * string with no special processing */ + cur.append(begin, it); + begin = ++it; + break; } - cur.append(begin, it); - if (!cur.empty()) res.push_back(cur); - return res; + } + cur.append(begin, it); + if (!cur.empty()) res.push_back(cur); + return res; } -static void _main(int argc, char * * argv) -{ - auto dryRun = false; - auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); - auto pure = false; - auto fromArgs = false; - auto packages = false; - // Same condition as bash uses for interactive shells - auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); - Strings attrPaths; - Strings left; - RepairFlag repair = NoRepair; - Path gcRoot; - BuildMode buildMode = bmNormal; - bool readStdin = false; - - std::string envCommand; // interactive shell - Strings envExclude; - - auto myName = runEnv ? "nix-shell" : "nix-build"; - - auto inShebang = false; - std::string script; - std::vector<string> savedArgs; - - AutoDelete tmpDir(createTempDir("", myName)); - - std::string outLink = "./result"; - - // List of environment variables kept for --pure - std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"}; - - Strings args; - for (int i = 1; i < argc; ++i) - args.push_back(argv[i]); - - // Heuristic to see if we're invoked as a shebang script, namely, - // if we have at least one argument, it's the name of an - // executable file, and it starts with "#!". - if (runEnv && argc > 1 && !std::regex_search(argv[1], std::regex("nix-shell"))) { - script = argv[1]; - try { - auto lines = tokenizeString<Strings>(readFile(script), "\n"); - if (std::regex_search(lines.front(), std::regex("^#!"))) { - lines.pop_front(); - inShebang = true; - for (int i = 2; i < argc; ++i) - savedArgs.push_back(argv[i]); - args.clear(); - for (auto line : lines) { - line = chomp(line); - std::smatch match; - if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$"))) - for (const auto & word : shellwords(match[1].str())) - args.push_back(word); - } - } - } catch (SysError &) { } +static void _main(int argc, char** argv) { + auto dryRun = false; + auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); + auto pure = false; + auto fromArgs = false; + auto packages = false; + // Same condition as bash uses for interactive shells + auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); + Strings attrPaths; + Strings left; + RepairFlag repair = NoRepair; + Path gcRoot; + BuildMode buildMode = bmNormal; + bool readStdin = false; + + std::string envCommand; // interactive shell + Strings envExclude; + + auto myName = runEnv ? "nix-shell" : "nix-build"; + + auto inShebang = false; + std::string script; + std::vector<string> savedArgs; + + AutoDelete tmpDir(createTempDir("", myName)); + + std::string outLink = "./result"; + + // List of environment variables kept for --pure + std::set<string> keepVars{ + "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", + "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"}; + + Strings args; + for (int i = 1; i < argc; ++i) args.push_back(argv[i]); + + // Heuristic to see if we're invoked as a shebang script, namely, + // if we have at least one argument, it's the name of an + // executable file, and it starts with "#!". + if (runEnv && argc > 1 && + !std::regex_search(argv[1], std::regex("nix-shell"))) { + script = argv[1]; + try { + auto lines = tokenizeString<Strings>(readFile(script), "\n"); + if (std::regex_search(lines.front(), std::regex("^#!"))) { + lines.pop_front(); + inShebang = true; + for (int i = 2; i < argc; ++i) savedArgs.push_back(argv[i]); + args.clear(); + for (auto line : lines) { + line = chomp(line); + std::smatch match; + if (std::regex_match(line, match, + std::regex("^#!\\s*nix-shell (.*)$"))) + for (const auto& word : shellwords(match[1].str())) + args.push_back(word); + } + } + } catch (SysError&) { } + } - struct MyArgs : LegacyArgs, MixEvalArgs - { - using LegacyArgs::LegacyArgs; - }; + struct MyArgs : LegacyArgs, MixEvalArgs { + using LegacyArgs::LegacyArgs; + }; - MyArgs myArgs(myName, [&](Strings::iterator & arg, const Strings::iterator & end) { + MyArgs myArgs( + myName, [&](Strings::iterator& arg, const Strings::iterator& end) { if (*arg == "--help") { - deletePath(tmpDir); - showManPage(myName); + deletePath(tmpDir); + showManPage(myName); } else if (*arg == "--version") - printVersion(myName); + printVersion(myName); else if (*arg == "--add-drv-link" || *arg == "--indirect") - ; // obsolete + ; // obsolete else if (*arg == "--no-out-link" || *arg == "--no-link") - outLink = (Path) tmpDir + "/result"; + outLink = (Path)tmpDir + "/result"; else if (*arg == "--attr" || *arg == "-A") - attrPaths.push_back(getArg(*arg, arg, end)); + attrPaths.push_back(getArg(*arg, arg, end)); else if (*arg == "--drv-link") - getArg(*arg, arg, end); // obsolete + getArg(*arg, arg, end); // obsolete else if (*arg == "--out-link" || *arg == "-o") - outLink = getArg(*arg, arg, end); + outLink = getArg(*arg, arg, end); else if (*arg == "--add-root") - gcRoot = getArg(*arg, arg, end); + gcRoot = getArg(*arg, arg, end); else if (*arg == "--dry-run") - dryRun = true; + dryRun = true; else if (*arg == "--repair") { - repair = Repair; - buildMode = bmRepair; + repair = Repair; + buildMode = bmRepair; } - else if (*arg == "--run-env") // obsolete - runEnv = true; + else if (*arg == "--run-env") // obsolete + runEnv = true; else if (*arg == "--command" || *arg == "--run") { - if (*arg == "--run") - interactive = false; - envCommand = getArg(*arg, arg, end) + "\nexit"; + if (*arg == "--run") interactive = false; + envCommand = getArg(*arg, arg, end) + "\nexit"; } else if (*arg == "--check") - buildMode = bmCheck; + buildMode = bmCheck; else if (*arg == "--exclude") - envExclude.push_back(getArg(*arg, arg, end)); + envExclude.push_back(getArg(*arg, arg, end)); else if (*arg == "--expr" || *arg == "-E") - fromArgs = true; + fromArgs = true; - else if (*arg == "--pure") pure = true; - else if (*arg == "--impure") pure = false; + else if (*arg == "--pure") + pure = true; + else if (*arg == "--impure") + pure = false; else if (*arg == "--packages" || *arg == "-p") - packages = true; + packages = true; else if (inShebang && *arg == "-i") { - auto interpreter = getArg(*arg, arg, end); - interactive = false; - auto execArgs = ""; - - // Überhack to support Perl. Perl examines the shebang and - // executes it unless it contains the string "perl" or "indir", - // or (undocumented) argv[0] does not contain "perl". Exploit - // the latter by doing "exec -a". - if (std::regex_search(interpreter, std::regex("perl"))) - execArgs = "-a PERL"; - - std::ostringstream joined; - for (const auto & i : savedArgs) - joined << shellEscape(i) << ' '; - - if (std::regex_search(interpreter, std::regex("ruby"))) { - // Hack for Ruby. Ruby also examines the shebang. It tries to - // read the shebang to understand which packages to read from. Since - // this is handled via nix-shell -p, we wrap our ruby script execution - // in ruby -e 'load' which ignores the shebangs. - envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str(); - } else { - envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str(); - } + auto interpreter = getArg(*arg, arg, end); + interactive = false; + auto execArgs = ""; + + // Überhack to support Perl. Perl examines the shebang and + // executes it unless it contains the string "perl" or "indir", + // or (undocumented) argv[0] does not contain "perl". Exploit + // the latter by doing "exec -a". + if (std::regex_search(interpreter, std::regex("perl"))) + execArgs = "-a PERL"; + + std::ostringstream joined; + for (const auto& i : savedArgs) joined << shellEscape(i) << ' '; + + if (std::regex_search(interpreter, std::regex("ruby"))) { + // Hack for Ruby. Ruby also examines the shebang. It tries to + // read the shebang to understand which packages to read from. Since + // this is handled via nix-shell -p, we wrap our ruby script + // execution in ruby -e 'load' which ignores the shebangs. + envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % + execArgs % interpreter % script % joined.str()) + .str(); + } else { + envCommand = (format("exec %1% %2% %3% %4%") % execArgs % + interpreter % script % joined.str()) + .str(); + } } else if (*arg == "--keep") - keepVars.insert(getArg(*arg, arg, end)); + keepVars.insert(getArg(*arg, arg, end)); else if (*arg == "-") - readStdin = true; + readStdin = true; else if (*arg != "" && arg->at(0) == '-') - return false; + return false; else - left.push_back(*arg); + left.push_back(*arg); return true; - }); + }); - myArgs.parseCmdline(args); + myArgs.parseCmdline(args); - initPlugins(); + initPlugins(); - if (packages && fromArgs) - throw UsageError("'-p' and '-E' are mutually exclusive"); + if (packages && fromArgs) + throw UsageError("'-p' and '-E' are mutually exclusive"); - auto store = openStore(); + auto store = openStore(); - auto state = std::make_unique<EvalState>(myArgs.searchPath, store); - state->repair = repair; + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + state->repair = repair; - Bindings & autoArgs = *myArgs.getAutoArgs(*state); + Bindings& autoArgs = *myArgs.getAutoArgs(*state); - if (packages) { - std::ostringstream joined; - joined << "with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ "; - for (const auto & i : left) - joined << '(' << i << ") "; - joined << "]; } \"\""; - fromArgs = true; - left = {joined.str()}; - } else if (!fromArgs) { - if (left.empty() && runEnv && pathExists("shell.nix")) - left = {"shell.nix"}; - if (left.empty()) - left = {"default.nix"}; - } + if (packages) { + std::ostringstream joined; + joined << "with import <nixpkgs> { }; (pkgs.runCommandCC or " + "pkgs.runCommand) \"shell\" { buildInputs = [ "; + for (const auto& i : left) joined << '(' << i << ") "; + joined << "]; } \"\""; + fromArgs = true; + left = {joined.str()}; + } else if (!fromArgs) { + if (left.empty() && runEnv && pathExists("shell.nix")) left = {"shell.nix"}; + if (left.empty()) left = {"default.nix"}; + } - if (runEnv) - setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1); - - DrvInfos drvs; - - /* Parse the expressions. */ - std::vector<Expr *> exprs; - - if (readStdin) - exprs = {state->parseStdin()}; - else - for (auto i : left) { - if (fromArgs) - exprs.push_back(state->parseExprFromString(i, absPath("."))); - else { - auto absolute = i; - try { - absolute = canonPath(absPath(i), true); - } catch (Error & e) {}; - if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) - drvs.push_back(DrvInfo(*state, store, absolute)); - else - /* If we're in a #! script, interpret filenames - relative to the script. */ - exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, - inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); - } - } + if (runEnv) setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1); - /* Evaluate them into derivations. */ - if (attrPaths.empty()) attrPaths = {""}; + DrvInfos drvs; - for (auto e : exprs) { - Value vRoot; - state->eval(e, vRoot); + /* Parse the expressions. */ + std::vector<Expr*> exprs; - for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(*state, i, autoArgs, vRoot)); - state->forceValue(v); - getDerivations(*state, v, "", autoArgs, drvs, false); - } + if (readStdin) + exprs = {state->parseStdin()}; + else + for (auto i : left) { + if (fromArgs) + exprs.push_back(state->parseExprFromString(i, absPath("."))); + else { + auto absolute = i; + try { + absolute = canonPath(absPath(i), true); + } catch (Error& e) { + }; + if (store->isStorePath(absolute) && + std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) + drvs.push_back(DrvInfo(*state, store, absolute)); + else + /* If we're in a #! script, interpret filenames + relative to the script. */ + exprs.push_back( + state->parseExprFromFile(resolveExprPath(state->checkSourcePath( + lookupFileArg(*state, inShebang && !packages + ? absPath(i, absPath(dirOf(script))) + : i))))); + } } - state->printStats(); + /* Evaluate them into derivations. */ + if (attrPaths.empty()) attrPaths = {""}; - auto buildPaths = [&](const PathSet & paths) { - /* Note: we do this even when !printMissing to efficiently - fetch binary cache data. */ - unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; - store->queryMissing(paths, - willBuild, willSubstitute, unknown, downloadSize, narSize); + for (auto e : exprs) { + Value vRoot; + state->eval(e, vRoot); - if (settings.printMissing) - printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize); + for (auto& i : attrPaths) { + Value& v(*findAlongAttrPath(*state, i, autoArgs, vRoot)); + state->forceValue(v); + getDerivations(*state, v, "", autoArgs, drvs, false); + } + } - if (!dryRun) - store->buildPaths(paths, buildMode); - }; + state->printStats(); - if (runEnv) { - if (drvs.size() != 1) - throw UsageError("nix-shell requires a single derivation"); + auto buildPaths = [&](const PathSet& paths) { + /* Note: we do this even when !printMissing to efficiently + fetch binary cache data. */ + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, + narSize); - auto & drvInfo = drvs.front(); - auto drv = store->derivationFromPath(drvInfo.queryDrvPath()); + if (settings.printMissing) + printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, + downloadSize, narSize); - PathSet pathsToBuild; + if (!dryRun) store->buildPaths(paths, buildMode); + }; - /* Figure out what bash shell to use. If $NIX_BUILD_SHELL - is not set, then build bashInteractive from - <nixpkgs>. */ - auto shell = getEnv("NIX_BUILD_SHELL", ""); + if (runEnv) { + if (drvs.size() != 1) + throw UsageError("nix-shell requires a single derivation"); - if (shell == "") { + auto& drvInfo = drvs.front(); + auto drv = store->derivationFromPath(drvInfo.queryDrvPath()); - try { - auto expr = state->parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath(".")); + PathSet pathsToBuild; - Value v; - state->eval(expr, v); + /* Figure out what bash shell to use. If $NIX_BUILD_SHELL + is not set, then build bashInteractive from + <nixpkgs>. */ + auto shell = getEnv("NIX_BUILD_SHELL", ""); - auto drv = getDerivation(*state, v, false); - if (!drv) - throw Error("the 'bashInteractive' attribute in <nixpkgs> did not evaluate to a derivation"); + if (shell == "") { + try { + auto expr = state->parseExprFromString( + "(import <nixpkgs> {}).bashInteractive", absPath(".")); - pathsToBuild.insert(drv->queryDrvPath()); + Value v; + state->eval(expr, v); - shell = drv->queryOutPath() + "/bin/bash"; + auto drv = getDerivation(*state, v, false); + if (!drv) + throw Error( + "the 'bashInteractive' attribute in <nixpkgs> did not evaluate " + "to a derivation"); - } catch (Error & e) { - printError("warning: %s; will use bash from your environment", e.what()); - shell = "bash"; - } - } + pathsToBuild.insert(drv->queryDrvPath()); - // Build or fetch all dependencies of the derivation. - for (const auto & input : drv.inputDrvs) - if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(input.first, std::regex(exclude)); })) - pathsToBuild.insert(makeDrvPathWithOutputs(input.first, input.second)); - for (const auto & src : drv.inputSrcs) - pathsToBuild.insert(src); + shell = drv->queryOutPath() + "/bin/bash"; - buildPaths(pathsToBuild); + } catch (Error& e) { + printError("warning: %s; will use bash from your environment", + e.what()); + shell = "bash"; + } + } - if (dryRun) return; + // Build or fetch all dependencies of the derivation. + for (const auto& input : drv.inputDrvs) + if (std::all_of(envExclude.cbegin(), envExclude.cend(), + [&](const string& exclude) { + return !std::regex_search(input.first, + std::regex(exclude)); + })) + pathsToBuild.insert(makeDrvPathWithOutputs(input.first, input.second)); + for (const auto& src : drv.inputSrcs) pathsToBuild.insert(src); - // Set the environment. - auto env = getEnv(); + buildPaths(pathsToBuild); - auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp")); + if (dryRun) return; - if (pure) { - decltype(env) newEnv; - for (auto & i : env) - if (keepVars.count(i.first)) - newEnv.emplace(i); - env = newEnv; - // NixOS hack: prevent /etc/bashrc from sourcing /etc/profile. - env["__ETC_PROFILE_SOURCED"] = "1"; - } + // Set the environment. + auto env = getEnv(); + + auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp")); - env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp; - env["NIX_STORE"] = store->storeDir; - env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); - - auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile", "")); - - bool keepTmp = false; - int fileNr = 0; - - for (auto & var : drv.env) - if (passAsFile.count(var.first)) { - keepTmp = true; - string fn = ".attr-" + std::to_string(fileNr++); - Path p = (Path) tmpDir + "/" + fn; - writeFile(p, var.second); - env[var.first + "Path"] = p; - } else - env[var.first] = var.second; - - restoreAffinity(); - - /* Run a shell using the derivation's environment. For - convenience, source $stdenv/setup to setup additional - environment variables and shell functions. Also don't - lose the current $PATH directories. */ - auto rcfile = (Path) tmpDir + "/rc"; - writeFile(rcfile, fmt( - (keepTmp ? "" : "rm -rf '%1%'; "s) + + if (pure) { + decltype(env) newEnv; + for (auto& i : env) + if (keepVars.count(i.first)) newEnv.emplace(i); + env = newEnv; + // NixOS hack: prevent /etc/bashrc from sourcing /etc/profile. + env["__ETC_PROFILE_SOURCED"] = "1"; + } + + env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = + env["TEMP"] = tmp; + env["NIX_STORE"] = store->storeDir; + env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); + + auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile", "")); + + bool keepTmp = false; + int fileNr = 0; + + for (auto& var : drv.env) + if (passAsFile.count(var.first)) { + keepTmp = true; + string fn = ".attr-" + std::to_string(fileNr++); + Path p = (Path)tmpDir + "/" + fn; + writeFile(p, var.second); + env[var.first + "Path"] = p; + } else + env[var.first] = var.second; + + restoreAffinity(); + + /* Run a shell using the derivation's environment. For + convenience, source $stdenv/setup to setup additional + environment variables and shell functions. Also don't + lose the current $PATH directories. */ + auto rcfile = (Path)tmpDir + "/rc"; + writeFile( + rcfile, + fmt((keepTmp ? "" : "rm -rf '%1%'; "s) + "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; " "%2%" "dontAddDisableDepTrack=1; " @@ -421,87 +432,81 @@ static void _main(int argc, char * * argv) "SHELL=%5%; " "set +e; " R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" - "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " + "if [ \"$(type -t runHook)\" = function ]; then runHook " + "shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " "unset TZ; %6%" "%7%", - (Path) tmpDir, - (pure ? "" : "p=$PATH; "), - (pure ? "" : "PATH=$PATH:$p; unset p; "), - dirOf(shell), - shell, - (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""), - envCommand)); + (Path)tmpDir, (pure ? "" : "p=$PATH; "), + (pure ? "" : "PATH=$PATH:$p; unset p; "), dirOf(shell), shell, + (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") + : ""), + envCommand)); - Strings envStrs; - for (auto & i : env) - envStrs.push_back(i.first + "=" + i.second); + Strings envStrs; + for (auto& i : env) envStrs.push_back(i.first + "=" + i.second); - auto args = interactive - ? Strings{"bash", "--rcfile", rcfile} - : Strings{"bash", rcfile}; + auto args = interactive ? Strings{"bash", "--rcfile", rcfile} + : Strings{"bash", rcfile}; - auto envPtrs = stringsToCharPtrs(envStrs); + auto envPtrs = stringsToCharPtrs(envStrs); - environ = envPtrs.data(); + environ = envPtrs.data(); - auto argPtrs = stringsToCharPtrs(args); + auto argPtrs = stringsToCharPtrs(args); - restoreSignals(); + restoreSignals(); - execvp(shell.c_str(), argPtrs.data()); + execvp(shell.c_str(), argPtrs.data()); - throw SysError("executing shell '%s'", shell); - } + throw SysError("executing shell '%s'", shell); + } - else { + else { + PathSet pathsToBuild; - PathSet pathsToBuild; + std::map<Path, Path> drvPrefixes; + std::map<Path, Path> resultSymlinks; + std::vector<Path> outPaths; - std::map<Path, Path> drvPrefixes; - std::map<Path, Path> resultSymlinks; - std::vector<Path> outPaths; + for (auto& drvInfo : drvs) { + auto drvPath = drvInfo.queryDrvPath(); + auto outPath = drvInfo.queryOutPath(); - for (auto & drvInfo : drvs) { - auto drvPath = drvInfo.queryDrvPath(); - auto outPath = drvInfo.queryOutPath(); + auto outputName = drvInfo.queryOutputName(); + if (outputName == "") + throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath); - auto outputName = drvInfo.queryOutputName(); - if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath); + pathsToBuild.insert(drvPath + "!" + outputName); - pathsToBuild.insert(drvPath + "!" + outputName); + std::string drvPrefix; + auto i = drvPrefixes.find(drvPath); + if (i != drvPrefixes.end()) + drvPrefix = i->second; + else { + drvPrefix = outLink; + if (drvPrefixes.size()) drvPrefix += fmt("-%d", drvPrefixes.size() + 1); + drvPrefixes[drvPath] = drvPrefix; + } - std::string drvPrefix; - auto i = drvPrefixes.find(drvPath); - if (i != drvPrefixes.end()) - drvPrefix = i->second; - else { - drvPrefix = outLink; - if (drvPrefixes.size()) - drvPrefix += fmt("-%d", drvPrefixes.size() + 1); - drvPrefixes[drvPath] = drvPrefix; - } + std::string symlink = drvPrefix; + if (outputName != "out") symlink += "-" + outputName; - std::string symlink = drvPrefix; - if (outputName != "out") symlink += "-" + outputName; - - resultSymlinks[symlink] = outPath; - outPaths.push_back(outPath); - } + resultSymlinks[symlink] = outPath; + outPaths.push_back(outPath); + } - buildPaths(pathsToBuild); + buildPaths(pathsToBuild); - if (dryRun) return; + if (dryRun) return; - for (auto & symlink : resultSymlinks) - if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) - store2->addPermRoot(symlink.second, absPath(symlink.first), true); + for (auto& symlink : resultSymlinks) + if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) + store2->addPermRoot(symlink.second, absPath(symlink.first), true); - for (auto & path : outPaths) - std::cout << path << '\n'; - } + for (auto& path : outPaths) std::cout << path << '\n'; + } } static RegisterLegacyCommand s1("nix-build", _main); diff --git a/third_party/nix/src/nix-channel/nix-channel.cc b/third_party/nix/src/nix-channel/nix-channel.cc index 70aa5c96669d..eda8d827ae30 100755..100644 --- a/third_party/nix/src/nix-channel/nix-channel.cc +++ b/third_party/nix/src/nix-channel/nix-channel.cc @@ -1,248 +1,250 @@ -#include "shared.hh" -#include "globals.hh" -#include "download.hh" -#include "store-api.hh" -#include "legacy.hh" - #include <fcntl.h> -#include <regex> #include <pwd.h> +#include <regex> +#include "download.hh" +#include "globals.hh" +#include "legacy.hh" +#include "shared.hh" +#include "store-api.hh" using namespace nix; -typedef std::map<string,string> Channels; +typedef std::map<string, string> Channels; static Channels channels; static Path channelsList; // Reads the list of channels. -static void readChannels() -{ - if (!pathExists(channelsList)) return; - auto channelsFile = readFile(channelsList); - - for (const auto & line : tokenizeString<std::vector<string>>(channelsFile, "\n")) { - chomp(line); - if (std::regex_search(line, std::regex("^\\s*\\#"))) - continue; - auto split = tokenizeString<std::vector<string>>(line, " "); - auto url = std::regex_replace(split[0], std::regex("/*$"), ""); - auto name = split.size() > 1 ? split[1] : baseNameOf(url); - channels[name] = url; - } +static void readChannels() { + if (!pathExists(channelsList)) return; + auto channelsFile = readFile(channelsList); + + for (const auto& line : + tokenizeString<std::vector<string>>(channelsFile, "\n")) { + chomp(line); + if (std::regex_search(line, std::regex("^\\s*\\#"))) continue; + auto split = tokenizeString<std::vector<string>>(line, " "); + auto url = std::regex_replace(split[0], std::regex("/*$"), ""); + auto name = split.size() > 1 ? split[1] : baseNameOf(url); + channels[name] = url; + } } // Writes the list of channels. -static void writeChannels() -{ - auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; - if (!channelsFD) - throw SysError(format("opening '%1%' for writing") % channelsList); - for (const auto & channel : channels) - writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); +static void writeChannels() { + auto channelsFD = AutoCloseFD{open( + channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; + if (!channelsFD) + throw SysError(format("opening '%1%' for writing") % channelsList); + for (const auto& channel : channels) + writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); } // Adds a channel. -static void addChannel(const string & url, const string & name) -{ - if (!regex_search(url, std::regex("^(file|http|https)://"))) - throw Error(format("invalid channel URL '%1%'") % url); - if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$"))) - throw Error(format("invalid channel identifier '%1%'") % name); - readChannels(); - channels[name] = url; - writeChannels(); +static void addChannel(const string& url, const string& name) { + if (!regex_search(url, std::regex("^(file|http|https)://"))) + throw Error(format("invalid channel URL '%1%'") % url); + if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$"))) + throw Error(format("invalid channel identifier '%1%'") % name); + readChannels(); + channels[name] = url; + writeChannels(); } static Path profile; // Remove a channel. -static void removeChannel(const string & name) -{ - readChannels(); - channels.erase(name); - writeChannels(); +static void removeChannel(const string& name) { + readChannels(); + channels.erase(name); + writeChannels(); - runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name }); + runProgram(settings.nixBinDir + "/nix-env", true, + {"--profile", profile, "--uninstall", name}); } static Path nixDefExpr; // Fetch Nix expressions and binary cache URLs from the subscribed channels. -static void update(const StringSet & channelNames) -{ - readChannels(); - - auto store = openStore(); - - // Download each channel. - Strings exprs; - for (const auto & channel : channels) { - auto name = channel.first; - auto url = channel.second; - if (!(channelNames.empty() || channelNames.count(name))) - continue; - - // We want to download the url to a file to see if it's a tarball while also checking if we - // got redirected in the process, so that we can grab the various parts of a nix channel - // definition from a consistent location if the redirect changes mid-download. - CachedDownloadRequest request(url); - request.ttl = 0; - auto dl = getDownloader(); - auto result = dl->downloadCached(store, request); - auto filename = result.path; - url = chomp(result.effectiveUri); - - // If the URL contains a version number, append it to the name - // attribute (so that "nix-env -q" on the channels profile - // shows something useful). - auto cname = name; - std::smatch match; - auto urlBase = baseNameOf(url); - if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) { - cname = cname + (string) match[1]; - } - - std::string extraAttrs; - - bool unpacked = false; - if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> " - "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); - unpacked = true; - } - - if (!unpacked) { - // Download the channel tarball. - try { - filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; - } catch (DownloadError & e) { - filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; - } - chomp(filename); - } +static void update(const StringSet& channelNames) { + readChannels(); + + auto store = openStore(); + + // Download each channel. + Strings exprs; + for (const auto& channel : channels) { + auto name = channel.first; + auto url = channel.second; + if (!(channelNames.empty() || channelNames.count(name))) continue; + + // We want to download the url to a file to see if it's a tarball while also + // checking if we got redirected in the process, so that we can grab the + // various parts of a nix channel definition from a consistent location if + // the redirect changes mid-download. + CachedDownloadRequest request(url); + request.ttl = 0; + auto dl = getDownloader(); + auto result = dl->downloadCached(store, request); + auto filename = result.path; + url = chomp(result.effectiveUri); + + // If the URL contains a version number, append it to the name + // attribute (so that "nix-env -q" on the channels profile + // shows something useful). + auto cname = name; + std::smatch match; + auto urlBase = baseNameOf(url); + if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) { + cname = cname + (string)match[1]; + } - // Regardless of where it came from, add the expression representing this channel to accumulated expression - exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }"); + std::string extraAttrs; + + bool unpacked = false; + if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { + runProgram(settings.nixBinDir + "/nix-build", false, + {"--no-out-link", "--expr", + "import <nix/unpack-channel.nix> " + "{ name = \"" + + cname + "\"; channelName = \"" + name + + "\"; src = builtins.storePath \"" + filename + "\"; }"}); + unpacked = true; } - // Unpack the channel tarballs into the Nix store and install them - // into the channels profile. - std::cerr << "unpacking channels...\n"; - Strings envArgs{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" }; - for (auto & expr : exprs) - envArgs.push_back(std::move(expr)); - envArgs.push_back("--quiet"); - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); - - // Make the channels appear in nix-env. - struct stat st; - if (lstat(nixDefExpr.c_str(), &st) == 0) { - if (S_ISLNK(st.st_mode)) - // old-skool ~/.nix-defexpr - if (unlink(nixDefExpr.c_str()) == -1) - throw SysError(format("unlinking %1%") % nixDefExpr); - } else if (errno != ENOENT) { - throw SysError(format("getting status of %1%") % nixDefExpr); + if (!unpacked) { + // Download the channel tarball. + try { + filename = dl->downloadCached( + store, CachedDownloadRequest(url + "/nixexprs.tar.xz")) + .path; + } catch (DownloadError& e) { + filename = + dl->downloadCached(store, + CachedDownloadRequest(url + "/nixexprs.tar.bz2")) + .path; + } + chomp(filename); } - createDirs(nixDefExpr); - auto channelLink = nixDefExpr + "/channels"; - replaceSymlink(profile, channelLink); + + // Regardless of where it came from, add the expression representing this + // channel to accumulated expression + exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + + "\"; src = builtins.storePath \"" + filename + "\"; " + + extraAttrs + " }"); + } + + // Unpack the channel tarballs into the Nix store and install them + // into the channels profile. + std::cerr << "unpacking channels...\n"; + Strings envArgs{"--profile", profile, + "--file", "<nix/unpack-channel.nix>", + "--install", "--from-expression"}; + for (auto& expr : exprs) envArgs.push_back(std::move(expr)); + envArgs.push_back("--quiet"); + runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + + // Make the channels appear in nix-env. + struct stat st; + if (lstat(nixDefExpr.c_str(), &st) == 0) { + if (S_ISLNK(st.st_mode)) + // old-skool ~/.nix-defexpr + if (unlink(nixDefExpr.c_str()) == -1) + throw SysError(format("unlinking %1%") % nixDefExpr); + } else if (errno != ENOENT) { + throw SysError(format("getting status of %1%") % nixDefExpr); + } + createDirs(nixDefExpr); + auto channelLink = nixDefExpr + "/channels"; + replaceSymlink(profile, channelLink); } -static int _main(int argc, char ** argv) -{ - { - // Figure out the name of the `.nix-channels' file to use - auto home = getHome(); - channelsList = home + "/.nix-channels"; - nixDefExpr = home + "/.nix-defexpr"; - - // Figure out the name of the channels profile. - profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); - - enum { - cNone, - cAdd, - cRemove, - cList, - cUpdate, - cRollback - } cmd = cNone; - std::vector<string> args; - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") { - showManPage("nix-channel"); - } else if (*arg == "--version") { - printVersion("nix-channel"); - } else if (*arg == "--add") { - cmd = cAdd; - } else if (*arg == "--remove") { - cmd = cRemove; - } else if (*arg == "--list") { - cmd = cList; - } else if (*arg == "--update") { - cmd = cUpdate; - } else if (*arg == "--rollback") { - cmd = cRollback; - } else { - args.push_back(std::move(*arg)); - } - return true; - }); - - initPlugins(); - - switch (cmd) { - case cNone: - throw UsageError("no command specified"); - case cAdd: - if (args.size() < 1 || args.size() > 2) - throw UsageError("'--add' requires one or two arguments"); - { - auto url = args[0]; - std::string name; - if (args.size() == 2) { - name = args[1]; - } else { - name = baseNameOf(url); - name = std::regex_replace(name, std::regex("-unstable$"), ""); - name = std::regex_replace(name, std::regex("-stable$"), ""); - } - addChannel(url, name); - } - break; - case cRemove: - if (args.size() != 1) - throw UsageError("'--remove' requires one argument"); - removeChannel(args[0]); - break; - case cList: - if (!args.empty()) - throw UsageError("'--list' expects no arguments"); - readChannels(); - for (const auto & channel : channels) - std::cout << channel.first << ' ' << channel.second << '\n'; - break; - case cUpdate: - update(StringSet(args.begin(), args.end())); - break; - case cRollback: - if (args.size() > 1) - throw UsageError("'--rollback' has at most one argument"); - Strings envArgs{"--profile", profile}; - if (args.size() == 1) { - envArgs.push_back("--switch-generation"); - envArgs.push_back(args[0]); - } else { - envArgs.push_back("--rollback"); - } - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); - break; +static int _main(int argc, char** argv) { + { + // Figure out the name of the `.nix-channels' file to use + auto home = getHome(); + channelsList = home + "/.nix-channels"; + nixDefExpr = home + "/.nix-defexpr"; + + // Figure out the name of the channels profile. + profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, + getUserName()); + + enum { cNone, cAdd, cRemove, cList, cUpdate, cRollback } cmd = cNone; + std::vector<string> args; + parseCmdLine(argc, argv, + [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") { + showManPage("nix-channel"); + } else if (*arg == "--version") { + printVersion("nix-channel"); + } else if (*arg == "--add") { + cmd = cAdd; + } else if (*arg == "--remove") { + cmd = cRemove; + } else if (*arg == "--list") { + cmd = cList; + } else if (*arg == "--update") { + cmd = cUpdate; + } else if (*arg == "--rollback") { + cmd = cRollback; + } else { + args.push_back(std::move(*arg)); + } + return true; + }); + + initPlugins(); + + switch (cmd) { + case cNone: + throw UsageError("no command specified"); + case cAdd: + if (args.size() < 1 || args.size() > 2) + throw UsageError("'--add' requires one or two arguments"); + { + auto url = args[0]; + std::string name; + if (args.size() == 2) { + name = args[1]; + } else { + name = baseNameOf(url); + name = std::regex_replace(name, std::regex("-unstable$"), ""); + name = std::regex_replace(name, std::regex("-stable$"), ""); + } + addChannel(url, name); } - - return 0; + break; + case cRemove: + if (args.size() != 1) + throw UsageError("'--remove' requires one argument"); + removeChannel(args[0]); + break; + case cList: + if (!args.empty()) throw UsageError("'--list' expects no arguments"); + readChannels(); + for (const auto& channel : channels) + std::cout << channel.first << ' ' << channel.second << '\n'; + break; + case cUpdate: + update(StringSet(args.begin(), args.end())); + break; + case cRollback: + if (args.size() > 1) + throw UsageError("'--rollback' has at most one argument"); + Strings envArgs{"--profile", profile}; + if (args.size() == 1) { + envArgs.push_back("--switch-generation"); + envArgs.push_back(args[0]); + } else { + envArgs.push_back("--rollback"); + } + runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + break; } + + return 0; + } } static RegisterLegacyCommand s1("nix-channel", _main); diff --git a/third_party/nix/src/nix-collect-garbage/nix-collect-garbage.cc b/third_party/nix/src/nix-collect-garbage/nix-collect-garbage.cc index d4060ac937fc..9c65b0877375 100644 --- a/third_party/nix/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/third_party/nix/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,97 +1,94 @@ -#include "store-api.hh" -#include "profiles.hh" -#include "shared.hh" +#include <cerrno> +#include <iostream> #include "globals.hh" #include "legacy.hh" - -#include <iostream> -#include <cerrno> +#include "profiles.hh" +#include "shared.hh" +#include "store-api.hh" using namespace nix; std::string deleteOlderThan; bool dryRun = false; - /* If `-d' was specified, remove all old generations of all profiles. * Of course, this makes rollbacks to before this point in time * impossible. */ -void removeOldGenerations(std::string dir) -{ - if (access(dir.c_str(), R_OK) != 0) return; - - bool canWrite = access(dir.c_str(), W_OK) == 0; - - for (auto & i : readDirectory(dir)) { - checkInterrupt(); - - auto path = dir + "/" + i.name; - auto type = i.type == DT_UNKNOWN ? getFileType(path) : i.type; - - if (type == DT_LNK && canWrite) { - std::string link; - try { - link = readLink(path); - } catch (SysError & e) { - if (e.errNo == ENOENT) continue; - } - if (link.find("link") != string::npos) { - printInfo(format("removing old generations of profile %1%") % path); - if (deleteOlderThan != "") - deleteGenerationsOlderThan(path, deleteOlderThan, dryRun); - else - deleteOldGenerations(path, dryRun); - } - } else if (type == DT_DIR) { - removeOldGenerations(path); - } +void removeOldGenerations(std::string dir) { + if (access(dir.c_str(), R_OK) != 0) return; + + bool canWrite = access(dir.c_str(), W_OK) == 0; + + for (auto& i : readDirectory(dir)) { + checkInterrupt(); + + auto path = dir + "/" + i.name; + auto type = i.type == DT_UNKNOWN ? getFileType(path) : i.type; + + if (type == DT_LNK && canWrite) { + std::string link; + try { + link = readLink(path); + } catch (SysError& e) { + if (e.errNo == ENOENT) continue; + } + if (link.find("link") != string::npos) { + printInfo(format("removing old generations of profile %1%") % path); + if (deleteOlderThan != "") + deleteGenerationsOlderThan(path, deleteOlderThan, dryRun); + else + deleteOldGenerations(path, dryRun); + } + } else if (type == DT_DIR) { + removeOldGenerations(path); } + } } -static int _main(int argc, char * * argv) -{ - { - bool removeOld = false; - - GCOptions options; - - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-collect-garbage"); - else if (*arg == "--version") - printVersion("nix-collect-garbage"); - else if (*arg == "--delete-old" || *arg == "-d") removeOld = true; - else if (*arg == "--delete-older-than") { - removeOld = true; - deleteOlderThan = getArg(*arg, arg, end); - } - else if (*arg == "--dry-run") dryRun = true; - else if (*arg == "--max-freed") { - long long maxFreed = getIntArg<long long>(*arg, arg, end, true); - options.maxFreed = maxFreed >= 0 ? maxFreed : 0; - } - else - return false; - return true; +static int _main(int argc, char** argv) { + { + bool removeOld = false; + + GCOptions options; + + parseCmdLine( + argc, argv, [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") + showManPage("nix-collect-garbage"); + else if (*arg == "--version") + printVersion("nix-collect-garbage"); + else if (*arg == "--delete-old" || *arg == "-d") + removeOld = true; + else if (*arg == "--delete-older-than") { + removeOld = true; + deleteOlderThan = getArg(*arg, arg, end); + } else if (*arg == "--dry-run") + dryRun = true; + else if (*arg == "--max-freed") { + long long maxFreed = getIntArg<long long>(*arg, arg, end, true); + options.maxFreed = maxFreed >= 0 ? maxFreed : 0; + } else + return false; + return true; }); - initPlugins(); - - auto profilesDir = settings.nixStateDir + "/profiles"; - if (removeOld) removeOldGenerations(profilesDir); + initPlugins(); - // Run the actual garbage collector. - if (!dryRun) { - auto store = openStore(); - options.action = GCOptions::gcDeleteDead; - GCResults results; - PrintFreed freed(true, results); - store->collectGarbage(options, results); - } + auto profilesDir = settings.nixStateDir + "/profiles"; + if (removeOld) removeOldGenerations(profilesDir); - return 0; + // Run the actual garbage collector. + if (!dryRun) { + auto store = openStore(); + options.action = GCOptions::gcDeleteDead; + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); } + + return 0; + } } static RegisterLegacyCommand s1("nix-collect-garbage", _main); diff --git a/third_party/nix/src/nix-copy-closure/nix-copy-closure.cc b/third_party/nix/src/nix-copy-closure/nix-copy-closure.cc index fdcde8b076b5..3cc29ae337ff 100755..100644 --- a/third_party/nix/src/nix-copy-closure/nix-copy-closure.cc +++ b/third_party/nix/src/nix-copy-closure/nix-copy-closure.cc @@ -1,68 +1,69 @@ +#include "legacy.hh" #include "shared.hh" #include "store-api.hh" -#include "legacy.hh" using namespace nix; -static int _main(int argc, char ** argv) -{ - { - auto gzip = false; - auto toMode = true; - auto includeOutputs = false; - auto dryRun = false; - auto useSubstitutes = NoSubstitute; - std::string sshHost; - PathSet storePaths; +static int _main(int argc, char** argv) { + { + auto gzip = false; + auto toMode = true; + auto includeOutputs = false; + auto dryRun = false; + auto useSubstitutes = NoSubstitute; + std::string sshHost; + PathSet storePaths; - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-copy-closure"); - else if (*arg == "--version") - printVersion("nix-copy-closure"); - else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") { - if (*arg != "--gzip") - printMsg(lvlError, format("Warning: '%1%' is not implemented, falling back to gzip") % *arg); - gzip = true; - } else if (*arg == "--from") - toMode = false; - else if (*arg == "--to") - toMode = true; - else if (*arg == "--include-outputs") - includeOutputs = true; - else if (*arg == "--show-progress") - printMsg(lvlError, "Warning: '--show-progress' is not implemented"); - else if (*arg == "--dry-run") - dryRun = true; - else if (*arg == "--use-substitutes" || *arg == "-s") - useSubstitutes = Substitute; - else if (sshHost.empty()) - sshHost = *arg; - else - storePaths.insert(*arg); - return true; + parseCmdLine( + argc, argv, [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") + showManPage("nix-copy-closure"); + else if (*arg == "--version") + printVersion("nix-copy-closure"); + else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") { + if (*arg != "--gzip") + printMsg(lvlError, format("Warning: '%1%' is not implemented, " + "falling back to gzip") % + *arg); + gzip = true; + } else if (*arg == "--from") + toMode = false; + else if (*arg == "--to") + toMode = true; + else if (*arg == "--include-outputs") + includeOutputs = true; + else if (*arg == "--show-progress") + printMsg(lvlError, "Warning: '--show-progress' is not implemented"); + else if (*arg == "--dry-run") + dryRun = true; + else if (*arg == "--use-substitutes" || *arg == "-s") + useSubstitutes = Substitute; + else if (sshHost.empty()) + sshHost = *arg; + else + storePaths.insert(*arg); + return true; }); - initPlugins(); + initPlugins(); - if (sshHost.empty()) - throw UsageError("no host name specified"); + if (sshHost.empty()) throw UsageError("no host name specified"); - auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : ""); - auto to = toMode ? openStore(remoteUri) : openStore(); - auto from = toMode ? openStore() : openStore(remoteUri); + auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : ""); + auto to = toMode ? openStore(remoteUri) : openStore(); + auto from = toMode ? openStore() : openStore(remoteUri); - PathSet storePaths2; - for (auto & path : storePaths) - storePaths2.insert(from->followLinksToStorePath(path)); + PathSet storePaths2; + for (auto& path : storePaths) + storePaths2.insert(from->followLinksToStorePath(path)); - PathSet closure; - from->computeFSClosure(storePaths2, closure, false, includeOutputs); + PathSet closure; + from->computeFSClosure(storePaths2, closure, false, includeOutputs); - copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); + copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); - return 0; - } + return 0; + } } static RegisterLegacyCommand s1("nix-copy-closure", _main); diff --git a/third_party/nix/src/nix-daemon/nix-daemon.cc b/third_party/nix/src/nix-daemon/nix-daemon.cc index cd18489b0cdb..799a4aadddcf 100644 --- a/third_party/nix/src/nix-daemon/nix-daemon.cc +++ b/third_party/nix/src/nix-daemon/nix-daemon.cc @@ -1,31 +1,29 @@ -#include "shared.hh" -#include "local-store.hh" -#include "util.hh" -#include "serialise.hh" -#include "worker-protocol.hh" -#include "archive.hh" +#include <errno.h> +#include <fcntl.h> +#include <grp.h> +#include <limits.h> +#include <pwd.h> +#include <signal.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/un.h> +#include <sys/wait.h> +#include <unistd.h> +#include <algorithm> +#include <cstring> #include "affinity.hh" -#include "globals.hh" -#include "monitor-fd.hh" +#include "archive.hh" #include "derivations.hh" #include "finally.hh" +#include "globals.hh" #include "legacy.hh" - -#include <algorithm> - -#include <cstring> -#include <unistd.h> -#include <signal.h> -#include <sys/types.h> -#include <sys/wait.h> -#include <sys/stat.h> -#include <sys/socket.h> -#include <sys/un.h> -#include <errno.h> -#include <pwd.h> -#include <grp.h> -#include <fcntl.h> -#include <limits.h> +#include "local-store.hh" +#include "monitor-fd.hh" +#include "serialise.hh" +#include "shared.hh" +#include "util.hh" +#include "worker-protocol.hh" #if __APPLE__ || __FreeBSD__ #include <sys/ucred.h> @@ -35,751 +33,718 @@ using namespace nix; #ifndef __linux__ #define SPLICE_F_MOVE 0 -static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) -{ - /* We ignore most parameters, we just have them for conformance with the linux syscall */ - std::vector<char> buf(8192); - auto read_count = read(fd_in, buf.data(), buf.size()); - if (read_count == -1) - return read_count; - auto write_count = decltype(read_count)(0); - while (write_count < read_count) { - auto res = write(fd_out, buf.data() + write_count, read_count - write_count); - if (res == -1) - return res; - write_count += res; - } - return read_count; +static ssize_t splice(int fd_in, void* off_in, int fd_out, void* off_out, + size_t len, unsigned int flags) { + /* We ignore most parameters, we just have them for conformance with the linux + * syscall */ + std::vector<char> buf(8192); + auto read_count = read(fd_in, buf.data(), buf.size()); + if (read_count == -1) return read_count; + auto write_count = decltype(read_count)(0); + while (write_count < read_count) { + auto res = + write(fd_out, buf.data() + write_count, read_count - write_count); + if (res == -1) return res; + write_count += res; + } + return read_count; } #endif static FdSource from(STDIN_FILENO); static FdSink to(STDOUT_FILENO); - -Sink & operator << (Sink & sink, const Logger::Fields & fields) -{ - sink << fields.size(); - for (auto & f : fields) { - sink << f.type; - if (f.type == Logger::Field::tInt) - sink << f.i; - else if (f.type == Logger::Field::tString) - sink << f.s; - else abort(); - } - return sink; +Sink& operator<<(Sink& sink, const Logger::Fields& fields) { + sink << fields.size(); + for (auto& f : fields) { + sink << f.type; + if (f.type == Logger::Field::tInt) + sink << f.i; + else if (f.type == Logger::Field::tString) + sink << f.s; + else + abort(); + } + return sink; } - /* Logger that forwards log messages to the client, *if* we're in a state where the protocol allows it (i.e., when canSendStderr is true). */ -struct TunnelLogger : public Logger -{ - struct State - { - bool canSendStderr = false; - std::vector<std::string> pendingMsgs; - }; +struct TunnelLogger : public Logger { + struct State { + bool canSendStderr = false; + std::vector<std::string> pendingMsgs; + }; - Sync<State> state_; + Sync<State> state_; - unsigned int clientVersion; + unsigned int clientVersion; - TunnelLogger(unsigned int clientVersion) : clientVersion(clientVersion) { } - - void enqueueMsg(const std::string & s) - { - auto state(state_.lock()); - - if (state->canSendStderr) { - assert(state->pendingMsgs.empty()); - try { - to(s); - to.flush(); - } catch (...) { - /* Write failed; that means that the other side is - gone. */ - state->canSendStderr = false; - throw; - } - } else - state->pendingMsgs.push_back(s); - } + TunnelLogger(unsigned int clientVersion) : clientVersion(clientVersion) {} - void log(Verbosity lvl, const FormatOrString & fs) override - { - if (lvl > verbosity) return; + void enqueueMsg(const std::string& s) { + auto state(state_.lock()); - StringSink buf; - buf << STDERR_NEXT << (fs.s + "\n"); - enqueueMsg(*buf.s); - } + if (state->canSendStderr) { + assert(state->pendingMsgs.empty()); + try { + to(s); + to.flush(); + } catch (...) { + /* Write failed; that means that the other side is + gone. */ + state->canSendStderr = false; + throw; + } + } else + state->pendingMsgs.push_back(s); + } - /* startWork() means that we're starting an operation for which we - want to send out stderr to the client. */ - void startWork() - { - auto state(state_.lock()); - state->canSendStderr = true; + void log(Verbosity lvl, const FormatOrString& fs) override { + if (lvl > verbosity) return; - for (auto & msg : state->pendingMsgs) - to(msg); + StringSink buf; + buf << STDERR_NEXT << (fs.s + "\n"); + enqueueMsg(*buf.s); + } - state->pendingMsgs.clear(); + /* startWork() means that we're starting an operation for which we + want to send out stderr to the client. */ + void startWork() { + auto state(state_.lock()); + state->canSendStderr = true; - to.flush(); - } + for (auto& msg : state->pendingMsgs) to(msg); - /* stopWork() means that we're done; stop sending stderr to the - client. */ - void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) - { - auto state(state_.lock()); + state->pendingMsgs.clear(); - state->canSendStderr = false; + to.flush(); + } - if (success) - to << STDERR_LAST; - else { - to << STDERR_ERROR << msg; - if (status != 0) to << status; - } - } + /* stopWork() means that we're done; stop sending stderr to the + client. */ + void stopWork(bool success = true, const string& msg = "", + unsigned int status = 0) { + auto state(state_.lock()); - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, - const std::string & s, const Fields & fields, ActivityId parent) override - { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) { - if (!s.empty()) - log(lvl, s + "..."); - return; - } + state->canSendStderr = false; - StringSink buf; - buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent; - enqueueMsg(*buf.s); + if (success) + to << STDERR_LAST; + else { + to << STDERR_ERROR << msg; + if (status != 0) to << status; } - - void stopActivity(ActivityId act) override - { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; - StringSink buf; - buf << STDERR_STOP_ACTIVITY << act; - enqueueMsg(*buf.s); + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string& s, const Fields& fields, + ActivityId parent) override { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) { + if (!s.empty()) log(lvl, s + "..."); + return; } - void result(ActivityId act, ResultType type, const Fields & fields) override - { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; - StringSink buf; - buf << STDERR_RESULT << act << type << fields; - enqueueMsg(*buf.s); - } + StringSink buf; + buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent; + enqueueMsg(*buf.s); + } + + void stopActivity(ActivityId act) override { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; + StringSink buf; + buf << STDERR_STOP_ACTIVITY << act; + enqueueMsg(*buf.s); + } + + void result(ActivityId act, ResultType type, const Fields& fields) override { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; + StringSink buf; + buf << STDERR_RESULT << act << type << fields; + enqueueMsg(*buf.s); + } }; - -struct TunnelSink : Sink -{ - Sink & to; - TunnelSink(Sink & to) : to(to) { } - virtual void operator () (const unsigned char * data, size_t len) - { - to << STDERR_WRITE; - writeString(data, len, to); - } +struct TunnelSink : Sink { + Sink& to; + TunnelSink(Sink& to) : to(to) {} + virtual void operator()(const unsigned char* data, size_t len) { + to << STDERR_WRITE; + writeString(data, len, to); + } }; +struct TunnelSource : BufferedSource { + Source& from; + TunnelSource(Source& from) : from(from) {} -struct TunnelSource : BufferedSource -{ - Source & from; - TunnelSource(Source & from) : from(from) { } -protected: - size_t readUnbuffered(unsigned char * data, size_t len) override - { - to << STDERR_READ << len; - to.flush(); - size_t n = readString(data, len, from); - if (n == 0) throw EndOfFile("unexpected end-of-file"); - return n; - } + protected: + size_t readUnbuffered(unsigned char* data, size_t len) override { + to << STDERR_READ << len; + to.flush(); + size_t n = readString(data, len, from); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; + } }; - /* If the NAR archive contains a single file at top-level, then save the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular; - string s; +struct RetrieveRegularNARSink : ParseSink { + bool regular; + string s; - RetrieveRegularNARSink() : regular(true) { } + RetrieveRegularNARSink() : regular(true) {} - void createDirectory(const Path & path) - { - regular = false; - } + void createDirectory(const Path& path) { regular = false; } - void receiveContents(unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } + void receiveContents(unsigned char* data, unsigned int len) { + s.append((const char*)data, len); + } - void createSymlink(const Path & path, const string & target) - { - regular = false; - } + void createSymlink(const Path& path, const string& target) { + regular = false; + } }; - -static void performOp(TunnelLogger * logger, ref<Store> store, - bool trusted, unsigned int clientVersion, - Source & from, Sink & to, unsigned int op) -{ - switch (op) { - +static void performOp(TunnelLogger* logger, ref<Store> store, bool trusted, + unsigned int clientVersion, Source& from, Sink& to, + unsigned int op) { + switch (op) { case wopIsValidPath: { - /* 'readStorePath' could raise an error leading to the connection - being closed. To be able to recover from an invalid path error, - call 'startWork' early, and do 'assertStorePath' afterwards so - that the 'Error' exception handler doesn't close the - connection. */ - Path path = readString(from); - logger->startWork(); - store->assertStorePath(path); - bool result = store->isValidPath(path); - logger->stopWork(); - to << result; - break; + /* 'readStorePath' could raise an error leading to the connection + being closed. To be able to recover from an invalid path error, + call 'startWork' early, and do 'assertStorePath' afterwards so + that the 'Error' exception handler doesn't close the + connection. */ + Path path = readString(from); + logger->startWork(); + store->assertStorePath(path); + bool result = store->isValidPath(path); + logger->stopWork(); + to << result; + break; } case wopQueryValidPaths: { - PathSet paths = readStorePaths<PathSet>(*store, from); - logger->startWork(); - PathSet res = store->queryValidPaths(paths); - logger->stopWork(); - to << res; - break; + PathSet paths = readStorePaths<PathSet>(*store, from); + logger->startWork(); + PathSet res = store->queryValidPaths(paths); + logger->stopWork(); + to << res; + break; } case wopHasSubstitutes: { - Path path = readStorePath(*store, from); - logger->startWork(); - PathSet res = store->querySubstitutablePaths({path}); - logger->stopWork(); - to << (res.find(path) != res.end()); - break; + Path path = readStorePath(*store, from); + logger->startWork(); + PathSet res = store->querySubstitutablePaths({path}); + logger->stopWork(); + to << (res.find(path) != res.end()); + break; } case wopQuerySubstitutablePaths: { - PathSet paths = readStorePaths<PathSet>(*store, from); - logger->startWork(); - PathSet res = store->querySubstitutablePaths(paths); - logger->stopWork(); - to << res; - break; + PathSet paths = readStorePaths<PathSet>(*store, from); + logger->startWork(); + PathSet res = store->querySubstitutablePaths(paths); + logger->stopWork(); + to << res; + break; } case wopQueryPathHash: { - Path path = readStorePath(*store, from); - logger->startWork(); - auto hash = store->queryPathInfo(path)->narHash; - logger->stopWork(); - to << hash.to_string(Base16, false); - break; + Path path = readStorePath(*store, from); + logger->startWork(); + auto hash = store->queryPathInfo(path)->narHash; + logger->stopWork(); + to << hash.to_string(Base16, false); + break; } case wopQueryReferences: case wopQueryReferrers: case wopQueryValidDerivers: case wopQueryDerivationOutputs: { - Path path = readStorePath(*store, from); - logger->startWork(); - PathSet paths; - if (op == wopQueryReferences) - paths = store->queryPathInfo(path)->references; - else if (op == wopQueryReferrers) - store->queryReferrers(path, paths); - else if (op == wopQueryValidDerivers) - paths = store->queryValidDerivers(path); - else paths = store->queryDerivationOutputs(path); - logger->stopWork(); - to << paths; - break; + Path path = readStorePath(*store, from); + logger->startWork(); + PathSet paths; + if (op == wopQueryReferences) + paths = store->queryPathInfo(path)->references; + else if (op == wopQueryReferrers) + store->queryReferrers(path, paths); + else if (op == wopQueryValidDerivers) + paths = store->queryValidDerivers(path); + else + paths = store->queryDerivationOutputs(path); + logger->stopWork(); + to << paths; + break; } case wopQueryDerivationOutputNames: { - Path path = readStorePath(*store, from); - logger->startWork(); - StringSet names; - names = store->queryDerivationOutputNames(path); - logger->stopWork(); - to << names; - break; + Path path = readStorePath(*store, from); + logger->startWork(); + StringSet names; + names = store->queryDerivationOutputNames(path); + logger->stopWork(); + to << names; + break; } case wopQueryDeriver: { - Path path = readStorePath(*store, from); - logger->startWork(); - auto deriver = store->queryPathInfo(path)->deriver; - logger->stopWork(); - to << deriver; - break; + Path path = readStorePath(*store, from); + logger->startWork(); + auto deriver = store->queryPathInfo(path)->deriver; + logger->stopWork(); + to << deriver; + break; } case wopQueryPathFromHashPart: { - string hashPart = readString(from); - logger->startWork(); - Path path = store->queryPathFromHashPart(hashPart); - logger->stopWork(); - to << path; - break; + string hashPart = readString(from); + logger->startWork(); + Path path = store->queryPathFromHashPart(hashPart); + logger->stopWork(); + to << path; + break; } case wopAddToStore: { - bool fixed, recursive; - std::string s, baseName; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; - /* Compatibility hack. */ - if (!fixed) { - s = "sha256"; - recursive = true; - } - HashType hashAlgo = parseHashType(s); - - TeeSource savedNAR(from); - RetrieveRegularNARSink savedRegular; - - if (recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNAR); - } else - parseDump(savedRegular, from); - - logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - - auto store2 = store.dynamic_pointer_cast<LocalStore>(); - if (!store2) throw Error("operation is only supported by LocalStore"); - - Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); - logger->stopWork(); - - to << path; - break; + bool fixed, recursive; + std::string s, baseName; + from >> baseName >> fixed /* obsolete */ >> recursive >> s; + /* Compatibility hack. */ + if (!fixed) { + s = "sha256"; + recursive = true; + } + HashType hashAlgo = parseHashType(s); + + TeeSource savedNAR(from); + RetrieveRegularNARSink savedRegular; + + if (recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNAR); + } else + parseDump(savedRegular, from); + + logger->startWork(); + if (!savedRegular.regular) throw Error("regular file expected"); + + auto store2 = store.dynamic_pointer_cast<LocalStore>(); + if (!store2) throw Error("operation is only supported by LocalStore"); + + Path path = store2->addToStoreFromDump( + recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, + hashAlgo); + logger->stopWork(); + + to << path; + break; } case wopAddTextToStore: { - string suffix = readString(from); - string s = readString(from); - PathSet refs = readStorePaths<PathSet>(*store, from); - logger->startWork(); - Path path = store->addTextToStore(suffix, s, refs, NoRepair); - logger->stopWork(); - to << path; - break; + string suffix = readString(from); + string s = readString(from); + PathSet refs = readStorePaths<PathSet>(*store, from); + logger->startWork(); + Path path = store->addTextToStore(suffix, s, refs, NoRepair); + logger->stopWork(); + to << path; + break; } case wopExportPath: { - Path path = readStorePath(*store, from); - readInt(from); // obsolete - logger->startWork(); - TunnelSink sink(to); - store->exportPath(path, sink); - logger->stopWork(); - to << 1; - break; + Path path = readStorePath(*store, from); + readInt(from); // obsolete + logger->startWork(); + TunnelSink sink(to); + store->exportPath(path, sink); + logger->stopWork(); + to << 1; + break; } case wopImportPaths: { - logger->startWork(); - TunnelSource source(from); - Paths paths = store->importPaths(source, nullptr, - trusted ? NoCheckSigs : CheckSigs); - logger->stopWork(); - to << paths; - break; + logger->startWork(); + TunnelSource source(from); + Paths paths = store->importPaths(source, nullptr, + trusted ? NoCheckSigs : CheckSigs); + logger->stopWork(); + to << paths; + break; } case wopBuildPaths: { - PathSet drvs = readStorePaths<PathSet>(*store, from); - BuildMode mode = bmNormal; - if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { - mode = (BuildMode) readInt(from); - - /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ - if (mode == bmRepair && !trusted) - throw Error("repairing is not allowed because you are not in 'trusted-users'"); - } - logger->startWork(); - store->buildPaths(drvs, mode); - logger->stopWork(); - to << 1; - break; + PathSet drvs = readStorePaths<PathSet>(*store, from); + BuildMode mode = bmNormal; + if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { + mode = (BuildMode)readInt(from); + + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ + if (mode == bmRepair && !trusted) + throw Error( + "repairing is not allowed because you are not in " + "'trusted-users'"); + } + logger->startWork(); + store->buildPaths(drvs, mode); + logger->stopWork(); + to << 1; + break; } case wopBuildDerivation: { - Path drvPath = readStorePath(*store, from); - BasicDerivation drv; - readDerivation(from, *store, drv); - BuildMode buildMode = (BuildMode) readInt(from); - logger->startWork(); - if (!trusted) - throw Error("you are not privileged to build derivations"); - auto res = store->buildDerivation(drvPath, drv, buildMode); - logger->stopWork(); - to << res.status << res.errorMsg; - break; + Path drvPath = readStorePath(*store, from); + BasicDerivation drv; + readDerivation(from, *store, drv); + BuildMode buildMode = (BuildMode)readInt(from); + logger->startWork(); + if (!trusted) throw Error("you are not privileged to build derivations"); + auto res = store->buildDerivation(drvPath, drv, buildMode); + logger->stopWork(); + to << res.status << res.errorMsg; + break; } case wopEnsurePath: { - Path path = readStorePath(*store, from); - logger->startWork(); - store->ensurePath(path); - logger->stopWork(); - to << 1; - break; + Path path = readStorePath(*store, from); + logger->startWork(); + store->ensurePath(path); + logger->stopWork(); + to << 1; + break; } case wopAddTempRoot: { - Path path = readStorePath(*store, from); - logger->startWork(); - store->addTempRoot(path); - logger->stopWork(); - to << 1; - break; + Path path = readStorePath(*store, from); + logger->startWork(); + store->addTempRoot(path); + logger->stopWork(); + to << 1; + break; } case wopAddIndirectRoot: { - Path path = absPath(readString(from)); - logger->startWork(); - store->addIndirectRoot(path); - logger->stopWork(); - to << 1; - break; + Path path = absPath(readString(from)); + logger->startWork(); + store->addIndirectRoot(path); + logger->stopWork(); + to << 1; + break; } case wopSyncWithGC: { - logger->startWork(); - store->syncWithGC(); - logger->stopWork(); - to << 1; - break; + logger->startWork(); + store->syncWithGC(); + logger->stopWork(); + to << 1; + break; } case wopFindRoots: { - logger->startWork(); - Roots roots = store->findRoots(!trusted); - logger->stopWork(); + logger->startWork(); + Roots roots = store->findRoots(!trusted); + logger->stopWork(); - size_t size = 0; - for (auto & i : roots) - size += i.second.size(); + size_t size = 0; + for (auto& i : roots) size += i.second.size(); - to << size; + to << size; - for (auto & [target, links] : roots) - for (auto & link : links) - to << link << target; + for (auto& [target, links] : roots) + for (auto& link : links) to << link << target; - break; + break; } case wopCollectGarbage: { - GCOptions options; - options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths<PathSet>(*store, from); - from >> options.ignoreLiveness >> options.maxFreed; - // obsolete fields - readInt(from); - readInt(from); - readInt(from); - - GCResults results; - - logger->startWork(); - if (options.ignoreLiveness) - throw Error("you are not allowed to ignore liveness"); - store->collectGarbage(options, results); - logger->stopWork(); - - to << results.paths << results.bytesFreed << 0 /* obsolete */; - - break; + GCOptions options; + options.action = (GCOptions::GCAction)readInt(from); + options.pathsToDelete = readStorePaths<PathSet>(*store, from); + from >> options.ignoreLiveness >> options.maxFreed; + // obsolete fields + readInt(from); + readInt(from); + readInt(from); + + GCResults results; + + logger->startWork(); + if (options.ignoreLiveness) + throw Error("you are not allowed to ignore liveness"); + store->collectGarbage(options, results); + logger->stopWork(); + + to << results.paths << results.bytesFreed << 0 /* obsolete */; + + break; } case wopSetOptions: { - settings.keepFailed = readInt(from); - settings.keepGoing = readInt(from); - settings.tryFallback = readInt(from); - verbosity = (Verbosity) readInt(from); - settings.maxBuildJobs.assign(readInt(from)); - settings.maxSilentTime = readInt(from); - readInt(from); // obsolete useBuildHook - settings.verboseBuild = lvlError == (Verbosity) readInt(from); - readInt(from); // obsolete logType - readInt(from); // obsolete printBuildTrace - settings.buildCores = readInt(from); - settings.useSubstitutes = readInt(from); - - StringMap overrides; - if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { - unsigned int n = readInt(from); - for (unsigned int i = 0; i < n; i++) { - string name = readString(from); - string value = readString(from); - overrides.emplace(name, value); - } + settings.keepFailed = readInt(from); + settings.keepGoing = readInt(from); + settings.tryFallback = readInt(from); + verbosity = (Verbosity)readInt(from); + settings.maxBuildJobs.assign(readInt(from)); + settings.maxSilentTime = readInt(from); + readInt(from); // obsolete useBuildHook + settings.verboseBuild = lvlError == (Verbosity)readInt(from); + readInt(from); // obsolete logType + readInt(from); // obsolete printBuildTrace + settings.buildCores = readInt(from); + settings.useSubstitutes = readInt(from); + + StringMap overrides; + if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { + unsigned int n = readInt(from); + for (unsigned int i = 0; i < n; i++) { + string name = readString(from); + string value = readString(from); + overrides.emplace(name, value); } + } + + logger->startWork(); + + for (auto& i : overrides) { + auto& name(i.first); + auto& value(i.second); + + auto setSubstituters = [&](Setting<Strings>& res) { + if (name != res.name && res.aliases.count(name) == 0) return false; + StringSet trusted = settings.trustedSubstituters; + for (auto& s : settings.substituters.get()) trusted.insert(s); + Strings subs; + auto ss = tokenizeString<Strings>(value); + for (auto& s : ss) + if (trusted.count(s)) + subs.push_back(s); + else + warn("ignoring untrusted substituter '%s'", s); + res = subs; + return true; + }; - logger->startWork(); - - for (auto & i : overrides) { - auto & name(i.first); - auto & value(i.second); - - auto setSubstituters = [&](Setting<Strings> & res) { - if (name != res.name && res.aliases.count(name) == 0) - return false; - StringSet trusted = settings.trustedSubstituters; - for (auto & s : settings.substituters.get()) - trusted.insert(s); - Strings subs; - auto ss = tokenizeString<Strings>(value); - for (auto & s : ss) - if (trusted.count(s)) - subs.push_back(s); - else - warn("ignoring untrusted substituter '%s'", s); - res = subs; - return true; - }; - - try { - if (name == "ssh-auth-sock") // obsolete - ; - else if (trusted - || name == settings.buildTimeout.name - || name == "connect-timeout" - || (name == "builders" && value == "")) - settings.set(name, value); - else if (setSubstituters(settings.substituters)) - ; - else if (setSubstituters(settings.extraSubstituters)) - ; - else - warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); - } catch (UsageError & e) { - warn(e.what()); - } + try { + if (name == "ssh-auth-sock") // obsolete + ; + else if (trusted || name == settings.buildTimeout.name || + name == "connect-timeout" || + (name == "builders" && value == "")) + settings.set(name, value); + else if (setSubstituters(settings.substituters)) + ; + else if (setSubstituters(settings.extraSubstituters)) + ; + else + warn( + "ignoring the user-specified setting '%s', because it is a " + "restricted setting and you are not a trusted user", + name); + } catch (UsageError& e) { + warn(e.what()); } + } - logger->stopWork(); - break; + logger->stopWork(); + break; } case wopQuerySubstitutablePathInfo: { - Path path = absPath(readString(from)); - logger->startWork(); - SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({path}, infos); - logger->stopWork(); - SubstitutablePathInfos::iterator i = infos.find(path); - if (i == infos.end()) - to << 0; - else { - to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize; - } - break; + Path path = absPath(readString(from)); + logger->startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos({path}, infos); + logger->stopWork(); + SubstitutablePathInfos::iterator i = infos.find(path); + if (i == infos.end()) + to << 0; + else { + to << 1 << i->second.deriver << i->second.references + << i->second.downloadSize << i->second.narSize; + } + break; } case wopQuerySubstitutablePathInfos: { - PathSet paths = readStorePaths<PathSet>(*store, from); - logger->startWork(); - SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(paths, infos); - logger->stopWork(); - to << infos.size(); - for (auto & i : infos) { - to << i.first << i.second.deriver << i.second.references - << i.second.downloadSize << i.second.narSize; - } - break; + PathSet paths = readStorePaths<PathSet>(*store, from); + logger->startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos(paths, infos); + logger->stopWork(); + to << infos.size(); + for (auto& i : infos) { + to << i.first << i.second.deriver << i.second.references + << i.second.downloadSize << i.second.narSize; + } + break; } case wopQueryAllValidPaths: { - logger->startWork(); - PathSet paths = store->queryAllValidPaths(); - logger->stopWork(); - to << paths; - break; + logger->startWork(); + PathSet paths = store->queryAllValidPaths(); + logger->stopWork(); + to << paths; + break; } case wopQueryPathInfo: { - Path path = readStorePath(*store, from); - std::shared_ptr<const ValidPathInfo> info; - logger->startWork(); - try { - info = store->queryPathInfo(path); - } catch (InvalidPath &) { - if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; - } - logger->stopWork(); - if (info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 17) - to << 1; - to << info->deriver << info->narHash.to_string(Base16, false) << info->references - << info->registrationTime << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { - to << info->ultimate - << info->sigs - << info->ca; - } - } else { - assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); - to << 0; + Path path = readStorePath(*store, from); + std::shared_ptr<const ValidPathInfo> info; + logger->startWork(); + try { + info = store->queryPathInfo(path); + } catch (InvalidPath&) { + if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + } + logger->stopWork(); + if (info) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; + to << info->deriver << info->narHash.to_string(Base16, false) + << info->references << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info->ultimate << info->sigs << info->ca; } - break; + } else { + assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); + to << 0; + } + break; } case wopOptimiseStore: - logger->startWork(); - store->optimiseStore(); - logger->stopWork(); - to << 1; - break; + logger->startWork(); + store->optimiseStore(); + logger->stopWork(); + to << 1; + break; case wopVerifyStore: { - bool checkContents, repair; - from >> checkContents >> repair; - logger->startWork(); - if (repair && !trusted) - throw Error("you are not privileged to repair paths"); - bool errors = store->verifyStore(checkContents, (RepairFlag) repair); - logger->stopWork(); - to << errors; - break; + bool checkContents, repair; + from >> checkContents >> repair; + logger->startWork(); + if (repair && !trusted) + throw Error("you are not privileged to repair paths"); + bool errors = store->verifyStore(checkContents, (RepairFlag)repair); + logger->stopWork(); + to << errors; + break; } case wopAddSignatures: { - Path path = readStorePath(*store, from); - StringSet sigs = readStrings<StringSet>(from); - logger->startWork(); - if (!trusted) - throw Error("you are not privileged to add signatures"); - store->addSignatures(path, sigs); - logger->stopWork(); - to << 1; - break; + Path path = readStorePath(*store, from); + StringSet sigs = readStrings<StringSet>(from); + logger->startWork(); + if (!trusted) throw Error("you are not privileged to add signatures"); + store->addSignatures(path, sigs); + logger->stopWork(); + to << 1; + break; } case wopNarFromPath: { - auto path = readStorePath(*store, from); - logger->startWork(); - logger->stopWork(); - dumpPath(path, to); - break; + auto path = readStorePath(*store, from); + logger->startWork(); + logger->stopWork(); + dumpPath(path, to); + break; } case wopAddToStoreNar: { - bool repair, dontCheckSigs; - ValidPathInfo info; - info.path = readStorePath(*store, from); - from >> info.deriver; - if (!info.deriver.empty()) - store->assertStorePath(info.deriver); - info.narHash = Hash(readString(from), htSHA256); - info.references = readStorePaths<PathSet>(*store, from); - from >> info.registrationTime >> info.narSize >> info.ultimate; - info.sigs = readStrings<StringSet>(from); - from >> info.ca >> repair >> dontCheckSigs; - if (!trusted && dontCheckSigs) - dontCheckSigs = false; - if (!trusted) - info.ultimate = false; - - std::string saved; - std::unique_ptr<Source> source; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique<TunnelSource>(from); - else { - TeeSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.source.data); - source = std::make_unique<StringSource>(saved); - } - - logger->startWork(); - - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, *source, (RepairFlag) repair, - dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); - - logger->stopWork(); - break; + bool repair, dontCheckSigs; + ValidPathInfo info; + info.path = readStorePath(*store, from); + from >> info.deriver; + if (!info.deriver.empty()) store->assertStorePath(info.deriver); + info.narHash = Hash(readString(from), htSHA256); + info.references = readStorePaths<PathSet>(*store, from); + from >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings<StringSet>(from); + from >> info.ca >> repair >> dontCheckSigs; + if (!trusted && dontCheckSigs) dontCheckSigs = false; + if (!trusted) info.ultimate = false; + + std::string saved; + std::unique_ptr<Source> source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique<TunnelSource>(from); + else { + TeeSink tee(from); + parseDump(tee, tee.source); + saved = std::move(*tee.source.data); + source = std::make_unique<StringSource>(saved); + } + + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag)repair, + dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + + logger->stopWork(); + break; } case wopQueryMissing: { - PathSet targets = readStorePaths<PathSet>(*store, from); - logger->startWork(); - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); - logger->stopWork(); - to << willBuild << willSubstitute << unknown << downloadSize << narSize; - break; + PathSet targets = readStorePaths<PathSet>(*store, from); + logger->startWork(); + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store->queryMissing(targets, willBuild, willSubstitute, unknown, + downloadSize, narSize); + logger->stopWork(); + to << willBuild << willSubstitute << unknown << downloadSize << narSize; + break; } default: - throw Error(format("invalid operation %1%") % op); - } + throw Error(format("invalid operation %1%") % op); + } } +static void processConnection(bool trusted, const std::string& userName, + uid_t userId) { + MonitorFdHup monitor(from.fd); -static void processConnection(bool trusted, - const std::string & userName, uid_t userId) -{ - MonitorFdHup monitor(from.fd); + /* Exchange the greeting. */ + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); + to << WORKER_MAGIC_2 << PROTOCOL_VERSION; + to.flush(); + unsigned int clientVersion = readInt(from); - /* Exchange the greeting. */ - unsigned int magic = readInt(from); - if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); - to << WORKER_MAGIC_2 << PROTOCOL_VERSION; - to.flush(); - unsigned int clientVersion = readInt(from); - - if (clientVersion < 0x10a) - throw Error("the Nix client version is too old"); - - auto tunnelLogger = new TunnelLogger(clientVersion); - auto prevLogger = nix::logger; - logger = tunnelLogger; + if (clientVersion < 0x10a) throw Error("the Nix client version is too old"); - unsigned int opCount = 0; + auto tunnelLogger = new TunnelLogger(clientVersion); + auto prevLogger = nix::logger; + logger = tunnelLogger; - Finally finally([&]() { - _isInterrupted = false; - prevLogger->log(lvlDebug, fmt("%d operations", opCount)); - }); + unsigned int opCount = 0; - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) - setAffinityTo(readInt(from)); + Finally finally([&]() { + _isInterrupted = false; + prevLogger->log(lvlDebug, fmt("%d operations", opCount)); + }); - readInt(from); // obsolete reserveSpace + if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) + setAffinityTo(readInt(from)); - /* Send startup error messages to the client. */ - tunnelLogger->startWork(); + readInt(from); // obsolete reserveSpace - try { + /* Send startup error messages to the client. */ + tunnelLogger->startWork(); - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ + try { + /* If we can't accept clientVersion, then throw an error + *here* (not above). */ #if 0 /* Prevent users from doing something very dangerous. */ @@ -788,124 +753,113 @@ static void processConnection(bool trusted, throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); #endif - /* Open the store. */ - Store::Params params; // FIXME: get params from somewhere - // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; - auto store = openStore(settings.storeUri, params); - - store->createUser(userName, userId); + /* Open the store. */ + Store::Params params; // FIXME: get params from somewhere + // Disable caching since the client already does that. + params["path-info-cache-size"] = "0"; + auto store = openStore(settings.storeUri, params); - tunnelLogger->stopWork(); - to.flush(); + store->createUser(userName, userId); - /* Process client requests. */ - while (true) { - WorkerOp op; - try { - op = (WorkerOp) readInt(from); - } catch (Interrupted & e) { - break; - } catch (EndOfFile & e) { - break; - } - - opCount++; - - try { - performOp(tunnelLogger, store, trusted, clientVersion, from, to, op); - } catch (Error & e) { - /* If we're not in a state where we can send replies, then - something went wrong processing the input of the - client. This can happen especially if I/O errors occur - during addTextToStore() / importPath(). If that - happens, just send the error message and exit. */ - bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; - tunnelLogger->stopWork(false, e.msg(), e.status); - if (!errorAllowed) throw; - } catch (std::bad_alloc & e) { - tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); - throw; - } - - to.flush(); + tunnelLogger->stopWork(); + to.flush(); - assert(!tunnelLogger->state_.lock()->canSendStderr); - }; + /* Process client requests. */ + while (true) { + WorkerOp op; + try { + op = (WorkerOp)readInt(from); + } catch (Interrupted& e) { + break; + } catch (EndOfFile& e) { + break; + } + + opCount++; + + try { + performOp(tunnelLogger, store, trusted, clientVersion, from, to, op); + } catch (Error& e) { + /* If we're not in a state where we can send replies, then + something went wrong processing the input of the + client. This can happen especially if I/O errors occur + during addTextToStore() / importPath(). If that + happens, just send the error message and exit. */ + bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; + tunnelLogger->stopWork(false, e.msg(), e.status); + if (!errorAllowed) throw; + } catch (std::bad_alloc& e) { + tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); + throw; + } + + to.flush(); + + assert(!tunnelLogger->state_.lock()->canSendStderr); + }; - } catch (std::exception & e) { - tunnelLogger->stopWork(false, e.what(), 1); - to.flush(); - return; - } + } catch (std::exception& e) { + tunnelLogger->stopWork(false, e.what(), 1); + to.flush(); + return; + } } - -static void sigChldHandler(int sigNo) -{ - // Ensure we don't modify errno of whatever we've interrupted - auto saved_errno = errno; - /* Reap all dead children. */ - while (waitpid(-1, 0, WNOHANG) > 0) ; - errno = saved_errno; +static void sigChldHandler(int sigNo) { + // Ensure we don't modify errno of whatever we've interrupted + auto saved_errno = errno; + /* Reap all dead children. */ + while (waitpid(-1, 0, WNOHANG) > 0) + ; + errno = saved_errno; } - -static void setSigChldAction(bool autoReap) -{ - struct sigaction act, oact; - act.sa_handler = autoReap ? sigChldHandler : SIG_DFL; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGCHLD, &act, &oact)) - throw SysError("setting SIGCHLD handler"); +static void setSigChldAction(bool autoReap) { + struct sigaction act, oact; + act.sa_handler = autoReap ? sigChldHandler : SIG_DFL; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, &oact)) + throw SysError("setting SIGCHLD handler"); } +bool matchUser(const string& user, const string& group, const Strings& users) { + if (find(users.begin(), users.end(), "*") != users.end()) return true; -bool matchUser(const string & user, const string & group, const Strings & users) -{ - if (find(users.begin(), users.end(), "*") != users.end()) - return true; + if (find(users.begin(), users.end(), user) != users.end()) return true; - if (find(users.begin(), users.end(), user) != users.end()) - return true; - - for (auto & i : users) - if (string(i, 0, 1) == "@") { - if (group == string(i, 1)) return true; - struct group * gr = getgrnam(i.c_str() + 1); - if (!gr) continue; - for (char * * mem = gr->gr_mem; *mem; mem++) - if (user == string(*mem)) return true; - } + for (auto& i : users) + if (string(i, 0, 1) == "@") { + if (group == string(i, 1)) return true; + struct group* gr = getgrnam(i.c_str() + 1); + if (!gr) continue; + for (char** mem = gr->gr_mem; *mem; mem++) + if (user == string(*mem)) return true; + } - return false; + return false; } - -struct PeerInfo -{ - bool pidKnown; - pid_t pid; - bool uidKnown; - uid_t uid; - bool gidKnown; - gid_t gid; +struct PeerInfo { + bool pidKnown; + pid_t pid; + bool uidKnown; + uid_t uid; + bool gidKnown; + gid_t gid; }; - /* Get the identity of the caller, if possible. */ -static PeerInfo getPeerInfo(int remote) -{ - PeerInfo peer = { false, 0, false, 0, false, 0 }; +static PeerInfo getPeerInfo(int remote) { + PeerInfo peer = {false, 0, false, 0, false, 0}; #if defined(SO_PEERCRED) - ucred cred; - socklen_t credLen = sizeof(cred); - if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) - throw SysError("getting peer credentials"); - peer = { true, cred.pid, true, cred.uid, true, cred.gid }; + ucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + peer = {true, cred.pid, true, cred.uid, true, cred.gid}; #elif defined(LOCAL_PEERCRED) @@ -913,237 +867,239 @@ static PeerInfo getPeerInfo(int remote) #define SOL_LOCAL 0 #endif - xucred cred; - socklen_t credLen = sizeof(cred); - if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == -1) - throw SysError("getting peer credentials"); - peer = { false, 0, true, cred.cr_uid, false, 0 }; + xucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + peer = {false, 0, true, cred.cr_uid, false, 0}; #endif - return peer; + return peer; } - #define SD_LISTEN_FDS_START 3 +static void daemonLoop(char** argv) { + if (chdir("/") == -1) throw SysError("cannot change current directory"); -static void daemonLoop(char * * argv) -{ - if (chdir("/") == -1) - throw SysError("cannot change current directory"); + /* Get rid of children automatically; don't let them become + zombies. */ + setSigChldAction(true); - /* Get rid of children automatically; don't let them become - zombies. */ - setSigChldAction(true); + AutoCloseFD fdSocket; - AutoCloseFD fdSocket; + /* Handle socket-based activation by systemd. */ + if (getEnv("LISTEN_FDS") != "") { + if (getEnv("LISTEN_PID") != std::to_string(getpid()) || + getEnv("LISTEN_FDS") != "1") + throw Error("unexpected systemd environment variables"); + fdSocket = SD_LISTEN_FDS_START; + } - /* Handle socket-based activation by systemd. */ - if (getEnv("LISTEN_FDS") != "") { - if (getEnv("LISTEN_PID") != std::to_string(getpid()) || getEnv("LISTEN_FDS") != "1") - throw Error("unexpected systemd environment variables"); - fdSocket = SD_LISTEN_FDS_START; - } + /* Otherwise, create and bind to a Unix domain socket. */ + else { + /* Create and bind to a Unix domain socket. */ + fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); + if (!fdSocket) throw SysError("cannot create Unix domain socket"); - /* Otherwise, create and bind to a Unix domain socket. */ - else { + string socketPath = settings.nixDaemonSocketFile; - /* Create and bind to a Unix domain socket. */ - fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); + createDirs(dirOf(socketPath)); - string socketPath = settings.nixDaemonSocketFile; + /* Urgh, sockaddr_un allows path names of only 108 characters. + So chdir to the socket directory so that we can pass a + relative path name. */ + if (chdir(dirOf(socketPath).c_str()) == -1) + throw SysError("cannot change current directory"); + Path socketPathRel = "./" + baseNameOf(socketPath); - createDirs(dirOf(socketPath)); + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (socketPathRel.size() >= sizeof(addr.sun_path)) + throw Error(format("socket path '%1%' is too long") % socketPathRel); + strcpy(addr.sun_path, socketPathRel.c_str()); - /* Urgh, sockaddr_un allows path names of only 108 characters. - So chdir to the socket directory so that we can pass a - relative path name. */ - if (chdir(dirOf(socketPath).c_str()) == -1) - throw SysError("cannot change current directory"); - Path socketPathRel = "./" + baseNameOf(socketPath); + unlink(socketPath.c_str()); - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPathRel.size() >= sizeof(addr.sun_path)) - throw Error(format("socket path '%1%' is too long") % socketPathRel); - strcpy(addr.sun_path, socketPathRel.c_str()); - - unlink(socketPath.c_str()); - - /* Make sure that the socket is created with 0666 permission - (everybody can connect --- provided they have access to the - directory containing the socket). */ - mode_t oldMode = umask(0111); - int res = bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)); - umask(oldMode); - if (res == -1) - throw SysError(format("cannot bind to socket '%1%'") % socketPath); - - if (chdir("/") == -1) /* back to the root */ - throw SysError("cannot change current directory"); - - if (listen(fdSocket.get(), 5) == -1) - throw SysError(format("cannot listen on socket '%1%'") % socketPath); - } - - closeOnExec(fdSocket.get()); - - /* Loop accepting connections. */ - while (1) { - - try { - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(fdSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - checkInterrupt(); - if (!remote) { - if (errno == EINTR) continue; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - bool trusted = false; - PeerInfo peer = getPeerInfo(remote.get()); - - struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; - string user = pw ? pw->pw_name : std::to_string(peer.uid); - - struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; - string group = gr ? gr->gr_name : std::to_string(peer.gid); - - Strings trustedUsers = settings.trustedUsers; - Strings allowedUsers = settings.allowedUsers; - - if (matchUser(user, group, trustedUsers)) - trusted = true; + /* Make sure that the socket is created with 0666 permission + (everybody can connect --- provided they have access to the + directory containing the socket). */ + mode_t oldMode = umask(0111); + int res = bind(fdSocket.get(), (struct sockaddr*)&addr, sizeof(addr)); + umask(oldMode); + if (res == -1) + throw SysError(format("cannot bind to socket '%1%'") % socketPath); - if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) - throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user); + if (chdir("/") == -1) /* back to the root */ + throw SysError("cannot change current directory"); - printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) - % (peer.pidKnown ? std::to_string(peer.pid) : "<unknown>") - % (peer.uidKnown ? user : "<unknown>")); + if (listen(fdSocket.get(), 5) == -1) + throw SysError(format("cannot listen on socket '%1%'") % socketPath); + } - /* Fork a child to handle the connection. */ - ProcessOptions options; - options.errorPrefix = "unexpected Nix daemon error: "; - options.dieWithParent = false; - options.runExitHandlers = true; - options.allowVfork = false; - startProcess([&]() { - fdSocket = -1; + closeOnExec(fdSocket.get()); - /* Background the daemon. */ - if (setsid() == -1) - throw SysError(format("creating a new session")); - - /* Restore normal handling of SIGCHLD. */ - setSigChldAction(false); - - /* For debugging, stuff the pid into argv[1]. */ - if (peer.pidKnown && argv[1]) { - string processName = std::to_string(peer.pid); - strncpy(argv[1], processName.c_str(), strlen(argv[1])); - } + /* Loop accepting connections. */ + while (1) { + try { + /* Accept a connection. */ + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = + accept(fdSocket.get(), (struct sockaddr*)&remoteAddr, &remoteAddrLen); + checkInterrupt(); + if (!remote) { + if (errno == EINTR) continue; + throw SysError("accepting connection"); + } + + closeOnExec(remote.get()); + + bool trusted = false; + PeerInfo peer = getPeerInfo(remote.get()); + + struct passwd* pw = peer.uidKnown ? getpwuid(peer.uid) : 0; + string user = pw ? pw->pw_name : std::to_string(peer.uid); + + struct group* gr = peer.gidKnown ? getgrgid(peer.gid) : 0; + string group = gr ? gr->gr_name : std::to_string(peer.gid); + + Strings trustedUsers = settings.trustedUsers; + Strings allowedUsers = settings.allowedUsers; + + if (matchUser(user, group, trustedUsers)) trusted = true; + + if ((!trusted && !matchUser(user, group, allowedUsers)) || + group == settings.buildUsersGroup) + throw Error( + format("user '%1%' is not allowed to connect to the Nix daemon") % + user); + + printInfo(format((string) "accepted connection from pid %1%, user %2%" + + (trusted ? " (trusted)" : "")) % + (peer.pidKnown ? std::to_string(peer.pid) : "<unknown>") % + (peer.uidKnown ? user : "<unknown>")); + + /* Fork a child to handle the connection. */ + ProcessOptions options; + options.errorPrefix = "unexpected Nix daemon error: "; + options.dieWithParent = false; + options.runExitHandlers = true; + options.allowVfork = false; + startProcess( + [&]() { + fdSocket = -1; + + /* Background the daemon. */ + if (setsid() == -1) + throw SysError(format("creating a new session")); + + /* Restore normal handling of SIGCHLD. */ + setSigChldAction(false); + + /* For debugging, stuff the pid into argv[1]. */ + if (peer.pidKnown && argv[1]) { + string processName = std::to_string(peer.pid); + strncpy(argv[1], processName.c_str(), strlen(argv[1])); + } - /* Handle the connection. */ - from.fd = remote.get(); - to.fd = remote.get(); - processConnection(trusted, user, peer.uid); + /* Handle the connection. */ + from.fd = remote.get(); + to.fd = remote.get(); + processConnection(trusted, user, peer.uid); - exit(0); - }, options); + exit(0); + }, + options); - } catch (Interrupted & e) { - return; - } catch (Error & e) { - printError(format("error processing connection: %1%") % e.msg()); - } + } catch (Interrupted& e) { + return; + } catch (Error& e) { + printError(format("error processing connection: %1%") % e.msg()); } + } } +static int _main(int argc, char** argv) { + { + auto stdio = false; + + parseCmdLine(argc, argv, + [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--daemon") + ; /* ignored for backwards compatibility */ + else if (*arg == "--help") + showManPage("nix-daemon"); + else if (*arg == "--version") + printVersion("nix-daemon"); + else if (*arg == "--stdio") + stdio = true; + else + return false; + return true; + }); + + initPlugins(); + + if (stdio) { + if (getStoreType() == tDaemon) { + /* Forward on this connection to the real daemon */ + auto socketPath = settings.nixDaemonSocketFile; + auto s = socket(PF_UNIX, SOCK_STREAM, 0); + if (s == -1) throw SysError("creating Unix domain socket"); + + auto socketDir = dirOf(socketPath); + if (chdir(socketDir.c_str()) == -1) + throw SysError(format("changing to socket directory '%1%'") % + socketDir); + + auto socketName = baseNameOf(socketPath); + auto addr = sockaddr_un{}; + addr.sun_family = AF_UNIX; + if (socketName.size() + 1 >= sizeof(addr.sun_path)) + throw Error(format("socket name %1% is too long") % socketName); + strcpy(addr.sun_path, socketName.c_str()); -static int _main(int argc, char * * argv) -{ - { - auto stdio = false; - - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--daemon") - ; /* ignored for backwards compatibility */ - else if (*arg == "--help") - showManPage("nix-daemon"); - else if (*arg == "--version") - printVersion("nix-daemon"); - else if (*arg == "--stdio") - stdio = true; - else return false; - return true; - }); - - initPlugins(); - - if (stdio) { - if (getStoreType() == tDaemon) { - /* Forward on this connection to the real daemon */ - auto socketPath = settings.nixDaemonSocketFile; - auto s = socket(PF_UNIX, SOCK_STREAM, 0); - if (s == -1) - throw SysError("creating Unix domain socket"); - - auto socketDir = dirOf(socketPath); - if (chdir(socketDir.c_str()) == -1) - throw SysError(format("changing to socket directory '%1%'") % socketDir); - - auto socketName = baseNameOf(socketPath); - auto addr = sockaddr_un{}; - addr.sun_family = AF_UNIX; - if (socketName.size() + 1 >= sizeof(addr.sun_path)) - throw Error(format("socket name %1% is too long") % socketName); - strcpy(addr.sun_path, socketName.c_str()); - - if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at %1%") % socketPath); - - auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(s, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(s, &fds)) { - auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return 0; - } - } - } else { - processConnection(true, "root", 0); - } - } else { - daemonLoop(argv); - } + if (connect(s, (struct sockaddr*)&addr, sizeof(addr)) == -1) + throw SysError(format("cannot connect to daemon at %1%") % + socketPath); - return 0; + auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(s, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(s, &fds)) { + auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, + SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, + SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return 0; + } + } + } else { + processConnection(true, "root", 0); + } + } else { + daemonLoop(argv); } + + return 0; + } } static RegisterLegacyCommand s1("nix-daemon", _main); diff --git a/third_party/nix/src/nix-env/nix-env.cc b/third_party/nix/src/nix-env/nix-env.cc index 3292f546ab42..f9a441e8e30b 100644 --- a/third_party/nix/src/nix-env/nix-env.cc +++ b/third_party/nix/src/nix-env/nix-env.cc @@ -1,812 +1,749 @@ +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <algorithm> +#include <cerrno> +#include <ctime> +#include <iostream> +#include <sstream> #include "attr-path.hh" #include "common-eval-args.hh" #include "derivations.hh" #include "eval.hh" #include "get-drvs.hh" #include "globals.hh" +#include "json.hh" +#include "legacy.hh" #include "names.hh" #include "profiles.hh" #include "shared.hh" #include "store-api.hh" #include "user-env.hh" #include "util.hh" -#include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" -#include "legacy.hh" - -#include <cerrno> -#include <ctime> -#include <algorithm> -#include <iostream> -#include <sstream> - -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> - using namespace nix; using std::cout; - typedef enum { - srcNixExprDrvs, - srcNixExprs, - srcStorePaths, - srcProfile, - srcAttrPath, - srcUnknown + srcNixExprDrvs, + srcNixExprs, + srcStorePaths, + srcProfile, + srcAttrPath, + srcUnknown } InstallSourceType; - -struct InstallSourceInfo -{ - InstallSourceType type; - Path nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ - Path profile; /* for srcProfile */ - string systemFilter; /* for srcNixExprDrvs */ - Bindings * autoArgs; +struct InstallSourceInfo { + InstallSourceType type; + Path nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ + Path profile; /* for srcProfile */ + string systemFilter; /* for srcNixExprDrvs */ + Bindings* autoArgs; }; - -struct Globals -{ - InstallSourceInfo instSource; - Path profile; - std::shared_ptr<EvalState> state; - bool dryRun; - bool preserveInstalled; - bool removeAll; - string forceName; - bool prebuiltOnly; +struct Globals { + InstallSourceInfo instSource; + Path profile; + std::shared_ptr<EvalState> state; + bool dryRun; + bool preserveInstalled; + bool removeAll; + string forceName; + bool prebuiltOnly; }; +typedef void (*Operation)(Globals& globals, Strings opFlags, Strings opArgs); -typedef void (* Operation) (Globals & globals, - Strings opFlags, Strings opArgs); - - -static string needArg(Strings::iterator & i, - Strings & args, const string & arg) -{ - if (i == args.end()) throw UsageError( - format("'%1%' requires an argument") % arg); - return *i++; +static string needArg(Strings::iterator& i, Strings& args, const string& arg) { + if (i == args.end()) + throw UsageError(format("'%1%' requires an argument") % arg); + return *i++; } - -static bool parseInstallSourceOptions(Globals & globals, - Strings::iterator & i, Strings & args, const string & arg) -{ - if (arg == "--from-expression" || arg == "-E") - globals.instSource.type = srcNixExprs; - else if (arg == "--from-profile") { - globals.instSource.type = srcProfile; - globals.instSource.profile = needArg(i, args, arg); - } - else if (arg == "--attr" || arg == "-A") - globals.instSource.type = srcAttrPath; - else return false; - return true; +static bool parseInstallSourceOptions(Globals& globals, Strings::iterator& i, + Strings& args, const string& arg) { + if (arg == "--from-expression" || arg == "-E") + globals.instSource.type = srcNixExprs; + else if (arg == "--from-profile") { + globals.instSource.type = srcProfile; + globals.instSource.profile = needArg(i, args, arg); + } else if (arg == "--attr" || arg == "-A") + globals.instSource.type = srcAttrPath; + else + return false; + return true; } - -static bool isNixExpr(const Path & path, struct stat & st) -{ - return S_ISREG(st.st_mode) || (S_ISDIR(st.st_mode) && pathExists(path + "/default.nix")); +static bool isNixExpr(const Path& path, struct stat& st) { + return S_ISREG(st.st_mode) || + (S_ISDIR(st.st_mode) && pathExists(path + "/default.nix")); } +static void getAllExprs(EvalState& state, const Path& path, StringSet& attrs, + Value& v) { + StringSet namesSorted; + for (auto& i : readDirectory(path)) namesSorted.insert(i.name); -static void getAllExprs(EvalState & state, - const Path & path, StringSet & attrs, Value & v) -{ - StringSet namesSorted; - for (auto & i : readDirectory(path)) namesSorted.insert(i.name); - - for (auto & i : namesSorted) { - /* Ignore the manifest.nix used by profiles. This is - necessary to prevent it from showing up in channels (which - are implemented using profiles). */ - if (i == "manifest.nix") continue; - - Path path2 = path + "/" + i; - - struct stat st; - if (stat(path2.c_str(), &st) == -1) - continue; // ignore dangling symlinks in ~/.nix-defexpr - - if (isNixExpr(path2, st) && (!S_ISREG(st.st_mode) || hasSuffix(path2, ".nix"))) { - /* Strip off the `.nix' filename suffix (if applicable), - otherwise the attribute cannot be selected with the - `-A' option. Useful if you want to stick a Nix - expression directly in ~/.nix-defexpr. */ - string attrName = i; - if (hasSuffix(attrName, ".nix")) - attrName = string(attrName, 0, attrName.size() - 4); - if (attrs.find(attrName) != attrs.end()) { - printError(format("warning: name collision in input Nix expressions, skipping '%1%'") % path2); - continue; - } - attrs.insert(attrName); - /* Load the expression on demand. */ - Value & vFun = state.getBuiltin("import"); - Value & vArg(*state.allocValue()); - mkString(vArg, path2); - if (v.attrs->size() == v.attrs->capacity()) - throw Error(format("too many Nix expressions in directory '%1%'") % path); - mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg); - } - else if (S_ISDIR(st.st_mode)) - /* `path2' is a directory (with no default.nix in it); - recurse into it. */ - getAllExprs(state, path2, attrs, v); - } -} + for (auto& i : namesSorted) { + /* Ignore the manifest.nix used by profiles. This is + necessary to prevent it from showing up in channels (which + are implemented using profiles). */ + if (i == "manifest.nix") continue; + Path path2 = path + "/" + i; -static void loadSourceExpr(EvalState & state, const Path & path, Value & v) -{ struct stat st; - if (stat(path.c_str(), &st) == -1) - throw SysError(format("getting information about '%1%'") % path); - - if (isNixExpr(path, st)) - state.evalFile(path, v); - - /* The path is a directory. Put the Nix expressions in the - directory in a set, with the file name of each expression as - the attribute name. Recurse into subdirectories (but keep the - set flat, not nested, to make it easier for a user to have a - ~/.nix-defexpr directory that includes some system-wide - directory). */ - else if (S_ISDIR(st.st_mode)) { - state.mkAttrs(v, 1024); - state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0); - StringSet attrs; - getAllExprs(state, path, attrs, v); - v.attrs->sort(); - } - - else throw Error("path '%s' is not a directory or a Nix expression", path); + if (stat(path2.c_str(), &st) == -1) + continue; // ignore dangling symlinks in ~/.nix-defexpr + + if (isNixExpr(path2, st) && + (!S_ISREG(st.st_mode) || hasSuffix(path2, ".nix"))) { + /* Strip off the `.nix' filename suffix (if applicable), + otherwise the attribute cannot be selected with the + `-A' option. Useful if you want to stick a Nix + expression directly in ~/.nix-defexpr. */ + string attrName = i; + if (hasSuffix(attrName, ".nix")) + attrName = string(attrName, 0, attrName.size() - 4); + if (attrs.find(attrName) != attrs.end()) { + printError(format("warning: name collision in input Nix expressions, " + "skipping '%1%'") % + path2); + continue; + } + attrs.insert(attrName); + /* Load the expression on demand. */ + Value& vFun = state.getBuiltin("import"); + Value& vArg(*state.allocValue()); + mkString(vArg, path2); + if (v.attrs->size() == v.attrs->capacity()) + throw Error(format("too many Nix expressions in directory '%1%'") % + path); + mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg); + } else if (S_ISDIR(st.st_mode)) + /* `path2' is a directory (with no default.nix in it); + recurse into it. */ + getAllExprs(state, path2, attrs, v); + } } +static void loadSourceExpr(EvalState& state, const Path& path, Value& v) { + struct stat st; + if (stat(path.c_str(), &st) == -1) + throw SysError(format("getting information about '%1%'") % path); + + if (isNixExpr(path, st)) state.evalFile(path, v); + + /* The path is a directory. Put the Nix expressions in the + directory in a set, with the file name of each expression as + the attribute name. Recurse into subdirectories (but keep the + set flat, not nested, to make it easier for a user to have a + ~/.nix-defexpr directory that includes some system-wide + directory). */ + else if (S_ISDIR(st.st_mode)) { + state.mkAttrs(v, 1024); + state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), + 0); + StringSet attrs; + getAllExprs(state, path, attrs, v); + v.attrs->sort(); + } + + else + throw Error("path '%s' is not a directory or a Nix expression", path); +} -static void loadDerivations(EvalState & state, Path nixExprPath, - string systemFilter, Bindings & autoArgs, - const string & pathPrefix, DrvInfos & elems) -{ - Value vRoot; - loadSourceExpr(state, nixExprPath, vRoot); +static void loadDerivations(EvalState& state, Path nixExprPath, + string systemFilter, Bindings& autoArgs, + const string& pathPrefix, DrvInfos& elems) { + Value vRoot; + loadSourceExpr(state, nixExprPath, vRoot); - Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot)); + Value& v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot)); - getDerivations(state, v, pathPrefix, autoArgs, elems, true); + getDerivations(state, v, pathPrefix, autoArgs, elems, true); - /* Filter out all derivations not applicable to the current - system. */ - for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { - j = i; j++; - if (systemFilter != "*" && i->querySystem() != systemFilter) - elems.erase(i); - } + /* Filter out all derivations not applicable to the current + system. */ + for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { + j = i; + j++; + if (systemFilter != "*" && i->querySystem() != systemFilter) elems.erase(i); + } } - -static long getPriority(EvalState & state, DrvInfo & drv) -{ - return drv.queryMetaInt("priority", 0); +static long getPriority(EvalState& state, DrvInfo& drv) { + return drv.queryMetaInt("priority", 0); } - -static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) -{ - return getPriority(state, drv2) - getPriority(state, drv1); +static long comparePriorities(EvalState& state, DrvInfo& drv1, DrvInfo& drv2) { + return getPriority(state, drv2) - getPriority(state, drv1); } - // FIXME: this function is rather slow since it checks a single path // at a time. -static bool isPrebuilt(EvalState & state, DrvInfo & elem) -{ - Path path = elem.queryOutPath(); - if (state.store->isValidPath(path)) return true; - PathSet ps = state.store->querySubstitutablePaths({path}); - return ps.find(path) != ps.end(); +static bool isPrebuilt(EvalState& state, DrvInfo& elem) { + Path path = elem.queryOutPath(); + if (state.store->isValidPath(path)) return true; + PathSet ps = state.store->querySubstitutablePaths({path}); + return ps.find(path) != ps.end(); } - -static void checkSelectorUse(DrvNames & selectors) -{ - /* Check that all selectors have been used. */ - for (auto & i : selectors) - if (i.hits == 0 && i.fullName != "*") - throw Error(format("selector '%1%' matches no derivations") % i.fullName); +static void checkSelectorUse(DrvNames& selectors) { + /* Check that all selectors have been used. */ + for (auto& i : selectors) + if (i.hits == 0 && i.fullName != "*") + throw Error(format("selector '%1%' matches no derivations") % i.fullName); } +static DrvInfos filterBySelector(EvalState& state, const DrvInfos& allElems, + const Strings& args, bool newestOnly) { + DrvNames selectors = drvNamesFromArgs(args); + if (selectors.empty()) selectors.push_back(DrvName("*")); + + DrvInfos elems; + set<unsigned int> done; + + for (auto& i : selectors) { + typedef list<std::pair<DrvInfo, unsigned int> > Matches; + Matches matches; + unsigned int n = 0; + for (DrvInfos::const_iterator j = allElems.begin(); j != allElems.end(); + ++j, ++n) { + DrvName drvName(j->queryName()); + if (i.matches(drvName)) { + i.hits++; + matches.push_back(std::pair<DrvInfo, unsigned int>(*j, n)); + } + } -static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, - const Strings & args, bool newestOnly) -{ - DrvNames selectors = drvNamesFromArgs(args); - if (selectors.empty()) - selectors.push_back(DrvName("*")); - - DrvInfos elems; - set<unsigned int> done; - - for (auto & i : selectors) { - typedef list<std::pair<DrvInfo, unsigned int> > Matches; - Matches matches; - unsigned int n = 0; - for (DrvInfos::const_iterator j = allElems.begin(); - j != allElems.end(); ++j, ++n) - { - DrvName drvName(j->queryName()); - if (i.matches(drvName)) { - i.hits++; - matches.push_back(std::pair<DrvInfo, unsigned int>(*j, n)); - } + /* If `newestOnly', if a selector matches multiple derivations + with the same name, pick the one matching the current + system. If there are still multiple derivations, pick the + one with the highest priority. If there are still multiple + derivations, pick the one with the highest version. + Finally, if there are still multiple derivations, + arbitrarily pick the first one. */ + if (newestOnly) { + /* Map from package names to derivations. */ + typedef map<string, std::pair<DrvInfo, unsigned int> > Newest; + Newest newest; + StringSet multiple; + + for (auto& j : matches) { + DrvName drvName(j.first.queryName()); + long d = 1; + + Newest::iterator k = newest.find(drvName.name); + + if (k != newest.end()) { + d = j.first.querySystem() == k->second.first.querySystem() + ? 0 + : j.first.querySystem() == settings.thisSystem + ? 1 + : k->second.first.querySystem() == settings.thisSystem + ? -1 + : 0; + if (d == 0) d = comparePriorities(state, j.first, k->second.first); + if (d == 0) + d = compareVersions(drvName.version, + DrvName(k->second.first.queryName()).version); } - /* If `newestOnly', if a selector matches multiple derivations - with the same name, pick the one matching the current - system. If there are still multiple derivations, pick the - one with the highest priority. If there are still multiple - derivations, pick the one with the highest version. - Finally, if there are still multiple derivations, - arbitrarily pick the first one. */ - if (newestOnly) { - - /* Map from package names to derivations. */ - typedef map<string, std::pair<DrvInfo, unsigned int> > Newest; - Newest newest; - StringSet multiple; - - for (auto & j : matches) { - DrvName drvName(j.first.queryName()); - long d = 1; - - Newest::iterator k = newest.find(drvName.name); - - if (k != newest.end()) { - d = j.first.querySystem() == k->second.first.querySystem() ? 0 : - j.first.querySystem() == settings.thisSystem ? 1 : - k->second.first.querySystem() == settings.thisSystem ? -1 : 0; - if (d == 0) - d = comparePriorities(state, j.first, k->second.first); - if (d == 0) - d = compareVersions(drvName.version, DrvName(k->second.first.queryName()).version); - } - - if (d > 0) { - newest.erase(drvName.name); - newest.insert(Newest::value_type(drvName.name, j)); - multiple.erase(j.first.queryName()); - } else if (d == 0) { - multiple.insert(j.first.queryName()); - } - } - - matches.clear(); - for (auto & j : newest) { - if (multiple.find(j.second.first.queryName()) != multiple.end()) - printInfo( - "warning: there are multiple derivations named '%1%'; using the first one", - j.second.first.queryName()); - matches.push_back(j.second); - } + if (d > 0) { + newest.erase(drvName.name); + newest.insert(Newest::value_type(drvName.name, j)); + multiple.erase(j.first.queryName()); + } else if (d == 0) { + multiple.insert(j.first.queryName()); } - - /* Insert only those elements in the final list that we - haven't inserted before. */ - for (auto & j : matches) - if (done.find(j.second) == done.end()) { - done.insert(j.second); - elems.push_back(j.first); - } + } + + matches.clear(); + for (auto& j : newest) { + if (multiple.find(j.second.first.queryName()) != multiple.end()) + printInfo( + "warning: there are multiple derivations named '%1%'; using the " + "first one", + j.second.first.queryName()); + matches.push_back(j.second); + } } - checkSelectorUse(selectors); + /* Insert only those elements in the final list that we + haven't inserted before. */ + for (auto& j : matches) + if (done.find(j.second) == done.end()) { + done.insert(j.second); + elems.push_back(j.first); + } + } - return elems; -} + checkSelectorUse(selectors); - -static bool isPath(const string & s) -{ - return s.find('/') != string::npos; + return elems; } +static bool isPath(const string& s) { return s.find('/') != string::npos; } + +static void queryInstSources(EvalState& state, InstallSourceInfo& instSource, + const Strings& args, DrvInfos& elems, + bool newestOnly) { + InstallSourceType type = instSource.type; + if (type == srcUnknown && args.size() > 0 && isPath(args.front())) + type = srcStorePaths; + + switch (type) { + /* Get the available user environment elements from the + derivations specified in a Nix expression, including only + those with names matching any of the names in `args'. */ + case srcUnknown: + case srcNixExprDrvs: { + /* Load the derivations from the (default or specified) + Nix expression. */ + DrvInfos allElems; + loadDerivations(state, instSource.nixExprPath, instSource.systemFilter, + *instSource.autoArgs, "", allElems); + + elems = filterBySelector(state, allElems, args, newestOnly); + + break; + } -static void queryInstSources(EvalState & state, - InstallSourceInfo & instSource, const Strings & args, - DrvInfos & elems, bool newestOnly) -{ - InstallSourceType type = instSource.type; - if (type == srcUnknown && args.size() > 0 && isPath(args.front())) - type = srcStorePaths; - - switch (type) { - - /* Get the available user environment elements from the - derivations specified in a Nix expression, including only - those with names matching any of the names in `args'. */ - case srcUnknown: - case srcNixExprDrvs: { - - /* Load the derivations from the (default or specified) - Nix expression. */ - DrvInfos allElems; - loadDerivations(state, instSource.nixExprPath, - instSource.systemFilter, *instSource.autoArgs, "", allElems); - - elems = filterBySelector(state, allElems, args, newestOnly); - - break; - } - - /* Get the available user environment elements from the Nix - expressions specified on the command line; these should be - functions that take the default Nix expression file as - argument, e.g., if the file is `./foo.nix', then the - argument `x: x.bar' is equivalent to `(x: x.bar) - (import ./foo.nix)' = `(import ./foo.nix).bar'. */ - case srcNixExprs: { - - Value vArg; - loadSourceExpr(state, instSource.nixExprPath, vArg); - - for (auto & i : args) { - Expr * eFun = state.parseExprFromString(i, absPath(".")); - Value vFun, vTmp; - state.eval(eFun, vFun); - mkApp(vTmp, vFun, vArg); - getDerivations(state, vTmp, "", *instSource.autoArgs, elems, true); - } - - break; - } - - /* The available user environment elements are specified as a - list of store paths (which may or may not be - derivations). */ - case srcStorePaths: { - - for (auto & i : args) { - Path path = state.store->followLinksToStorePath(i); - - string name = baseNameOf(path); - string::size_type dash = name.find('-'); - if (dash != string::npos) - name = string(name, dash + 1); - - DrvInfo elem(state, "", nullptr); - elem.setName(name); - - if (isDerivation(path)) { - elem.setDrvPath(path); - elem.setOutPath(state.store->derivationFromPath(path).findOutput("out")); - if (name.size() >= drvExtension.size() && - string(name, name.size() - drvExtension.size()) == drvExtension) - name = string(name, 0, name.size() - drvExtension.size()); - } - else elem.setOutPath(path); - - elems.push_back(elem); - } - - break; - } - - /* Get the available user environment elements from another - user environment. These are then filtered as in the - `srcNixExprDrvs' case. */ - case srcProfile: { - elems = filterBySelector(state, - queryInstalled(state, instSource.profile), - args, newestOnly); - break; - } - - case srcAttrPath: { - Value vRoot; - loadSourceExpr(state, instSource.nixExprPath, vRoot); - for (auto & i : args) { - Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot)); - getDerivations(state, v, "", *instSource.autoArgs, elems, true); - } - break; - } + /* Get the available user environment elements from the Nix + expressions specified on the command line; these should be + functions that take the default Nix expression file as + argument, e.g., if the file is `./foo.nix', then the + argument `x: x.bar' is equivalent to `(x: x.bar) + (import ./foo.nix)' = `(import ./foo.nix).bar'. */ + case srcNixExprs: { + Value vArg; + loadSourceExpr(state, instSource.nixExprPath, vArg); + + for (auto& i : args) { + Expr* eFun = state.parseExprFromString(i, absPath(".")); + Value vFun, vTmp; + state.eval(eFun, vFun); + mkApp(vTmp, vFun, vArg); + getDerivations(state, vTmp, "", *instSource.autoArgs, elems, true); + } + + break; } -} + /* The available user environment elements are specified as a + list of store paths (which may or may not be + derivations). */ + case srcStorePaths: { + for (auto& i : args) { + Path path = state.store->followLinksToStorePath(i); + + string name = baseNameOf(path); + string::size_type dash = name.find('-'); + if (dash != string::npos) name = string(name, dash + 1); + + DrvInfo elem(state, "", nullptr); + elem.setName(name); + + if (isDerivation(path)) { + elem.setDrvPath(path); + elem.setOutPath( + state.store->derivationFromPath(path).findOutput("out")); + if (name.size() >= drvExtension.size() && + string(name, name.size() - drvExtension.size()) == drvExtension) + name = string(name, 0, name.size() - drvExtension.size()); + } else + elem.setOutPath(path); + + elems.push_back(elem); + } + + break; + } -static void printMissing(EvalState & state, DrvInfos & elems) -{ - PathSet targets; - for (auto & i : elems) { - Path drvPath = i.queryDrvPath(); - if (drvPath != "") - targets.insert(drvPath); - else - targets.insert(i.queryOutPath()); + /* Get the available user environment elements from another + user environment. These are then filtered as in the + `srcNixExprDrvs' case. */ + case srcProfile: { + elems = filterBySelector(state, queryInstalled(state, instSource.profile), + args, newestOnly); + break; } - printMissing(state.store, targets); + case srcAttrPath: { + Value vRoot; + loadSourceExpr(state, instSource.nixExprPath, vRoot); + for (auto& i : args) { + Value& v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot)); + getDerivations(state, v, "", *instSource.autoArgs, elems, true); + } + break; + } + } } - -static bool keep(DrvInfo & drv) -{ - return drv.queryMetaBool("keep", false); +static void printMissing(EvalState& state, DrvInfos& elems) { + PathSet targets; + for (auto& i : elems) { + Path drvPath = i.queryDrvPath(); + if (drvPath != "") + targets.insert(drvPath); + else + targets.insert(i.queryOutPath()); + } + + printMissing(state.store, targets); } +static bool keep(DrvInfo& drv) { return drv.queryMetaBool("keep", false); } + +static void installDerivations(Globals& globals, const Strings& args, + const Path& profile) { + debug(format("installing derivations")); + + /* Get the set of user environment elements to be installed. */ + DrvInfos newElems, newElemsTmp; + queryInstSources(*globals.state, globals.instSource, args, newElemsTmp, true); + + /* If --prebuilt-only is given, filter out source-only packages. */ + for (auto& i : newElemsTmp) + if (!globals.prebuiltOnly || isPrebuilt(*globals.state, i)) + newElems.push_back(i); + + StringSet newNames; + for (auto& i : newElems) { + /* `forceName' is a hack to get package names right in some + one-click installs, namely those where the name used in the + path is not the one we want (e.g., `java-front' versus + `java-front-0.9pre15899'). */ + if (globals.forceName != "") i.setName(globals.forceName); + newNames.insert(DrvName(i.queryName()).name); + } + + while (true) { + string lockToken = optimisticLockProfile(profile); + + DrvInfos allElems(newElems); + + /* Add in the already installed derivations, unless they have + the same name as a to-be-installed element. */ + if (!globals.removeAll) { + DrvInfos installedElems = queryInstalled(*globals.state, profile); + + for (auto& i : installedElems) { + DrvName drvName(i.queryName()); + if (!globals.preserveInstalled && + newNames.find(drvName.name) != newNames.end() && !keep(i)) + printInfo("replacing old '%s'", i.queryName()); + else + allElems.push_back(i); + } -static void installDerivations(Globals & globals, - const Strings & args, const Path & profile) -{ - debug(format("installing derivations")); - - /* Get the set of user environment elements to be installed. */ - DrvInfos newElems, newElemsTmp; - queryInstSources(*globals.state, globals.instSource, args, newElemsTmp, true); - - /* If --prebuilt-only is given, filter out source-only packages. */ - for (auto & i : newElemsTmp) - if (!globals.prebuiltOnly || isPrebuilt(*globals.state, i)) - newElems.push_back(i); - - StringSet newNames; - for (auto & i : newElems) { - /* `forceName' is a hack to get package names right in some - one-click installs, namely those where the name used in the - path is not the one we want (e.g., `java-front' versus - `java-front-0.9pre15899'). */ - if (globals.forceName != "") - i.setName(globals.forceName); - newNames.insert(DrvName(i.queryName()).name); + for (auto& i : newElems) printInfo("installing '%s'", i.queryName()); } + printMissing(*globals.state, newElems); - while (true) { - string lockToken = optimisticLockProfile(profile); - - DrvInfos allElems(newElems); - - /* Add in the already installed derivations, unless they have - the same name as a to-be-installed element. */ - if (!globals.removeAll) { - DrvInfos installedElems = queryInstalled(*globals.state, profile); - - for (auto & i : installedElems) { - DrvName drvName(i.queryName()); - if (!globals.preserveInstalled && - newNames.find(drvName.name) != newNames.end() && - !keep(i)) - printInfo("replacing old '%s'", i.queryName()); - else - allElems.push_back(i); - } - - for (auto & i : newElems) - printInfo("installing '%s'", i.queryName()); - } - - printMissing(*globals.state, newElems); - - if (globals.dryRun) return; + if (globals.dryRun) return; - if (createUserEnv(*globals.state, allElems, - profile, settings.envKeepDerivations, lockToken)) break; - } + if (createUserEnv(*globals.state, allElems, profile, + settings.envKeepDerivations, lockToken)) + break; + } } - -static void opInstall(Globals & globals, Strings opFlags, Strings opArgs) -{ - for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { - string arg = *i++; - if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; - else if (arg == "--preserve-installed" || arg == "-P") - globals.preserveInstalled = true; - else if (arg == "--remove-all" || arg == "-r") - globals.removeAll = true; - else throw UsageError(format("unknown flag '%1%'") % arg); - } - - installDerivations(globals, opArgs, globals.profile); +static void opInstall(Globals& globals, Strings opFlags, Strings opArgs) { + for (Strings::iterator i = opFlags.begin(); i != opFlags.end();) { + string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) + ; + else if (arg == "--preserve-installed" || arg == "-P") + globals.preserveInstalled = true; + else if (arg == "--remove-all" || arg == "-r") + globals.removeAll = true; + else + throw UsageError(format("unknown flag '%1%'") % arg); + } + + installDerivations(globals, opArgs, globals.profile); } - typedef enum { utLt, utLeq, utEq, utAlways } UpgradeType; +static void upgradeDerivations(Globals& globals, const Strings& args, + UpgradeType upgradeType) { + debug(format("upgrading derivations")); -static void upgradeDerivations(Globals & globals, - const Strings & args, UpgradeType upgradeType) -{ - debug(format("upgrading derivations")); - - /* Upgrade works as follows: we take all currently installed - derivations, and for any derivation matching any selector, look - for a derivation in the input Nix expression that has the same - name and a higher version number. */ - - while (true) { - string lockToken = optimisticLockProfile(globals.profile); + /* Upgrade works as follows: we take all currently installed + derivations, and for any derivation matching any selector, look + for a derivation in the input Nix expression that has the same + name and a higher version number. */ - DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); + while (true) { + string lockToken = optimisticLockProfile(globals.profile); - /* Fetch all derivations from the input file. */ - DrvInfos availElems; - queryInstSources(*globals.state, globals.instSource, args, availElems, false); + DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); - /* Go through all installed derivations. */ - DrvInfos newElems; - for (auto & i : installedElems) { - DrvName drvName(i.queryName()); + /* Fetch all derivations from the input file. */ + DrvInfos availElems; + queryInstSources(*globals.state, globals.instSource, args, availElems, + false); - try { + /* Go through all installed derivations. */ + DrvInfos newElems; + for (auto& i : installedElems) { + DrvName drvName(i.queryName()); - if (keep(i)) { - newElems.push_back(i); - continue; - } - - /* Find the derivation in the input Nix expression - with the same name that satisfies the version - constraints specified by upgradeType. If there are - multiple matches, take the one with the highest - priority. If there are still multiple matches, - take the one with the highest version. - Do not upgrade if it would decrease the priority. */ - DrvInfos::iterator bestElem = availElems.end(); - string bestVersion; - for (auto j = availElems.begin(); j != availElems.end(); ++j) { - if (comparePriorities(*globals.state, i, *j) > 0) - continue; - DrvName newName(j->queryName()); - if (newName.name == drvName.name) { - int d = compareVersions(drvName.version, newName.version); - if ((upgradeType == utLt && d < 0) || - (upgradeType == utLeq && d <= 0) || - (upgradeType == utEq && d == 0) || - upgradeType == utAlways) - { - long d2 = -1; - if (bestElem != availElems.end()) { - d2 = comparePriorities(*globals.state, *bestElem, *j); - if (d2 == 0) d2 = compareVersions(bestVersion, newName.version); - } - if (d2 < 0 && (!globals.prebuiltOnly || isPrebuilt(*globals.state, *j))) { - bestElem = j; - bestVersion = newName.version; - } - } - } - } + try { + if (keep(i)) { + newElems.push_back(i); + continue; + } - if (bestElem != availElems.end() && - i.queryOutPath() != - bestElem->queryOutPath()) - { - const char * action = compareVersions(drvName.version, bestVersion) <= 0 - ? "upgrading" : "downgrading"; - printInfo("%1% '%2%' to '%3%'", - action, i.queryName(), bestElem->queryName()); - newElems.push_back(*bestElem); - } else newElems.push_back(i); - - } catch (Error & e) { - e.addPrefix(fmt("while trying to find an upgrade for '%s':\n", i.queryName())); - throw; + /* Find the derivation in the input Nix expression + with the same name that satisfies the version + constraints specified by upgradeType. If there are + multiple matches, take the one with the highest + priority. If there are still multiple matches, + take the one with the highest version. + Do not upgrade if it would decrease the priority. */ + DrvInfos::iterator bestElem = availElems.end(); + string bestVersion; + for (auto j = availElems.begin(); j != availElems.end(); ++j) { + if (comparePriorities(*globals.state, i, *j) > 0) continue; + DrvName newName(j->queryName()); + if (newName.name == drvName.name) { + int d = compareVersions(drvName.version, newName.version); + if ((upgradeType == utLt && d < 0) || + (upgradeType == utLeq && d <= 0) || + (upgradeType == utEq && d == 0) || upgradeType == utAlways) { + long d2 = -1; + if (bestElem != availElems.end()) { + d2 = comparePriorities(*globals.state, *bestElem, *j); + if (d2 == 0) d2 = compareVersions(bestVersion, newName.version); + } + if (d2 < 0 && + (!globals.prebuiltOnly || isPrebuilt(*globals.state, *j))) { + bestElem = j; + bestVersion = newName.version; + } } + } } - printMissing(*globals.state, newElems); - - if (globals.dryRun) return; - - if (createUserEnv(*globals.state, newElems, - globals.profile, settings.envKeepDerivations, lockToken)) break; + if (bestElem != availElems.end() && + i.queryOutPath() != bestElem->queryOutPath()) { + const char* action = + compareVersions(drvName.version, bestVersion) <= 0 + ? "upgrading" + : "downgrading"; + printInfo("%1% '%2%' to '%3%'", action, i.queryName(), + bestElem->queryName()); + newElems.push_back(*bestElem); + } else + newElems.push_back(i); + + } catch (Error& e) { + e.addPrefix( + fmt("while trying to find an upgrade for '%s':\n", i.queryName())); + throw; + } } -} + printMissing(*globals.state, newElems); -static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs) -{ - UpgradeType upgradeType = utLt; - for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { - string arg = *i++; - if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; - else if (arg == "--lt") upgradeType = utLt; - else if (arg == "--leq") upgradeType = utLeq; - else if (arg == "--eq") upgradeType = utEq; - else if (arg == "--always") upgradeType = utAlways; - else throw UsageError(format("unknown flag '%1%'") % arg); - } + if (globals.dryRun) return; - upgradeDerivations(globals, opArgs, upgradeType); + if (createUserEnv(*globals.state, newElems, globals.profile, + settings.envKeepDerivations, lockToken)) + break; + } } - -static void setMetaFlag(EvalState & state, DrvInfo & drv, - const string & name, const string & value) -{ - Value * v = state.allocValue(); - mkString(*v, value.c_str()); - drv.setMeta(name, v); +static void opUpgrade(Globals& globals, Strings opFlags, Strings opArgs) { + UpgradeType upgradeType = utLt; + for (Strings::iterator i = opFlags.begin(); i != opFlags.end();) { + string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) + ; + else if (arg == "--lt") + upgradeType = utLt; + else if (arg == "--leq") + upgradeType = utLeq; + else if (arg == "--eq") + upgradeType = utEq; + else if (arg == "--always") + upgradeType = utAlways; + else + throw UsageError(format("unknown flag '%1%'") % arg); + } + + upgradeDerivations(globals, opArgs, upgradeType); } +static void setMetaFlag(EvalState& state, DrvInfo& drv, const string& name, + const string& value) { + Value* v = state.allocValue(); + mkString(*v, value.c_str()); + drv.setMeta(name, v); +} -static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); - if (opArgs.size() < 2) - throw UsageError("not enough arguments to '--set-flag'"); - - Strings::iterator arg = opArgs.begin(); - string flagName = *arg++; - string flagValue = *arg++; - DrvNames selectors = drvNamesFromArgs(Strings(arg, opArgs.end())); - - while (true) { - string lockToken = optimisticLockProfile(globals.profile); - - DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); - - /* Update all matching derivations. */ - for (auto & i : installedElems) { - DrvName drvName(i.queryName()); - for (auto & j : selectors) - if (j.matches(drvName)) { - printInfo("setting flag on '%1%'", i.queryName()); - j.hits++; - setMetaFlag(*globals.state, i, flagName, flagValue); - break; - } +static void opSetFlag(Globals& globals, Strings opFlags, Strings opArgs) { + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + if (opArgs.size() < 2) + throw UsageError("not enough arguments to '--set-flag'"); + + Strings::iterator arg = opArgs.begin(); + string flagName = *arg++; + string flagValue = *arg++; + DrvNames selectors = drvNamesFromArgs(Strings(arg, opArgs.end())); + + while (true) { + string lockToken = optimisticLockProfile(globals.profile); + + DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); + + /* Update all matching derivations. */ + for (auto& i : installedElems) { + DrvName drvName(i.queryName()); + for (auto& j : selectors) + if (j.matches(drvName)) { + printInfo("setting flag on '%1%'", i.queryName()); + j.hits++; + setMetaFlag(*globals.state, i, flagName, flagValue); + break; } + } - checkSelectorUse(selectors); + checkSelectorUse(selectors); - /* Write the new user environment. */ - if (createUserEnv(*globals.state, installedElems, - globals.profile, settings.envKeepDerivations, lockToken)) break; - } + /* Write the new user environment. */ + if (createUserEnv(*globals.state, installedElems, globals.profile, + settings.envKeepDerivations, lockToken)) + break; + } } +static void opSet(Globals& globals, Strings opFlags, Strings opArgs) { + auto store2 = globals.state->store.dynamic_pointer_cast<LocalFSStore>(); + if (!store2) throw Error("--set is not supported for this Nix store"); -static void opSet(Globals & globals, Strings opFlags, Strings opArgs) -{ - auto store2 = globals.state->store.dynamic_pointer_cast<LocalFSStore>(); - if (!store2) throw Error("--set is not supported for this Nix store"); + for (Strings::iterator i = opFlags.begin(); i != opFlags.end();) { + string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) + ; + else + throw UsageError(format("unknown flag '%1%'") % arg); + } - for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { - string arg = *i++; - if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; - else throw UsageError(format("unknown flag '%1%'") % arg); - } - - DrvInfos elems; - queryInstSources(*globals.state, globals.instSource, opArgs, elems, true); + DrvInfos elems; + queryInstSources(*globals.state, globals.instSource, opArgs, elems, true); - if (elems.size() != 1) - throw Error("--set requires exactly one derivation"); + if (elems.size() != 1) throw Error("--set requires exactly one derivation"); - DrvInfo & drv(elems.front()); + DrvInfo& drv(elems.front()); - if (globals.forceName != "") - drv.setName(globals.forceName); + if (globals.forceName != "") drv.setName(globals.forceName); - if (drv.queryDrvPath() != "") { - PathSet paths = {drv.queryDrvPath()}; - printMissing(globals.state->store, paths); - if (globals.dryRun) return; - globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); - } - else { - printMissing(globals.state->store, {drv.queryOutPath()}); - if (globals.dryRun) return; - globals.state->store->ensurePath(drv.queryOutPath()); - } + if (drv.queryDrvPath() != "") { + PathSet paths = {drv.queryDrvPath()}; + printMissing(globals.state->store, paths); + if (globals.dryRun) return; + globals.state->store->buildPaths( + paths, globals.state->repair ? bmRepair : bmNormal); + } else { + printMissing(globals.state->store, {drv.queryOutPath()}); + if (globals.dryRun) return; + globals.state->store->ensurePath(drv.queryOutPath()); + } - debug(format("switching to new user environment")); - Path generation = createGeneration(ref<LocalFSStore>(store2), globals.profile, drv.queryOutPath()); - switchLink(globals.profile, generation); + debug(format("switching to new user environment")); + Path generation = createGeneration(ref<LocalFSStore>(store2), globals.profile, + drv.queryOutPath()); + switchLink(globals.profile, generation); } - -static void uninstallDerivations(Globals & globals, Strings & selectors, - Path & profile) -{ - while (true) { - string lockToken = optimisticLockProfile(profile); - - DrvInfos installedElems = queryInstalled(*globals.state, profile); - DrvInfos newElems; - - for (auto & i : installedElems) { - DrvName drvName(i.queryName()); - bool found = false; - for (auto & j : selectors) - /* !!! the repeated calls to followLinksToStorePath() - are expensive, should pre-compute them. */ - if ((isPath(j) && i.queryOutPath() == globals.state->store->followLinksToStorePath(j)) - || DrvName(j).matches(drvName)) - { - printInfo("uninstalling '%s'", i.queryName()); - found = true; - break; - } - if (!found) newElems.push_back(i); +static void uninstallDerivations(Globals& globals, Strings& selectors, + Path& profile) { + while (true) { + string lockToken = optimisticLockProfile(profile); + + DrvInfos installedElems = queryInstalled(*globals.state, profile); + DrvInfos newElems; + + for (auto& i : installedElems) { + DrvName drvName(i.queryName()); + bool found = false; + for (auto& j : selectors) + /* !!! the repeated calls to followLinksToStorePath() + are expensive, should pre-compute them. */ + if ((isPath(j) && + i.queryOutPath() == + globals.state->store->followLinksToStorePath(j)) || + DrvName(j).matches(drvName)) { + printInfo("uninstalling '%s'", i.queryName()); + found = true; + break; } - - if (globals.dryRun) return; - - if (createUserEnv(*globals.state, newElems, - profile, settings.envKeepDerivations, lockToken)) break; + if (!found) newElems.push_back(i); } -} + if (globals.dryRun) return; -static void opUninstall(Globals & globals, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); - uninstallDerivations(globals, opArgs, globals.profile); + if (createUserEnv(*globals.state, newElems, profile, + settings.envKeepDerivations, lockToken)) + break; + } } - -static bool cmpChars(char a, char b) -{ - return toupper(a) < toupper(b); +static void opUninstall(Globals& globals, Strings opFlags, Strings opArgs) { + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + uninstallDerivations(globals, opArgs, globals.profile); } +static bool cmpChars(char a, char b) { return toupper(a) < toupper(b); } -static bool cmpElemByName(const DrvInfo & a, const DrvInfo & b) -{ - auto a_name = a.queryName(); - auto b_name = b.queryName(); - return lexicographical_compare( - a_name.begin(), a_name.end(), - b_name.begin(), b_name.end(), cmpChars); +static bool cmpElemByName(const DrvInfo& a, const DrvInfo& b) { + auto a_name = a.queryName(); + auto b_name = b.queryName(); + return lexicographical_compare(a_name.begin(), a_name.end(), b_name.begin(), + b_name.end(), cmpChars); } - typedef list<Strings> Table; - -void printTable(Table & table) -{ - auto nrColumns = table.size() > 0 ? table.front().size() : 0; - - vector<size_t> widths; - widths.resize(nrColumns); - - for (auto & i : table) { - assert(i.size() == nrColumns); - Strings::iterator j; - size_t column; - for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) - if (j->size() > widths[column]) widths[column] = j->size(); - } - - for (auto & i : table) { - Strings::iterator j; - size_t column; - for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) { - string s = *j; - replace(s.begin(), s.end(), '\n', ' '); - cout << s; - if (column < nrColumns - 1) - cout << string(widths[column] - s.size() + 2, ' '); - } - cout << std::endl; +void printTable(Table& table) { + auto nrColumns = table.size() > 0 ? table.front().size() : 0; + + vector<size_t> widths; + widths.resize(nrColumns); + + for (auto& i : table) { + assert(i.size() == nrColumns); + Strings::iterator j; + size_t column; + for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) + if (j->size() > widths[column]) widths[column] = j->size(); + } + + for (auto& i : table) { + Strings::iterator j; + size_t column; + for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) { + string s = *j; + replace(s.begin(), s.end(), '\n', ' '); + cout << s; + if (column < nrColumns - 1) + cout << string(widths[column] - s.size() + 2, ' '); } + cout << std::endl; + } } - /* This function compares the version of an element against the versions in the given set of elements. `cvLess' means that only lower versions are in the set, `cvEqual' means that at most an @@ -816,642 +753,647 @@ void printTable(Table & table) typedef enum { cvLess, cvEqual, cvGreater, cvUnavail } VersionDiff; -static VersionDiff compareVersionAgainstSet( - const DrvInfo & elem, const DrvInfos & elems, string & version) -{ - DrvName name(elem.queryName()); - - VersionDiff diff = cvUnavail; - version = "?"; - - for (auto & i : elems) { - DrvName name2(i.queryName()); - if (name.name == name2.name) { - int d = compareVersions(name.version, name2.version); - if (d < 0) { - diff = cvGreater; - version = name2.version; - } - else if (diff != cvGreater && d == 0) { - diff = cvEqual; - version = name2.version; - } - else if (diff != cvGreater && diff != cvEqual && d > 0) { - diff = cvLess; - if (version == "" || compareVersions(version, name2.version) < 0) - version = name2.version; - } - } +static VersionDiff compareVersionAgainstSet(const DrvInfo& elem, + const DrvInfos& elems, + string& version) { + DrvName name(elem.queryName()); + + VersionDiff diff = cvUnavail; + version = "?"; + + for (auto& i : elems) { + DrvName name2(i.queryName()); + if (name.name == name2.name) { + int d = compareVersions(name.version, name2.version); + if (d < 0) { + diff = cvGreater; + version = name2.version; + } else if (diff != cvGreater && d == 0) { + diff = cvEqual; + version = name2.version; + } else if (diff != cvGreater && diff != cvEqual && d > 0) { + diff = cvLess; + if (version == "" || compareVersions(version, name2.version) < 0) + version = name2.version; + } } + } - return diff; + return diff; } - -static void queryJSON(Globals & globals, vector<DrvInfo> & elems) -{ - JSONObject topObj(cout, true); - for (auto & i : elems) { - JSONObject pkgObj = topObj.object(i.attrPath); - - auto drvName = DrvName(i.queryName()); - pkgObj.attr("name", drvName.fullName); - pkgObj.attr("pname", drvName.name); - pkgObj.attr("version", drvName.version); - pkgObj.attr("system", i.querySystem()); - - JSONObject metaObj = pkgObj.object("meta"); - StringSet metaNames = i.queryMetaNames(); - for (auto & j : metaNames) { - auto placeholder = metaObj.placeholder(j); - Value * v = i.queryMeta(j); - if (!v) { - printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); - placeholder.write(nullptr); - } else { - PathSet context; - printValueAsJSON(*globals.state, true, *v, placeholder, context); - } - } +static void queryJSON(Globals& globals, vector<DrvInfo>& elems) { + JSONObject topObj(cout, true); + for (auto& i : elems) { + JSONObject pkgObj = topObj.object(i.attrPath); + + auto drvName = DrvName(i.queryName()); + pkgObj.attr("name", drvName.fullName); + pkgObj.attr("pname", drvName.name); + pkgObj.attr("version", drvName.version); + pkgObj.attr("system", i.querySystem()); + + JSONObject metaObj = pkgObj.object("meta"); + StringSet metaNames = i.queryMetaNames(); + for (auto& j : metaNames) { + auto placeholder = metaObj.placeholder(j); + Value* v = i.queryMeta(j); + if (!v) { + printError("derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j); + placeholder.write(nullptr); + } else { + PathSet context; + printValueAsJSON(*globals.state, true, *v, placeholder, context); + } } + } } +static void opQuery(Globals& globals, Strings opFlags, Strings opArgs) { + Strings remaining; + string attrPath; + + bool printStatus = false; + bool printName = true; + bool printAttrPath = false; + bool printSystem = false; + bool printDrvPath = false; + bool printOutPath = false; + bool printDescription = false; + bool printMeta = false; + bool compareVersions = false; + bool xmlOutput = false; + bool jsonOutput = false; + + enum { sInstalled, sAvailable } source = sInstalled; + + settings.readOnlyMode = true; /* makes evaluation a bit faster */ + + for (Strings::iterator i = opFlags.begin(); i != opFlags.end();) { + string arg = *i++; + if (arg == "--status" || arg == "-s") + printStatus = true; + else if (arg == "--no-name") + printName = false; + else if (arg == "--system") + printSystem = true; + else if (arg == "--description") + printDescription = true; + else if (arg == "--compare-versions" || arg == "-c") + compareVersions = true; + else if (arg == "--drv-path") + printDrvPath = true; + else if (arg == "--out-path") + printOutPath = true; + else if (arg == "--meta") + printMeta = true; + else if (arg == "--installed") + source = sInstalled; + else if (arg == "--available" || arg == "-a") + source = sAvailable; + else if (arg == "--xml") + xmlOutput = true; + else if (arg == "--json") + jsonOutput = true; + else if (arg == "--attr-path" || arg == "-P") + printAttrPath = true; + else if (arg == "--attr" || arg == "-A") + attrPath = needArg(i, opFlags, arg); + else + throw UsageError(format("unknown flag '%1%'") % arg); + } + + /* Obtain derivation information from the specified source. */ + DrvInfos availElems, installedElems; + + if (source == sInstalled || compareVersions || printStatus) + installedElems = queryInstalled(*globals.state, globals.profile); + + if (source == sAvailable || compareVersions) + loadDerivations(*globals.state, globals.instSource.nixExprPath, + globals.instSource.systemFilter, + *globals.instSource.autoArgs, attrPath, availElems); + + DrvInfos elems_ = filterBySelector( + *globals.state, source == sInstalled ? installedElems : availElems, + opArgs, false); + + DrvInfos& otherElems(source == sInstalled ? availElems : installedElems); + + /* Sort them by name. */ + /* !!! */ + vector<DrvInfo> elems; + for (auto& i : elems_) elems.push_back(i); + sort(elems.begin(), elems.end(), cmpElemByName); + + /* We only need to know the installed paths when we are querying + the status of the derivation. */ + PathSet installed; /* installed paths */ + + if (printStatus) { + for (auto& i : installedElems) installed.insert(i.queryOutPath()); + } + + /* Query which paths have substitutes. */ + PathSet validPaths, substitutablePaths; + if (printStatus || globals.prebuiltOnly) { + PathSet paths; + for (auto& i : elems) try { + paths.insert(i.queryOutPath()); + } catch (AssertionError& e) { + printMsg( + lvlTalkative, + "skipping derivation named '%s' which gives an assertion failure", + i.queryName()); + i.setFailed(); + } + validPaths = globals.state->store->queryValidPaths(paths); + substitutablePaths = globals.state->store->querySubstitutablePaths(paths); + } + + /* Print the desired columns, or XML output. */ + if (jsonOutput) { + queryJSON(globals, elems); + return; + } + + bool tty = isatty(STDOUT_FILENO); + RunPager pager; + + Table table; + std::ostringstream dummy; + XMLWriter xml(true, *(xmlOutput ? &cout : &dummy)); + XMLOpenElement xmlRoot(xml, "items"); + + for (auto& i : elems) { + try { + if (i.hasFailed()) continue; + + // Activity act(*logger, lvlDebug, format("outputting query result '%1%'") + // % i.attrPath); + + if (globals.prebuiltOnly && + validPaths.find(i.queryOutPath()) == validPaths.end() && + substitutablePaths.find(i.queryOutPath()) == substitutablePaths.end()) + continue; + + /* For table output. */ + Strings columns; + + /* For XML output. */ + XMLAttrs attrs; + + if (printStatus) { + Path outPath = i.queryOutPath(); + bool hasSubs = + substitutablePaths.find(outPath) != substitutablePaths.end(); + bool isInstalled = installed.find(outPath) != installed.end(); + bool isValid = validPaths.find(outPath) != validPaths.end(); + if (xmlOutput) { + attrs["installed"] = isInstalled ? "1" : "0"; + attrs["valid"] = isValid ? "1" : "0"; + attrs["substitutable"] = hasSubs ? "1" : "0"; + } else + columns.push_back((string)(isInstalled ? "I" : "-") + + (isValid ? "P" : "-") + (hasSubs ? "S" : "-")); + } + + if (xmlOutput) + attrs["attrPath"] = i.attrPath; + else if (printAttrPath) + columns.push_back(i.attrPath); + + if (xmlOutput) { + auto drvName = DrvName(i.queryName()); + attrs["name"] = drvName.fullName; + attrs["pname"] = drvName.name; + attrs["version"] = drvName.version; + } else if (printName) { + columns.push_back(i.queryName()); + } + + if (compareVersions) { + /* Compare this element against the versions of the + same named packages in either the set of available + elements, or the set of installed elements. !!! + This is O(N * M), should be O(N * lg M). */ + string version; + VersionDiff diff = compareVersionAgainstSet(i, otherElems, version); + + char ch; + switch (diff) { + case cvLess: + ch = '>'; + break; + case cvEqual: + ch = '='; + break; + case cvGreater: + ch = '<'; + break; + case cvUnavail: + ch = '-'; + break; + default: + abort(); + } -static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) -{ - Strings remaining; - string attrPath; - - bool printStatus = false; - bool printName = true; - bool printAttrPath = false; - bool printSystem = false; - bool printDrvPath = false; - bool printOutPath = false; - bool printDescription = false; - bool printMeta = false; - bool compareVersions = false; - bool xmlOutput = false; - bool jsonOutput = false; - - enum { sInstalled, sAvailable } source = sInstalled; - - settings.readOnlyMode = true; /* makes evaluation a bit faster */ - - for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { - string arg = *i++; - if (arg == "--status" || arg == "-s") printStatus = true; - else if (arg == "--no-name") printName = false; - else if (arg == "--system") printSystem = true; - else if (arg == "--description") printDescription = true; - else if (arg == "--compare-versions" || arg == "-c") compareVersions = true; - else if (arg == "--drv-path") printDrvPath = true; - else if (arg == "--out-path") printOutPath = true; - else if (arg == "--meta") printMeta = true; - else if (arg == "--installed") source = sInstalled; - else if (arg == "--available" || arg == "-a") source = sAvailable; - else if (arg == "--xml") xmlOutput = true; - else if (arg == "--json") jsonOutput = true; - else if (arg == "--attr-path" || arg == "-P") printAttrPath = true; - else if (arg == "--attr" || arg == "-A") - attrPath = needArg(i, opFlags, arg); - else - throw UsageError(format("unknown flag '%1%'") % arg); - } - - - /* Obtain derivation information from the specified source. */ - DrvInfos availElems, installedElems; - - if (source == sInstalled || compareVersions || printStatus) - installedElems = queryInstalled(*globals.state, globals.profile); - - if (source == sAvailable || compareVersions) - loadDerivations(*globals.state, globals.instSource.nixExprPath, - globals.instSource.systemFilter, *globals.instSource.autoArgs, - attrPath, availElems); - - DrvInfos elems_ = filterBySelector(*globals.state, - source == sInstalled ? installedElems : availElems, - opArgs, false); - - DrvInfos & otherElems(source == sInstalled ? availElems : installedElems); - - - /* Sort them by name. */ - /* !!! */ - vector<DrvInfo> elems; - for (auto & i : elems_) elems.push_back(i); - sort(elems.begin(), elems.end(), cmpElemByName); - - - /* We only need to know the installed paths when we are querying - the status of the derivation. */ - PathSet installed; /* installed paths */ - - if (printStatus) { - for (auto & i : installedElems) - installed.insert(i.queryOutPath()); - } - - - /* Query which paths have substitutes. */ - PathSet validPaths, substitutablePaths; - if (printStatus || globals.prebuiltOnly) { - PathSet paths; - for (auto & i : elems) - try { - paths.insert(i.queryOutPath()); - } catch (AssertionError & e) { - printMsg(lvlTalkative, "skipping derivation named '%s' which gives an assertion failure", i.queryName()); - i.setFailed(); - } - validPaths = globals.state->store->queryValidPaths(paths); - substitutablePaths = globals.state->store->querySubstitutablePaths(paths); - } - - - /* Print the desired columns, or XML output. */ - if (jsonOutput) { - queryJSON(globals, elems); - return; - } - - bool tty = isatty(STDOUT_FILENO); - RunPager pager; - - Table table; - std::ostringstream dummy; - XMLWriter xml(true, *(xmlOutput ? &cout : &dummy)); - XMLOpenElement xmlRoot(xml, "items"); - - for (auto & i : elems) { - try { - if (i.hasFailed()) continue; - - //Activity act(*logger, lvlDebug, format("outputting query result '%1%'") % i.attrPath); - - if (globals.prebuiltOnly && - validPaths.find(i.queryOutPath()) == validPaths.end() && - substitutablePaths.find(i.queryOutPath()) == substitutablePaths.end()) - continue; - - /* For table output. */ - Strings columns; - - /* For XML output. */ - XMLAttrs attrs; - - if (printStatus) { - Path outPath = i.queryOutPath(); - bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); - bool isInstalled = installed.find(outPath) != installed.end(); - bool isValid = validPaths.find(outPath) != validPaths.end(); - if (xmlOutput) { - attrs["installed"] = isInstalled ? "1" : "0"; - attrs["valid"] = isValid ? "1" : "0"; - attrs["substitutable"] = hasSubs ? "1" : "0"; - } else - columns.push_back( - (string) (isInstalled ? "I" : "-") - + (isValid ? "P" : "-") - + (hasSubs ? "S" : "-")); - } - - if (xmlOutput) - attrs["attrPath"] = i.attrPath; - else if (printAttrPath) - columns.push_back(i.attrPath); - - if (xmlOutput) { - auto drvName = DrvName(i.queryName()); - attrs["name"] = drvName.fullName; - attrs["pname"] = drvName.name; - attrs["version"] = drvName.version; - } else if (printName) { - columns.push_back(i.queryName()); - } - - if (compareVersions) { - /* Compare this element against the versions of the - same named packages in either the set of available - elements, or the set of installed elements. !!! - This is O(N * M), should be O(N * lg M). */ - string version; - VersionDiff diff = compareVersionAgainstSet(i, otherElems, version); - - char ch; - switch (diff) { - case cvLess: ch = '>'; break; - case cvEqual: ch = '='; break; - case cvGreater: ch = '<'; break; - case cvUnavail: ch = '-'; break; - default: abort(); - } - - if (xmlOutput) { - if (diff != cvUnavail) { - attrs["versionDiff"] = ch; - attrs["maxComparedVersion"] = version; - } - } else { - string column = (string) "" + ch + " " + version; - if (diff == cvGreater && tty) - column = ANSI_RED + column + ANSI_NORMAL; - columns.push_back(column); - } - } - - if (xmlOutput) { - if (i.querySystem() != "") attrs["system"] = i.querySystem(); - } - else if (printSystem) - columns.push_back(i.querySystem()); - - if (printDrvPath) { - string drvPath = i.queryDrvPath(); - if (xmlOutput) { - if (drvPath != "") attrs["drvPath"] = drvPath; - } else - columns.push_back(drvPath == "" ? "-" : drvPath); + if (xmlOutput) { + if (diff != cvUnavail) { + attrs["versionDiff"] = ch; + attrs["maxComparedVersion"] = version; + } + } else { + string column = (string) "" + ch + " " + version; + if (diff == cvGreater && tty) + column = ANSI_RED + column + ANSI_NORMAL; + columns.push_back(column); + } + } + + if (xmlOutput) { + if (i.querySystem() != "") attrs["system"] = i.querySystem(); + } else if (printSystem) + columns.push_back(i.querySystem()); + + if (printDrvPath) { + string drvPath = i.queryDrvPath(); + if (xmlOutput) { + if (drvPath != "") attrs["drvPath"] = drvPath; + } else + columns.push_back(drvPath == "" ? "-" : drvPath); + } + + if (printOutPath && !xmlOutput) { + DrvInfo::Outputs outputs = i.queryOutputs(); + string s; + for (auto& j : outputs) { + if (!s.empty()) s += ';'; + if (j.first != "out") { + s += j.first; + s += "="; + } + s += j.second; + } + columns.push_back(s); + } + + if (printDescription) { + string descr = i.queryMetaString("description"); + if (xmlOutput) { + if (descr != "") attrs["description"] = descr; + } else + columns.push_back(descr); + } + + if (xmlOutput) { + if (printOutPath || printMeta) { + XMLOpenElement item(xml, "item", attrs); + if (printOutPath) { + DrvInfo::Outputs outputs = i.queryOutputs(); + for (auto& j : outputs) { + XMLAttrs attrs2; + attrs2["name"] = j.first; + attrs2["path"] = j.second; + xml.writeEmptyElement("output", attrs2); } - - if (printOutPath && !xmlOutput) { - DrvInfo::Outputs outputs = i.queryOutputs(); - string s; - for (auto & j : outputs) { - if (!s.empty()) s += ';'; - if (j.first != "out") { s += j.first; s += "="; } - s += j.second; + } + if (printMeta) { + StringSet metaNames = i.queryMetaNames(); + for (auto& j : metaNames) { + XMLAttrs attrs2; + attrs2["name"] = j; + Value* v = i.queryMeta(j); + if (!v) + printError("derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j); + else { + if (v->type == tString) { + attrs2["type"] = "string"; + attrs2["value"] = v->string.s; + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tInt) { + attrs2["type"] = "int"; + attrs2["value"] = (format("%1%") % v->integer).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tFloat) { + attrs2["type"] = "float"; + attrs2["value"] = (format("%1%") % v->fpoint).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tBool) { + attrs2["type"] = "bool"; + attrs2["value"] = v->boolean ? "true" : "false"; + xml.writeEmptyElement("meta", attrs2); + } else if (v->isList()) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + for (unsigned int j = 0; j < v->listSize(); ++j) { + if (v->listElems()[j]->type != tString) continue; + XMLAttrs attrs3; + attrs3["value"] = v->listElems()[j]->string.s; + xml.writeEmptyElement("string", attrs3); + } + } else if (v->type == tAttrs) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + Bindings& attrs = *v->attrs; + for (auto& i : attrs) { + Attr& a(*attrs.find(i.name)); + if (a.value->type != tString) continue; + XMLAttrs attrs3; + attrs3["type"] = i.name; + attrs3["value"] = a.value->string.s; + xml.writeEmptyElement("string", attrs3); + } } - columns.push_back(s); - } - - if (printDescription) { - string descr = i.queryMetaString("description"); - if (xmlOutput) { - if (descr != "") attrs["description"] = descr; - } else - columns.push_back(descr); + } } - - if (xmlOutput) { - if (printOutPath || printMeta) { - XMLOpenElement item(xml, "item", attrs); - if (printOutPath) { - DrvInfo::Outputs outputs = i.queryOutputs(); - for (auto & j : outputs) { - XMLAttrs attrs2; - attrs2["name"] = j.first; - attrs2["path"] = j.second; - xml.writeEmptyElement("output", attrs2); - } - } - if (printMeta) { - StringSet metaNames = i.queryMetaNames(); - for (auto & j : metaNames) { - XMLAttrs attrs2; - attrs2["name"] = j; - Value * v = i.queryMeta(j); - if (!v) - printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); - else { - if (v->type == tString) { - attrs2["type"] = "string"; - attrs2["value"] = v->string.s; - xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tInt) { - attrs2["type"] = "int"; - attrs2["value"] = (format("%1%") % v->integer).str(); - xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tFloat) { - attrs2["type"] = "float"; - attrs2["value"] = (format("%1%") % v->fpoint).str(); - xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tBool) { - attrs2["type"] = "bool"; - attrs2["value"] = v->boolean ? "true" : "false"; - xml.writeEmptyElement("meta", attrs2); - } else if (v->isList()) { - attrs2["type"] = "strings"; - XMLOpenElement m(xml, "meta", attrs2); - for (unsigned int j = 0; j < v->listSize(); ++j) { - if (v->listElems()[j]->type != tString) continue; - XMLAttrs attrs3; - attrs3["value"] = v->listElems()[j]->string.s; - xml.writeEmptyElement("string", attrs3); - } - } else if (v->type == tAttrs) { - attrs2["type"] = "strings"; - XMLOpenElement m(xml, "meta", attrs2); - Bindings & attrs = *v->attrs; - for (auto &i : attrs) { - Attr & a(*attrs.find(i.name)); - if(a.value->type != tString) continue; - XMLAttrs attrs3; - attrs3["type"] = i.name; - attrs3["value"] = a.value->string.s; - xml.writeEmptyElement("string", attrs3); - } - } - } - } - } - } else - xml.writeEmptyElement("item", attrs); - } else - table.push_back(columns); - - cout.flush(); - - } catch (AssertionError & e) { - printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName()); - } catch (Error & e) { - e.addPrefix(fmt("while querying the derivation named '%1%':\n", i.queryName())); - throw; - } + } + } else + xml.writeEmptyElement("item", attrs); + } else + table.push_back(columns); + + cout.flush(); + + } catch (AssertionError& e) { + printMsg( + lvlTalkative, + "skipping derivation named '%1%' which gives an assertion failure", + i.queryName()); + } catch (Error& e) { + e.addPrefix( + fmt("while querying the derivation named '%1%':\n", i.queryName())); + throw; } + } - if (!xmlOutput) printTable(table); + if (!xmlOutput) printTable(table); } +static void opSwitchProfile(Globals& globals, Strings opFlags, Strings opArgs) { + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + if (opArgs.size() != 1) + throw UsageError(format("exactly one argument expected")); -static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); - if (opArgs.size() != 1) - throw UsageError(format("exactly one argument expected")); - - Path profile = absPath(opArgs.front()); - Path profileLink = getHome() + "/.nix-profile"; + Path profile = absPath(opArgs.front()); + Path profileLink = getHome() + "/.nix-profile"; - switchLink(profileLink, profile); + switchLink(profileLink, profile); } - static const int prevGen = -2; +static void switchGeneration(Globals& globals, int dstGen) { + PathLocks lock; + lockProfile(lock, globals.profile); -static void switchGeneration(Globals & globals, int dstGen) -{ - PathLocks lock; - lockProfile(lock, globals.profile); - - int curGen; - Generations gens = findGenerations(globals.profile, curGen); + int curGen; + Generations gens = findGenerations(globals.profile, curGen); - Generation dst; - for (auto & i : gens) - if ((dstGen == prevGen && i.number < curGen) || - (dstGen >= 0 && i.number == dstGen)) - dst = i; + Generation dst; + for (auto& i : gens) + if ((dstGen == prevGen && i.number < curGen) || + (dstGen >= 0 && i.number == dstGen)) + dst = i; - if (!dst) { - if (dstGen == prevGen) - throw Error(format("no generation older than the current (%1%) exists") - % curGen); - else - throw Error(format("generation %1% does not exist") % dstGen); - } + if (!dst) { + if (dstGen == prevGen) + throw Error(format("no generation older than the current (%1%) exists") % + curGen); + else + throw Error(format("generation %1% does not exist") % dstGen); + } - printInfo(format("switching from generation %1% to %2%") - % curGen % dst.number); + printInfo(format("switching from generation %1% to %2%") % curGen % + dst.number); - if (globals.dryRun) return; + if (globals.dryRun) return; - switchLink(globals.profile, dst.path); + switchLink(globals.profile, dst.path); } +static void opSwitchGeneration(Globals& globals, Strings opFlags, + Strings opArgs) { + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + if (opArgs.size() != 1) + throw UsageError(format("exactly one argument expected")); -static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); - if (opArgs.size() != 1) - throw UsageError(format("exactly one argument expected")); + int dstGen; + if (!string2Int(opArgs.front(), dstGen)) + throw UsageError(format("expected a generation number")); - int dstGen; - if (!string2Int(opArgs.front(), dstGen)) - throw UsageError(format("expected a generation number")); - - switchGeneration(globals, dstGen); + switchGeneration(globals, dstGen); } +static void opRollback(Globals& globals, Strings opFlags, Strings opArgs) { + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + if (opArgs.size() != 0) throw UsageError(format("no arguments expected")); -static void opRollback(Globals & globals, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); - if (opArgs.size() != 0) - throw UsageError(format("no arguments expected")); - - switchGeneration(globals, prevGen); + switchGeneration(globals, prevGen); } +static void opListGenerations(Globals& globals, Strings opFlags, + Strings opArgs) { + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + if (opArgs.size() != 0) throw UsageError(format("no arguments expected")); -static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); - if (opArgs.size() != 0) - throw UsageError(format("no arguments expected")); + PathLocks lock; + lockProfile(lock, globals.profile); - PathLocks lock; - lockProfile(lock, globals.profile); + int curGen; + Generations gens = findGenerations(globals.profile, curGen); - int curGen; - Generations gens = findGenerations(globals.profile, curGen); + RunPager pager; - RunPager pager; - - for (auto & i : gens) { - tm t; - if (!localtime_r(&i.creationTime, &t)) throw Error("cannot convert time"); - cout << format("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||\n") - % i.number - % (t.tm_year + 1900) % (t.tm_mon + 1) % t.tm_mday - % t.tm_hour % t.tm_min % t.tm_sec - % (i.number == curGen ? "(current)" : ""); - } + for (auto& i : gens) { + tm t; + if (!localtime_r(&i.creationTime, &t)) throw Error("cannot convert time"); + cout << format("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||\n") % + i.number % (t.tm_year + 1900) % (t.tm_mon + 1) % t.tm_mday % + t.tm_hour % t.tm_min % t.tm_sec % + (i.number == curGen ? "(current)" : ""); + } } - -static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); - - if (opArgs.size() == 1 && opArgs.front() == "old") { - deleteOldGenerations(globals.profile, globals.dryRun); - } else if (opArgs.size() == 1 && opArgs.front().find('d') != string::npos) { - deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); - } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) { - if(opArgs.front().size() < 2) - throw Error(format("invalid number of generations ‘%1%’") % opArgs.front()); - string str_max = string(opArgs.front(), 1, opArgs.front().size()); - int max; - if (!string2Int(str_max, max) || max == 0) - throw Error(format("invalid number of generations to keep ‘%1%’") % opArgs.front()); - deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); - } else { - std::set<unsigned int> gens; - for (auto & i : opArgs) { - unsigned int n; - if (!string2Int(i, n)) - throw UsageError(format("invalid generation number '%1%'") % i); - gens.insert(n); - } - deleteGenerations(globals.profile, gens, globals.dryRun); +static void opDeleteGenerations(Globals& globals, Strings opFlags, + Strings opArgs) { + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + + if (opArgs.size() == 1 && opArgs.front() == "old") { + deleteOldGenerations(globals.profile, globals.dryRun); + } else if (opArgs.size() == 1 && opArgs.front().find('d') != string::npos) { + deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); + } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) { + if (opArgs.front().size() < 2) + throw Error(format("invalid number of generations ‘%1%’") % + opArgs.front()); + string str_max = string(opArgs.front(), 1, opArgs.front().size()); + int max; + if (!string2Int(str_max, max) || max == 0) + throw Error(format("invalid number of generations to keep ‘%1%’") % + opArgs.front()); + deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); + } else { + std::set<unsigned int> gens; + for (auto& i : opArgs) { + unsigned int n; + if (!string2Int(i, n)) + throw UsageError(format("invalid generation number '%1%'") % i); + gens.insert(n); } + deleteGenerations(globals.profile, gens, globals.dryRun); + } } - -static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) -{ - printVersion("nix-env"); +static void opVersion(Globals& globals, Strings opFlags, Strings opArgs) { + printVersion("nix-env"); } +static int _main(int argc, char** argv) { + { + Strings opFlags, opArgs; + Operation op = 0; + RepairFlag repair = NoRepair; + string file; + + Globals globals; + + globals.instSource.type = srcUnknown; + globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; + globals.instSource.systemFilter = "*"; + + if (!pathExists(globals.instSource.nixExprPath)) { + try { + createDirs(globals.instSource.nixExprPath); + replaceSymlink(fmt("%s/profiles/per-user/%s/channels", + settings.nixStateDir, getUserName()), + globals.instSource.nixExprPath + "/channels"); + if (getuid() != 0) + replaceSymlink( + fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), + globals.instSource.nixExprPath + "/channels_root"); + } catch (Error&) { + } + } -static int _main(int argc, char * * argv) -{ - { - Strings opFlags, opArgs; - Operation op = 0; - RepairFlag repair = NoRepair; - string file; - - Globals globals; - - globals.instSource.type = srcUnknown; - globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; - globals.instSource.systemFilter = "*"; - - if (!pathExists(globals.instSource.nixExprPath)) { - try { - createDirs(globals.instSource.nixExprPath); - replaceSymlink( - fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), - globals.instSource.nixExprPath + "/channels"); - if (getuid() != 0) - replaceSymlink( - fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), - globals.instSource.nixExprPath + "/channels_root"); - } catch (Error &) { } - } - - globals.dryRun = false; - globals.preserveInstalled = false; - globals.removeAll = false; - globals.prebuiltOnly = false; - - struct MyArgs : LegacyArgs, MixEvalArgs - { - using LegacyArgs::LegacyArgs; - }; - - MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { - Operation oldOp = op; - - if (*arg == "--help") - showManPage("nix-env"); - else if (*arg == "--version") - op = opVersion; - else if (*arg == "--install" || *arg == "-i") - op = opInstall; - else if (*arg == "--force-name") // undocumented flag for nix-install-package - globals.forceName = getArg(*arg, arg, end); - else if (*arg == "--uninstall" || *arg == "-e") - op = opUninstall; - else if (*arg == "--upgrade" || *arg == "-u") - op = opUpgrade; - else if (*arg == "--set-flag") - op = opSetFlag; - else if (*arg == "--set") - op = opSet; - else if (*arg == "--query" || *arg == "-q") - op = opQuery; - else if (*arg == "--profile" || *arg == "-p") - globals.profile = absPath(getArg(*arg, arg, end)); - else if (*arg == "--file" || *arg == "-f") - file = getArg(*arg, arg, end); - else if (*arg == "--switch-profile" || *arg == "-S") - op = opSwitchProfile; - else if (*arg == "--switch-generation" || *arg == "-G") - op = opSwitchGeneration; - else if (*arg == "--rollback") - op = opRollback; - else if (*arg == "--list-generations") - op = opListGenerations; - else if (*arg == "--delete-generations") - op = opDeleteGenerations; - else if (*arg == "--dry-run") { - printInfo("(dry run; not doing anything)"); - globals.dryRun = true; - } - else if (*arg == "--system-filter") - globals.instSource.systemFilter = getArg(*arg, arg, end); - else if (*arg == "--prebuilt-only" || *arg == "-b") - globals.prebuiltOnly = true; - else if (*arg == "--repair") - repair = Repair; - else if (*arg != "" && arg->at(0) == '-') { - opFlags.push_back(*arg); - /* FIXME: hacky */ - if (*arg == "--from-profile" || - (op == opQuery && (*arg == "--attr" || *arg == "-A"))) - opFlags.push_back(getArg(*arg, arg, end)); - } - else - opArgs.push_back(*arg); - - if (oldOp && oldOp != op) - throw UsageError("only one operation may be specified"); - - return true; - }); - - myArgs.parseCmdline(argvToStrings(argc, argv)); - - initPlugins(); - - if (!op) throw UsageError("no operation specified"); - - auto store = openStore(); - - globals.state = std::shared_ptr<EvalState>(new EvalState(myArgs.searchPath, store)); - globals.state->repair = repair; - - if (file != "") - globals.instSource.nixExprPath = lookupFileArg(*globals.state, file); - - globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); - - if (globals.profile == "") - globals.profile = getEnv("NIX_PROFILE", ""); - - if (globals.profile == "") { - Path profileLink = getHome() + "/.nix-profile"; - try { - if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); - } - globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); - } catch (Error &) { - globals.profile = profileLink; - } + globals.dryRun = false; + globals.preserveInstalled = false; + globals.removeAll = false; + globals.prebuiltOnly = false; + + struct MyArgs : LegacyArgs, MixEvalArgs { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator& arg, + const Strings::iterator& end) { + Operation oldOp = op; + + if (*arg == "--help") + showManPage("nix-env"); + else if (*arg == "--version") + op = opVersion; + else if (*arg == "--install" || *arg == "-i") + op = opInstall; + else if (*arg == + "--force-name") // undocumented flag for nix-install-package + globals.forceName = getArg(*arg, arg, end); + else if (*arg == "--uninstall" || *arg == "-e") + op = opUninstall; + else if (*arg == "--upgrade" || *arg == "-u") + op = opUpgrade; + else if (*arg == "--set-flag") + op = opSetFlag; + else if (*arg == "--set") + op = opSet; + else if (*arg == "--query" || *arg == "-q") + op = opQuery; + else if (*arg == "--profile" || *arg == "-p") + globals.profile = absPath(getArg(*arg, arg, end)); + else if (*arg == "--file" || *arg == "-f") + file = getArg(*arg, arg, end); + else if (*arg == "--switch-profile" || *arg == "-S") + op = opSwitchProfile; + else if (*arg == "--switch-generation" || *arg == "-G") + op = opSwitchGeneration; + else if (*arg == "--rollback") + op = opRollback; + else if (*arg == "--list-generations") + op = opListGenerations; + else if (*arg == "--delete-generations") + op = opDeleteGenerations; + else if (*arg == "--dry-run") { + printInfo("(dry run; not doing anything)"); + globals.dryRun = true; + } else if (*arg == "--system-filter") + globals.instSource.systemFilter = getArg(*arg, arg, end); + else if (*arg == "--prebuilt-only" || *arg == "-b") + globals.prebuiltOnly = true; + else if (*arg == "--repair") + repair = Repair; + else if (*arg != "" && arg->at(0) == '-') { + opFlags.push_back(*arg); + /* FIXME: hacky */ + if (*arg == "--from-profile" || + (op == opQuery && (*arg == "--attr" || *arg == "-A"))) + opFlags.push_back(getArg(*arg, arg, end)); + } else + opArgs.push_back(*arg); + + if (oldOp && oldOp != op) + throw UsageError("only one operation may be specified"); + + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (!op) throw UsageError("no operation specified"); + + auto store = openStore(); + + globals.state = + std::shared_ptr<EvalState>(new EvalState(myArgs.searchPath, store)); + globals.state->repair = repair; + + if (file != "") + globals.instSource.nixExprPath = lookupFileArg(*globals.state, file); + + globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); + + if (globals.profile == "") globals.profile = getEnv("NIX_PROFILE", ""); + + if (globals.profile == "") { + Path profileLink = getHome() + "/.nix-profile"; + try { + if (!pathExists(profileLink)) { + replaceSymlink(getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", + settings.nixStateDir, getUserName()), + profileLink); } + globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error&) { + globals.profile = profileLink; + } + } - op(globals, opFlags, opArgs); + op(globals, opFlags, opArgs); - globals.state->printStats(); + globals.state->printStats(); - return 0; - } + return 0; + } } static RegisterLegacyCommand s1("nix-env", _main); diff --git a/third_party/nix/src/nix-env/user-env.cc b/third_party/nix/src/nix-env/user-env.cc index 7b9a88281171..c1c174bcc529 100644 --- a/third_party/nix/src/nix-env/user-env.cc +++ b/third_party/nix/src/nix-env/user-env.cc @@ -1,156 +1,152 @@ #include "user-env.hh" -#include "util.hh" #include "derivations.hh" -#include "store-api.hh" -#include "globals.hh" -#include "shared.hh" -#include "eval.hh" #include "eval-inline.hh" +#include "eval.hh" +#include "globals.hh" #include "profiles.hh" - +#include "shared.hh" +#include "store-api.hh" +#include "util.hh" namespace nix { - -DrvInfos queryInstalled(EvalState & state, const Path & userEnv) -{ - DrvInfos elems; - Path manifestFile = userEnv + "/manifest.nix"; - if (pathExists(manifestFile)) { - Value v; - state.evalFile(manifestFile, v); - Bindings & bindings(*state.allocBindings(0)); - getDerivations(state, v, "", bindings, elems, false); - } - return elems; +DrvInfos queryInstalled(EvalState& state, const Path& userEnv) { + DrvInfos elems; + Path manifestFile = userEnv + "/manifest.nix"; + if (pathExists(manifestFile)) { + Value v; + state.evalFile(manifestFile, v); + Bindings& bindings(*state.allocBindings(0)); + getDerivations(state, v, "", bindings, elems, false); + } + return elems; } - -bool createUserEnv(EvalState & state, DrvInfos & elems, - const Path & profile, bool keepDerivations, - const string & lockToken) -{ - /* Build the components in the user environment, if they don't - exist already. */ - PathSet drvsToBuild; - for (auto & i : elems) - if (i.queryDrvPath() != "") - drvsToBuild.insert(i.queryDrvPath()); - - debug(format("building user environment dependencies")); - state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); - - /* Construct the whole top level derivation. */ - PathSet references; - Value manifest; - state.mkList(manifest, elems.size()); - unsigned int n = 0; - for (auto & i : elems) { - /* Create a pseudo-derivation containing the name, system, - output paths, and optionally the derivation path, as well - as the meta attributes. */ - Path drvPath = keepDerivations ? i.queryDrvPath() : ""; - - Value & v(*state.allocValue()); - manifest.listElems()[n++] = &v; - state.mkAttrs(v, 16); - - mkString(*state.allocAttr(v, state.sType), "derivation"); - mkString(*state.allocAttr(v, state.sName), i.queryName()); - auto system = i.querySystem(); - if (!system.empty()) - mkString(*state.allocAttr(v, state.sSystem), system); - mkString(*state.allocAttr(v, state.sOutPath), i.queryOutPath()); - if (drvPath != "") - mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); - - // Copy each output meant for installation. - DrvInfo::Outputs outputs = i.queryOutputs(true); - Value & vOutputs = *state.allocAttr(v, state.sOutputs); - state.mkList(vOutputs, outputs.size()); - unsigned int m = 0; - for (auto & j : outputs) { - mkString(*(vOutputs.listElems()[m++] = state.allocValue()), j.first); - Value & vOutputs = *state.allocAttr(v, state.symbols.create(j.first)); - state.mkAttrs(vOutputs, 2); - mkString(*state.allocAttr(vOutputs, state.sOutPath), j.second); - - /* This is only necessary when installing store paths, e.g., - `nix-env -i /nix/store/abcd...-foo'. */ - state.store->addTempRoot(j.second); - state.store->ensurePath(j.second); - - references.insert(j.second); - } - - // Copy the meta attributes. - Value & vMeta = *state.allocAttr(v, state.sMeta); - state.mkAttrs(vMeta, 16); - StringSet metaNames = i.queryMetaNames(); - for (auto & j : metaNames) { - Value * v = i.queryMeta(j); - if (!v) continue; - vMeta.attrs->push_back(Attr(state.symbols.create(j), v)); - } - vMeta.attrs->sort(); - v.attrs->sort(); - - if (drvPath != "") references.insert(drvPath); +bool createUserEnv(EvalState& state, DrvInfos& elems, const Path& profile, + bool keepDerivations, const string& lockToken) { + /* Build the components in the user environment, if they don't + exist already. */ + PathSet drvsToBuild; + for (auto& i : elems) + if (i.queryDrvPath() != "") drvsToBuild.insert(i.queryDrvPath()); + + debug(format("building user environment dependencies")); + state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); + + /* Construct the whole top level derivation. */ + PathSet references; + Value manifest; + state.mkList(manifest, elems.size()); + unsigned int n = 0; + for (auto& i : elems) { + /* Create a pseudo-derivation containing the name, system, + output paths, and optionally the derivation path, as well + as the meta attributes. */ + Path drvPath = keepDerivations ? i.queryDrvPath() : ""; + + Value& v(*state.allocValue()); + manifest.listElems()[n++] = &v; + state.mkAttrs(v, 16); + + mkString(*state.allocAttr(v, state.sType), "derivation"); + mkString(*state.allocAttr(v, state.sName), i.queryName()); + auto system = i.querySystem(); + if (!system.empty()) mkString(*state.allocAttr(v, state.sSystem), system); + mkString(*state.allocAttr(v, state.sOutPath), i.queryOutPath()); + if (drvPath != "") + mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); + + // Copy each output meant for installation. + DrvInfo::Outputs outputs = i.queryOutputs(true); + Value& vOutputs = *state.allocAttr(v, state.sOutputs); + state.mkList(vOutputs, outputs.size()); + unsigned int m = 0; + for (auto& j : outputs) { + mkString(*(vOutputs.listElems()[m++] = state.allocValue()), j.first); + Value& vOutputs = *state.allocAttr(v, state.symbols.create(j.first)); + state.mkAttrs(vOutputs, 2); + mkString(*state.allocAttr(vOutputs, state.sOutPath), j.second); + + /* This is only necessary when installing store paths, e.g., + `nix-env -i /nix/store/abcd...-foo'. */ + state.store->addTempRoot(j.second); + state.store->ensurePath(j.second); + + references.insert(j.second); } - /* Also write a copy of the list of user environment elements to - the store; we need it for future modifications of the - environment. */ - Path manifestFile = state.store->addTextToStore("env-manifest.nix", - (format("%1%") % manifest).str(), references); - - /* Get the environment builder expression. */ - Value envBuilder; - state.evalFile(state.findFile("nix/buildenv.nix"), envBuilder); - - /* Construct a Nix expression that calls the user environment - builder with the manifest as argument. */ - Value args, topLevel; - state.mkAttrs(args, 3); - mkString(*state.allocAttr(args, state.symbols.create("manifest")), - manifestFile, {manifestFile}); - args.attrs->push_back(Attr(state.symbols.create("derivations"), &manifest)); - args.attrs->sort(); - mkApp(topLevel, envBuilder, args); - - /* Evaluate it. */ - debug("evaluating user environment builder"); - state.forceValue(topLevel); - PathSet context; - Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); - Path topLevelDrv = state.coerceToPath(aDrvPath.pos ? *(aDrvPath.pos) : noPos, *(aDrvPath.value), context); - Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); - Path topLevelOut = state.coerceToPath(aOutPath.pos ? *(aOutPath.pos) : noPos, *(aOutPath.value), context); - - /* Realise the resulting store expression. */ - debug("building user environment"); - state.store->buildPaths({topLevelDrv}, state.repair ? bmRepair : bmNormal); - - /* Switch the current user environment to the output path. */ - auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); - - if (store2) { - PathLocks lock; - lockProfile(lock, profile); - - Path lockTokenCur = optimisticLockProfile(profile); - if (lockToken != lockTokenCur) { - printError(format("profile '%1%' changed while we were busy; restarting") % profile); - return false; - } - - debug(format("switching to new user environment")); - Path generation = createGeneration(ref<LocalFSStore>(store2), profile, topLevelOut); - switchLink(profile, generation); + // Copy the meta attributes. + Value& vMeta = *state.allocAttr(v, state.sMeta); + state.mkAttrs(vMeta, 16); + StringSet metaNames = i.queryMetaNames(); + for (auto& j : metaNames) { + Value* v = i.queryMeta(j); + if (!v) continue; + vMeta.attrs->push_back(Attr(state.symbols.create(j), v)); + } + vMeta.attrs->sort(); + v.attrs->sort(); + + if (drvPath != "") references.insert(drvPath); + } + + /* Also write a copy of the list of user environment elements to + the store; we need it for future modifications of the + environment. */ + Path manifestFile = state.store->addTextToStore( + "env-manifest.nix", (format("%1%") % manifest).str(), references); + + /* Get the environment builder expression. */ + Value envBuilder; + state.evalFile(state.findFile("nix/buildenv.nix"), envBuilder); + + /* Construct a Nix expression that calls the user environment + builder with the manifest as argument. */ + Value args, topLevel; + state.mkAttrs(args, 3); + mkString(*state.allocAttr(args, state.symbols.create("manifest")), + manifestFile, {manifestFile}); + args.attrs->push_back(Attr(state.symbols.create("derivations"), &manifest)); + args.attrs->sort(); + mkApp(topLevel, envBuilder, args); + + /* Evaluate it. */ + debug("evaluating user environment builder"); + state.forceValue(topLevel); + PathSet context; + Attr& aDrvPath(*topLevel.attrs->find(state.sDrvPath)); + Path topLevelDrv = state.coerceToPath(aDrvPath.pos ? *(aDrvPath.pos) : noPos, + *(aDrvPath.value), context); + Attr& aOutPath(*topLevel.attrs->find(state.sOutPath)); + Path topLevelOut = state.coerceToPath(aOutPath.pos ? *(aOutPath.pos) : noPos, + *(aOutPath.value), context); + + /* Realise the resulting store expression. */ + debug("building user environment"); + state.store->buildPaths({topLevelDrv}, state.repair ? bmRepair : bmNormal); + + /* Switch the current user environment to the output path. */ + auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); + + if (store2) { + PathLocks lock; + lockProfile(lock, profile); + + Path lockTokenCur = optimisticLockProfile(profile); + if (lockToken != lockTokenCur) { + printError( + format("profile '%1%' changed while we were busy; restarting") % + profile); + return false; } - return true; -} - + debug(format("switching to new user environment")); + Path generation = + createGeneration(ref<LocalFSStore>(store2), profile, topLevelOut); + switchLink(profile, generation); + } + return true; } + +} // namespace nix diff --git a/third_party/nix/src/nix-env/user-env.hh b/third_party/nix/src/nix-env/user-env.hh index f188efe9b4a9..6111b21c353d 100644 --- a/third_party/nix/src/nix-env/user-env.hh +++ b/third_party/nix/src/nix-env/user-env.hh @@ -4,10 +4,9 @@ namespace nix { -DrvInfos queryInstalled(EvalState & state, const Path & userEnv); +DrvInfos queryInstalled(EvalState& state, const Path& userEnv); -bool createUserEnv(EvalState & state, DrvInfos & elems, - const Path & profile, bool keepDerivations, - const string & lockToken); +bool createUserEnv(EvalState& state, DrvInfos& elems, const Path& profile, + bool keepDerivations, const string& lockToken); -} +} // namespace nix diff --git a/third_party/nix/src/nix-instantiate/nix-instantiate.cc b/third_party/nix/src/nix-instantiate/nix-instantiate.cc index a736caa8f056..cd1ebce97b0c 100644 --- a/third_party/nix/src/nix-instantiate/nix-instantiate.cc +++ b/third_party/nix/src/nix-instantiate/nix-instantiate.cc @@ -1,196 +1,191 @@ -#include "globals.hh" -#include "shared.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "get-drvs.hh" +#include <iostream> +#include <map> #include "attr-path.hh" -#include "value-to-xml.hh" -#include "value-to-json.hh" -#include "util.hh" -#include "store-api.hh" #include "common-eval-args.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "globals.hh" #include "legacy.hh" - -#include <map> -#include <iostream> - +#include "shared.hh" +#include "store-api.hh" +#include "util.hh" +#include "value-to-json.hh" +#include "value-to-xml.hh" using namespace nix; - static Path gcRoot; static int rootNr = 0; static bool indirectRoot = false; - enum OutputKind { okPlain, okXML, okJSON }; - -void processExpr(EvalState & state, const Strings & attrPaths, - bool parseOnly, bool strict, Bindings & autoArgs, - bool evalOnly, OutputKind output, bool location, Expr * e) -{ - if (parseOnly) { - std::cout << format("%1%\n") % *e; - return; - } - - Value vRoot; - state.eval(e, vRoot); - - for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot)); - state.forceValue(v); - - PathSet context; - if (evalOnly) { - Value vRes; - if (autoArgs.empty()) - vRes = v; - else - state.autoCallFunction(autoArgs, v, vRes); - if (output == okXML) - printValueAsXML(state, strict, location, vRes, std::cout, context); - else if (output == okJSON) - printValueAsJSON(state, strict, vRes, std::cout, context); - else { - if (strict) state.forceValueDeep(vRes); - std::cout << vRes << std::endl; - } - } else { - DrvInfos drvs; - getDerivations(state, v, "", autoArgs, drvs, false); - for (auto & i : drvs) { - Path drvPath = i.queryDrvPath(); - - /* What output do we want? */ - string outputName = i.queryOutputName(); - if (outputName == "") - throw Error(format("derivation '%1%' lacks an 'outputName' attribute ") % drvPath); - - if (gcRoot == "") - printGCWarning(); - else { - Path rootName = indirectRoot ? absPath(gcRoot) : gcRoot; - if (++rootNr > 1) rootName += "-" + std::to_string(rootNr); - auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); - if (store2) - drvPath = store2->addPermRoot(drvPath, rootName, indirectRoot); - } - std::cout << format("%1%%2%\n") % drvPath % (outputName != "out" ? "!" + outputName : ""); - } +void processExpr(EvalState& state, const Strings& attrPaths, bool parseOnly, + bool strict, Bindings& autoArgs, bool evalOnly, + OutputKind output, bool location, Expr* e) { + if (parseOnly) { + std::cout << format("%1%\n") % *e; + return; + } + + Value vRoot; + state.eval(e, vRoot); + + for (auto& i : attrPaths) { + Value& v(*findAlongAttrPath(state, i, autoArgs, vRoot)); + state.forceValue(v); + + PathSet context; + if (evalOnly) { + Value vRes; + if (autoArgs.empty()) + vRes = v; + else + state.autoCallFunction(autoArgs, v, vRes); + if (output == okXML) + printValueAsXML(state, strict, location, vRes, std::cout, context); + else if (output == okJSON) + printValueAsJSON(state, strict, vRes, std::cout, context); + else { + if (strict) state.forceValueDeep(vRes); + std::cout << vRes << std::endl; + } + } else { + DrvInfos drvs; + getDerivations(state, v, "", autoArgs, drvs, false); + for (auto& i : drvs) { + Path drvPath = i.queryDrvPath(); + + /* What output do we want? */ + string outputName = i.queryOutputName(); + if (outputName == "") + throw Error( + format("derivation '%1%' lacks an 'outputName' attribute ") % + drvPath); + + if (gcRoot == "") + printGCWarning(); + else { + Path rootName = indirectRoot ? absPath(gcRoot) : gcRoot; + if (++rootNr > 1) rootName += "-" + std::to_string(rootNr); + auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); + if (store2) + drvPath = store2->addPermRoot(drvPath, rootName, indirectRoot); } + std::cout << format("%1%%2%\n") % drvPath % + (outputName != "out" ? "!" + outputName : ""); + } } + } } +static int _main(int argc, char** argv) { + { + Strings files; + bool readStdin = false; + bool fromArgs = false; + bool findFile = false; + bool evalOnly = false; + bool parseOnly = false; + OutputKind outputKind = okPlain; + bool xmlOutputSourceLocation = true; + bool strict = false; + Strings attrPaths; + bool wantsReadWrite = false; + RepairFlag repair = NoRepair; + + struct MyArgs : LegacyArgs, MixEvalArgs { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), + [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") + showManPage("nix-instantiate"); + else if (*arg == "--version") + printVersion("nix-instantiate"); + else if (*arg == "-") + readStdin = true; + else if (*arg == "--expr" || *arg == "-E") + fromArgs = true; + else if (*arg == "--eval" || *arg == "--eval-only") + evalOnly = true; + else if (*arg == "--read-write-mode") + wantsReadWrite = true; + else if (*arg == "--parse" || *arg == "--parse-only") + parseOnly = evalOnly = true; + else if (*arg == "--find-file") + findFile = true; + else if (*arg == "--attr" || *arg == "-A") + attrPaths.push_back(getArg(*arg, arg, end)); + else if (*arg == "--add-root") + gcRoot = getArg(*arg, arg, end); + else if (*arg == "--indirect") + indirectRoot = true; + else if (*arg == "--xml") + outputKind = okXML; + else if (*arg == "--json") + outputKind = okJSON; + else if (*arg == "--no-location") + xmlOutputSourceLocation = false; + else if (*arg == "--strict") + strict = true; + else if (*arg == "--repair") + repair = Repair; + else if (*arg == "--dry-run") + settings.readOnlyMode = true; + else if (*arg != "" && arg->at(0) == '-') + return false; + else + files.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (evalOnly && !wantsReadWrite) settings.readOnlyMode = true; + + auto store = openStore(); + + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + state->repair = repair; + + Bindings& autoArgs = *myArgs.getAutoArgs(*state); + + if (attrPaths.empty()) attrPaths = {""}; + + if (findFile) { + for (auto& i : files) { + Path p = state->findFile(i); + if (p == "") throw Error(format("unable to find '%1%'") % i); + std::cout << p << std::endl; + } + return 0; + } -static int _main(int argc, char * * argv) -{ - { - Strings files; - bool readStdin = false; - bool fromArgs = false; - bool findFile = false; - bool evalOnly = false; - bool parseOnly = false; - OutputKind outputKind = okPlain; - bool xmlOutputSourceLocation = true; - bool strict = false; - Strings attrPaths; - bool wantsReadWrite = false; - RepairFlag repair = NoRepair; - - struct MyArgs : LegacyArgs, MixEvalArgs - { - using LegacyArgs::LegacyArgs; - }; - - MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-instantiate"); - else if (*arg == "--version") - printVersion("nix-instantiate"); - else if (*arg == "-") - readStdin = true; - else if (*arg == "--expr" || *arg == "-E") - fromArgs = true; - else if (*arg == "--eval" || *arg == "--eval-only") - evalOnly = true; - else if (*arg == "--read-write-mode") - wantsReadWrite = true; - else if (*arg == "--parse" || *arg == "--parse-only") - parseOnly = evalOnly = true; - else if (*arg == "--find-file") - findFile = true; - else if (*arg == "--attr" || *arg == "-A") - attrPaths.push_back(getArg(*arg, arg, end)); - else if (*arg == "--add-root") - gcRoot = getArg(*arg, arg, end); - else if (*arg == "--indirect") - indirectRoot = true; - else if (*arg == "--xml") - outputKind = okXML; - else if (*arg == "--json") - outputKind = okJSON; - else if (*arg == "--no-location") - xmlOutputSourceLocation = false; - else if (*arg == "--strict") - strict = true; - else if (*arg == "--repair") - repair = Repair; - else if (*arg == "--dry-run") - settings.readOnlyMode = true; - else if (*arg != "" && arg->at(0) == '-') - return false; - else - files.push_back(*arg); - return true; - }); - - myArgs.parseCmdline(argvToStrings(argc, argv)); - - initPlugins(); - - if (evalOnly && !wantsReadWrite) - settings.readOnlyMode = true; - - auto store = openStore(); - - auto state = std::make_unique<EvalState>(myArgs.searchPath, store); - state->repair = repair; - - Bindings & autoArgs = *myArgs.getAutoArgs(*state); - - if (attrPaths.empty()) attrPaths = {""}; - - if (findFile) { - for (auto & i : files) { - Path p = state->findFile(i); - if (p == "") throw Error(format("unable to find '%1%'") % i); - std::cout << p << std::endl; - } - return 0; - } - - if (readStdin) { - Expr * e = state->parseStdin(); - processExpr(*state, attrPaths, parseOnly, strict, autoArgs, - evalOnly, outputKind, xmlOutputSourceLocation, e); - } else if (files.empty() && !fromArgs) - files.push_back("./default.nix"); - - for (auto & i : files) { - Expr * e = fromArgs - ? state->parseExprFromString(i, absPath(".")) - : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i)))); - processExpr(*state, attrPaths, parseOnly, strict, autoArgs, - evalOnly, outputKind, xmlOutputSourceLocation, e); - } + if (readStdin) { + Expr* e = state->parseStdin(); + processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, + outputKind, xmlOutputSourceLocation, e); + } else if (files.empty() && !fromArgs) + files.push_back("./default.nix"); + + for (auto& i : files) { + Expr* e = fromArgs + ? state->parseExprFromString(i, absPath(".")) + : state->parseExprFromFile(resolveExprPath( + state->checkSourcePath(lookupFileArg(*state, i)))); + processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, + outputKind, xmlOutputSourceLocation, e); + } - state->printStats(); + state->printStats(); - return 0; - } + return 0; + } } static RegisterLegacyCommand s1("nix-instantiate", _main); diff --git a/third_party/nix/src/nix-prefetch-url/nix-prefetch-url.cc b/third_party/nix/src/nix-prefetch-url/nix-prefetch-url.cc index f54706a8a011..223f65c4a8e4 100644 --- a/third_party/nix/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/third_party/nix/src/nix-prefetch-url/nix-prefetch-url.cc @@ -1,236 +1,224 @@ -#include "hash.hh" -#include "shared.hh" +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <iostream> +#include "attr-path.hh" +#include "common-eval-args.hh" #include "download.hh" -#include "store-api.hh" -#include "eval.hh" #include "eval-inline.hh" -#include "common-eval-args.hh" -#include "attr-path.hh" -#include "legacy.hh" +#include "eval.hh" #include "finally.hh" +#include "hash.hh" +#include "legacy.hh" #include "progress-bar.hh" - -#include <iostream> - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> +#include "shared.hh" +#include "store-api.hh" using namespace nix; - /* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of mirrors defined in Nixpkgs. */ -string resolveMirrorUri(EvalState & state, string uri) -{ - if (string(uri, 0, 9) != "mirror://") return uri; +string resolveMirrorUri(EvalState& state, string uri) { + if (string(uri, 0, 9) != "mirror://") return uri; + + string s(uri, 9); + auto p = s.find('/'); + if (p == string::npos) throw Error("invalid mirror URI"); + string mirrorName(s, 0, p); + + Value vMirrors; + state.eval( + state.parseExprFromString( + "import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), + vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) + throw Error(format("unknown mirror name '%1%'") % mirrorName); + state.forceList(*mirrorList->value); + + if (mirrorList->value->listSize() < 1) + throw Error(format("mirror URI '%1%' did not expand to anything") % uri); + + string mirror = state.forceString(*mirrorList->value->listElems()[0]); + return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); +} - string s(uri, 9); - auto p = s.find('/'); - if (p == string::npos) throw Error("invalid mirror URI"); - string mirrorName(s, 0, p); +static int _main(int argc, char** argv) { + { + HashType ht = htSHA256; + std::vector<string> args; + bool printPath = getEnv("PRINT_PATH") != ""; + bool fromExpr = false; + string attrPath; + bool unpack = false; + string name; + + struct MyArgs : LegacyArgs, MixEvalArgs { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), + [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") + showManPage("nix-prefetch-url"); + else if (*arg == "--version") + printVersion("nix-prefetch-url"); + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + if (ht == htUnknown) + throw UsageError(format("unknown hash type '%1%'") % s); + } else if (*arg == "--print-path") + printPath = true; + else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } else if (*arg == "--unpack") + unpack = true; + else if (*arg == "--name") + name = getArg(*arg, arg, end); + else if (*arg != "" && arg->at(0) == '-') + return false; + else + args.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (args.size() > 2) throw UsageError("too many arguments"); + + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) startProgressBar(); + + auto store = openStore(); + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + + Bindings& autoArgs = *myArgs.getAutoArgs(*state); + + /* If -A is given, get the URI from the specified Nix + expression. */ + string uri; + if (!fromExpr) { + if (args.empty()) throw UsageError("you must specify a URI"); + uri = args[0]; + } else { + Path path = + resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); + Value vRoot; + state->evalFile(path, vRoot); + Value& v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot)); + state->forceAttrs(v); + + /* Extract the URI. */ + auto attr = v.attrs->find(state->symbols.create("urls")); + if (attr == v.attrs->end()) + throw Error("attribute set does not contain a 'urls' attribute"); + state->forceList(*attr->value); + if (attr->value->listSize() < 1) throw Error("'urls' list is empty"); + uri = state->forceString(*attr->value->listElems()[0]); + + /* Extract the hash mode. */ + attr = v.attrs->find(state->symbols.create("outputHashMode")); + if (attr == v.attrs->end()) + printInfo("warning: this does not look like a fetchurl call"); + else + unpack = state->forceString(*attr->value) == "recursive"; + + /* Extract the name. */ + if (name.empty()) { + attr = v.attrs->find(state->symbols.create("name")); + if (attr != v.attrs->end()) name = state->forceString(*attr->value); + } + } - Value vMirrors; - state.eval(state.parseExprFromString("import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), vMirrors); - state.forceAttrs(vMirrors); + /* Figure out a name in the Nix store. */ + if (name.empty()) name = baseNameOf(uri); + if (name.empty()) + throw Error(format("cannot figure out file name for '%1%'") % uri); + + /* If an expected hash is given, the file may already exist in + the store. */ + Hash hash, expectedHash(ht); + Path storePath; + if (args.size() == 2) { + expectedHash = Hash(args[1], ht); + storePath = store->makeFixedOutputPath(unpack, expectedHash, name); + if (store->isValidPath(storePath)) + hash = expectedHash; + else + storePath.clear(); + } - auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); - if (mirrorList == vMirrors.attrs->end()) - throw Error(format("unknown mirror name '%1%'") % mirrorName); - state.forceList(*mirrorList->value); + if (storePath.empty()) { + auto actualUri = resolveMirrorUri(*state, uri); + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path)tmpDir + "/tmp"; + + /* Download the file. */ + { + AutoCloseFD fd = + open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + DownloadRequest req(actualUri); + req.decompress = false; + getDownloader()->download(std::move(req), sink); + } + + /* Optionally unpack the file. */ + if (unpack) { + printInfo("unpacking..."); + Path unpacked = (Path)tmpDir + "/unpacked"; + createDirs(unpacked); + if (hasSuffix(baseNameOf(uri), ".zip")) + runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); + else + // FIXME: this requires GNU tar for decompression. + runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}); + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) + tmpFile = unpacked + "/" + entries[0].name; + else + tmpFile = unpacked; + } + + /* FIXME: inefficient; addToStore() will also hash + this. */ + hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); + + if (expectedHash != Hash(ht) && expectedHash != hash) + throw Error(format("hash mismatch for '%1%'") % uri); + + /* Copy the file to the Nix store. FIXME: if RemoteStore + implemented addToStoreFromDump() and downloadFile() + supported a sink, we could stream the download directly + into the Nix store. */ + storePath = store->addToStore(name, tmpFile, unpack, ht); + + assert(storePath == store->makeFixedOutputPath(unpack, hash, name)); + } - if (mirrorList->value->listSize() < 1) - throw Error(format("mirror URI '%1%' did not expand to anything") % uri); + stopProgressBar(); - string mirror = state.forceString(*mirrorList->value->listElems()[0]); - return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); -} + if (!printPath) printInfo(format("path is '%1%'") % storePath); + std::cout << printHash16or32(hash) << std::endl; + if (printPath) std::cout << storePath << std::endl; -static int _main(int argc, char * * argv) -{ - { - HashType ht = htSHA256; - std::vector<string> args; - bool printPath = getEnv("PRINT_PATH") != ""; - bool fromExpr = false; - string attrPath; - bool unpack = false; - string name; - - struct MyArgs : LegacyArgs, MixEvalArgs - { - using LegacyArgs::LegacyArgs; - }; - - MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-prefetch-url"); - else if (*arg == "--version") - printVersion("nix-prefetch-url"); - else if (*arg == "--type") { - string s = getArg(*arg, arg, end); - ht = parseHashType(s); - if (ht == htUnknown) - throw UsageError(format("unknown hash type '%1%'") % s); - } - else if (*arg == "--print-path") - printPath = true; - else if (*arg == "--attr" || *arg == "-A") { - fromExpr = true; - attrPath = getArg(*arg, arg, end); - } - else if (*arg == "--unpack") - unpack = true; - else if (*arg == "--name") - name = getArg(*arg, arg, end); - else if (*arg != "" && arg->at(0) == '-') - return false; - else - args.push_back(*arg); - return true; - }); - - myArgs.parseCmdline(argvToStrings(argc, argv)); - - initPlugins(); - - if (args.size() > 2) - throw UsageError("too many arguments"); - - Finally f([]() { stopProgressBar(); }); - - if (isatty(STDERR_FILENO)) - startProgressBar(); - - auto store = openStore(); - auto state = std::make_unique<EvalState>(myArgs.searchPath, store); - - Bindings & autoArgs = *myArgs.getAutoArgs(*state); - - /* If -A is given, get the URI from the specified Nix - expression. */ - string uri; - if (!fromExpr) { - if (args.empty()) - throw UsageError("you must specify a URI"); - uri = args[0]; - } else { - Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); - Value vRoot; - state->evalFile(path, vRoot); - Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot)); - state->forceAttrs(v); - - /* Extract the URI. */ - auto attr = v.attrs->find(state->symbols.create("urls")); - if (attr == v.attrs->end()) - throw Error("attribute set does not contain a 'urls' attribute"); - state->forceList(*attr->value); - if (attr->value->listSize() < 1) - throw Error("'urls' list is empty"); - uri = state->forceString(*attr->value->listElems()[0]); - - /* Extract the hash mode. */ - attr = v.attrs->find(state->symbols.create("outputHashMode")); - if (attr == v.attrs->end()) - printInfo("warning: this does not look like a fetchurl call"); - else - unpack = state->forceString(*attr->value) == "recursive"; - - /* Extract the name. */ - if (name.empty()) { - attr = v.attrs->find(state->symbols.create("name")); - if (attr != v.attrs->end()) - name = state->forceString(*attr->value); - } - } - - /* Figure out a name in the Nix store. */ - if (name.empty()) - name = baseNameOf(uri); - if (name.empty()) - throw Error(format("cannot figure out file name for '%1%'") % uri); - - /* If an expected hash is given, the file may already exist in - the store. */ - Hash hash, expectedHash(ht); - Path storePath; - if (args.size() == 2) { - expectedHash = Hash(args[1], ht); - storePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(storePath)) - hash = expectedHash; - else - storePath.clear(); - } - - if (storePath.empty()) { - - auto actualUri = resolveMirrorUri(*state, uri); - - AutoDelete tmpDir(createTempDir(), true); - Path tmpFile = (Path) tmpDir + "/tmp"; - - /* Download the file. */ - { - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); - if (!fd) throw SysError("creating temporary file '%s'", tmpFile); - - FdSink sink(fd.get()); - - DownloadRequest req(actualUri); - req.decompress = false; - getDownloader()->download(std::move(req), sink); - } - - /* Optionally unpack the file. */ - if (unpack) { - printInfo("unpacking..."); - Path unpacked = (Path) tmpDir + "/unpacked"; - createDirs(unpacked); - if (hasSuffix(baseNameOf(uri), ".zip")) - runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); - else - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}); - - /* If the archive unpacks to a single file/directory, then use - that as the top-level. */ - auto entries = readDirectory(unpacked); - if (entries.size() == 1) - tmpFile = unpacked + "/" + entries[0].name; - else - tmpFile = unpacked; - } - - /* FIXME: inefficient; addToStore() will also hash - this. */ - hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); - - if (expectedHash != Hash(ht) && expectedHash != hash) - throw Error(format("hash mismatch for '%1%'") % uri); - - /* Copy the file to the Nix store. FIXME: if RemoteStore - implemented addToStoreFromDump() and downloadFile() - supported a sink, we could stream the download directly - into the Nix store. */ - storePath = store->addToStore(name, tmpFile, unpack, ht); - - assert(storePath == store->makeFixedOutputPath(unpack, hash, name)); - } - - stopProgressBar(); - - if (!printPath) - printInfo(format("path is '%1%'") % storePath); - - std::cout << printHash16or32(hash) << std::endl; - if (printPath) - std::cout << storePath << std::endl; - - return 0; - } + return 0; + } } static RegisterLegacyCommand s1("nix-prefetch-url", _main); diff --git a/third_party/nix/src/nix-store/dotgraph.cc b/third_party/nix/src/nix-store/dotgraph.cc index abdfa5e58f93..daec3da311d4 100644 --- a/third_party/nix/src/nix-store/dotgraph.cc +++ b/third_party/nix/src/nix-store/dotgraph.cc @@ -1,56 +1,41 @@ #include "dotgraph.hh" -#include "util.hh" -#include "store-api.hh" - #include <iostream> - +#include "store-api.hh" +#include "util.hh" using std::cout; namespace nix { +static string dotQuote(const string& s) { return "\"" + s + "\""; } -static string dotQuote(const string & s) -{ - return "\"" + s + "\""; +static string nextColour() { + static int n = 0; + static string colours[] = {"black", "red", "green", + "blue", "magenta", "burlywood"}; + return colours[n++ % (sizeof(colours) / sizeof(string))]; } - -static string nextColour() -{ - static int n = 0; - static string colours[] = - { "black", "red", "green", "blue" - , "magenta", "burlywood" }; - return colours[n++ % (sizeof(colours) / sizeof(string))]; +static string makeEdge(const string& src, const string& dst) { + format f = format("%1% -> %2% [color = %3%];\n") % dotQuote(src) % + dotQuote(dst) % dotQuote(nextColour()); + return f.str(); } - -static string makeEdge(const string & src, const string & dst) -{ - format f = format("%1% -> %2% [color = %3%];\n") - % dotQuote(src) % dotQuote(dst) % dotQuote(nextColour()); - return f.str(); -} - - -static string makeNode(const string & id, const string & label, - const string & colour) -{ - format f = format("%1% [label = %2%, shape = box, " - "style = filled, fillcolor = %3%];\n") - % dotQuote(id) % dotQuote(label) % dotQuote(colour); - return f.str(); +static string makeNode(const string& id, const string& label, + const string& colour) { + format f = format( + "%1% [label = %2%, shape = box, " + "style = filled, fillcolor = %3%];\n") % + dotQuote(id) % dotQuote(label) % dotQuote(colour); + return f.str(); } - -static string symbolicName(const string & path) -{ - string p = baseNameOf(path); - return string(p, p.find('-') + 1); +static string symbolicName(const string& path) { + string p = baseNameOf(path); + return string(p, p.find('-') + 1); } - #if 0 string pathLabel(const Path & nePath, const string & elemPath) { @@ -92,29 +77,27 @@ void printClosure(const Path & nePath, const StoreExpr & fs) } #endif +void printDotGraph(ref<Store> store, const PathSet& roots) { + PathSet workList(roots); + PathSet doneSet; -void printDotGraph(ref<Store> store, const PathSet & roots) -{ - PathSet workList(roots); - PathSet doneSet; + cout << "digraph G {\n"; - cout << "digraph G {\n"; + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); - while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); + if (doneSet.find(path) != doneSet.end()) continue; + doneSet.insert(path); - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); + cout << makeNode(path, symbolicName(path), "#ff0000"); - cout << makeNode(path, symbolicName(path), "#ff0000"); - - for (auto & p : store->queryPathInfo(path)->references) { - if (p != path) { - workList.insert(p); - cout << makeEdge(p, path); - } - } + for (auto& p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(p, path); + } + } #if 0 StoreExpr ne = storeExprFromPath(path); @@ -146,10 +129,9 @@ void printDotGraph(ref<Store> store, const PathSet & roots) cout << makeNode(path, label, colour); #endif - } + } - cout << "}\n"; + cout << "}\n"; } - -} +} // namespace nix diff --git a/third_party/nix/src/nix-store/dotgraph.hh b/third_party/nix/src/nix-store/dotgraph.hh index e2b5fc72fbe1..c7069451e114 100644 --- a/third_party/nix/src/nix-store/dotgraph.hh +++ b/third_party/nix/src/nix-store/dotgraph.hh @@ -6,6 +6,6 @@ namespace nix { class Store; -void printDotGraph(ref<Store> store, const PathSet & roots); +void printDotGraph(ref<Store> store, const PathSet& roots); -} +} // namespace nix diff --git a/third_party/nix/src/nix-store/graphml.cc b/third_party/nix/src/nix-store/graphml.cc index 670fbe227a4c..299468ab9d5e 100644 --- a/third_party/nix/src/nix-store/graphml.cc +++ b/third_party/nix/src/nix-store/graphml.cc @@ -1,90 +1,76 @@ #include "graphml.hh" -#include "util.hh" -#include "store-api.hh" -#include "derivations.hh" - #include <iostream> - +#include "derivations.hh" +#include "store-api.hh" +#include "util.hh" using std::cout; namespace nix { - -static inline const string & xmlQuote(const string & s) -{ - // Luckily, store paths shouldn't contain any character that needs to be - // quoted. - return s; +static inline const string& xmlQuote(const string& s) { + // Luckily, store paths shouldn't contain any character that needs to be + // quoted. + return s; } - -static string symbolicName(const string & path) -{ - string p = baseNameOf(path); - return string(p, p.find('-') + 1); +static string symbolicName(const string& path) { + string p = baseNameOf(path); + return string(p, p.find('-') + 1); } - -static string makeEdge(const string & src, const string & dst) -{ - return fmt(" <edge source=\"%1%\" target=\"%2%\"/>\n", - xmlQuote(src), xmlQuote(dst)); +static string makeEdge(const string& src, const string& dst) { + return fmt(" <edge source=\"%1%\" target=\"%2%\"/>\n", xmlQuote(src), + xmlQuote(dst)); } - -static string makeNode(const ValidPathInfo & info) -{ - return fmt( - " <node id=\"%1%\">\n" - " <data key=\"narSize\">%2%</data>\n" - " <data key=\"name\">%3%</data>\n" - " <data key=\"type\">%4%</data>\n" - " </node>\n", - info.path, - info.narSize, - symbolicName(info.path), - (isDerivation(info.path) ? "derivation" : "output-path")); +static string makeNode(const ValidPathInfo& info) { + return fmt( + " <node id=\"%1%\">\n" + " <data key=\"narSize\">%2%</data>\n" + " <data key=\"name\">%3%</data>\n" + " <data key=\"type\">%4%</data>\n" + " </node>\n", + info.path, info.narSize, symbolicName(info.path), + (isDerivation(info.path) ? "derivation" : "output-path")); } - -void printGraphML(ref<Store> store, const PathSet & roots) -{ - PathSet workList(roots); - PathSet doneSet; - std::pair<PathSet::iterator,bool> ret; - - cout << "<?xml version='1.0' encoding='utf-8'?>\n" - << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n" - << " xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n" - << " xsi:schemaLocation='http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd'>\n" - << "<key id='narSize' for='node' attr.name='narSize' attr.type='int'/>" - << "<key id='name' for='node' attr.name='name' attr.type='string'/>" - << "<key id='type' for='node' attr.name='type' attr.type='string'/>" - << "<graph id='G' edgedefault='directed'>\n"; - - while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); - - ret = doneSet.insert(path); - if (ret.second == false) continue; - - ValidPathInfo info = *(store->queryPathInfo(path)); - cout << makeNode(info); - - for (auto & p : store->queryPathInfo(path)->references) { - if (p != path) { - workList.insert(p); - cout << makeEdge(path, p); - } - } - +void printGraphML(ref<Store> store, const PathSet& roots) { + PathSet workList(roots); + PathSet doneSet; + std::pair<PathSet::iterator, bool> ret; + + cout << "<?xml version='1.0' encoding='utf-8'?>\n" + << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n" + << " xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n" + << " " + "xsi:schemaLocation='http://graphml.graphdrawing.org/xmlns/1.0/" + "graphml.xsd'>\n" + << "<key id='narSize' for='node' attr.name='narSize' attr.type='int'/>" + << "<key id='name' for='node' attr.name='name' attr.type='string'/>" + << "<key id='type' for='node' attr.name='type' attr.type='string'/>" + << "<graph id='G' edgedefault='directed'>\n"; + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + ret = doneSet.insert(path); + if (ret.second == false) continue; + + ValidPathInfo info = *(store->queryPathInfo(path)); + cout << makeNode(info); + + for (auto& p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(path, p); + } } + } - cout << "</graph>\n"; - cout << "</graphml>\n"; + cout << "</graph>\n"; + cout << "</graphml>\n"; } - -} +} // namespace nix diff --git a/third_party/nix/src/nix-store/graphml.hh b/third_party/nix/src/nix-store/graphml.hh index b78df1e49a67..c330dc6abc43 100644 --- a/third_party/nix/src/nix-store/graphml.hh +++ b/third_party/nix/src/nix-store/graphml.hh @@ -6,6 +6,6 @@ namespace nix { class Store; -void printGraphML(ref<Store> store, const PathSet & roots); +void printGraphML(ref<Store> store, const PathSet& roots); -} +} // namespace nix diff --git a/third_party/nix/src/nix-store/nix-store.cc b/third_party/nix/src/nix-store/nix-store.cc index 0cbceb02f31e..7f576b1b3ed9 100644 --- a/third_party/nix/src/nix-store/nix-store.cc +++ b/third_party/nix/src/nix-store/nix-store.cc @@ -1,36 +1,31 @@ +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <algorithm> +#include <cstdio> +#include <iostream> #include "archive.hh" #include "derivations.hh" #include "dotgraph.hh" #include "globals.hh" +#include "graphml.hh" +#include "legacy.hh" #include "local-store.hh" #include "monitor-fd.hh" #include "serve-protocol.hh" #include "shared.hh" #include "util.hh" #include "worker-protocol.hh" -#include "graphml.hh" -#include "legacy.hh" - -#include <iostream> -#include <algorithm> -#include <cstdio> - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> #if HAVE_SODIUM #include <sodium.h> #endif - using namespace nix; using std::cin; using std::cout; - -typedef void (* Operation) (Strings opFlags, Strings opArgs); - +typedef void (*Operation)(Strings opFlags, Strings opArgs); static Path gcRoot; static int rootNr = 0; @@ -38,198 +33,197 @@ static bool indirectRoot = false; static bool noOutput = false; static std::shared_ptr<Store> store; - -ref<LocalStore> ensureLocalStore() -{ - auto store2 = std::dynamic_pointer_cast<LocalStore>(store); - if (!store2) throw Error("you don't have sufficient rights to use this command"); - return ref<LocalStore>(store2); +ref<LocalStore> ensureLocalStore() { + auto store2 = std::dynamic_pointer_cast<LocalStore>(store); + if (!store2) + throw Error("you don't have sufficient rights to use this command"); + return ref<LocalStore>(store2); } - -static Path useDeriver(Path path) -{ - if (isDerivation(path)) return path; - Path drvPath = store->queryPathInfo(path)->deriver; - if (drvPath == "") - throw Error(format("deriver of path '%1%' is not known") % path); - return drvPath; +static Path useDeriver(Path path) { + if (isDerivation(path)) return path; + Path drvPath = store->queryPathInfo(path)->deriver; + if (drvPath == "") + throw Error(format("deriver of path '%1%' is not known") % path); + return drvPath; } - /* Realise the given path. For a derivation that means build it; for other paths it means ensure their validity. */ -static PathSet realisePath(Path path, bool build = true) -{ - DrvPathWithOutputs p = parseDrvPathWithOutputs(path); - - auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store); - - if (isDerivation(p.first)) { - if (build) store->buildPaths({path}); - Derivation drv = store->derivationFromPath(p.first); - rootNr++; - - if (p.second.empty()) - for (auto & i : drv.outputs) p.second.insert(i.first); - - PathSet outputs; - for (auto & j : p.second) { - DerivationOutputs::iterator i = drv.outputs.find(j); - if (i == drv.outputs.end()) - throw Error(format("derivation '%1%' does not have an output named '%2%'") % p.first % j); - Path outPath = i->second.path; - if (store2) { - if (gcRoot == "") - printGCWarning(); - else { - Path rootName = gcRoot; - if (rootNr > 1) rootName += "-" + std::to_string(rootNr); - if (i->first != "out") rootName += "-" + i->first; - outPath = store2->addPermRoot(outPath, rootName, indirectRoot); - } - } - outputs.insert(outPath); +static PathSet realisePath(Path path, bool build = true) { + DrvPathWithOutputs p = parseDrvPathWithOutputs(path); + + auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store); + + if (isDerivation(p.first)) { + if (build) store->buildPaths({path}); + Derivation drv = store->derivationFromPath(p.first); + rootNr++; + + if (p.second.empty()) + for (auto& i : drv.outputs) p.second.insert(i.first); + + PathSet outputs; + for (auto& j : p.second) { + DerivationOutputs::iterator i = drv.outputs.find(j); + if (i == drv.outputs.end()) + throw Error( + format("derivation '%1%' does not have an output named '%2%'") % + p.first % j); + Path outPath = i->second.path; + if (store2) { + if (gcRoot == "") + printGCWarning(); + else { + Path rootName = gcRoot; + if (rootNr > 1) rootName += "-" + std::to_string(rootNr); + if (i->first != "out") rootName += "-" + i->first; + outPath = store2->addPermRoot(outPath, rootName, indirectRoot); } - return outputs; + } + outputs.insert(outPath); } - - else { - if (build) store->ensurePath(path); - else if (!store->isValidPath(path)) throw Error(format("path '%1%' does not exist and cannot be created") % path); - if (store2) { - if (gcRoot == "") - printGCWarning(); - else { - Path rootName = gcRoot; - rootNr++; - if (rootNr > 1) rootName += "-" + std::to_string(rootNr); - path = store2->addPermRoot(path, rootName, indirectRoot); - } - } - return {path}; + return outputs; + } + + else { + if (build) + store->ensurePath(path); + else if (!store->isValidPath(path)) + throw Error(format("path '%1%' does not exist and cannot be created") % + path); + if (store2) { + if (gcRoot == "") + printGCWarning(); + else { + Path rootName = gcRoot; + rootNr++; + if (rootNr > 1) rootName += "-" + std::to_string(rootNr); + path = store2->addPermRoot(path, rootName, indirectRoot); + } } + return {path}; + } } - /* Realise the given paths. */ -static void opRealise(Strings opFlags, Strings opArgs) -{ - bool dryRun = false; - BuildMode buildMode = bmNormal; - bool ignoreUnknown = false; - - for (auto & i : opFlags) - if (i == "--dry-run") dryRun = true; - else if (i == "--repair") buildMode = bmRepair; - else if (i == "--check") buildMode = bmCheck; - else if (i == "--ignore-unknown") ignoreUnknown = true; - else throw UsageError(format("unknown flag '%1%'") % i); - - Paths paths; - for (auto & i : opArgs) { - DrvPathWithOutputs p = parseDrvPathWithOutputs(i); - paths.push_back(makeDrvPathWithOutputs(store->followLinksToStorePath(p.first), p.second)); +static void opRealise(Strings opFlags, Strings opArgs) { + bool dryRun = false; + BuildMode buildMode = bmNormal; + bool ignoreUnknown = false; + + for (auto& i : opFlags) + if (i == "--dry-run") + dryRun = true; + else if (i == "--repair") + buildMode = bmRepair; + else if (i == "--check") + buildMode = bmCheck; + else if (i == "--ignore-unknown") + ignoreUnknown = true; + else + throw UsageError(format("unknown flag '%1%'") % i); + + Paths paths; + for (auto& i : opArgs) { + DrvPathWithOutputs p = parseDrvPathWithOutputs(i); + paths.push_back(makeDrvPathWithOutputs( + store->followLinksToStorePath(p.first), p.second)); + } + + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + store->queryMissing(PathSet(paths.begin(), paths.end()), willBuild, + willSubstitute, unknown, downloadSize, narSize); + + if (ignoreUnknown) { + Paths paths2; + for (auto& i : paths) + if (unknown.find(i) == unknown.end()) paths2.push_back(i); + paths = paths2; + unknown = PathSet(); + } + + if (settings.printMissing) + printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, + downloadSize, narSize); + + if (dryRun) return; + + /* Build all paths at the same time to exploit parallelism. */ + store->buildPaths(PathSet(paths.begin(), paths.end()), buildMode); + + if (!ignoreUnknown) + for (auto& i : paths) { + PathSet paths = realisePath(i, false); + if (!noOutput) + for (auto& j : paths) cout << format("%1%\n") % j; } - - unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; - store->queryMissing(PathSet(paths.begin(), paths.end()), - willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (ignoreUnknown) { - Paths paths2; - for (auto & i : paths) - if (unknown.find(i) == unknown.end()) paths2.push_back(i); - paths = paths2; - unknown = PathSet(); - } - - if (settings.printMissing) - printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (dryRun) return; - - /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(PathSet(paths.begin(), paths.end()), buildMode); - - if (!ignoreUnknown) - for (auto & i : paths) { - PathSet paths = realisePath(i, false); - if (!noOutput) - for (auto & j : paths) - cout << format("%1%\n") % j; - } } - /* Add files to the Nix store and print the resulting paths. */ -static void opAdd(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); +static void opAdd(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) throw UsageError("unknown flag"); - for (auto & i : opArgs) - cout << format("%1%\n") % store->addToStore(baseNameOf(i), i); + for (auto& i : opArgs) + cout << format("%1%\n") % store->addToStore(baseNameOf(i), i); } - /* Preload the output of a fixed-output derivation into the Nix store. */ -static void opAddFixed(Strings opFlags, Strings opArgs) -{ - bool recursive = false; +static void opAddFixed(Strings opFlags, Strings opArgs) { + bool recursive = false; - for (auto & i : opFlags) - if (i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag '%1%'") % i); + for (auto& i : opFlags) + if (i == "--recursive") + recursive = true; + else + throw UsageError(format("unknown flag '%1%'") % i); - if (opArgs.empty()) - throw UsageError("first argument must be hash algorithm"); + if (opArgs.empty()) throw UsageError("first argument must be hash algorithm"); - HashType hashAlgo = parseHashType(opArgs.front()); - opArgs.pop_front(); + HashType hashAlgo = parseHashType(opArgs.front()); + opArgs.pop_front(); - for (auto & i : opArgs) - cout << format("%1%\n") % store->addToStore(baseNameOf(i), i, recursive, hashAlgo); + for (auto& i : opArgs) + cout << format("%1%\n") % + store->addToStore(baseNameOf(i), i, recursive, hashAlgo); } - /* Hack to support caching in `nix-prefetch-url'. */ -static void opPrintFixedPath(Strings opFlags, Strings opArgs) -{ - bool recursive = false; +static void opPrintFixedPath(Strings opFlags, Strings opArgs) { + bool recursive = false; - for (auto i : opFlags) - if (i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag '%1%'") % i); + for (auto i : opFlags) + if (i == "--recursive") + recursive = true; + else + throw UsageError(format("unknown flag '%1%'") % i); - if (opArgs.size() != 3) - throw UsageError(format("'--print-fixed-path' requires three arguments")); + if (opArgs.size() != 3) + throw UsageError(format("'--print-fixed-path' requires three arguments")); - Strings::iterator i = opArgs.begin(); - HashType hashAlgo = parseHashType(*i++); - string hash = *i++; - string name = *i++; + Strings::iterator i = opArgs.begin(); + HashType hashAlgo = parseHashType(*i++); + string hash = *i++; + string name = *i++; - cout << format("%1%\n") % - store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name); + cout << format("%1%\n") % + store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name); } - -static PathSet maybeUseOutputs(const Path & storePath, bool useOutput, bool forceRealise) -{ - if (forceRealise) realisePath(storePath); - if (useOutput && isDerivation(storePath)) { - Derivation drv = store->derivationFromPath(storePath); - PathSet outputs; - for (auto & i : drv.outputs) - outputs.insert(i.second.path); - return outputs; - } - else return {storePath}; +static PathSet maybeUseOutputs(const Path& storePath, bool useOutput, + bool forceRealise) { + if (forceRealise) realisePath(storePath); + if (useOutput && isDerivation(storePath)) { + Derivation drv = store->derivationFromPath(storePath); + PathSet outputs; + for (auto& i : drv.outputs) outputs.insert(i.second.path); + return outputs; + } else + return {storePath}; } - /* Some code to print a tree representation of a derivation dependency graph. Topological sorting is used to keep the tree relatively flat. */ @@ -238,873 +232,870 @@ const string treeConn = "+---"; const string treeLine = "| "; const string treeNull = " "; - -static void printTree(const Path & path, - const string & firstPad, const string & tailPad, PathSet & done) -{ - if (done.find(path) != done.end()) { - cout << format("%1%%2% [...]\n") % firstPad % path; - return; - } - done.insert(path); - - cout << format("%1%%2%\n") % firstPad % path; - - auto references = store->queryPathInfo(path)->references; - - /* Topologically sort under the relation A < B iff A \in - closure(B). That is, if derivation A is an (possibly indirect) - input of B, then A is printed first. This has the effect of - flattening the tree, preventing deeply nested structures. */ - Paths sorted = store->topoSortPaths(references); - reverse(sorted.begin(), sorted.end()); - - for (auto i = sorted.begin(); i != sorted.end(); ++i) { - auto j = i; ++j; - printTree(*i, tailPad + treeConn, - j == sorted.end() ? tailPad + treeNull : tailPad + treeLine, - done); - } +static void printTree(const Path& path, const string& firstPad, + const string& tailPad, PathSet& done) { + if (done.find(path) != done.end()) { + cout << format("%1%%2% [...]\n") % firstPad % path; + return; + } + done.insert(path); + + cout << format("%1%%2%\n") % firstPad % path; + + auto references = store->queryPathInfo(path)->references; + + /* Topologically sort under the relation A < B iff A \in + closure(B). That is, if derivation A is an (possibly indirect) + input of B, then A is printed first. This has the effect of + flattening the tree, preventing deeply nested structures. */ + Paths sorted = store->topoSortPaths(references); + reverse(sorted.begin(), sorted.end()); + + for (auto i = sorted.begin(); i != sorted.end(); ++i) { + auto j = i; + ++j; + printTree(*i, tailPad + treeConn, + j == sorted.end() ? tailPad + treeNull : tailPad + treeLine, + done); + } } - /* Perform various sorts of queries. */ -static void opQuery(Strings opFlags, Strings opArgs) -{ - enum QueryType - { qDefault, qOutputs, qRequisites, qReferences, qReferrers - , qReferrersClosure, qDeriver, qBinding, qHash, qSize - , qTree, qGraph, qGraphML, qResolve, qRoots }; - QueryType query = qDefault; - bool useOutput = false; - bool includeOutputs = false; - bool forceRealise = false; - string bindingName; - - for (auto & i : opFlags) { - QueryType prev = query; - if (i == "--outputs") query = qOutputs; - else if (i == "--requisites" || i == "-R") query = qRequisites; - else if (i == "--references") query = qReferences; - else if (i == "--referrers" || i == "--referers") query = qReferrers; - else if (i == "--referrers-closure" || i == "--referers-closure") query = qReferrersClosure; - else if (i == "--deriver" || i == "-d") query = qDeriver; - else if (i == "--binding" || i == "-b") { - if (opArgs.size() == 0) - throw UsageError("expected binding name"); - bindingName = opArgs.front(); - opArgs.pop_front(); - query = qBinding; - } - else if (i == "--hash") query = qHash; - else if (i == "--size") query = qSize; - else if (i == "--tree") query = qTree; - else if (i == "--graph") query = qGraph; - else if (i == "--graphml") query = qGraphML; - else if (i == "--resolve") query = qResolve; - else if (i == "--roots") query = qRoots; - else if (i == "--use-output" || i == "-u") useOutput = true; - else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; - else if (i == "--include-outputs") includeOutputs = true; - else throw UsageError(format("unknown flag '%1%'") % i); - if (prev != qDefault && prev != query) - throw UsageError(format("query type '%1%' conflicts with earlier flag") % i); +static void opQuery(Strings opFlags, Strings opArgs) { + enum QueryType { + qDefault, + qOutputs, + qRequisites, + qReferences, + qReferrers, + qReferrersClosure, + qDeriver, + qBinding, + qHash, + qSize, + qTree, + qGraph, + qGraphML, + qResolve, + qRoots + }; + QueryType query = qDefault; + bool useOutput = false; + bool includeOutputs = false; + bool forceRealise = false; + string bindingName; + + for (auto& i : opFlags) { + QueryType prev = query; + if (i == "--outputs") + query = qOutputs; + else if (i == "--requisites" || i == "-R") + query = qRequisites; + else if (i == "--references") + query = qReferences; + else if (i == "--referrers" || i == "--referers") + query = qReferrers; + else if (i == "--referrers-closure" || i == "--referers-closure") + query = qReferrersClosure; + else if (i == "--deriver" || i == "-d") + query = qDeriver; + else if (i == "--binding" || i == "-b") { + if (opArgs.size() == 0) throw UsageError("expected binding name"); + bindingName = opArgs.front(); + opArgs.pop_front(); + query = qBinding; + } else if (i == "--hash") + query = qHash; + else if (i == "--size") + query = qSize; + else if (i == "--tree") + query = qTree; + else if (i == "--graph") + query = qGraph; + else if (i == "--graphml") + query = qGraphML; + else if (i == "--resolve") + query = qResolve; + else if (i == "--roots") + query = qRoots; + else if (i == "--use-output" || i == "-u") + useOutput = true; + else if (i == "--force-realise" || i == "--force-realize" || i == "-f") + forceRealise = true; + else if (i == "--include-outputs") + includeOutputs = true; + else + throw UsageError(format("unknown flag '%1%'") % i); + if (prev != qDefault && prev != query) + throw UsageError(format("query type '%1%' conflicts with earlier flag") % + i); + } + + if (query == qDefault) query = qOutputs; + + RunPager pager; + + switch (query) { + case qOutputs: { + for (auto& i : opArgs) { + i = store->followLinksToStorePath(i); + if (forceRealise) realisePath(i); + Derivation drv = store->derivationFromPath(i); + for (auto& j : drv.outputs) cout << format("%1%\n") % j.second.path; + } + break; } - if (query == qDefault) query = qOutputs; - - RunPager pager; - - switch (query) { - - case qOutputs: { - for (auto & i : opArgs) { - i = store->followLinksToStorePath(i); - if (forceRealise) realisePath(i); - Derivation drv = store->derivationFromPath(i); - for (auto & j : drv.outputs) - cout << format("%1%\n") % j.second.path; - } - break; - } - - case qRequisites: - case qReferences: - case qReferrers: - case qReferrersClosure: { - PathSet paths; - for (auto & i : opArgs) { - PathSet ps = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); - for (auto & j : ps) { - if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); - else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->references) - paths.insert(p); - } - else if (query == qReferrers) store->queryReferrers(j, paths); - else if (query == qReferrersClosure) store->computeFSClosure(j, paths, true); - } - } - Paths sorted = store->topoSortPaths(paths); - for (Paths::reverse_iterator i = sorted.rbegin(); - i != sorted.rend(); ++i) - cout << format("%s\n") % *i; - break; - } - - case qDeriver: - for (auto & i : opArgs) { - Path deriver = store->queryPathInfo(store->followLinksToStorePath(i))->deriver; - cout << format("%1%\n") % - (deriver == "" ? "unknown-deriver" : deriver); - } - break; - - case qBinding: - for (auto & i : opArgs) { - Path path = useDeriver(store->followLinksToStorePath(i)); - Derivation drv = store->derivationFromPath(path); - StringPairs::iterator j = drv.env.find(bindingName); - if (j == drv.env.end()) - throw Error(format("derivation '%1%' has no environment binding named '%2%'") - % path % bindingName); - cout << format("%1%\n") % j->second; - } - break; - - case qHash: - case qSize: - for (auto & i : opArgs) { - PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); - for (auto & j : paths) { - auto info = store->queryPathInfo(j); - if (query == qHash) { - assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32)); - } else if (query == qSize) - cout << fmt("%d\n", info->narSize); - } - } - break; - - case qTree: { - PathSet done; - for (auto & i : opArgs) - printTree(store->followLinksToStorePath(i), "", "", done); - break; - } - - case qGraph: { - PathSet roots; - for (auto & i : opArgs) { - PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); - roots.insert(paths.begin(), paths.end()); - } - printDotGraph(ref<Store>(store), roots); - break; - } - - case qGraphML: { - PathSet roots; - for (auto & i : opArgs) { - PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); - roots.insert(paths.begin(), paths.end()); - } - printGraphML(ref<Store>(store), roots); - break; + case qRequisites: + case qReferences: + case qReferrers: + case qReferrersClosure: { + PathSet paths; + for (auto& i : opArgs) { + PathSet ps = maybeUseOutputs(store->followLinksToStorePath(i), + useOutput, forceRealise); + for (auto& j : ps) { + if (query == qRequisites) + store->computeFSClosure(j, paths, false, includeOutputs); + else if (query == qReferences) { + for (auto& p : store->queryPathInfo(j)->references) paths.insert(p); + } else if (query == qReferrers) + store->queryReferrers(j, paths); + else if (query == qReferrersClosure) + store->computeFSClosure(j, paths, true); } + } + Paths sorted = store->topoSortPaths(paths); + for (Paths::reverse_iterator i = sorted.rbegin(); i != sorted.rend(); ++i) + cout << format("%s\n") % *i; + break; + } - case qResolve: { - for (auto & i : opArgs) - cout << format("%1%\n") % store->followLinksToStorePath(i); - break; + case qDeriver: + for (auto& i : opArgs) { + Path deriver = + store->queryPathInfo(store->followLinksToStorePath(i))->deriver; + cout << format("%1%\n") % (deriver == "" ? "unknown-deriver" : deriver); + } + break; + + case qBinding: + for (auto& i : opArgs) { + Path path = useDeriver(store->followLinksToStorePath(i)); + Derivation drv = store->derivationFromPath(path); + StringPairs::iterator j = drv.env.find(bindingName); + if (j == drv.env.end()) + throw Error( + format( + "derivation '%1%' has no environment binding named '%2%'") % + path % bindingName); + cout << format("%1%\n") % j->second; + } + break; + + case qHash: + case qSize: + for (auto& i : opArgs) { + PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), + useOutput, forceRealise); + for (auto& j : paths) { + auto info = store->queryPathInfo(j); + if (query == qHash) { + assert(info->narHash.type == htSHA256); + cout << fmt("%s\n", info->narHash.to_string(Base32)); + } else if (query == qSize) + cout << fmt("%d\n", info->narSize); } - - case qRoots: { - PathSet referrers; - for (auto & i : opArgs) { - store->computeFSClosure( - maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise), - referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); - } - Roots roots = store->findRoots(false); - for (auto & [target, links] : roots) - if (referrers.find(target) != referrers.end()) - for (auto & link : links) - cout << format("%1% -> %2%\n") % link % target; - break; - } - - default: - abort(); + } + break; + + case qTree: { + PathSet done; + for (auto& i : opArgs) + printTree(store->followLinksToStorePath(i), "", "", done); + break; } -} - -static void opPrintEnv(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (opArgs.size() != 1) throw UsageError("'--print-env' requires one derivation store path"); + case qGraph: { + PathSet roots; + for (auto& i : opArgs) { + PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), + useOutput, forceRealise); + roots.insert(paths.begin(), paths.end()); + } + printDotGraph(ref<Store>(store), roots); + break; + } - Path drvPath = opArgs.front(); - Derivation drv = store->derivationFromPath(drvPath); + case qGraphML: { + PathSet roots; + for (auto& i : opArgs) { + PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), + useOutput, forceRealise); + roots.insert(paths.begin(), paths.end()); + } + printGraphML(ref<Store>(store), roots); + break; + } - /* Print each environment variable in the derivation in a format - that can be sourced by the shell. */ - for (auto & i : drv.env) - cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); + case qResolve: { + for (auto& i : opArgs) + cout << format("%1%\n") % store->followLinksToStorePath(i); + break; + } - /* Also output the arguments. This doesn't preserve whitespace in - arguments. */ - cout << "export _args; _args='"; - bool first = true; - for (auto & i : drv.args) { - if (!first) cout << ' '; - first = false; - cout << shellEscape(i); + case qRoots: { + PathSet referrers; + for (auto& i : opArgs) { + store->computeFSClosure( + maybeUseOutputs(store->followLinksToStorePath(i), useOutput, + forceRealise), + referrers, true, settings.gcKeepOutputs, + settings.gcKeepDerivations); + } + Roots roots = store->findRoots(false); + for (auto& [target, links] : roots) + if (referrers.find(target) != referrers.end()) + for (auto& link : links) + cout << format("%1% -> %2%\n") % link % target; + break; } - cout << "'\n"; + + default: + abort(); + } } +static void opPrintEnv(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (opArgs.size() != 1) + throw UsageError("'--print-env' requires one derivation store path"); + + Path drvPath = opArgs.front(); + Derivation drv = store->derivationFromPath(drvPath); + + /* Print each environment variable in the derivation in a format + that can be sourced by the shell. */ + for (auto& i : drv.env) + cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); + + /* Also output the arguments. This doesn't preserve whitespace in + arguments. */ + cout << "export _args; _args='"; + bool first = true; + for (auto& i : drv.args) { + if (!first) cout << ' '; + first = false; + cout << shellEscape(i); + } + cout << "'\n"; +} -static void opReadLog(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); +static void opReadLog(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) throw UsageError("unknown flag"); - RunPager pager; + RunPager pager; - for (auto & i : opArgs) { - auto path = store->followLinksToStorePath(i); - auto log = store->getBuildLog(path); - if (!log) - throw Error("build log of derivation '%s' is not available", path); - std::cout << *log; - } + for (auto& i : opArgs) { + auto path = store->followLinksToStorePath(i); + auto log = store->getBuildLog(path); + if (!log) + throw Error("build log of derivation '%s' is not available", path); + std::cout << *log; + } } - -static void opDumpDB(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) { - for (auto & i : opArgs) - i = store->followLinksToStorePath(i); - for (auto & i : opArgs) - cout << store->makeValidityRegistration({i}, true, true); - } else { - PathSet validPaths = store->queryAllValidPaths(); - for (auto & i : validPaths) - cout << store->makeValidityRegistration({i}, true, true); - } +static void opDumpDB(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (!opArgs.empty()) { + for (auto& i : opArgs) i = store->followLinksToStorePath(i); + for (auto& i : opArgs) + cout << store->makeValidityRegistration({i}, true, true); + } else { + PathSet validPaths = store->queryAllValidPaths(); + for (auto& i : validPaths) + cout << store->makeValidityRegistration({i}, true, true); + } } - -static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) -{ - ValidPathInfos infos; - - while (1) { - ValidPathInfo info = decodeValidPathInfo(cin, hashGiven); - if (info.path == "") break; - if (!store->isValidPath(info.path) || reregister) { - /* !!! races */ - if (canonicalise) - canonicalisePathMetaData(info.path, -1); - if (!hashGiven) { - HashResult hash = hashPath(htSHA256, info.path); - info.narHash = hash.first; - info.narSize = hash.second; - } - infos.push_back(info); - } +static void registerValidity(bool reregister, bool hashGiven, + bool canonicalise) { + ValidPathInfos infos; + + while (1) { + ValidPathInfo info = decodeValidPathInfo(cin, hashGiven); + if (info.path == "") break; + if (!store->isValidPath(info.path) || reregister) { + /* !!! races */ + if (canonicalise) canonicalisePathMetaData(info.path, -1); + if (!hashGiven) { + HashResult hash = hashPath(htSHA256, info.path); + info.narHash = hash.first; + info.narSize = hash.second; + } + infos.push_back(info); } + } - ensureLocalStore()->registerValidPaths(infos); + ensureLocalStore()->registerValidPaths(infos); } - -static void opLoadDB(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - registerValidity(true, true, false); +static void opLoadDB(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (!opArgs.empty()) throw UsageError("no arguments expected"); + registerValidity(true, true, false); } +static void opRegisterValidity(Strings opFlags, Strings opArgs) { + bool reregister = false; // !!! maybe this should be the default + bool hashGiven = false; -static void opRegisterValidity(Strings opFlags, Strings opArgs) -{ - bool reregister = false; // !!! maybe this should be the default - bool hashGiven = false; + for (auto& i : opFlags) + if (i == "--reregister") + reregister = true; + else if (i == "--hash-given") + hashGiven = true; + else + throw UsageError(format("unknown flag '%1%'") % i); - for (auto & i : opFlags) - if (i == "--reregister") reregister = true; - else if (i == "--hash-given") hashGiven = true; - else throw UsageError(format("unknown flag '%1%'") % i); + if (!opArgs.empty()) throw UsageError("no arguments expected"); - if (!opArgs.empty()) throw UsageError("no arguments expected"); - - registerValidity(reregister, hashGiven, true); + registerValidity(reregister, hashGiven, true); } - -static void opCheckValidity(Strings opFlags, Strings opArgs) -{ - bool printInvalid = false; - - for (auto & i : opFlags) - if (i == "--print-invalid") printInvalid = true; - else throw UsageError(format("unknown flag '%1%'") % i); - - for (auto & i : opArgs) { - Path path = store->followLinksToStorePath(i); - if (!store->isValidPath(path)) { - if (printInvalid) - cout << format("%1%\n") % path; - else - throw Error(format("path '%1%' is not valid") % path); - } +static void opCheckValidity(Strings opFlags, Strings opArgs) { + bool printInvalid = false; + + for (auto& i : opFlags) + if (i == "--print-invalid") + printInvalid = true; + else + throw UsageError(format("unknown flag '%1%'") % i); + + for (auto& i : opArgs) { + Path path = store->followLinksToStorePath(i); + if (!store->isValidPath(path)) { + if (printInvalid) + cout << format("%1%\n") % path; + else + throw Error(format("path '%1%' is not valid") % path); } + } } +static void opGC(Strings opFlags, Strings opArgs) { + bool printRoots = false; + GCOptions options; + options.action = GCOptions::gcDeleteDead; + + GCResults results; + + /* Do what? */ + for (auto i = opFlags.begin(); i != opFlags.end(); ++i) + if (*i == "--print-roots") + printRoots = true; + else if (*i == "--print-live") + options.action = GCOptions::gcReturnLive; + else if (*i == "--print-dead") + options.action = GCOptions::gcReturnDead; + else if (*i == "--delete") + options.action = GCOptions::gcDeleteDead; + else if (*i == "--max-freed") { + long long maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true); + options.maxFreed = maxFreed >= 0 ? maxFreed : 0; + } else + throw UsageError(format("bad sub-operation '%1%' in GC") % *i); + + if (!opArgs.empty()) throw UsageError("no arguments expected"); + + if (printRoots) { + Roots roots = store->findRoots(false); + std::set<std::pair<Path, Path>> roots2; + // Transpose and sort the roots. + for (auto& [target, links] : roots) + for (auto& link : links) roots2.emplace(link, target); + for (auto& [link, target] : roots2) + std::cout << link << " -> " << target << "\n"; + } + + else { + PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + store->collectGarbage(options, results); -static void opGC(Strings opFlags, Strings opArgs) -{ - bool printRoots = false; - GCOptions options; - options.action = GCOptions::gcDeleteDead; - - GCResults results; - - /* Do what? */ - for (auto i = opFlags.begin(); i != opFlags.end(); ++i) - if (*i == "--print-roots") printRoots = true; - else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; - else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; - else if (*i == "--delete") options.action = GCOptions::gcDeleteDead; - else if (*i == "--max-freed") { - long long maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true); - options.maxFreed = maxFreed >= 0 ? maxFreed : 0; - } - else throw UsageError(format("bad sub-operation '%1%' in GC") % *i); - - if (!opArgs.empty()) throw UsageError("no arguments expected"); - - if (printRoots) { - Roots roots = store->findRoots(false); - std::set<std::pair<Path, Path>> roots2; - // Transpose and sort the roots. - for (auto & [target, links] : roots) - for (auto & link : links) - roots2.emplace(link, target); - for (auto & [link, target] : roots2) - std::cout << link << " -> " << target << "\n"; - } - - else { - PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); - store->collectGarbage(options, results); - - if (options.action != GCOptions::gcDeleteDead) - for (auto & i : results.paths) - cout << i << std::endl; - } + if (options.action != GCOptions::gcDeleteDead) + for (auto& i : results.paths) cout << i << std::endl; + } } - /* Remove paths from the Nix store if possible (i.e., if they do not have any remaining referrers and are not reachable from any GC roots). */ -static void opDelete(Strings opFlags, Strings opArgs) -{ - GCOptions options; - options.action = GCOptions::gcDeleteSpecific; - - for (auto & i : opFlags) - if (i == "--ignore-liveness") options.ignoreLiveness = true; - else throw UsageError(format("unknown flag '%1%'") % i); - - for (auto & i : opArgs) - options.pathsToDelete.insert(store->followLinksToStorePath(i)); - - GCResults results; - PrintFreed freed(true, results); - store->collectGarbage(options, results); +static void opDelete(Strings opFlags, Strings opArgs) { + GCOptions options; + options.action = GCOptions::gcDeleteSpecific; + + for (auto& i : opFlags) + if (i == "--ignore-liveness") + options.ignoreLiveness = true; + else + throw UsageError(format("unknown flag '%1%'") % i); + + for (auto& i : opArgs) + options.pathsToDelete.insert(store->followLinksToStorePath(i)); + + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); } - /* Dump a path as a Nix archive. The archive is written to standard output. */ -static void opDump(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (opArgs.size() != 1) throw UsageError("only one argument allowed"); - - FdSink sink(STDOUT_FILENO); - string path = *opArgs.begin(); - dumpPath(path, sink); - sink.flush(); +static void opDump(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (opArgs.size() != 1) throw UsageError("only one argument allowed"); + + FdSink sink(STDOUT_FILENO); + string path = *opArgs.begin(); + dumpPath(path, sink); + sink.flush(); } - /* Restore a value from a Nix archive. The archive is read from standard input. */ -static void opRestore(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (opArgs.size() != 1) throw UsageError("only one argument allowed"); +static void opRestore(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (opArgs.size() != 1) throw UsageError("only one argument allowed"); - FdSource source(STDIN_FILENO); - restorePath(*opArgs.begin(), source); + FdSource source(STDIN_FILENO); + restorePath(*opArgs.begin(), source); } +static void opExport(Strings opFlags, Strings opArgs) { + for (auto& i : opFlags) throw UsageError(format("unknown flag '%1%'") % i); -static void opExport(Strings opFlags, Strings opArgs) -{ - for (auto & i : opFlags) - throw UsageError(format("unknown flag '%1%'") % i); + for (auto& i : opArgs) i = store->followLinksToStorePath(i); - for (auto & i : opArgs) - i = store->followLinksToStorePath(i); - - FdSink sink(STDOUT_FILENO); - store->exportPaths(opArgs, sink); - sink.flush(); + FdSink sink(STDOUT_FILENO); + store->exportPaths(opArgs, sink); + sink.flush(); } +static void opImport(Strings opFlags, Strings opArgs) { + for (auto& i : opFlags) throw UsageError(format("unknown flag '%1%'") % i); -static void opImport(Strings opFlags, Strings opArgs) -{ - for (auto & i : opFlags) - throw UsageError(format("unknown flag '%1%'") % i); - - if (!opArgs.empty()) throw UsageError("no arguments expected"); + if (!opArgs.empty()) throw UsageError("no arguments expected"); - FdSource source(STDIN_FILENO); - Paths paths = store->importPaths(source, nullptr, NoCheckSigs); + FdSource source(STDIN_FILENO); + Paths paths = store->importPaths(source, nullptr, NoCheckSigs); - for (auto & i : paths) - cout << format("%1%\n") % i << std::flush; + for (auto& i : paths) cout << format("%1%\n") % i << std::flush; } - /* Initialise the Nix databases. */ -static void opInit(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - /* Doesn't do anything right now; database tables are initialised - automatically. */ +static void opInit(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (!opArgs.empty()) throw UsageError("no arguments expected"); + /* Doesn't do anything right now; database tables are initialised + automatically. */ } - /* Verify the consistency of the Nix environment. */ -static void opVerify(Strings opFlags, Strings opArgs) -{ - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - - bool checkContents = false; - RepairFlag repair = NoRepair; - - for (auto & i : opFlags) - if (i == "--check-contents") checkContents = true; - else if (i == "--repair") repair = Repair; - else throw UsageError(format("unknown flag '%1%'") % i); - - if (store->verifyStore(checkContents, repair)) { - printError("warning: not all errors were fixed"); - throw Exit(1); - } +static void opVerify(Strings opFlags, Strings opArgs) { + if (!opArgs.empty()) throw UsageError("no arguments expected"); + + bool checkContents = false; + RepairFlag repair = NoRepair; + + for (auto& i : opFlags) + if (i == "--check-contents") + checkContents = true; + else if (i == "--repair") + repair = Repair; + else + throw UsageError(format("unknown flag '%1%'") % i); + + if (store->verifyStore(checkContents, repair)) { + printError("warning: not all errors were fixed"); + throw Exit(1); + } } - /* Verify whether the contents of the given store path have not changed. */ -static void opVerifyPath(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) - throw UsageError("no flags expected"); - - int status = 0; - - for (auto & i : opArgs) { - Path path = store->followLinksToStorePath(i); - printMsg(lvlTalkative, format("checking path '%1%'...") % path); - auto info = store->queryPathInfo(path); - HashSink sink(info->narHash.type); - store->narFromPath(path, sink); - auto current = sink.finish(); - if (current.first != info->narHash) { - printError( - format("path '%1%' was modified! expected hash '%2%', got '%3%'") - % path % info->narHash.to_string() % current.first.to_string()); - status = 1; - } +static void opVerifyPath(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) throw UsageError("no flags expected"); + + int status = 0; + + for (auto& i : opArgs) { + Path path = store->followLinksToStorePath(i); + printMsg(lvlTalkative, format("checking path '%1%'...") % path); + auto info = store->queryPathInfo(path); + HashSink sink(info->narHash.type); + store->narFromPath(path, sink); + auto current = sink.finish(); + if (current.first != info->narHash) { + printError( + format("path '%1%' was modified! expected hash '%2%', got '%3%'") % + path % info->narHash.to_string() % current.first.to_string()); + status = 1; } + } - throw Exit(status); + throw Exit(status); } - /* Repair the contents of the given path by redownloading it using a substituter (if available). */ -static void opRepairPath(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) - throw UsageError("no flags expected"); - - for (auto & i : opArgs) { - Path path = store->followLinksToStorePath(i); - ensureLocalStore()->repairPath(path); - } +static void opRepairPath(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) throw UsageError("no flags expected"); + + for (auto& i : opArgs) { + Path path = store->followLinksToStorePath(i); + ensureLocalStore()->repairPath(path); + } } /* Optimise the disk space usage of the Nix store by hard-linking files with the same contents. */ -static void opOptimise(Strings opFlags, Strings opArgs) -{ - if (!opArgs.empty() || !opFlags.empty()) - throw UsageError("no arguments expected"); +static void opOptimise(Strings opFlags, Strings opArgs) { + if (!opArgs.empty() || !opFlags.empty()) + throw UsageError("no arguments expected"); - store->optimiseStore(); + store->optimiseStore(); } /* Serve the nix store in a way usable by a restricted ssh user. */ -static void opServe(Strings opFlags, Strings opArgs) -{ - bool writeAllowed = false; - for (auto & i : opFlags) - if (i == "--write") writeAllowed = true; - else throw UsageError(format("unknown flag '%1%'") % i); - - if (!opArgs.empty()) throw UsageError("no arguments expected"); - - FdSource in(STDIN_FILENO); - FdSink out(STDOUT_FILENO); - - /* Exchange the greeting. */ - unsigned int magic = readInt(in); - if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); - out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; - out.flush(); - unsigned int clientVersion = readInt(in); - - auto getBuildSettings = [&]() { - // FIXME: changing options here doesn't work if we're - // building through the daemon. - verbosity = lvlError; - settings.keepLog = false; - settings.useSubstitutes = false; - settings.maxSilentTime = readInt(in); - settings.buildTimeout = readInt(in); - if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - settings.maxLogSize = readNum<unsigned long>(in); - if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - settings.buildRepeat = readInt(in); - settings.enforceDeterminism = readInt(in); - settings.runDiffHook = true; - } - settings.printRepeatedBuilds = false; - }; - - while (true) { - ServeCommand cmd; - try { - cmd = (ServeCommand) readInt(in); - } catch (EndOfFile & e) { - break; - } - - switch (cmd) { - - case cmdQueryValidPaths: { - bool lock = readInt(in); - bool substitute = readInt(in); - PathSet paths = readStorePaths<PathSet>(*store, in); - if (lock && writeAllowed) - for (auto & path : paths) - store->addTempRoot(path); - - /* If requested, substitute missing paths. This - implements nix-copy-closure's --use-substitutes - flag. */ - if (substitute && writeAllowed) { - /* Filter out .drv files (we don't want to build anything). */ - PathSet paths2; - for (auto & path : paths) - if (!isDerivation(path)) paths2.insert(path); - unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; - store->queryMissing(PathSet(paths2.begin(), paths2.end()), - willBuild, willSubstitute, unknown, downloadSize, narSize); - /* FIXME: should use ensurePath(), but it only - does one path at a time. */ - if (!willSubstitute.empty()) - try { - store->buildPaths(willSubstitute); - } catch (Error & e) { - printError(format("warning: %1%") % e.msg()); - } - } - - out << store->queryValidPaths(paths); - break; - } - - case cmdQueryPathInfos: { - PathSet paths = readStorePaths<PathSet>(*store, in); - // !!! Maybe we want a queryPathInfos? - for (auto & i : paths) { - try { - auto info = store->queryPathInfo(i); - out << info->path << info->deriver << info->references; - // !!! Maybe we want compression? - out << info->narSize // downloadSize - << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string() : "") << info->ca << info->sigs; - } catch (InvalidPath &) { - } - } - out << ""; - break; - } - - case cmdDumpStorePath: - store->narFromPath(readStorePath(*store, in), out); - break; - - case cmdImportPaths: { - if (!writeAllowed) throw Error("importing paths is not allowed"); - store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking? - out << 1; // indicate success - break; - } +static void opServe(Strings opFlags, Strings opArgs) { + bool writeAllowed = false; + for (auto& i : opFlags) + if (i == "--write") + writeAllowed = true; + else + throw UsageError(format("unknown flag '%1%'") % i); + + if (!opArgs.empty()) throw UsageError("no arguments expected"); + + FdSource in(STDIN_FILENO); + FdSink out(STDOUT_FILENO); + + /* Exchange the greeting. */ + unsigned int magic = readInt(in); + if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); + out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; + out.flush(); + unsigned int clientVersion = readInt(in); + + auto getBuildSettings = [&]() { + // FIXME: changing options here doesn't work if we're + // building through the daemon. + verbosity = lvlError; + settings.keepLog = false; + settings.useSubstitutes = false; + settings.maxSilentTime = readInt(in); + settings.buildTimeout = readInt(in); + if (GET_PROTOCOL_MINOR(clientVersion) >= 2) + settings.maxLogSize = readNum<unsigned long>(in); + if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { + settings.buildRepeat = readInt(in); + settings.enforceDeterminism = readInt(in); + settings.runDiffHook = true; + } + settings.printRepeatedBuilds = false; + }; + + while (true) { + ServeCommand cmd; + try { + cmd = (ServeCommand)readInt(in); + } catch (EndOfFile& e) { + break; + } - case cmdExportPaths: { - readInt(in); // obsolete - store->exportPaths(readStorePaths<Paths>(*store, in), out); - break; + switch (cmd) { + case cmdQueryValidPaths: { + bool lock = readInt(in); + bool substitute = readInt(in); + PathSet paths = readStorePaths<PathSet>(*store, in); + if (lock && writeAllowed) + for (auto& path : paths) store->addTempRoot(path); + + /* If requested, substitute missing paths. This + implements nix-copy-closure's --use-substitutes + flag. */ + if (substitute && writeAllowed) { + /* Filter out .drv files (we don't want to build anything). */ + PathSet paths2; + for (auto& path : paths) + if (!isDerivation(path)) paths2.insert(path); + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + store->queryMissing(PathSet(paths2.begin(), paths2.end()), willBuild, + willSubstitute, unknown, downloadSize, narSize); + /* FIXME: should use ensurePath(), but it only + does one path at a time. */ + if (!willSubstitute.empty()) try { + store->buildPaths(willSubstitute); + } catch (Error& e) { + printError(format("warning: %1%") % e.msg()); } + } - case cmdBuildPaths: { - - if (!writeAllowed) throw Error("building paths is not allowed"); - PathSet paths = readStorePaths<PathSet>(*store, in); - - getBuildSettings(); - - try { - MonitorFdHup monitor(in.fd); - store->buildPaths(paths); - out << 0; - } catch (Error & e) { - assert(e.status); - out << e.status << e.msg(); - } - break; - } + out << store->queryValidPaths(paths); + break; + } + + case cmdQueryPathInfos: { + PathSet paths = readStorePaths<PathSet>(*store, in); + // !!! Maybe we want a queryPathInfos? + for (auto& i : paths) { + try { + auto info = store->queryPathInfo(i); + out << info->path << info->deriver << info->references; + // !!! Maybe we want compression? + out << info->narSize // downloadSize + << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 4) + out << (info->narHash ? info->narHash.to_string() : "") + << info->ca << info->sigs; + } catch (InvalidPath&) { + } + } + out << ""; + break; + } + + case cmdDumpStorePath: + store->narFromPath(readStorePath(*store, in), out); + break; + + case cmdImportPaths: { + if (!writeAllowed) throw Error("importing paths is not allowed"); + store->importPaths(in, nullptr, + NoCheckSigs); // FIXME: should we skip sig checking? + out << 1; // indicate success + break; + } + + case cmdExportPaths: { + readInt(in); // obsolete + store->exportPaths(readStorePaths<Paths>(*store, in), out); + break; + } + + case cmdBuildPaths: { + if (!writeAllowed) throw Error("building paths is not allowed"); + PathSet paths = readStorePaths<PathSet>(*store, in); + + getBuildSettings(); - case cmdBuildDerivation: { /* Used by hydra-queue-runner. */ + try { + MonitorFdHup monitor(in.fd); + store->buildPaths(paths); + out << 0; + } catch (Error& e) { + assert(e.status); + out << e.status << e.msg(); + } + break; + } - if (!writeAllowed) throw Error("building paths is not allowed"); + case cmdBuildDerivation: { /* Used by hydra-queue-runner. */ - Path drvPath = readStorePath(*store, in); // informational only - BasicDerivation drv; - readDerivation(in, *store, drv); + if (!writeAllowed) throw Error("building paths is not allowed"); - getBuildSettings(); + Path drvPath = readStorePath(*store, in); // informational only + BasicDerivation drv; + readDerivation(in, *store, drv); - MonitorFdHup monitor(in.fd); - auto status = store->buildDerivation(drvPath, drv); + getBuildSettings(); - out << status.status << status.errorMsg; + MonitorFdHup monitor(in.fd); + auto status = store->buildDerivation(drvPath, drv); - if (GET_PROTOCOL_MINOR(clientVersion) >= 3) - out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; + out << status.status << status.errorMsg; - break; - } + if (GET_PROTOCOL_MINOR(clientVersion) >= 3) + out << status.timesBuilt << status.isNonDeterministic + << status.startTime << status.stopTime; - case cmdQueryClosure: { - bool includeOutputs = readInt(in); - PathSet closure; - store->computeFSClosure(readStorePaths<PathSet>(*store, in), - closure, false, includeOutputs); - out << closure; - break; - } + break; + } - case cmdAddToStoreNar: { - if (!writeAllowed) throw Error("importing paths is not allowed"); + case cmdQueryClosure: { + bool includeOutputs = readInt(in); + PathSet closure; + store->computeFSClosure(readStorePaths<PathSet>(*store, in), closure, + false, includeOutputs); + out << closure; + break; + } - ValidPathInfo info; - info.path = readStorePath(*store, in); - in >> info.deriver; - if (!info.deriver.empty()) - store->assertStorePath(info.deriver); - info.narHash = Hash(readString(in), htSHA256); - info.references = readStorePaths<PathSet>(*store, in); - in >> info.registrationTime >> info.narSize >> info.ultimate; - info.sigs = readStrings<StringSet>(in); - in >> info.ca; + case cmdAddToStoreNar: { + if (!writeAllowed) throw Error("importing paths is not allowed"); - if (info.narSize == 0) { - throw Error("narInfo is too old and missing the narSize field"); - } + ValidPathInfo info; + info.path = readStorePath(*store, in); + in >> info.deriver; + if (!info.deriver.empty()) store->assertStorePath(info.deriver); + info.narHash = Hash(readString(in), htSHA256); + info.references = readStorePaths<PathSet>(*store, in); + in >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings<StringSet>(in); + in >> info.ca; - SizedSource sizedSource(in, info.narSize); + if (info.narSize == 0) { + throw Error("narInfo is too old and missing the narSize field"); + } - store->addToStore(info, sizedSource, NoRepair, NoCheckSigs); + SizedSource sizedSource(in, info.narSize); - // consume all the data that has been sent before continuing. - sizedSource.drainAll(); + store->addToStore(info, sizedSource, NoRepair, NoCheckSigs); - out << 1; // indicate success + // consume all the data that has been sent before continuing. + sizedSource.drainAll(); - break; - } + out << 1; // indicate success - default: - throw Error(format("unknown serve command %1%") % cmd); - } + break; + } - out.flush(); + default: + throw Error(format("unknown serve command %1%") % cmd); } -} + out.flush(); + } +} -static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) -{ - for (auto & i : opFlags) - throw UsageError(format("unknown flag '%1%'") % i); +static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) { + for (auto& i : opFlags) throw UsageError(format("unknown flag '%1%'") % i); - if (opArgs.size() != 3) throw UsageError("three arguments expected"); - auto i = opArgs.begin(); - string keyName = *i++; - string secretKeyFile = *i++; - string publicKeyFile = *i++; + if (opArgs.size() != 3) throw UsageError("three arguments expected"); + auto i = opArgs.begin(); + string keyName = *i++; + string secretKeyFile = *i++; + string publicKeyFile = *i++; #if HAVE_SODIUM - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; - unsigned char sk[crypto_sign_SECRETKEYBYTES]; - if (crypto_sign_keypair(pk, sk) != 0) - throw Error("key generation failed"); - - writeFile(publicKeyFile, keyName + ":" + base64Encode(string((char *) pk, crypto_sign_PUBLICKEYBYTES))); - umask(0077); - writeFile(secretKeyFile, keyName + ":" + base64Encode(string((char *) sk, crypto_sign_SECRETKEYBYTES))); + if (sodium_init() == -1) throw Error("could not initialise libsodium"); + + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + unsigned char sk[crypto_sign_SECRETKEYBYTES]; + if (crypto_sign_keypair(pk, sk) != 0) throw Error("key generation failed"); + + writeFile(publicKeyFile, + keyName + ":" + + base64Encode(string((char*)pk, crypto_sign_PUBLICKEYBYTES))); + umask(0077); + writeFile(secretKeyFile, + keyName + ":" + + base64Encode(string((char*)sk, crypto_sign_SECRETKEYBYTES))); #else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); + throw Error( + "Nix was not compiled with libsodium, required for signed binary cache " + "support"); #endif } - -static void opVersion(Strings opFlags, Strings opArgs) -{ - printVersion("nix-store"); +static void opVersion(Strings opFlags, Strings opArgs) { + printVersion("nix-store"); } - /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ -static int _main(int argc, char * * argv) -{ - { - Strings opFlags, opArgs; - Operation op = 0; - - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - Operation oldOp = op; - - if (*arg == "--help") - showManPage("nix-store"); - else if (*arg == "--version") - op = opVersion; - else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") - op = opRealise; - else if (*arg == "--add" || *arg == "-A") - op = opAdd; - else if (*arg == "--add-fixed") - op = opAddFixed; - else if (*arg == "--print-fixed-path") - op = opPrintFixedPath; - else if (*arg == "--delete") - op = opDelete; - else if (*arg == "--query" || *arg == "-q") - op = opQuery; - else if (*arg == "--print-env") - op = opPrintEnv; - else if (*arg == "--read-log" || *arg == "-l") - op = opReadLog; - else if (*arg == "--dump-db") - op = opDumpDB; - else if (*arg == "--load-db") - op = opLoadDB; - else if (*arg == "--register-validity") - op = opRegisterValidity; - else if (*arg == "--check-validity") - op = opCheckValidity; - else if (*arg == "--gc") - op = opGC; - else if (*arg == "--dump") - op = opDump; - else if (*arg == "--restore") - op = opRestore; - else if (*arg == "--export") - op = opExport; - else if (*arg == "--import") - op = opImport; - else if (*arg == "--init") - op = opInit; - else if (*arg == "--verify") - op = opVerify; - else if (*arg == "--verify-path") - op = opVerifyPath; - else if (*arg == "--repair-path") - op = opRepairPath; - else if (*arg == "--optimise" || *arg == "--optimize") - op = opOptimise; - else if (*arg == "--serve") - op = opServe; - else if (*arg == "--generate-binary-cache-key") - op = opGenerateBinaryCacheKey; - else if (*arg == "--add-root") - gcRoot = absPath(getArg(*arg, arg, end)); - else if (*arg == "--indirect") - indirectRoot = true; - else if (*arg == "--no-output") - noOutput = true; - else if (*arg != "" && arg->at(0) == '-') { - opFlags.push_back(*arg); - if (*arg == "--max-freed" || *arg == "--max-links" || *arg == "--max-atime") /* !!! hack */ - opFlags.push_back(getArg(*arg, arg, end)); - } - else - opArgs.push_back(*arg); - - if (oldOp && oldOp != op) - throw UsageError("only one operation may be specified"); - - return true; +static int _main(int argc, char** argv) { + { + Strings opFlags, opArgs; + Operation op = 0; + + parseCmdLine( + argc, argv, [&](Strings::iterator& arg, const Strings::iterator& end) { + Operation oldOp = op; + + if (*arg == "--help") + showManPage("nix-store"); + else if (*arg == "--version") + op = opVersion; + else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") + op = opRealise; + else if (*arg == "--add" || *arg == "-A") + op = opAdd; + else if (*arg == "--add-fixed") + op = opAddFixed; + else if (*arg == "--print-fixed-path") + op = opPrintFixedPath; + else if (*arg == "--delete") + op = opDelete; + else if (*arg == "--query" || *arg == "-q") + op = opQuery; + else if (*arg == "--print-env") + op = opPrintEnv; + else if (*arg == "--read-log" || *arg == "-l") + op = opReadLog; + else if (*arg == "--dump-db") + op = opDumpDB; + else if (*arg == "--load-db") + op = opLoadDB; + else if (*arg == "--register-validity") + op = opRegisterValidity; + else if (*arg == "--check-validity") + op = opCheckValidity; + else if (*arg == "--gc") + op = opGC; + else if (*arg == "--dump") + op = opDump; + else if (*arg == "--restore") + op = opRestore; + else if (*arg == "--export") + op = opExport; + else if (*arg == "--import") + op = opImport; + else if (*arg == "--init") + op = opInit; + else if (*arg == "--verify") + op = opVerify; + else if (*arg == "--verify-path") + op = opVerifyPath; + else if (*arg == "--repair-path") + op = opRepairPath; + else if (*arg == "--optimise" || *arg == "--optimize") + op = opOptimise; + else if (*arg == "--serve") + op = opServe; + else if (*arg == "--generate-binary-cache-key") + op = opGenerateBinaryCacheKey; + else if (*arg == "--add-root") + gcRoot = absPath(getArg(*arg, arg, end)); + else if (*arg == "--indirect") + indirectRoot = true; + else if (*arg == "--no-output") + noOutput = true; + else if (*arg != "" && arg->at(0) == '-') { + opFlags.push_back(*arg); + if (*arg == "--max-freed" || *arg == "--max-links" || + *arg == "--max-atime") /* !!! hack */ + opFlags.push_back(getArg(*arg, arg, end)); + } else + opArgs.push_back(*arg); + + if (oldOp && oldOp != op) + throw UsageError("only one operation may be specified"); + + return true; }); - initPlugins(); + initPlugins(); - if (!op) throw UsageError("no operation specified"); + if (!op) throw UsageError("no operation specified"); - if (op != opDump && op != opRestore) /* !!! hack */ - store = openStore(); + if (op != opDump && op != opRestore) /* !!! hack */ + store = openStore(); - op(opFlags, opArgs); + op(opFlags, opArgs); - return 0; - } + return 0; + } } static RegisterLegacyCommand s1("nix-store", _main); diff --git a/third_party/nix/src/nix/add-to-store.cc b/third_party/nix/src/nix/add-to-store.cc index e86b96e3f3f2..cc85cc02ea6e 100644 --- a/third_party/nix/src/nix/add-to-store.cc +++ b/third_party/nix/src/nix/add-to-store.cc @@ -1,61 +1,47 @@ +#include "archive.hh" #include "command.hh" #include "common-args.hh" #include "store-api.hh" -#include "archive.hh" using namespace nix; -struct CmdAddToStore : MixDryRun, StoreCommand -{ - Path path; - std::optional<std::string> namePart; - - CmdAddToStore() - { - expectArg("path", &path); - - mkFlag() - .longName("name") - .shortName('n') - .description("name component of the store path") - .labels({"name"}) - .dest(&namePart); - } - - std::string name() override - { - return "add-to-store"; - } - - std::string description() override - { - return "add a path to the Nix store"; - } - - Examples examples() override - { - return { - }; - } - - void run(ref<Store> store) override - { - if (!namePart) namePart = baseNameOf(path); - - StringSink sink; - dumpPath(path, sink); - - ValidPathInfo info; - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.path = store->makeFixedOutputPath(true, info.narHash, *namePart); - info.ca = makeFixedOutputCA(true, info.narHash); - - if (!dryRun) - store->addToStore(info, sink.s); - - std::cout << fmt("%s\n", info.path); - } +struct CmdAddToStore : MixDryRun, StoreCommand { + Path path; + std::optional<std::string> namePart; + + CmdAddToStore() { + expectArg("path", &path); + + mkFlag() + .longName("name") + .shortName('n') + .description("name component of the store path") + .labels({"name"}) + .dest(&namePart); + } + + std::string name() override { return "add-to-store"; } + + std::string description() override { return "add a path to the Nix store"; } + + Examples examples() override { return {}; } + + void run(ref<Store> store) override { + if (!namePart) namePart = baseNameOf(path); + + StringSink sink; + dumpPath(path, sink); + + ValidPathInfo info; + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.path = store->makeFixedOutputPath(true, info.narHash, *namePart); + info.ca = makeFixedOutputCA(true, info.narHash); + + if (!dryRun) store->addToStore(info, sink.s); + + std::cout << fmt("%s\n", info.path); + } }; static RegisterCommand r1(make_ref<CmdAddToStore>()); diff --git a/third_party/nix/src/nix/build.cc b/third_party/nix/src/nix/build.cc index b329ac38ac2b..e23f63199fa2 100644 --- a/third_party/nix/src/nix/build.cc +++ b/third_party/nix/src/nix/build.cc @@ -5,68 +5,56 @@ using namespace nix; -struct CmdBuild : MixDryRun, InstallablesCommand -{ - Path outLink = "result"; - - CmdBuild() - { - mkFlag() - .longName("out-link") - .shortName('o') - .description("path of the symlink to the build result") - .labels({"path"}) - .dest(&outLink); - - mkFlag() - .longName("no-link") - .description("do not create a symlink to the build result") - .set(&outLink, Path("")); - } - - std::string name() override - { - return "build"; - } - - std::string description() override - { - return "build a derivation or fetch a store path"; - } - - Examples examples() override - { - return { - Example{ - "To build and run GNU Hello from NixOS 17.03:", - "nix build -f channel:nixos-17.03 hello; ./result/bin/hello" - }, - Example{ - "To build the build.x86_64-linux attribute from release.nix:", - "nix build -f release.nix build.x86_64-linux" - }, - }; - } - - void run(ref<Store> store) override - { - auto buildables = build(store, dryRun ? DryRun : Build, installables); - - if (dryRun) return; - - for (size_t i = 0; i < buildables.size(); ++i) { - auto & b(buildables[i]); - - if (outLink != "") - for (auto & output : b.outputs) - if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) { - std::string symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store2->addPermRoot(output.second, absPath(symlink), true); - } - } +struct CmdBuild : MixDryRun, InstallablesCommand { + Path outLink = "result"; + + CmdBuild() { + mkFlag() + .longName("out-link") + .shortName('o') + .description("path of the symlink to the build result") + .labels({"path"}) + .dest(&outLink); + + mkFlag() + .longName("no-link") + .description("do not create a symlink to the build result") + .set(&outLink, Path("")); + } + + std::string name() override { return "build"; } + + std::string description() override { + return "build a derivation or fetch a store path"; + } + + Examples examples() override { + return { + Example{"To build and run GNU Hello from NixOS 17.03:", + "nix build -f channel:nixos-17.03 hello; ./result/bin/hello"}, + Example{"To build the build.x86_64-linux attribute from release.nix:", + "nix build -f release.nix build.x86_64-linux"}, + }; + } + + void run(ref<Store> store) override { + auto buildables = build(store, dryRun ? DryRun : Build, installables); + + if (dryRun) return; + + for (size_t i = 0; i < buildables.size(); ++i) { + auto& b(buildables[i]); + + if (outLink != "") + for (auto& output : b.outputs) + if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store2->addPermRoot(output.second, absPath(symlink), true); + } } + } }; static RegisterCommand r1(make_ref<CmdBuild>()); diff --git a/third_party/nix/src/nix/cat.cc b/third_party/nix/src/nix/cat.cc index a35f640d8840..ea9fd0d942fd 100644 --- a/third_party/nix/src/nix/cat.cc +++ b/third_party/nix/src/nix/cat.cc @@ -1,73 +1,53 @@ #include "command.hh" -#include "store-api.hh" #include "fs-accessor.hh" #include "nar-accessor.hh" +#include "store-api.hh" using namespace nix; -struct MixCat : virtual Args -{ - std::string path; +struct MixCat : virtual Args { + std::string path; - void cat(ref<FSAccessor> accessor) - { - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) - throw Error(format("path '%1%' does not exist") % path); - if (st.type != FSAccessor::Type::tRegular) - throw Error(format("path '%1%' is not a regular file") % path); + void cat(ref<FSAccessor> accessor) { + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) + throw Error(format("path '%1%' does not exist") % path); + if (st.type != FSAccessor::Type::tRegular) + throw Error(format("path '%1%' is not a regular file") % path); - std::cout << accessor->readFile(path); - } + std::cout << accessor->readFile(path); + } }; -struct CmdCatStore : StoreCommand, MixCat -{ - CmdCatStore() - { - expectArg("path", &path); - } +struct CmdCatStore : StoreCommand, MixCat { + CmdCatStore() { expectArg("path", &path); } - std::string name() override - { - return "cat-store"; - } + std::string name() override { return "cat-store"; } - std::string description() override - { - return "print the contents of a store file on stdout"; - } + std::string description() override { + return "print the contents of a store file on stdout"; + } - void run(ref<Store> store) override - { - cat(store->getFSAccessor()); - } + void run(ref<Store> store) override { cat(store->getFSAccessor()); } }; -struct CmdCatNar : StoreCommand, MixCat -{ - Path narPath; +struct CmdCatNar : StoreCommand, MixCat { + Path narPath; - CmdCatNar() - { - expectArg("nar", &narPath); - expectArg("path", &path); - } + CmdCatNar() { + expectArg("nar", &narPath); + expectArg("path", &path); + } - std::string name() override - { - return "cat-nar"; - } + std::string name() override { return "cat-nar"; } - std::string description() override - { - return "print the contents of a file inside a NAR file"; - } + std::string description() override { + return "print the contents of a file inside a NAR file"; + } - void run(ref<Store> store) override - { - cat(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); - } + void run(ref<Store> store) override { + cat(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); + } }; static RegisterCommand r1(make_ref<CmdCatStore>()); diff --git a/third_party/nix/src/nix/command.cc b/third_party/nix/src/nix/command.cc index 3d7d582d6f5e..5565e49f56f2 100644 --- a/third_party/nix/src/nix/command.cc +++ b/third_party/nix/src/nix/command.cc @@ -1,61 +1,58 @@ #include "command.hh" -#include "store-api.hh" #include "derivations.hh" +#include "store-api.hh" namespace nix { -Commands * RegisterCommand::commands = 0; +Commands* RegisterCommand::commands = 0; -void Command::printHelp(const string & programName, std::ostream & out) -{ - Args::printHelp(programName, out); +void Command::printHelp(const string& programName, std::ostream& out) { + Args::printHelp(programName, out); - auto exs = examples(); - if (!exs.empty()) { - out << "\n"; - out << "Examples:\n"; - for (auto & ex : exs) - out << "\n" - << " " << ex.description << "\n" // FIXME: wrap - << " $ " << ex.command << "\n"; - } + auto exs = examples(); + if (!exs.empty()) { + out << "\n"; + out << "Examples:\n"; + for (auto& ex : exs) + out << "\n" + << " " << ex.description << "\n" // FIXME: wrap + << " $ " << ex.command << "\n"; + } } -MultiCommand::MultiCommand(const Commands & _commands) - : commands(_commands) -{ - expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) { +MultiCommand::MultiCommand(const Commands& _commands) : commands(_commands) { + expectedArgs.push_back(ExpectedArg{ + "command", 1, true, [=](std::vector<std::string> ss) { assert(!command); auto i = commands.find(ss[0]); if (i == commands.end()) - throw UsageError("'%s' is not a recognised command", ss[0]); + throw UsageError("'%s' is not a recognised command", ss[0]); command = i->second; - }}); + }}); } -void MultiCommand::printHelp(const string & programName, std::ostream & out) -{ - if (command) { - command->printHelp(programName + " " + command->name(), out); - return; - } +void MultiCommand::printHelp(const string& programName, std::ostream& out) { + if (command) { + command->printHelp(programName + " " + command->name(), out); + return; + } - out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n"; + out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n"; - out << "\n"; - out << "Common flags:\n"; - printFlags(out); + out << "\n"; + out << "Common flags:\n"; + printFlags(out); - out << "\n"; - out << "Available commands:\n"; + out << "\n"; + out << "Available commands:\n"; - Table2 table; - for (auto & command : commands) { - auto descr = command.second->description(); - if (!descr.empty()) - table.push_back(std::make_pair(command.second->name(), descr)); - } - printTable(out, table); + Table2 table; + for (auto& command : commands) { + auto descr = command.second->description(); + if (!descr.empty()) + table.push_back(std::make_pair(command.second->name(), descr)); + } + printTable(out, table); #if 0 out << "\n"; @@ -63,94 +60,77 @@ void MultiCommand::printHelp(const string & programName, std::ostream & out) #endif } -bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) -{ - if (Args::processFlag(pos, end)) return true; - if (command && command->processFlag(pos, end)) return true; - return false; +bool MultiCommand::processFlag(Strings::iterator& pos, Strings::iterator end) { + if (Args::processFlag(pos, end)) return true; + if (command && command->processFlag(pos, end)) return true; + return false; } -bool MultiCommand::processArgs(const Strings & args, bool finish) -{ - if (command) - return command->processArgs(args, finish); - else - return Args::processArgs(args, finish); +bool MultiCommand::processArgs(const Strings& args, bool finish) { + if (command) + return command->processArgs(args, finish); + else + return Args::processArgs(args, finish); } -StoreCommand::StoreCommand() -{ -} +StoreCommand::StoreCommand() {} -ref<Store> StoreCommand::getStore() -{ - if (!_store) - _store = createStore(); - return ref<Store>(_store); +ref<Store> StoreCommand::getStore() { + if (!_store) _store = createStore(); + return ref<Store>(_store); } -ref<Store> StoreCommand::createStore() -{ - return openStore(); -} +ref<Store> StoreCommand::createStore() { return openStore(); } -void StoreCommand::run() -{ - run(getStore()); -} +void StoreCommand::run() { run(getStore()); } -StorePathsCommand::StorePathsCommand(bool recursive) - : recursive(recursive) -{ - if (recursive) - mkFlag() - .longName("no-recursive") - .description("apply operation to specified paths only") - .set(&this->recursive, false); - else - mkFlag() - .longName("recursive") - .shortName('r') - .description("apply operation to closure of the specified paths") - .set(&this->recursive, true); - - mkFlag(0, "all", "apply operation to the entire store", &all); +StorePathsCommand::StorePathsCommand(bool recursive) : recursive(recursive) { + if (recursive) + mkFlag() + .longName("no-recursive") + .description("apply operation to specified paths only") + .set(&this->recursive, false); + else + mkFlag() + .longName("recursive") + .shortName('r') + .description("apply operation to closure of the specified paths") + .set(&this->recursive, true); + + mkFlag(0, "all", "apply operation to the entire store", &all); } -void StorePathsCommand::run(ref<Store> store) -{ - Paths storePaths; +void StorePathsCommand::run(ref<Store> store) { + Paths storePaths; - if (all) { - if (installables.size()) - throw UsageError("'--all' does not expect arguments"); - for (auto & p : store->queryAllValidPaths()) - storePaths.push_back(p); - } + if (all) { + if (installables.size()) + throw UsageError("'--all' does not expect arguments"); + for (auto& p : store->queryAllValidPaths()) storePaths.push_back(p); + } - else { - for (auto & p : toStorePaths(store, NoBuild, installables)) - storePaths.push_back(p); + else { + for (auto& p : toStorePaths(store, NoBuild, installables)) + storePaths.push_back(p); - if (recursive) { - PathSet closure; - store->computeFSClosure(PathSet(storePaths.begin(), storePaths.end()), - closure, false, false); - storePaths = Paths(closure.begin(), closure.end()); - } + if (recursive) { + PathSet closure; + store->computeFSClosure(PathSet(storePaths.begin(), storePaths.end()), + closure, false, false); + storePaths = Paths(closure.begin(), closure.end()); } + } - run(store, storePaths); + run(store, storePaths); } -void StorePathCommand::run(ref<Store> store) -{ - auto storePaths = toStorePaths(store, NoBuild, installables); +void StorePathCommand::run(ref<Store> store) { + auto storePaths = toStorePaths(store, NoBuild, installables); - if (storePaths.size() != 1) - throw UsageError("this command requires exactly one store path"); + if (storePaths.size() != 1) + throw UsageError("this command requires exactly one store path"); - run(store, *storePaths.begin()); + run(store, *storePaths.begin()); } -} +} // namespace nix diff --git a/third_party/nix/src/nix/command.hh b/third_party/nix/src/nix/command.hh index 97a6fee7fd27..cc82c4b1aadd 100644 --- a/third_party/nix/src/nix/command.hh +++ b/third_party/nix/src/nix/command.hh @@ -13,202 +13,177 @@ class EvalState; /* A command is an argument parser that can be executed by calling its run() method. */ -struct Command : virtual Args -{ - virtual std::string name() = 0; - virtual void prepare() { }; - virtual void run() = 0; +struct Command : virtual Args { + virtual std::string name() = 0; + virtual void prepare(){}; + virtual void run() = 0; - struct Example - { - std::string description; - std::string command; - }; + struct Example { + std::string description; + std::string command; + }; - typedef std::list<Example> Examples; + typedef std::list<Example> Examples; - virtual Examples examples() { return Examples(); } + virtual Examples examples() { return Examples(); } - void printHelp(const string & programName, std::ostream & out) override; + void printHelp(const string& programName, std::ostream& out) override; }; class Store; /* A command that require a Nix store. */ -struct StoreCommand : virtual Command -{ - StoreCommand(); - void run() override; - ref<Store> getStore(); - virtual ref<Store> createStore(); - virtual void run(ref<Store>) = 0; - -private: - std::shared_ptr<Store> _store; +struct StoreCommand : virtual Command { + StoreCommand(); + void run() override; + ref<Store> getStore(); + virtual ref<Store> createStore(); + virtual void run(ref<Store>) = 0; + + private: + std::shared_ptr<Store> _store; }; -struct Buildable -{ - Path drvPath; // may be empty - std::map<std::string, Path> outputs; +struct Buildable { + Path drvPath; // may be empty + std::map<std::string, Path> outputs; }; typedef std::vector<Buildable> Buildables; -struct Installable -{ - virtual std::string what() = 0; +struct Installable { + virtual std::string what() = 0; - virtual Buildables toBuildables() - { - throw Error("argument '%s' cannot be built", what()); - } + virtual Buildables toBuildables() { + throw Error("argument '%s' cannot be built", what()); + } - Buildable toBuildable(); + Buildable toBuildable(); - virtual Value * toValue(EvalState & state) - { - throw Error("argument '%s' cannot be evaluated", what()); - } + virtual Value* toValue(EvalState& state) { + throw Error("argument '%s' cannot be evaluated", what()); + } }; -struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs -{ - Path file; +struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs { + Path file; - SourceExprCommand(); + SourceExprCommand(); - /* Return a value representing the Nix expression from which we - are installing. This is either the file specified by ‘--file’, - or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs - = import ...; bla = import ...; }’. */ - Value * getSourceExpr(EvalState & state); + /* Return a value representing the Nix expression from which we + are installing. This is either the file specified by ‘--file’, + or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs + = import ...; bla = import ...; }’. */ + Value* getSourceExpr(EvalState& state); - ref<EvalState> getEvalState(); + ref<EvalState> getEvalState(); -private: + private: + std::shared_ptr<EvalState> evalState; - std::shared_ptr<EvalState> evalState; - - Value * vSourceExpr = 0; + Value* vSourceExpr = 0; }; enum RealiseMode { Build, NoBuild, DryRun }; /* A command that operates on a list of "installables", which can be store paths, attribute paths, Nix expressions, etc. */ -struct InstallablesCommand : virtual Args, SourceExprCommand -{ - std::vector<std::shared_ptr<Installable>> installables; - - InstallablesCommand() - { - expectArgs("installables", &_installables); - } +struct InstallablesCommand : virtual Args, SourceExprCommand { + std::vector<std::shared_ptr<Installable>> installables; - void prepare() override; + InstallablesCommand() { expectArgs("installables", &_installables); } - virtual bool useDefaultInstallables() { return true; } + void prepare() override; -private: + virtual bool useDefaultInstallables() { return true; } - std::vector<std::string> _installables; + private: + std::vector<std::string> _installables; }; -struct InstallableCommand : virtual Args, SourceExprCommand -{ - std::shared_ptr<Installable> installable; - - InstallableCommand() - { - expectArg("installable", &_installable); - } +struct InstallableCommand : virtual Args, SourceExprCommand { + std::shared_ptr<Installable> installable; - void prepare() override; + InstallableCommand() { expectArg("installable", &_installable); } -private: + void prepare() override; - std::string _installable; + private: + std::string _installable; }; /* A command that operates on zero or more store paths. */ -struct StorePathsCommand : public InstallablesCommand -{ -private: +struct StorePathsCommand : public InstallablesCommand { + private: + bool recursive = false; + bool all = false; - bool recursive = false; - bool all = false; + public: + StorePathsCommand(bool recursive = false); -public: + using StoreCommand::run; - StorePathsCommand(bool recursive = false); + virtual void run(ref<Store> store, Paths storePaths) = 0; - using StoreCommand::run; + void run(ref<Store> store) override; - virtual void run(ref<Store> store, Paths storePaths) = 0; - - void run(ref<Store> store) override; - - bool useDefaultInstallables() override { return !all; } + bool useDefaultInstallables() override { return !all; } }; /* A command that operates on exactly one store path. */ -struct StorePathCommand : public InstallablesCommand -{ - using StoreCommand::run; +struct StorePathCommand : public InstallablesCommand { + using StoreCommand::run; - virtual void run(ref<Store> store, const Path & storePath) = 0; + virtual void run(ref<Store> store, const Path& storePath) = 0; - void run(ref<Store> store) override; + void run(ref<Store> store) override; }; typedef std::map<std::string, ref<Command>> Commands; /* An argument parser that supports multiple subcommands, i.e. ‘<command> <subcommand>’. */ -class MultiCommand : virtual Args -{ -public: - Commands commands; +class MultiCommand : virtual Args { + public: + Commands commands; - std::shared_ptr<Command> command; + std::shared_ptr<Command> command; - MultiCommand(const Commands & commands); + MultiCommand(const Commands& commands); - void printHelp(const string & programName, std::ostream & out) override; + void printHelp(const string& programName, std::ostream& out) override; - bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + bool processFlag(Strings::iterator& pos, Strings::iterator end) override; - bool processArgs(const Strings & args, bool finish) override; + bool processArgs(const Strings& args, bool finish) override; }; /* A helper class for registering commands globally. */ -struct RegisterCommand -{ - static Commands * commands; - - RegisterCommand(ref<Command> command) - { - if (!commands) commands = new Commands; - commands->emplace(command->name(), command); - } +struct RegisterCommand { + static Commands* commands; + + RegisterCommand(ref<Command> command) { + if (!commands) commands = new Commands; + commands->emplace(command->name(), command); + } }; -std::shared_ptr<Installable> parseInstallable( - SourceExprCommand & cmd, ref<Store> store, const std::string & installable, - bool useDefaultInstallables); +std::shared_ptr<Installable> parseInstallable(SourceExprCommand& cmd, + ref<Store> store, + const std::string& installable, + bool useDefaultInstallables); Buildables build(ref<Store> store, RealiseMode mode, - std::vector<std::shared_ptr<Installable>> installables); + std::vector<std::shared_ptr<Installable>> installables); PathSet toStorePaths(ref<Store> store, RealiseMode mode, - std::vector<std::shared_ptr<Installable>> installables); + std::vector<std::shared_ptr<Installable>> installables); Path toStorePath(ref<Store> store, RealiseMode mode, - std::shared_ptr<Installable> installable); + std::shared_ptr<Installable> installable); PathSet toDerivations(ref<Store> store, - std::vector<std::shared_ptr<Installable>> installables, - bool useDeriver = false); + std::vector<std::shared_ptr<Installable>> installables, + bool useDeriver = false); -} +} // namespace nix diff --git a/third_party/nix/src/nix/copy.cc b/third_party/nix/src/nix/copy.cc index 12a9f9cd3372..030a13a3d019 100644 --- a/third_party/nix/src/nix/copy.cc +++ b/third_party/nix/src/nix/copy.cc @@ -1,100 +1,84 @@ +#include <atomic> #include "command.hh" #include "shared.hh" #include "store-api.hh" #include "sync.hh" #include "thread-pool.hh" -#include <atomic> - using namespace nix; -struct CmdCopy : StorePathsCommand -{ - std::string srcUri, dstUri; - - CheckSigsFlag checkSigs = CheckSigs; - - SubstituteFlag substitute = NoSubstitute; - - CmdCopy() - : StorePathsCommand(true) - { - mkFlag() - .longName("from") - .labels({"store-uri"}) - .description("URI of the source Nix store") - .dest(&srcUri); - mkFlag() - .longName("to") - .labels({"store-uri"}) - .description("URI of the destination Nix store") - .dest(&dstUri); - - mkFlag() - .longName("no-check-sigs") - .description("do not require that paths are signed by trusted keys") - .set(&checkSigs, NoCheckSigs); - - mkFlag() - .longName("substitute-on-destination") - .shortName('s') - .description("whether to try substitutes on the destination store (only supported by SSH)") - .set(&substitute, Substitute); - } - - std::string name() override - { - return "copy"; - } - - std::string description() override - { - return "copy paths between Nix stores"; - } - - Examples examples() override - { - return { - Example{ - "To copy Firefox from the local store to a binary cache in file:///tmp/cache:", - "nix copy --to file:///tmp/cache $(type -p firefox)" - }, - Example{ - "To copy the entire current NixOS system closure to another machine via SSH:", - "nix copy --to ssh://server /run/current-system" - }, - Example{ - "To copy a closure from another machine via SSH:", - "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" - }, +struct CmdCopy : StorePathsCommand { + std::string srcUri, dstUri; + + CheckSigsFlag checkSigs = CheckSigs; + + SubstituteFlag substitute = NoSubstitute; + + CmdCopy() : StorePathsCommand(true) { + mkFlag() + .longName("from") + .labels({"store-uri"}) + .description("URI of the source Nix store") + .dest(&srcUri); + mkFlag() + .longName("to") + .labels({"store-uri"}) + .description("URI of the destination Nix store") + .dest(&dstUri); + + mkFlag() + .longName("no-check-sigs") + .description("do not require that paths are signed by trusted keys") + .set(&checkSigs, NoCheckSigs); + + mkFlag() + .longName("substitute-on-destination") + .shortName('s') + .description( + "whether to try substitutes on the destination store (only " + "supported by SSH)") + .set(&substitute, Substitute); + } + + std::string name() override { return "copy"; } + + std::string description() override { return "copy paths between Nix stores"; } + + Examples examples() override { + return { + Example{"To copy Firefox from the local store to a binary cache in " + "file:///tmp/cache:", + "nix copy --to file:///tmp/cache $(type -p firefox)"}, + Example{"To copy the entire current NixOS system closure to another " + "machine via SSH:", + "nix copy --to ssh://server /run/current-system"}, + Example{"To copy a closure from another machine via SSH:", + "nix copy --from ssh://server " + "/nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2"}, #ifdef ENABLE_S3 - Example{ - "To copy Hello to an S3 binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello" - }, - Example{ - "To copy Hello to an S3-compatible binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello" - }, + Example{"To copy Hello to an S3 binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello"}, + Example{"To copy Hello to an S3-compatible binary cache:", + "nix copy --to " + "s3://my-bucket?region=eu-west-1&endpoint=example.com " + "nixpkgs.hello"}, #endif - }; - } + }; + } - ref<Store> createStore() override - { - return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); - } + ref<Store> createStore() override { + return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); + } - void run(ref<Store> srcStore, Paths storePaths) override - { - if (srcUri.empty() && dstUri.empty()) - throw UsageError("you must pass '--from' and/or '--to'"); + void run(ref<Store> srcStore, Paths storePaths) override { + if (srcUri.empty() && dstUri.empty()) + throw UsageError("you must pass '--from' and/or '--to'"); - ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()), - NoRepair, checkSigs, substitute); - } + copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()), + NoRepair, checkSigs, substitute); + } }; static RegisterCommand r1(make_ref<CmdCopy>()); diff --git a/third_party/nix/src/nix/doctor.cc b/third_party/nix/src/nix/doctor.cc index 7b5444619470..93016e007a84 100644 --- a/third_party/nix/src/nix/doctor.cc +++ b/third_party/nix/src/nix/doctor.cc @@ -6,119 +6,121 @@ using namespace nix; -std::string formatProtocol(unsigned int proto) -{ - if (proto) { - auto major = GET_PROTOCOL_MAJOR(proto) >> 8; - auto minor = GET_PROTOCOL_MINOR(proto); - return (format("%1%.%2%") % major % minor).str(); - } - return "unknown"; +std::string formatProtocol(unsigned int proto) { + if (proto) { + auto major = GET_PROTOCOL_MAJOR(proto) >> 8; + auto minor = GET_PROTOCOL_MINOR(proto); + return (format("%1%.%2%") % major % minor).str(); + } + return "unknown"; } -struct CmdDoctor : StoreCommand -{ - bool success = true; +struct CmdDoctor : StoreCommand { + bool success = true; - std::string name() override - { - return "doctor"; - } + std::string name() override { return "doctor"; } - std::string description() override - { - return "check your system for potential problems"; - } + std::string description() override { + return "check your system for potential problems"; + } - void run(ref<Store> store) override - { - std::cout << "Store uri: " << store->getUri() << std::endl; - std::cout << std::endl; + void run(ref<Store> store) override { + std::cout << "Store uri: " << store->getUri() << std::endl; + std::cout << std::endl; - auto type = getStoreType(); + auto type = getStoreType(); - if (type < tOther) { - success &= checkNixInPath(); - success &= checkProfileRoots(store); - } - success &= checkStoreProtocol(store->getProtocol()); - - if (!success) - throw Exit(2); + if (type < tOther) { + success &= checkNixInPath(); + success &= checkProfileRoots(store); } + success &= checkStoreProtocol(store->getProtocol()); - bool checkNixInPath() - { - PathSet dirs; - - for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) - if (pathExists(dir + "/nix-env")) - dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); - - if (dirs.size() != 1) { - std::cout << "Warning: multiple versions of nix found in PATH." << std::endl; - std::cout << std::endl; - for (auto & dir : dirs) - std::cout << " " << dir << std::endl; - std::cout << std::endl; - return false; - } + if (!success) throw Exit(2); + } + + bool checkNixInPath() { + PathSet dirs; - return true; + for (auto& dir : tokenizeString<Strings>(getEnv("PATH"), ":")) + if (pathExists(dir + "/nix-env")) + dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); + + if (dirs.size() != 1) { + std::cout << "Warning: multiple versions of nix found in PATH." + << std::endl; + std::cout << std::endl; + for (auto& dir : dirs) std::cout << " " << dir << std::endl; + std::cout << std::endl; + return false; } - bool checkProfileRoots(ref<Store> store) - { - PathSet dirs; + return true; + } - for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) { - Path profileDir = dirOf(dir); - try { - Path userEnv = canonPath(profileDir, true); + bool checkProfileRoots(ref<Store> store) { + PathSet dirs; - if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) { - while (profileDir.find("/profiles/") == std::string::npos && isLink(profileDir)) - profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + for (auto& dir : tokenizeString<Strings>(getEnv("PATH"), ":")) { + Path profileDir = dirOf(dir); + try { + Path userEnv = canonPath(profileDir, true); - if (profileDir.find("/profiles/") == std::string::npos) - dirs.insert(dir); - } - } catch (SysError &) {} - } + if (store->isStorePath(userEnv) && + hasSuffix(userEnv, "user-environment")) { + while (profileDir.find("/profiles/") == std::string::npos && + isLink(profileDir)) + profileDir = absPath(readLink(profileDir), dirOf(profileDir)); - if (!dirs.empty()) { - std::cout << "Warning: found profiles outside of " << settings.nixStateDir << "/profiles." << std::endl; - std::cout << "The generation this profile points to might not have a gcroot and could be" << std::endl; - std::cout << "garbage collected, resulting in broken symlinks." << std::endl; - std::cout << std::endl; - for (auto & dir : dirs) - std::cout << " " << dir << std::endl; - std::cout << std::endl; - return false; + if (profileDir.find("/profiles/") == std::string::npos) + dirs.insert(dir); } - - return true; + } catch (SysError&) { + } } - bool checkStoreProtocol(unsigned int storeProto) - { - unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto) - ? SERVE_PROTOCOL_VERSION - : PROTOCOL_VERSION; - - if (clientProto != storeProto) { - std::cout << "Warning: protocol version of this client does not match the store." << std::endl; - std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl; - std::cout << "sync with the daemon." << std::endl; - std::cout << std::endl; - std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl; - std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl; - std::cout << std::endl; - return false; - } + if (!dirs.empty()) { + std::cout << "Warning: found profiles outside of " << settings.nixStateDir + << "/profiles." << std::endl; + std::cout << "The generation this profile points to might not have a " + "gcroot and could be" + << std::endl; + std::cout << "garbage collected, resulting in broken symlinks." + << std::endl; + std::cout << std::endl; + for (auto& dir : dirs) std::cout << " " << dir << std::endl; + std::cout << std::endl; + return false; + } - return true; + return true; + } + + bool checkStoreProtocol(unsigned int storeProto) { + unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == + GET_PROTOCOL_MAJOR(storeProto) + ? SERVE_PROTOCOL_VERSION + : PROTOCOL_VERSION; + + if (clientProto != storeProto) { + std::cout << "Warning: protocol version of this client does not match " + "the store." + << std::endl; + std::cout << "While this is not necessarily a problem it's recommended " + "to keep the client in" + << std::endl; + std::cout << "sync with the daemon." << std::endl; + std::cout << std::endl; + std::cout << "Client protocol: " << formatProtocol(clientProto) + << std::endl; + std::cout << "Store protocol: " << formatProtocol(storeProto) + << std::endl; + std::cout << std::endl; + return false; } + + return true; + } }; static RegisterCommand r1(make_ref<CmdDoctor>()); diff --git a/third_party/nix/src/nix/dump-path.cc b/third_party/nix/src/nix/dump-path.cc index f411c0cb7c89..5e93a3a44b83 100644 --- a/third_party/nix/src/nix/dump-path.cc +++ b/third_party/nix/src/nix/dump-path.cc @@ -3,34 +3,26 @@ using namespace nix; -struct CmdDumpPath : StorePathCommand -{ - std::string name() override - { - return "dump-path"; - } +struct CmdDumpPath : StorePathCommand { + std::string name() override { return "dump-path"; } - std::string description() override - { - return "dump a store path to stdout (in NAR format)"; - } + std::string description() override { + return "dump a store path to stdout (in NAR format)"; + } - Examples examples() override - { - return { - Example{ - "To get a NAR from the binary cache https://cache.nixos.org/:", - "nix dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" - }, - }; - } + Examples examples() override { + return { + Example{"To get a NAR from the binary cache https://cache.nixos.org/:", + "nix dump-path --store https://cache.nixos.org/ " + "/nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25"}, + }; + } - void run(ref<Store> store, const Path & storePath) override - { - FdSink sink(STDOUT_FILENO); - store->narFromPath(storePath, sink); - sink.flush(); - } + void run(ref<Store> store, const Path& storePath) override { + FdSink sink(STDOUT_FILENO); + store->narFromPath(storePath, sink); + sink.flush(); + } }; static RegisterCommand r1(make_ref<CmdDumpPath>()); diff --git a/third_party/nix/src/nix/edit.cc b/third_party/nix/src/nix/edit.cc index a6169f1183cd..8f31e19a1049 100644 --- a/third_party/nix/src/nix/edit.cc +++ b/third_party/nix/src/nix/edit.cc @@ -1,81 +1,72 @@ +#include <unistd.h> +#include "attr-path.hh" #include "command.hh" -#include "shared.hh" #include "eval.hh" -#include "attr-path.hh" #include "progress-bar.hh" - -#include <unistd.h> +#include "shared.hh" using namespace nix; -struct CmdEdit : InstallableCommand -{ - std::string name() override - { - return "edit"; - } +struct CmdEdit : InstallableCommand { + std::string name() override { return "edit"; } - std::string description() override - { - return "open the Nix expression of a Nix package in $EDITOR"; - } + std::string description() override { + return "open the Nix expression of a Nix package in $EDITOR"; + } - Examples examples() override - { - return { - Example{ - "To open the Nix expression of the GNU Hello package:", - "nix edit nixpkgs.hello" - }, - }; - } + Examples examples() override { + return { + Example{"To open the Nix expression of the GNU Hello package:", + "nix edit nixpkgs.hello"}, + }; + } - void run(ref<Store> store) override - { - auto state = getEvalState(); + void run(ref<Store> store) override { + auto state = getEvalState(); - auto v = installable->toValue(*state); + auto v = installable->toValue(*state); - Value * v2; - try { - auto dummyArgs = state->allocBindings(0); - v2 = findAlongAttrPath(*state, "meta.position", *dummyArgs, *v); - } catch (Error &) { - throw Error("package '%s' has no source location information", installable->what()); - } + Value* v2; + try { + auto dummyArgs = state->allocBindings(0); + v2 = findAlongAttrPath(*state, "meta.position", *dummyArgs, *v); + } catch (Error&) { + throw Error("package '%s' has no source location information", + installable->what()); + } - auto pos = state->forceString(*v2); - debug("position is %s", pos); + auto pos = state->forceString(*v2); + debug("position is %s", pos); - auto colon = pos.rfind(':'); - if (colon == std::string::npos) - throw Error("cannot parse meta.position attribute '%s'", pos); + auto colon = pos.rfind(':'); + if (colon == std::string::npos) + throw Error("cannot parse meta.position attribute '%s'", pos); - std::string filename(pos, 0, colon); - int lineno; - try { - lineno = std::stoi(std::string(pos, colon + 1)); - } catch (std::invalid_argument & e) { - throw Error("cannot parse line number '%s'", pos); - } + std::string filename(pos, 0, colon); + int lineno; + try { + lineno = std::stoi(std::string(pos, colon + 1)); + } catch (std::invalid_argument& e) { + throw Error("cannot parse line number '%s'", pos); + } - auto editor = getEnv("EDITOR", "cat"); + auto editor = getEnv("EDITOR", "cat"); - auto args = tokenizeString<Strings>(editor); + auto args = tokenizeString<Strings>(editor); - if (editor.find("emacs") != std::string::npos || - editor.find("nano") != std::string::npos || - editor.find("vim") != std::string::npos) - args.push_back(fmt("+%d", lineno)); + if (editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos) + args.push_back(fmt("+%d", lineno)); - args.push_back(filename); + args.push_back(filename); - stopProgressBar(); + stopProgressBar(); - execvp(args.front().c_str(), stringsToCharPtrs(args).data()); + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); - throw SysError("cannot run editor '%s'", editor); - } + throw SysError("cannot run editor '%s'", editor); + } }; static RegisterCommand r1(make_ref<CmdEdit>()); diff --git a/third_party/nix/src/nix/eval.cc b/third_party/nix/src/nix/eval.cc index b7058361cbec..34d91ba1f0a9 100644 --- a/third_party/nix/src/nix/eval.cc +++ b/third_party/nix/src/nix/eval.cc @@ -1,77 +1,57 @@ +#include "eval.hh" #include "command.hh" #include "common-args.hh" +#include "json.hh" +#include "progress-bar.hh" #include "shared.hh" #include "store-api.hh" -#include "eval.hh" -#include "json.hh" #include "value-to-json.hh" -#include "progress-bar.hh" using namespace nix; -struct CmdEval : MixJSON, InstallableCommand -{ - bool raw = false; +struct CmdEval : MixJSON, InstallableCommand { + bool raw = false; - CmdEval() - { - mkFlag(0, "raw", "print strings unquoted", &raw); - } + CmdEval() { mkFlag(0, "raw", "print strings unquoted", &raw); } - std::string name() override - { - return "eval"; - } + std::string name() override { return "eval"; } - std::string description() override - { - return "evaluate a Nix expression"; - } + std::string description() override { return "evaluate a Nix expression"; } - Examples examples() override - { - return { - Example{ - "To evaluate a Nix expression given on the command line:", - "nix eval '(1 + 2)'" - }, - Example{ - "To evaluate a Nix expression from a file or URI:", - "nix eval -f channel:nixos-17.09 hello.name" - }, - Example{ - "To get the current version of Nixpkgs:", - "nix eval --raw nixpkgs.lib.nixpkgsVersion" - }, - Example{ - "To print the store path of the Hello package:", - "nix eval --raw nixpkgs.hello" - }, - }; - } + Examples examples() override { + return { + Example{"To evaluate a Nix expression given on the command line:", + "nix eval '(1 + 2)'"}, + Example{"To evaluate a Nix expression from a file or URI:", + "nix eval -f channel:nixos-17.09 hello.name"}, + Example{"To get the current version of Nixpkgs:", + "nix eval --raw nixpkgs.lib.nixpkgsVersion"}, + Example{"To print the store path of the Hello package:", + "nix eval --raw nixpkgs.hello"}, + }; + } - void run(ref<Store> store) override - { - if (raw && json) - throw UsageError("--raw and --json are mutually exclusive"); + void run(ref<Store> store) override { + if (raw && json) + throw UsageError("--raw and --json are mutually exclusive"); - auto state = getEvalState(); + auto state = getEvalState(); - auto v = installable->toValue(*state); - PathSet context; + auto v = installable->toValue(*state); + PathSet context; - stopProgressBar(); + stopProgressBar(); - if (raw) { - std::cout << state->coerceToString(noPos, *v, context); - } else if (json) { - JSONPlaceholder jsonOut(std::cout); - printValueAsJSON(*state, true, *v, jsonOut, context); - } else { - state->forceValueDeep(*v); - std::cout << *v << "\n"; - } + if (raw) { + std::cout << state->coerceToString(noPos, *v, context); + } else if (json) { + JSONPlaceholder jsonOut(std::cout); + printValueAsJSON(*state, true, *v, jsonOut, context); + } else { + state->forceValueDeep(*v); + std::cout << *v << "\n"; } + } }; static RegisterCommand r1(make_ref<CmdEval>()); diff --git a/third_party/nix/src/nix/hash.cc b/third_party/nix/src/nix/hash.cc index af4105e28904..11fefb7939f1 100644 --- a/third_party/nix/src/nix/hash.cc +++ b/third_party/nix/src/nix/hash.cc @@ -1,94 +1,78 @@ -#include "command.hh" #include "hash.hh" +#include "command.hh" #include "legacy.hh" #include "shared.hh" using namespace nix; -struct CmdHash : Command -{ - enum Mode { mFile, mPath }; - Mode mode; - Base base = SRI; - bool truncate = false; - HashType ht = htSHA256; - std::vector<std::string> paths; - - CmdHash(Mode mode) : mode(mode) - { - mkFlag(0, "sri", "print hash in SRI format", &base, SRI); - mkFlag(0, "base64", "print hash in base-64", &base, Base64); - mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); - mkFlag(0, "base16", "print hash in base-16", &base, Base16); - mkFlag() - .longName("type") - .mkHashTypeFlag(&ht); - expectArgs("paths", &paths); - } - - std::string name() override - { - return mode == mFile ? "hash-file" : "hash-path"; - } - - std::string description() override - { - return mode == mFile - ? "print cryptographic hash of a regular file" - : "print cryptographic hash of the NAR serialisation of a path"; - } - - void run() override - { - for (auto path : paths) { - Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; - if (truncate && h.hashSize > 20) h = compressHash(h, 20); - std::cout << format("%1%\n") % - h.to_string(base, base == SRI); - } +struct CmdHash : Command { + enum Mode { mFile, mPath }; + Mode mode; + Base base = SRI; + bool truncate = false; + HashType ht = htSHA256; + std::vector<std::string> paths; + + CmdHash(Mode mode) : mode(mode) { + mkFlag(0, "sri", "print hash in SRI format", &base, SRI); + mkFlag(0, "base64", "print hash in base-64", &base, Base64); + mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); + mkFlag(0, "base16", "print hash in base-16", &base, Base16); + mkFlag().longName("type").mkHashTypeFlag(&ht); + expectArgs("paths", &paths); + } + + std::string name() override { + return mode == mFile ? "hash-file" : "hash-path"; + } + + std::string description() override { + return mode == mFile + ? "print cryptographic hash of a regular file" + : "print cryptographic hash of the NAR serialisation of a path"; + } + + void run() override { + for (auto path : paths) { + Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; + if (truncate && h.hashSize > 20) h = compressHash(h, 20); + std::cout << format("%1%\n") % h.to_string(base, base == SRI); } + } }; static RegisterCommand r1(make_ref<CmdHash>(CmdHash::mFile)); static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath)); -struct CmdToBase : Command -{ - Base base; - HashType ht = htUnknown; - std::vector<std::string> args; - - CmdToBase(Base base) : base(base) - { - mkFlag() - .longName("type") - .mkHashTypeFlag(&ht); - expectArgs("strings", &args); - } - - std::string name() override - { - return - base == Base16 ? "to-base16" : - base == Base32 ? "to-base32" : - base == Base64 ? "to-base64" : - "to-sri"; - } - - std::string description() override - { - return fmt("convert a hash to %s representation", - base == Base16 ? "base-16" : - base == Base32 ? "base-32" : - base == Base64 ? "base-64" : - "SRI"); - } - - void run() override - { - for (auto s : args) - std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI)); - } +struct CmdToBase : Command { + Base base; + HashType ht = htUnknown; + std::vector<std::string> args; + + CmdToBase(Base base) : base(base) { + mkFlag().longName("type").mkHashTypeFlag(&ht); + expectArgs("strings", &args); + } + + std::string name() override { + return base == Base16 + ? "to-base16" + : base == Base32 ? "to-base32" + : base == Base64 ? "to-base64" : "to-sri"; + } + + std::string description() override { + return fmt( + "convert a hash to %s representation", + base == Base16 + ? "base-16" + : base == Base32 ? "base-32" : base == Base64 ? "base-64" : "SRI"); + } + + void run() override { + for (auto s : args) + std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI)); + } }; static RegisterCommand r3(make_ref<CmdToBase>(Base16)); @@ -97,55 +81,59 @@ static RegisterCommand r5(make_ref<CmdToBase>(Base64)); static RegisterCommand r6(make_ref<CmdToBase>(SRI)); /* Legacy nix-hash command. */ -static int compatNixHash(int argc, char * * argv) -{ - HashType ht = htMD5; - bool flat = false; - bool base32 = false; - bool truncate = false; - enum { opHash, opTo32, opTo16 } op = opHash; - std::vector<std::string> ss; - - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-hash"); - else if (*arg == "--version") - printVersion("nix-hash"); - else if (*arg == "--flat") flat = true; - else if (*arg == "--base32") base32 = true; - else if (*arg == "--truncate") truncate = true; - else if (*arg == "--type") { - string s = getArg(*arg, arg, end); - ht = parseHashType(s); - if (ht == htUnknown) - throw UsageError(format("unknown hash type '%1%'") % s); - } - else if (*arg == "--to-base16") op = opTo16; - else if (*arg == "--to-base32") op = opTo32; - else if (*arg != "" && arg->at(0) == '-') - return false; - else - ss.push_back(*arg); - return true; - }); - - if (op == opHash) { - CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); - cmd.ht = ht; - cmd.base = base32 ? Base32 : Base16; - cmd.truncate = truncate; - cmd.paths = ss; - cmd.run(); - } - - else { - CmdToBase cmd(op == opTo32 ? Base32 : Base16); - cmd.args = ss; - cmd.ht = ht; - cmd.run(); - } - - return 0; +static int compatNixHash(int argc, char** argv) { + HashType ht = htMD5; + bool flat = false; + bool base32 = false; + bool truncate = false; + enum { opHash, opTo32, opTo16 } op = opHash; + std::vector<std::string> ss; + + parseCmdLine(argc, argv, + [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") + showManPage("nix-hash"); + else if (*arg == "--version") + printVersion("nix-hash"); + else if (*arg == "--flat") + flat = true; + else if (*arg == "--base32") + base32 = true; + else if (*arg == "--truncate") + truncate = true; + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + if (ht == htUnknown) + throw UsageError(format("unknown hash type '%1%'") % s); + } else if (*arg == "--to-base16") + op = opTo16; + else if (*arg == "--to-base32") + op = opTo32; + else if (*arg != "" && arg->at(0) == '-') + return false; + else + ss.push_back(*arg); + return true; + }); + + if (op == opHash) { + CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); + cmd.ht = ht; + cmd.base = base32 ? Base32 : Base16; + cmd.truncate = truncate; + cmd.paths = ss; + cmd.run(); + } + + else { + CmdToBase cmd(op == opTo32 ? Base32 : Base16); + cmd.args = ss; + cmd.ht = ht; + cmd.run(); + } + + return 0; } static RegisterLegacyCommand s1("nix-hash", compatNixHash); diff --git a/third_party/nix/src/nix/installables.cc b/third_party/nix/src/nix/installables.cc index 0e8bba39de20..633b6d58e170 100644 --- a/third_party/nix/src/nix/installables.cc +++ b/third_party/nix/src/nix/installables.cc @@ -1,190 +1,179 @@ -#include "command.hh" +#include <regex> #include "attr-path.hh" +#include "command.hh" #include "common-eval-args.hh" #include "derivations.hh" #include "eval-inline.hh" #include "eval.hh" #include "get-drvs.hh" -#include "store-api.hh" #include "shared.hh" - -#include <regex> +#include "store-api.hh" namespace nix { -SourceExprCommand::SourceExprCommand() -{ - mkFlag() - .shortName('f') - .longName("file") - .label("file") - .description("evaluate FILE rather than the default") - .dest(&file); +SourceExprCommand::SourceExprCommand() { + mkFlag() + .shortName('f') + .longName("file") + .label("file") + .description("evaluate FILE rather than the default") + .dest(&file); } -Value * SourceExprCommand::getSourceExpr(EvalState & state) -{ - if (vSourceExpr) return vSourceExpr; +Value* SourceExprCommand::getSourceExpr(EvalState& state) { + if (vSourceExpr) return vSourceExpr; - auto sToplevel = state.symbols.create("_toplevel"); + auto sToplevel = state.symbols.create("_toplevel"); - vSourceExpr = state.allocValue(); + vSourceExpr = state.allocValue(); - if (file != "") - state.evalFile(lookupFileArg(state, file), *vSourceExpr); + if (file != "") + state.evalFile(lookupFileArg(state, file), *vSourceExpr); - else { + else { + /* Construct the installation source from $NIX_PATH. */ - /* Construct the installation source from $NIX_PATH. */ + auto searchPath = state.getSearchPath(); - auto searchPath = state.getSearchPath(); + state.mkAttrs(*vSourceExpr, 1024); - state.mkAttrs(*vSourceExpr, 1024); + mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); - mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); + std::unordered_set<std::string> seen; - std::unordered_set<std::string> seen; + auto addEntry = [&](const std::string& name) { + if (name == "") return; + if (!seen.insert(name).second) return; + Value* v1 = state.allocValue(); + mkPrimOpApp(*v1, state.getBuiltin("findFile"), + state.getBuiltin("nixPath")); + Value* v2 = state.allocValue(); + mkApp(*v2, *v1, mkString(*state.allocValue(), name)); + mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)), + state.getBuiltin("import"), *v2); + }; - auto addEntry = [&](const std::string & name) { - if (name == "") return; - if (!seen.insert(name).second) return; - Value * v1 = state.allocValue(); - mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); - Value * v2 = state.allocValue(); - mkApp(*v2, *v1, mkString(*state.allocValue(), name)); - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)), - state.getBuiltin("import"), *v2); - }; + for (auto& i : searchPath) /* Hack to handle channels. */ + if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { + for (auto& j : readDirectory(i.second)) + if (j.name != "manifest.nix" && + pathExists(fmt("%s/%s/default.nix", i.second, j.name))) + addEntry(j.name); + } else + addEntry(i.first); - for (auto & i : searchPath) - /* Hack to handle channels. */ - if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { - for (auto & j : readDirectory(i.second)) - if (j.name != "manifest.nix" - && pathExists(fmt("%s/%s/default.nix", i.second, j.name))) - addEntry(j.name); - } else - addEntry(i.first); + vSourceExpr->attrs->sort(); + } - vSourceExpr->attrs->sort(); - } - - return vSourceExpr; + return vSourceExpr; } -ref<EvalState> SourceExprCommand::getEvalState() -{ - if (!evalState) - evalState = std::make_shared<EvalState>(searchPath, getStore()); - return ref<EvalState>(evalState); +ref<EvalState> SourceExprCommand::getEvalState() { + if (!evalState) + evalState = std::make_shared<EvalState>(searchPath, getStore()); + return ref<EvalState>(evalState); } -Buildable Installable::toBuildable() -{ - auto buildables = toBuildables(); - if (buildables.size() != 1) - throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size()); - return std::move(buildables[0]); +Buildable Installable::toBuildable() { + auto buildables = toBuildables(); + if (buildables.size() != 1) + throw Error( + "installable '%s' evaluates to %d derivations, where only one is " + "expected", + what(), buildables.size()); + return std::move(buildables[0]); } -struct InstallableStorePath : Installable -{ - Path storePath; +struct InstallableStorePath : Installable { + Path storePath; - InstallableStorePath(const Path & storePath) : storePath(storePath) { } + InstallableStorePath(const Path& storePath) : storePath(storePath) {} - std::string what() override { return storePath; } + std::string what() override { return storePath; } - Buildables toBuildables() override - { - return {{isDerivation(storePath) ? storePath : "", {{"out", storePath}}}}; - } + Buildables toBuildables() override { + return {{isDerivation(storePath) ? storePath : "", {{"out", storePath}}}}; + } }; -struct InstallableValue : Installable -{ - SourceExprCommand & cmd; +struct InstallableValue : Installable { + SourceExprCommand& cmd; - InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } + InstallableValue(SourceExprCommand& cmd) : cmd(cmd) {} - Buildables toBuildables() override - { - auto state = cmd.getEvalState(); + Buildables toBuildables() override { + auto state = cmd.getEvalState(); - auto v = toValue(*state); + auto v = toValue(*state); - Bindings & autoArgs = *cmd.getAutoArgs(*state); + Bindings& autoArgs = *cmd.getAutoArgs(*state); - DrvInfos drvs; - getDerivations(*state, *v, "", autoArgs, drvs, false); + DrvInfos drvs; + getDerivations(*state, *v, "", autoArgs, drvs, false); - Buildables res; + Buildables res; - PathSet drvPaths; - - for (auto & drv : drvs) { - Buildable b{drv.queryDrvPath()}; - drvPaths.insert(b.drvPath); + PathSet drvPaths; - auto outputName = drv.queryOutputName(); - if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath); + for (auto& drv : drvs) { + Buildable b{drv.queryDrvPath()}; + drvPaths.insert(b.drvPath); - b.outputs.emplace(outputName, drv.queryOutPath()); + auto outputName = drv.queryOutputName(); + if (outputName == "") + throw Error("derivation '%s' lacks an 'outputName' attribute", + b.drvPath); - res.push_back(std::move(b)); - } + b.outputs.emplace(outputName, drv.queryOutPath()); - // Hack to recognize .all: if all drvs have the same drvPath, - // merge the buildables. - if (drvPaths.size() == 1) { - Buildable b{*drvPaths.begin()}; - for (auto & b2 : res) - b.outputs.insert(b2.outputs.begin(), b2.outputs.end()); - return {b}; - } else - return res; + res.push_back(std::move(b)); } + + // Hack to recognize .all: if all drvs have the same drvPath, + // merge the buildables. + if (drvPaths.size() == 1) { + Buildable b{*drvPaths.begin()}; + for (auto& b2 : res) + b.outputs.insert(b2.outputs.begin(), b2.outputs.end()); + return {b}; + } else + return res; + } }; -struct InstallableExpr : InstallableValue -{ - std::string text; +struct InstallableExpr : InstallableValue { + std::string text; - InstallableExpr(SourceExprCommand & cmd, const std::string & text) - : InstallableValue(cmd), text(text) { } + InstallableExpr(SourceExprCommand& cmd, const std::string& text) + : InstallableValue(cmd), text(text) {} - std::string what() override { return text; } + std::string what() override { return text; } - Value * toValue(EvalState & state) override - { - auto v = state.allocValue(); - state.eval(state.parseExprFromString(text, absPath(".")), *v); - return v; - } + Value* toValue(EvalState& state) override { + auto v = state.allocValue(); + state.eval(state.parseExprFromString(text, absPath(".")), *v); + return v; + } }; -struct InstallableAttrPath : InstallableValue -{ - std::string attrPath; +struct InstallableAttrPath : InstallableValue { + std::string attrPath; - InstallableAttrPath(SourceExprCommand & cmd, const std::string & attrPath) - : InstallableValue(cmd), attrPath(attrPath) - { } + InstallableAttrPath(SourceExprCommand& cmd, const std::string& attrPath) + : InstallableValue(cmd), attrPath(attrPath) {} - std::string what() override { return attrPath; } + std::string what() override { return attrPath; } - Value * toValue(EvalState & state) override - { - auto source = cmd.getSourceExpr(state); + Value* toValue(EvalState& state) override { + auto source = cmd.getSourceExpr(state); - Bindings & autoArgs = *cmd.getAutoArgs(state); + Bindings& autoArgs = *cmd.getAutoArgs(state); - Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source); - state.forceValue(*v); + Value* v = findAlongAttrPath(state, attrPath, autoArgs, *source); + state.forceValue(*v); - return v; - } + return v; + } }; // FIXME: extend @@ -192,136 +181,127 @@ std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); static std::vector<std::shared_ptr<Installable>> parseInstallables( - SourceExprCommand & cmd, ref<Store> store, std::vector<std::string> ss, bool useDefaultInstallables) -{ - std::vector<std::shared_ptr<Installable>> result; - - if (ss.empty() && useDefaultInstallables) { - if (cmd.file == "") - cmd.file = "."; - ss = {""}; - } + SourceExprCommand& cmd, ref<Store> store, std::vector<std::string> ss, + bool useDefaultInstallables) { + std::vector<std::shared_ptr<Installable>> result; - for (auto & s : ss) { + if (ss.empty() && useDefaultInstallables) { + if (cmd.file == "") cmd.file = "."; + ss = {""}; + } - if (s.compare(0, 1, "(") == 0) - result.push_back(std::make_shared<InstallableExpr>(cmd, s)); + for (auto& s : ss) { + if (s.compare(0, 1, "(") == 0) + result.push_back(std::make_shared<InstallableExpr>(cmd, s)); - else if (s.find("/") != std::string::npos) { + else if (s.find("/") != std::string::npos) { + auto path = store->toStorePath(store->followLinksToStore(s)); - auto path = store->toStorePath(store->followLinksToStore(s)); + if (store->isStorePath(path)) + result.push_back(std::make_shared<InstallableStorePath>(path)); + } - if (store->isStorePath(path)) - result.push_back(std::make_shared<InstallableStorePath>(path)); - } + else if (s == "" || std::regex_match(s, attrPathRegex)) + result.push_back(std::make_shared<InstallableAttrPath>(cmd, s)); - else if (s == "" || std::regex_match(s, attrPathRegex)) - result.push_back(std::make_shared<InstallableAttrPath>(cmd, s)); + else + throw UsageError("don't know what to do with argument '%s'", s); + } - else - throw UsageError("don't know what to do with argument '%s'", s); - } - - return result; + return result; } -std::shared_ptr<Installable> parseInstallable( - SourceExprCommand & cmd, ref<Store> store, const std::string & installable, - bool useDefaultInstallables) -{ - auto installables = parseInstallables(cmd, store, {installable}, false); - assert(installables.size() == 1); - return installables.front(); +std::shared_ptr<Installable> parseInstallable(SourceExprCommand& cmd, + ref<Store> store, + const std::string& installable, + bool useDefaultInstallables) { + auto installables = parseInstallables(cmd, store, {installable}, false); + assert(installables.size() == 1); + return installables.front(); } Buildables build(ref<Store> store, RealiseMode mode, - std::vector<std::shared_ptr<Installable>> installables) -{ - if (mode != Build) - settings.readOnlyMode = true; - - Buildables buildables; - - PathSet pathsToBuild; - - for (auto & i : installables) { - for (auto & b : i->toBuildables()) { - if (b.drvPath != "") { - StringSet outputNames; - for (auto & output : b.outputs) - outputNames.insert(output.first); - pathsToBuild.insert( - b.drvPath + "!" + concatStringsSep(",", outputNames)); - } else - for (auto & output : b.outputs) - pathsToBuild.insert(output.second); - buildables.push_back(std::move(b)); - } + std::vector<std::shared_ptr<Installable>> installables) { + if (mode != Build) settings.readOnlyMode = true; + + Buildables buildables; + + PathSet pathsToBuild; + + for (auto& i : installables) { + for (auto& b : i->toBuildables()) { + if (b.drvPath != "") { + StringSet outputNames; + for (auto& output : b.outputs) outputNames.insert(output.first); + pathsToBuild.insert(b.drvPath + "!" + + concatStringsSep(",", outputNames)); + } else + for (auto& output : b.outputs) pathsToBuild.insert(output.second); + buildables.push_back(std::move(b)); } + } - if (mode == DryRun) - printMissing(store, pathsToBuild, lvlError); - else if (mode == Build) - store->buildPaths(pathsToBuild); + if (mode == DryRun) + printMissing(store, pathsToBuild, lvlError); + else if (mode == Build) + store->buildPaths(pathsToBuild); - return buildables; + return buildables; } PathSet toStorePaths(ref<Store> store, RealiseMode mode, - std::vector<std::shared_ptr<Installable>> installables) -{ - PathSet outPaths; + std::vector<std::shared_ptr<Installable>> installables) { + PathSet outPaths; - for (auto & b : build(store, mode, installables)) - for (auto & output : b.outputs) - outPaths.insert(output.second); + for (auto& b : build(store, mode, installables)) + for (auto& output : b.outputs) outPaths.insert(output.second); - return outPaths; + return outPaths; } Path toStorePath(ref<Store> store, RealiseMode mode, - std::shared_ptr<Installable> installable) -{ - auto paths = toStorePaths(store, mode, {installable}); + std::shared_ptr<Installable> installable) { + auto paths = toStorePaths(store, mode, {installable}); - if (paths.size() != 1) - throw Error("argument '%s' should evaluate to one store path", installable->what()); + if (paths.size() != 1) + throw Error("argument '%s' should evaluate to one store path", + installable->what()); - return *paths.begin(); + return *paths.begin(); } PathSet toDerivations(ref<Store> store, - std::vector<std::shared_ptr<Installable>> installables, bool useDeriver) -{ - PathSet drvPaths; - - for (auto & i : installables) - for (auto & b : i->toBuildables()) { - if (b.drvPath.empty()) { - if (!useDeriver) - throw Error("argument '%s' did not evaluate to a derivation", i->what()); - for (auto & output : b.outputs) { - auto derivers = store->queryValidDerivers(output.second); - if (derivers.empty()) - throw Error("'%s' does not have a known deriver", i->what()); - // FIXME: use all derivers? - drvPaths.insert(*derivers.begin()); - } - } else - drvPaths.insert(b.drvPath); + std::vector<std::shared_ptr<Installable>> installables, + bool useDeriver) { + PathSet drvPaths; + + for (auto& i : installables) + for (auto& b : i->toBuildables()) { + if (b.drvPath.empty()) { + if (!useDeriver) + throw Error("argument '%s' did not evaluate to a derivation", + i->what()); + for (auto& output : b.outputs) { + auto derivers = store->queryValidDerivers(output.second); + if (derivers.empty()) + throw Error("'%s' does not have a known deriver", i->what()); + // FIXME: use all derivers? + drvPaths.insert(*derivers.begin()); } + } else + drvPaths.insert(b.drvPath); + } - return drvPaths; + return drvPaths; } -void InstallablesCommand::prepare() -{ - installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables()); +void InstallablesCommand::prepare() { + installables = parseInstallables(*this, getStore(), _installables, + useDefaultInstallables()); } -void InstallableCommand::prepare() -{ - installable = parseInstallable(*this, getStore(), _installable, false); +void InstallableCommand::prepare() { + installable = parseInstallable(*this, getStore(), _installable, false); } -} +} // namespace nix diff --git a/third_party/nix/src/nix/legacy.cc b/third_party/nix/src/nix/legacy.cc index 6df09ee37a5e..2860afe53a49 100644 --- a/third_party/nix/src/nix/legacy.cc +++ b/third_party/nix/src/nix/legacy.cc @@ -2,6 +2,6 @@ namespace nix { -RegisterLegacyCommand::Commands * RegisterLegacyCommand::commands = 0; +RegisterLegacyCommand::Commands* RegisterLegacyCommand::commands = 0; } diff --git a/third_party/nix/src/nix/legacy.hh b/third_party/nix/src/nix/legacy.hh index f503b0da3e1a..09a102218132 100644 --- a/third_party/nix/src/nix/legacy.hh +++ b/third_party/nix/src/nix/legacy.hh @@ -6,18 +6,16 @@ namespace nix { -typedef std::function<void(int, char * *)> MainFunction; +typedef std::function<void(int, char**)> MainFunction; -struct RegisterLegacyCommand -{ - typedef std::map<std::string, MainFunction> Commands; - static Commands * commands; +struct RegisterLegacyCommand { + typedef std::map<std::string, MainFunction> Commands; + static Commands* commands; - RegisterLegacyCommand(const std::string & name, MainFunction fun) - { - if (!commands) commands = new Commands; - (*commands)[name] = fun; - } + RegisterLegacyCommand(const std::string& name, MainFunction fun) { + if (!commands) commands = new Commands; + (*commands)[name] = fun; + } }; -} +} // namespace nix diff --git a/third_party/nix/src/nix/log.cc b/third_party/nix/src/nix/log.cc index f07ec4e93a16..350d5c4a4999 100644 --- a/third_party/nix/src/nix/log.cc +++ b/third_party/nix/src/nix/log.cc @@ -1,71 +1,59 @@ #include "command.hh" #include "common-args.hh" +#include "progress-bar.hh" #include "shared.hh" #include "store-api.hh" -#include "progress-bar.hh" using namespace nix; -struct CmdLog : InstallableCommand -{ - CmdLog() - { - } - - std::string name() override - { - return "log"; - } - - std::string description() override - { - return "show the build log of the specified packages or paths, if available"; - } - - Examples examples() override - { - return { - Example{ - "To get the build log of GNU Hello:", - "nix log nixpkgs.hello" - }, - Example{ - "To get the build log of a specific path:", - "nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1" - }, - Example{ - "To get a build log from a specific binary cache:", - "nix log --store https://cache.nixos.org nixpkgs.hello" - }, - }; +struct CmdLog : InstallableCommand { + CmdLog() {} + + std::string name() override { return "log"; } + + std::string description() override { + return "show the build log of the specified packages or paths, if " + "available"; + } + + Examples examples() override { + return { + Example{"To get the build log of GNU Hello:", "nix log nixpkgs.hello"}, + Example{ + "To get the build log of a specific path:", + "nix log " + "/nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1"}, + Example{"To get a build log from a specific binary cache:", + "nix log --store https://cache.nixos.org nixpkgs.hello"}, + }; + } + + void run(ref<Store> store) override { + settings.readOnlyMode = true; + + auto subs = getDefaultSubstituters(); + + subs.push_front(store); + + auto b = installable->toBuildable(); + + RunPager pager; + for (auto& sub : subs) { + auto log = b.drvPath != "" ? sub->getBuildLog(b.drvPath) : nullptr; + for (auto& output : b.outputs) { + if (log) break; + log = sub->getBuildLog(output.second); + } + if (!log) continue; + stopProgressBar(); + printInfo("got build log for '%s' from '%s'", installable->what(), + sub->getUri()); + std::cout << *log; + return; } - void run(ref<Store> store) override - { - settings.readOnlyMode = true; - - auto subs = getDefaultSubstituters(); - - subs.push_front(store); - - auto b = installable->toBuildable(); - - RunPager pager; - for (auto & sub : subs) { - auto log = b.drvPath != "" ? sub->getBuildLog(b.drvPath) : nullptr; - for (auto & output : b.outputs) { - if (log) break; - log = sub->getBuildLog(output.second); - } - if (!log) continue; - stopProgressBar(); - printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); - std::cout << *log; - return; - } - - throw Error("build log of '%s' is not available", installable->what()); - } + throw Error("build log of '%s' is not available", installable->what()); + } }; static RegisterCommand r1(make_ref<CmdLog>()); diff --git a/third_party/nix/src/nix/ls.cc b/third_party/nix/src/nix/ls.cc index d089be42fb20..b23f83eb7824 100644 --- a/third_party/nix/src/nix/ls.cc +++ b/third_party/nix/src/nix/ls.cc @@ -1,155 +1,128 @@ #include "command.hh" -#include "store-api.hh" -#include "fs-accessor.hh" -#include "nar-accessor.hh" #include "common-args.hh" +#include "fs-accessor.hh" #include "json.hh" +#include "nar-accessor.hh" +#include "store-api.hh" using namespace nix; -struct MixLs : virtual Args, MixJSON -{ - std::string path; - - bool recursive = false; - bool verbose = false; - bool showDirectory = false; - - MixLs() - { - mkFlag('R', "recursive", "list subdirectories recursively", &recursive); - mkFlag('l', "long", "show more file information", &verbose); - mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); - } - - void listText(ref<FSAccessor> accessor) - { - std::function<void(const FSAccessor::Stat &, const Path &, const std::string &, bool)> doPath; - - auto showFile = [&](const Path & curPath, const std::string & relPath) { - if (verbose) { - auto st = accessor->stat(curPath); - std::string tp = - st.type == FSAccessor::Type::tRegular ? - (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : - st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : - "dr-xr-xr-x"; - std::cout << - (format("%s %20d %s") % tp % st.fileSize % relPath); - if (st.type == FSAccessor::Type::tSymlink) - std::cout << " -> " << accessor->readLink(curPath) - ; - std::cout << "\n"; - if (recursive && st.type == FSAccessor::Type::tDirectory) - doPath(st, curPath, relPath, false); - } else { - std::cout << relPath << "\n"; - if (recursive) { - auto st = accessor->stat(curPath); - if (st.type == FSAccessor::Type::tDirectory) - doPath(st, curPath, relPath, false); - } - } - }; - - doPath = [&](const FSAccessor::Stat & st, const Path & curPath, - const std::string & relPath, bool showDirectory) - { - if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { - auto names = accessor->readDirectory(curPath); - for (auto & name : names) - showFile(curPath + "/" + name, relPath + "/" + name); - } else - showFile(curPath, relPath); - }; - - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) - throw Error(format("path '%1%' does not exist") % path); - doPath(st, path, - st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path), - showDirectory); - } - - void list(ref<FSAccessor> accessor) - { - if (path == "/") path = ""; - - if (json) { - JSONPlaceholder jsonRoot(std::cout); - listNar(jsonRoot, accessor, path, recursive); - } else - listText(accessor); - } +struct MixLs : virtual Args, MixJSON { + std::string path; + + bool recursive = false; + bool verbose = false; + bool showDirectory = false; + + MixLs() { + mkFlag('R', "recursive", "list subdirectories recursively", &recursive); + mkFlag('l', "long", "show more file information", &verbose); + mkFlag('d', "directory", "show directories rather than their contents", + &showDirectory); + } + + void listText(ref<FSAccessor> accessor) { + std::function<void(const FSAccessor::Stat&, const Path&, const std::string&, + bool)> + doPath; + + auto showFile = [&](const Path& curPath, const std::string& relPath) { + if (verbose) { + auto st = accessor->stat(curPath); + std::string tp = st.type == FSAccessor::Type::tRegular + ? (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") + : st.type == FSAccessor::Type::tSymlink + ? "lrwxrwxrwx" + : "dr-xr-xr-x"; + std::cout << (format("%s %20d %s") % tp % st.fileSize % relPath); + if (st.type == FSAccessor::Type::tSymlink) + std::cout << " -> " << accessor->readLink(curPath); + std::cout << "\n"; + if (recursive && st.type == FSAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); + } else { + std::cout << relPath << "\n"; + if (recursive) { + auto st = accessor->stat(curPath); + if (st.type == FSAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); + } + } + }; + + doPath = [&](const FSAccessor::Stat& st, const Path& curPath, + const std::string& relPath, bool showDirectory) { + if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + auto names = accessor->readDirectory(curPath); + for (auto& name : names) + showFile(curPath + "/" + name, relPath + "/" + name); + } else + showFile(curPath, relPath); + }; + + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) + throw Error(format("path '%1%' does not exist") % path); + doPath(st, path, + st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path), + showDirectory); + } + + void list(ref<FSAccessor> accessor) { + if (path == "/") path = ""; + + if (json) { + JSONPlaceholder jsonRoot(std::cout); + listNar(jsonRoot, accessor, path, recursive); + } else + listText(accessor); + } }; -struct CmdLsStore : StoreCommand, MixLs -{ - CmdLsStore() - { - expectArg("path", &path); - } - - Examples examples() override - { - return { - Example{ - "To list the contents of a store path in a binary cache:", - "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" - }, - }; - } - - std::string name() override - { - return "ls-store"; - } - - std::string description() override - { - return "show information about a store path"; - } - - void run(ref<Store> store) override - { - list(store->getFSAccessor()); - } +struct CmdLsStore : StoreCommand, MixLs { + CmdLsStore() { expectArg("path", &path); } + + Examples examples() override { + return { + Example{"To list the contents of a store path in a binary cache:", + "nix ls-store --store https://cache.nixos.org/ -lR " + "/nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10"}, + }; + } + + std::string name() override { return "ls-store"; } + + std::string description() override { + return "show information about a store path"; + } + + void run(ref<Store> store) override { list(store->getFSAccessor()); } }; -struct CmdLsNar : Command, MixLs -{ - Path narPath; - - CmdLsNar() - { - expectArg("nar", &narPath); - expectArg("path", &path); - } - - Examples examples() override - { - return { - Example{ - "To list a specific file in a NAR:", - "nix ls-nar -l hello.nar /bin/hello" - }, - }; - } - - std::string name() override - { - return "ls-nar"; - } - - std::string description() override - { - return "show information about the contents of a NAR file"; - } - - void run() override - { - list(makeNarAccessor(make_ref<std::string>(readFile(narPath, true)))); - } +struct CmdLsNar : Command, MixLs { + Path narPath; + + CmdLsNar() { + expectArg("nar", &narPath); + expectArg("path", &path); + } + + Examples examples() override { + return { + Example{"To list a specific file in a NAR:", + "nix ls-nar -l hello.nar /bin/hello"}, + }; + } + + std::string name() override { return "ls-nar"; } + + std::string description() override { + return "show information about the contents of a NAR file"; + } + + void run() override { + list(makeNarAccessor(make_ref<std::string>(readFile(narPath, true)))); + } }; static RegisterCommand r1(make_ref<CmdLsStore>()); diff --git a/third_party/nix/src/nix/main.cc b/third_party/nix/src/nix/main.cc index bfc0a3def042..79730c58b994 100644 --- a/third_party/nix/src/nix/main.cc +++ b/third_party/nix/src/nix/main.cc @@ -1,181 +1,172 @@ +#include <ifaddrs.h> +#include <netdb.h> +#include <netinet/in.h> +#include <sys/socket.h> +#include <sys/types.h> #include <algorithm> - #include "command.hh" #include "common-args.hh" +#include "download.hh" #include "eval.hh" +#include "finally.hh" #include "globals.hh" #include "legacy.hh" +#include "progress-bar.hh" #include "shared.hh" #include "store-api.hh" -#include "progress-bar.hh" -#include "download.hh" -#include "finally.hh" - -#include <sys/types.h> -#include <sys/socket.h> - -#include <netinet/in.h> - -#include <ifaddrs.h> -#include <netdb.h> -#include <netinet/in.h> extern std::string chrootHelperName; -void chrootHelper(int argc, char * * argv); +void chrootHelper(int argc, char** argv); namespace nix { /* Check if we have a non-loopback/link-local network interface. */ -static bool haveInternet() -{ - struct ifaddrs * addrs; +static bool haveInternet() { + struct ifaddrs* addrs; - if (getifaddrs(&addrs)) - return true; + if (getifaddrs(&addrs)) return true; + + Finally free([&]() { freeifaddrs(addrs); }); - Finally free([&]() { freeifaddrs(addrs); }); - - for (auto i = addrs; i; i = i->ifa_next) { - if (!i->ifa_addr) continue; - if (i->ifa_addr->sa_family == AF_INET) { - if (ntohl(((sockaddr_in *) i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) { - return true; - } - } else if (i->ifa_addr->sa_family == AF_INET6) { - if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr) && - !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr)) - return true; - } + for (auto i = addrs; i; i = i->ifa_next) { + if (!i->ifa_addr) continue; + if (i->ifa_addr->sa_family == AF_INET) { + if (ntohl(((sockaddr_in*)i->ifa_addr)->sin_addr.s_addr) != + INADDR_LOOPBACK) { + return true; + } + } else if (i->ifa_addr->sa_family == AF_INET6) { + if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6*)i->ifa_addr)->sin6_addr) && + !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6*)i->ifa_addr)->sin6_addr)) + return true; } + } - return false; + return false; } std::string programPath; -struct NixArgs : virtual MultiCommand, virtual MixCommonArgs -{ - bool printBuildLogs = false; - bool useNet = true; - - NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") - { - mkFlag() - .longName("help") - .description("show usage information") - .handler([&]() { showHelpAndExit(); }); - - mkFlag() - .longName("help-config") - .description("show configuration options") - .handler([&]() { - std::cout << "The following configuration options are available:\n\n"; - Table2 tbl; - std::map<std::string, Config::SettingInfo> settings; - globalConfig.getSettings(settings); - for (const auto & s : settings) - tbl.emplace_back(s.first, s.second.description); - printTable(std::cout, tbl); - throw Exit(); - }); - - mkFlag() - .longName("print-build-logs") - .shortName('L') - .description("print full build logs on stderr") - .set(&printBuildLogs, true); - - mkFlag() - .longName("version") - .description("show version information") - .handler([&]() { printVersion(programName); }); - - mkFlag() - .longName("no-net") - .description("disable substituters and consider all previously downloaded files up-to-date") - .handler([&]() { useNet = false; }); - } - - void printFlags(std::ostream & out) override - { - Args::printFlags(out); - std::cout << - "\n" - "In addition, most configuration settings can be overriden using '--<name> <value>'.\n" - "Boolean settings can be overriden using '--<name>' or '--no-<name>'. See 'nix\n" - "--help-config' for a list of configuration settings.\n"; - } - - void showHelpAndExit() - { - printHelp(programName, std::cout); - std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; - throw Exit(); - } +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { + bool printBuildLogs = false; + bool useNet = true; + + NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { + mkFlag() + .longName("help") + .description("show usage information") + .handler([&]() { showHelpAndExit(); }); + + mkFlag() + .longName("help-config") + .description("show configuration options") + .handler([&]() { + std::cout << "The following configuration options are available:\n\n"; + Table2 tbl; + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (const auto& s : settings) + tbl.emplace_back(s.first, s.second.description); + printTable(std::cout, tbl); + throw Exit(); + }); + + mkFlag() + .longName("print-build-logs") + .shortName('L') + .description("print full build logs on stderr") + .set(&printBuildLogs, true); + + mkFlag() + .longName("version") + .description("show version information") + .handler([&]() { printVersion(programName); }); + + mkFlag() + .longName("no-net") + .description( + "disable substituters and consider all previously downloaded files " + "up-to-date") + .handler([&]() { useNet = false; }); + } + + void printFlags(std::ostream& out) override { + Args::printFlags(out); + std::cout << "\n" + "In addition, most configuration settings can be overriden " + "using '--<name> <value>'.\n" + "Boolean settings can be overriden using '--<name>' or " + "'--no-<name>'. See 'nix\n" + "--help-config' for a list of configuration settings.\n"; + } + + void showHelpAndExit() { + printHelp(programName, std::cout); + std::cout + << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; + throw Exit(); + } }; -void mainWrapped(int argc, char * * argv) -{ - /* The chroot helper needs to be run before any threads have been - started. */ - if (argc > 0 && argv[0] == chrootHelperName) { - chrootHelper(argc, argv); - return; - } +void mainWrapped(int argc, char** argv) { + /* The chroot helper needs to be run before any threads have been + started. */ + if (argc > 0 && argv[0] == chrootHelperName) { + chrootHelper(argc, argv); + return; + } - initNix(); - initGC(); + initNix(); + initGC(); - programPath = argv[0]; - string programName = baseNameOf(programPath); + programPath = argv[0]; + string programName = baseNameOf(programPath); - { - auto legacy = (*RegisterLegacyCommand::commands)[programName]; - if (legacy) return legacy(argc, argv); - } + { + auto legacy = (*RegisterLegacyCommand::commands)[programName]; + if (legacy) return legacy(argc, argv); + } - verbosity = lvlWarn; - settings.verboseBuild = false; + verbosity = lvlWarn; + settings.verboseBuild = false; - NixArgs args; + NixArgs args; - args.parseCmdline(argvToStrings(argc, argv)); + args.parseCmdline(argvToStrings(argc, argv)); - initPlugins(); + initPlugins(); - if (!args.command) args.showHelpAndExit(); + if (!args.command) args.showHelpAndExit(); - Finally f([]() { stopProgressBar(); }); + Finally f([]() { stopProgressBar(); }); - startProgressBar(args.printBuildLogs); + startProgressBar(args.printBuildLogs); - if (args.useNet && !haveInternet()) { - warn("you don't have Internet access; disabling some network-dependent features"); - args.useNet = false; - } + if (args.useNet && !haveInternet()) { + warn( + "you don't have Internet access; disabling some network-dependent " + "features"); + args.useNet = false; + } - if (!args.useNet) { - // FIXME: should check for command line overrides only. - if (!settings.useSubstitutes.overriden) - settings.useSubstitutes = false; - if (!settings.tarballTtl.overriden) - settings.tarballTtl = std::numeric_limits<unsigned int>::max(); - if (!downloadSettings.tries.overriden) - downloadSettings.tries = 0; - if (!downloadSettings.connectTimeout.overriden) - downloadSettings.connectTimeout = 1; - } + if (!args.useNet) { + // FIXME: should check for command line overrides only. + if (!settings.useSubstitutes.overriden) settings.useSubstitutes = false; + if (!settings.tarballTtl.overriden) + settings.tarballTtl = std::numeric_limits<unsigned int>::max(); + if (!downloadSettings.tries.overriden) downloadSettings.tries = 0; + if (!downloadSettings.connectTimeout.overriden) + downloadSettings.connectTimeout = 1; + } - args.command->prepare(); - args.command->run(); + args.command->prepare(); + args.command->run(); } -} +} // namespace nix -int main(int argc, char * * argv) -{ - return nix::handleExceptions(argv[0], [&]() { - nix::mainWrapped(argc, argv); - }); +int main(int argc, char** argv) { + return nix::handleExceptions(argv[0], + [&]() { nix::mainWrapped(argc, argv); }); } diff --git a/third_party/nix/src/nix/optimise-store.cc b/third_party/nix/src/nix/optimise-store.cc index 725fb75a19ec..49871df680bc 100644 --- a/third_party/nix/src/nix/optimise-store.cc +++ b/third_party/nix/src/nix/optimise-store.cc @@ -1,41 +1,26 @@ +#include <atomic> #include "command.hh" #include "shared.hh" #include "store-api.hh" -#include <atomic> - using namespace nix; -struct CmdOptimiseStore : StoreCommand -{ - CmdOptimiseStore() - { - } +struct CmdOptimiseStore : StoreCommand { + CmdOptimiseStore() {} - std::string name() override - { - return "optimise-store"; - } + std::string name() override { return "optimise-store"; } - std::string description() override - { - return "replace identical files in the store by hard links"; - } + std::string description() override { + return "replace identical files in the store by hard links"; + } - Examples examples() override - { - return { - Example{ - "To optimise the Nix store:", - "nix optimise-store" - }, - }; - } + Examples examples() override { + return { + Example{"To optimise the Nix store:", "nix optimise-store"}, + }; + } - void run(ref<Store> store) override - { - store->optimiseStore(); - } + void run(ref<Store> store) override { store->optimiseStore(); } }; static RegisterCommand r1(make_ref<CmdOptimiseStore>()); diff --git a/third_party/nix/src/nix/path-info.cc b/third_party/nix/src/nix/path-info.cc index dea5f0557b81..bd12be5c2311 100644 --- a/third_party/nix/src/nix/path-info.cc +++ b/third_party/nix/src/nix/path-info.cc @@ -1,133 +1,118 @@ +#include <algorithm> +#include <array> #include "command.hh" +#include "common-args.hh" +#include "json.hh" #include "shared.hh" #include "store-api.hh" -#include "json.hh" -#include "common-args.hh" - -#include <algorithm> -#include <array> using namespace nix; -struct CmdPathInfo : StorePathsCommand, MixJSON -{ - bool showSize = false; - bool showClosureSize = false; - bool humanReadable = false; - bool showSigs = false; - - CmdPathInfo() - { - mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); - mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); - mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable); - mkFlag(0, "sigs", "show signatures", &showSigs); +struct CmdPathInfo : StorePathsCommand, MixJSON { + bool showSize = false; + bool showClosureSize = false; + bool humanReadable = false; + bool showSigs = false; + + CmdPathInfo() { + mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); + mkFlag('S', "closure-size", + "print sum size of the NAR dumps of the closure of each path", + &showClosureSize); + mkFlag('h', "human-readable", + "with -s and -S, print sizes like 1K 234M 5.67G etc.", + &humanReadable); + mkFlag(0, "sigs", "show signatures", &showSigs); + } + + std::string name() override { return "path-info"; } + + std::string description() override { + return "query information about store paths"; + } + + Examples examples() override { + return { + Example{"To show the closure sizes of every path in the current NixOS " + "system closure, sorted by size:", + "nix path-info -rS /run/current-system | sort -nk2"}, + Example{"To show a package's closure size and all its dependencies " + "with human readable sizes:", + "nix path-info -rsSh nixpkgs.rust"}, + Example{"To check the existence of a path in a binary cache:", + "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store " + "https://cache.nixos.org/"}, + Example{"To print the 10 most recently added paths (using --json and " + "the jq(1) command):", + "nix path-info --json --all | jq -r " + "'sort_by(.registrationTime)[-11:-1][].path'"}, + Example{"To show the size of the entire Nix store:", + "nix path-info --json --all | jq 'map(.narSize) | add'"}, + Example{"To show every path whose closure is bigger than 1 GB, sorted " + "by closure size:", + "nix path-info --json --all -S | jq 'map(select(.closureSize > " + "1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'"}, + }; + } + + void printSize(unsigned long long value) { + if (!humanReadable) { + std::cout << fmt("\t%11d", value); + return; } - std::string name() override - { - return "path-info"; + static const std::array<char, 9> idents{ + {' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'}}; + size_t power = 0; + double res = value; + while (res > 1024 && power < idents.size()) { + ++power; + res /= 1024; } - - std::string description() override - { - return "query information about store paths"; + std::cout << fmt("\t%6.1f%c", res, idents.at(power)); + } + + void run(ref<Store> store, Paths storePaths) override { + size_t pathLen = 0; + for (auto& storePath : storePaths) + pathLen = std::max(pathLen, storePath.size()); + + if (json) { + JSONPlaceholder jsonRoot(std::cout); + store->pathInfoToJSON(jsonRoot, + // FIXME: preserve order? + PathSet(storePaths.begin(), storePaths.end()), true, + showClosureSize, AllowInvalid); } - Examples examples() override - { - return { - Example{ - "To show the closure sizes of every path in the current NixOS system closure, sorted by size:", - "nix path-info -rS /run/current-system | sort -nk2" - }, - Example{ - "To show a package's closure size and all its dependencies with human readable sizes:", - "nix path-info -rsSh nixpkgs.rust" - }, - Example{ - "To check the existence of a path in a binary cache:", - "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/" - }, - Example{ - "To print the 10 most recently added paths (using --json and the jq(1) command):", - "nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path'" - }, - Example{ - "To show the size of the entire Nix store:", - "nix path-info --json --all | jq 'map(.narSize) | add'" - }, - Example{ - "To show every path whose closure is bigger than 1 GB, sorted by closure size:", - "nix path-info --json --all -S | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'" - }, - }; - } + else { + for (auto storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + storePath = info->path; // FIXME: screws up padding - void printSize(unsigned long long value) - { - if (!humanReadable) { - std::cout << fmt("\t%11d", value); - return; - } + std::cout << storePath; - static const std::array<char, 9> idents{{ - ' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y' - }}; - size_t power = 0; - double res = value; - while (res > 1024 && power < idents.size()) { - ++power; - res /= 1024; - } - std::cout << fmt("\t%6.1f%c", res, idents.at(power)); - } - - void run(ref<Store> store, Paths storePaths) override - { - size_t pathLen = 0; - for (auto & storePath : storePaths) - pathLen = std::max(pathLen, storePath.size()); - - if (json) { - JSONPlaceholder jsonRoot(std::cout); - store->pathInfoToJSON(jsonRoot, - // FIXME: preserve order? - PathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, AllowInvalid); - } - - else { - - for (auto storePath : storePaths) { - auto info = store->queryPathInfo(storePath); - storePath = info->path; // FIXME: screws up padding + if (showSize || showClosureSize || showSigs) + std::cout << std::string( + std::max(0, (int)pathLen - (int)storePath.size()), ' '); - std::cout << storePath; + if (showSize) printSize(info->narSize); - if (showSize || showClosureSize || showSigs) - std::cout << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' '); - - if (showSize) - printSize(info->narSize); - - if (showClosureSize) - printSize(store->getClosureSize(storePath).first); - - if (showSigs) { - std::cout << '\t'; - Strings ss; - if (info->ultimate) ss.push_back("ultimate"); - if (info->ca != "") ss.push_back("ca:" + info->ca); - for (auto & sig : info->sigs) ss.push_back(sig); - std::cout << concatStringsSep(" ", ss); - } - - std::cout << std::endl; - } + if (showClosureSize) printSize(store->getClosureSize(storePath).first); + if (showSigs) { + std::cout << '\t'; + Strings ss; + if (info->ultimate) ss.push_back("ultimate"); + if (info->ca != "") ss.push_back("ca:" + info->ca); + for (auto& sig : info->sigs) ss.push_back(sig); + std::cout << concatStringsSep(" ", ss); } + + std::cout << std::endl; + } } + } }; static RegisterCommand r1(make_ref<CmdPathInfo>()); diff --git a/third_party/nix/src/nix/ping-store.cc b/third_party/nix/src/nix/ping-store.cc index 310942574a2a..9e9ee15c8966 100644 --- a/third_party/nix/src/nix/ping-store.cc +++ b/third_party/nix/src/nix/ping-store.cc @@ -4,32 +4,22 @@ using namespace nix; -struct CmdPingStore : StoreCommand -{ - std::string name() override - { - return "ping-store"; - } +struct CmdPingStore : StoreCommand { + std::string name() override { return "ping-store"; } - std::string description() override - { - return "test whether a store can be opened"; - } + std::string description() override { + return "test whether a store can be opened"; + } - Examples examples() override - { - return { - Example{ - "To test whether connecting to a remote Nix store via SSH works:", - "nix ping-store --store ssh://mac1" - }, - }; - } + Examples examples() override { + return { + Example{ + "To test whether connecting to a remote Nix store via SSH works:", + "nix ping-store --store ssh://mac1"}, + }; + } - void run(ref<Store> store) override - { - store->connect(); - } + void run(ref<Store> store) override { store->connect(); } }; static RegisterCommand r1(make_ref<CmdPingStore>()); diff --git a/third_party/nix/src/nix/progress-bar.cc b/third_party/nix/src/nix/progress-bar.cc index 98049b5daf9c..d3fca01c38a1 100644 --- a/third_party/nix/src/nix/progress-bar.cc +++ b/third_party/nix/src/nix/progress-bar.cc @@ -1,454 +1,444 @@ #include "progress-bar.hh" -#include "util.hh" -#include "sync.hh" -#include "store-api.hh" -#include "names.hh" - #include <atomic> #include <map> #include <thread> +#include "names.hh" +#include "store-api.hh" +#include "sync.hh" +#include "util.hh" namespace nix { -static std::string getS(const std::vector<Logger::Field> & fields, size_t n) -{ - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tString); - return fields[n].s; +static std::string getS(const std::vector<Logger::Field>& fields, size_t n) { + assert(n < fields.size()); + assert(fields[n].type == Logger::Field::tString); + return fields[n].s; } -static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n) -{ - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tInt); - return fields[n].i; +static uint64_t getI(const std::vector<Logger::Field>& fields, size_t n) { + assert(n < fields.size()); + assert(fields[n].type == Logger::Field::tInt); + return fields[n].i; } -class ProgressBar : public Logger -{ -private: - - struct ActInfo - { - std::string s, lastLine, phase; - ActivityType type = actUnknown; - uint64_t done = 0; - uint64_t expected = 0; - uint64_t running = 0; - uint64_t failed = 0; - std::map<ActivityType, uint64_t> expectedByType; - bool visible = true; - ActivityId parent; - std::optional<std::string> name; - }; - - struct ActivitiesByType - { - std::map<ActivityId, std::list<ActInfo>::iterator> its; - uint64_t done = 0; - uint64_t expected = 0; - uint64_t failed = 0; - }; - - struct State - { - std::list<ActInfo> activities; - std::map<ActivityId, std::list<ActInfo>::iterator> its; - - std::map<ActivityType, ActivitiesByType> activitiesByType; - - uint64_t filesLinked = 0, bytesLinked = 0; +class ProgressBar : public Logger { + private: + struct ActInfo { + std::string s, lastLine, phase; + ActivityType type = actUnknown; + uint64_t done = 0; + uint64_t expected = 0; + uint64_t running = 0; + uint64_t failed = 0; + std::map<ActivityType, uint64_t> expectedByType; + bool visible = true; + ActivityId parent; + std::optional<std::string> name; + }; + + struct ActivitiesByType { + std::map<ActivityId, std::list<ActInfo>::iterator> its; + uint64_t done = 0; + uint64_t expected = 0; + uint64_t failed = 0; + }; + + struct State { + std::list<ActInfo> activities; + std::map<ActivityId, std::list<ActInfo>::iterator> its; + + std::map<ActivityType, ActivitiesByType> activitiesByType; + + uint64_t filesLinked = 0, bytesLinked = 0; + + uint64_t corruptedPaths = 0, untrustedPaths = 0; + + bool active = true; + bool haveUpdate = true; + }; + + Sync<State> state_; + + std::thread updateThread; + + std::condition_variable quitCV, updateCV; + + bool printBuildLogs; + bool isTTY; + + public: + ProgressBar(bool printBuildLogs, bool isTTY) + : printBuildLogs(printBuildLogs), isTTY(isTTY) { + state_.lock()->active = isTTY; + updateThread = std::thread([&]() { + auto state(state_.lock()); + while (state->active) { + if (!state->haveUpdate) state.wait(updateCV); + draw(*state); + state.wait_for(quitCV, std::chrono::milliseconds(50)); + } + }); + } + + ~ProgressBar() { + stop(); + updateThread.join(); + } + + void stop() { + auto state(state_.lock()); + if (!state->active) return; + state->active = false; + std::string status = getStatus(*state); + writeToStderr("\r\e[K"); + if (status != "") writeToStderr("[" + status + "]\n"); + updateCV.notify_one(); + quitCV.notify_one(); + } + + void log(Verbosity lvl, const FormatOrString& fs) override { + auto state(state_.lock()); + log(*state, lvl, fs.s); + } + + void log(State& state, Verbosity lvl, const std::string& s) { + if (state.active) { + writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); + draw(state); + } else { + auto s2 = s + ANSI_NORMAL "\n"; + if (!isTTY) s2 = filterANSIEscapes(s2, true); + writeToStderr(s2); + } + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string& s, const Fields& fields, + ActivityId parent) override { + auto state(state_.lock()); + + if (lvl <= verbosity && !s.empty()) log(*state, lvl, s + "..."); + + state->activities.emplace_back(ActInfo()); + auto i = std::prev(state->activities.end()); + i->s = s; + i->type = type; + i->parent = parent; + state->its.emplace(act, i); + state->activitiesByType[type].its.emplace(act, i); + + if (type == actBuild) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) name.resize(name.size() - 4); + i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); + auto machineName = getS(fields, 1); + if (machineName != "") + i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName); + auto curRound = getI(fields, 2); + auto nrRounds = getI(fields, 3); + if (nrRounds != 1) i->s += fmt(" (round %d/%d)", curRound, nrRounds); + i->name = DrvName(name).name; + } - uint64_t corruptedPaths = 0, untrustedPaths = 0; + if (type == actSubstitute) { + auto name = storePathToName(getS(fields, 0)); + auto sub = getS(fields, 1); + i->s = fmt(hasPrefix(sub, "local") + ? "copying " ANSI_BOLD "%s" ANSI_NORMAL " from %s" + : "fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", + name, sub); + } - bool active = true; - bool haveUpdate = true; - }; + if (type == actPostBuildHook) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) name.resize(name.size() - 4); + i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); + i->name = DrvName(name).name; + } - Sync<State> state_; + if (type == actQueryPathInfo) { + auto name = storePathToName(getS(fields, 0)); + i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, + getS(fields, 1)); + } - std::thread updateThread; + if ((type == actDownload && hasAncestor(*state, actCopyPath, parent)) || + (type == actDownload && + hasAncestor(*state, actQueryPathInfo, parent)) || + (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) + i->visible = false; + + update(*state); + } + + /* Check whether an activity has an ancestore with the specified + type. */ + bool hasAncestor(State& state, ActivityType type, ActivityId act) { + while (act != 0) { + auto i = state.its.find(act); + if (i == state.its.end()) break; + if (i->second->type == type) return true; + act = i->second->parent; + } + return false; + } - std::condition_variable quitCV, updateCV; + void stopActivity(ActivityId act) override { + auto state(state_.lock()); - bool printBuildLogs; - bool isTTY; + auto i = state->its.find(act); + if (i != state->its.end()) { + auto& actByType = state->activitiesByType[i->second->type]; + actByType.done += i->second->done; + actByType.failed += i->second->failed; -public: + for (auto& j : i->second->expectedByType) + state->activitiesByType[j.first].expected -= j.second; - ProgressBar(bool printBuildLogs, bool isTTY) - : printBuildLogs(printBuildLogs) - , isTTY(isTTY) - { - state_.lock()->active = isTTY; - updateThread = std::thread([&]() { - auto state(state_.lock()); - while (state->active) { - if (!state->haveUpdate) - state.wait(updateCV); - draw(*state); - state.wait_for(quitCV, std::chrono::milliseconds(50)); - } - }); + actByType.its.erase(act); + state->activities.erase(i->second); + state->its.erase(i); } - ~ProgressBar() - { - stop(); - updateThread.join(); - } + update(*state); + } - void stop() - { - auto state(state_.lock()); - if (!state->active) return; - state->active = false; - std::string status = getStatus(*state); - writeToStderr("\r\e[K"); - if (status != "") - writeToStderr("[" + status + "]\n"); - updateCV.notify_one(); - quitCV.notify_one(); - } + void result(ActivityId act, ResultType type, + const std::vector<Field>& fields) override { + auto state(state_.lock()); - void log(Verbosity lvl, const FormatOrString & fs) override - { - auto state(state_.lock()); - log(*state, lvl, fs.s); + if (type == resFileLinked) { + state->filesLinked++; + state->bytesLinked += getI(fields, 0); + update(*state); } - void log(State & state, Verbosity lvl, const std::string & s) - { - if (state.active) { - writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); - draw(state); + else if (type == resBuildLogLine || type == resPostBuildLogLine) { + auto lastLine = trim(getS(fields, 0)); + if (!lastLine.empty()) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo info = *i->second; + if (printBuildLogs) { + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; + } + log(*state, lvlInfo, + ANSI_FAINT + info.name.value_or("unnamed") + suffix + + ANSI_NORMAL + lastLine); } else { - auto s2 = s + ANSI_NORMAL "\n"; - if (!isTTY) s2 = filterANSIEscapes(s2, true); - writeToStderr(s2); + state->activities.erase(i->second); + info.lastLine = lastLine; + state->activities.emplace_back(info); + i->second = std::prev(state->activities.end()); + update(*state); } + } } - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, - const std::string & s, const Fields & fields, ActivityId parent) override - { - auto state(state_.lock()); - - if (lvl <= verbosity && !s.empty()) - log(*state, lvl, s + "..."); - - state->activities.emplace_back(ActInfo()); - auto i = std::prev(state->activities.end()); - i->s = s; - i->type = type; - i->parent = parent; - state->its.emplace(act, i); - state->activitiesByType[type].its.emplace(act, i); - - if (type == actBuild) { - auto name = storePathToName(getS(fields, 0)); - if (hasSuffix(name, ".drv")) - name.resize(name.size() - 4); - i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); - auto machineName = getS(fields, 1); - if (machineName != "") - i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName); - auto curRound = getI(fields, 2); - auto nrRounds = getI(fields, 3); - if (nrRounds != 1) - i->s += fmt(" (round %d/%d)", curRound, nrRounds); - i->name = DrvName(name).name; - } - - if (type == actSubstitute) { - auto name = storePathToName(getS(fields, 0)); - auto sub = getS(fields, 1); - i->s = fmt( - hasPrefix(sub, "local") - ? "copying " ANSI_BOLD "%s" ANSI_NORMAL " from %s" - : "fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", - name, sub); - } - - if (type == actPostBuildHook) { - auto name = storePathToName(getS(fields, 0)); - if (hasSuffix(name, ".drv")) - name.resize(name.size() - 4); - i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); - i->name = DrvName(name).name; - } - - if (type == actQueryPathInfo) { - auto name = storePathToName(getS(fields, 0)); - i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); - } - - if ((type == actDownload && hasAncestor(*state, actCopyPath, parent)) - || (type == actDownload && hasAncestor(*state, actQueryPathInfo, parent)) - || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) - i->visible = false; + else if (type == resUntrustedPath) { + state->untrustedPaths++; + update(*state); + } - update(*state); + else if (type == resCorruptedPath) { + state->corruptedPaths++; + update(*state); } - /* Check whether an activity has an ancestore with the specified - type. */ - bool hasAncestor(State & state, ActivityType type, ActivityId act) - { - while (act != 0) { - auto i = state.its.find(act); - if (i == state.its.end()) break; - if (i->second->type == type) return true; - act = i->second->parent; - } - return false; + else if (type == resSetPhase) { + auto i = state->its.find(act); + assert(i != state->its.end()); + i->second->phase = getS(fields, 0); + update(*state); } - void stopActivity(ActivityId act) override - { - auto state(state_.lock()); + else if (type == resProgress) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo& actInfo = *i->second; + actInfo.done = getI(fields, 0); + actInfo.expected = getI(fields, 1); + actInfo.running = getI(fields, 2); + actInfo.failed = getI(fields, 3); + update(*state); + } - auto i = state->its.find(act); - if (i != state->its.end()) { + else if (type == resSetExpected) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo& actInfo = *i->second; + auto type = (ActivityType)getI(fields, 0); + auto& j = actInfo.expectedByType[type]; + state->activitiesByType[type].expected -= j; + j = getI(fields, 1); + state->activitiesByType[type].expected += j; + update(*state); + } + } - auto & actByType = state->activitiesByType[i->second->type]; - actByType.done += i->second->done; - actByType.failed += i->second->failed; + void update(State& state) { + state.haveUpdate = true; + updateCV.notify_one(); + } - for (auto & j : i->second->expectedByType) - state->activitiesByType[j.first].expected -= j.second; + void draw(State& state) { + state.haveUpdate = false; + if (!state.active) return; - actByType.its.erase(act); - state->activities.erase(i->second); - state->its.erase(i); - } + std::string line; - update(*state); + std::string status = getStatus(state); + if (!status.empty()) { + line += '['; + line += status; + line += "]"; } - void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override - { - auto state(state_.lock()); + if (!state.activities.empty()) { + if (!status.empty()) line += " "; + auto i = state.activities.rbegin(); - if (type == resFileLinked) { - state->filesLinked++; - state->bytesLinked += getI(fields, 0); - update(*state); - } + while (i != state.activities.rend() && + (!i->visible || (i->s.empty() && i->lastLine.empty()))) + ++i; - else if (type == resBuildLogLine || type == resPostBuildLogLine) { - auto lastLine = trim(getS(fields, 0)); - if (!lastLine.empty()) { - auto i = state->its.find(act); - assert(i != state->its.end()); - ActInfo info = *i->second; - if (printBuildLogs) { - auto suffix = "> "; - if (type == resPostBuildLogLine) { - suffix = " (post)> "; - } - log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); - } else { - state->activities.erase(i->second); - info.lastLine = lastLine; - state->activities.emplace_back(info); - i->second = std::prev(state->activities.end()); - update(*state); - } - } + if (i != state.activities.rend()) { + line += i->s; + if (!i->phase.empty()) { + line += " ("; + line += i->phase; + line += ")"; } - - else if (type == resUntrustedPath) { - state->untrustedPaths++; - update(*state); - } - - else if (type == resCorruptedPath) { - state->corruptedPaths++; - update(*state); + if (!i->lastLine.empty()) { + if (!i->s.empty()) line += ": "; + line += i->lastLine; } + } + } - else if (type == resSetPhase) { - auto i = state->its.find(act); - assert(i != state->its.end()); - i->second->phase = getS(fields, 0); - update(*state); - } + auto width = getWindowSize().second; + if (width <= 0) width = std::numeric_limits<decltype(width)>::max(); + + writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K"); + } + + std::string getStatus(State& state) { + auto MiB = 1024.0 * 1024.0; + + std::string res; + + auto renderActivity = [&](ActivityType type, const std::string& itemFmt, + const std::string& numberFmt = "%d", + double unit = 1) { + auto& act = state.activitiesByType[type]; + uint64_t done = act.done, expected = act.done, running = 0, + failed = act.failed; + for (auto& j : act.its) { + done += j.second->done; + expected += j.second->expected; + running += j.second->running; + failed += j.second->failed; + } + + expected = std::max(expected, act.expected); + + std::string s; + + if (running || done || expected || failed) { + if (running) + if (expected != 0) + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + + numberFmt + ANSI_NORMAL "/" + numberFmt, + running / unit, done / unit, expected / unit); + else + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + + numberFmt + ANSI_NORMAL, + running / unit, done / unit); + else if (expected != done) + if (expected != 0) + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + done / unit, expected / unit); + else + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit); + else + s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, + done / unit); + s = fmt(itemFmt, s); + + if (failed) + s += fmt(" (" ANSI_RED "%d failed" ANSI_NORMAL ")", failed / unit); + } + + return s; + }; - else if (type == resProgress) { - auto i = state->its.find(act); - assert(i != state->its.end()); - ActInfo & actInfo = *i->second; - actInfo.done = getI(fields, 0); - actInfo.expected = getI(fields, 1); - actInfo.running = getI(fields, 2); - actInfo.failed = getI(fields, 3); - update(*state); - } + auto showActivity = [&](ActivityType type, const std::string& itemFmt, + const std::string& numberFmt = "%d", + double unit = 1) { + auto s = renderActivity(type, itemFmt, numberFmt, unit); + if (s.empty()) return; + if (!res.empty()) res += ", "; + res += s; + }; - else if (type == resSetExpected) { - auto i = state->its.find(act); - assert(i != state->its.end()); - ActInfo & actInfo = *i->second; - auto type = (ActivityType) getI(fields, 0); - auto & j = actInfo.expectedByType[type]; - state->activitiesByType[type].expected -= j; - j = getI(fields, 1); - state->activitiesByType[type].expected += j; - update(*state); - } + showActivity(actBuilds, "%s built"); + + auto s1 = renderActivity(actCopyPaths, "%s copied"); + auto s2 = renderActivity(actCopyPath, "%s MiB", "%.1f", MiB); + + if (!s1.empty() || !s2.empty()) { + if (!res.empty()) res += ", "; + if (s1.empty()) + res += "0 copied"; + else + res += s1; + if (!s2.empty()) { + res += " ("; + res += s2; + res += ')'; + } } - void update(State & state) - { - state.haveUpdate = true; - updateCV.notify_one(); - } + showActivity(actDownload, "%s MiB DL", "%.1f", MiB); - void draw(State & state) { - state.haveUpdate = false; - if (!state.active) return; - - std::string line; - - std::string status = getStatus(state); - if (!status.empty()) { - line += '['; - line += status; - line += "]"; - } - - if (!state.activities.empty()) { - if (!status.empty()) line += " "; - auto i = state.activities.rbegin(); - - while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->lastLine.empty()))) - ++i; - - if (i != state.activities.rend()) { - line += i->s; - if (!i->phase.empty()) { - line += " ("; - line += i->phase; - line += ")"; - } - if (!i->lastLine.empty()) { - if (!i->s.empty()) line += ": "; - line += i->lastLine; - } - } - } - - auto width = getWindowSize().second; - if (width <= 0) width = std::numeric_limits<decltype(width)>::max(); - - writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K"); + auto s = renderActivity(actOptimiseStore, "%s paths optimised"); + if (s != "") { + s += fmt(", %.1f MiB / %d inodes freed", state.bytesLinked / MiB, + state.filesLinked); + if (!res.empty()) res += ", "; + res += s; + } } - std::string getStatus(State & state) - { - auto MiB = 1024.0 * 1024.0; - - std::string res; - - auto renderActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) { - auto & act = state.activitiesByType[type]; - uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed; - for (auto & j : act.its) { - done += j.second->done; - expected += j.second->expected; - running += j.second->running; - failed += j.second->failed; - } - - expected = std::max(expected, act.expected); - - std::string s; - - if (running || done || expected || failed) { - if (running) - if (expected != 0) - s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - running / unit, done / unit, expected / unit); - else - s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL, - running / unit, done / unit); - else if (expected != done) - if (expected != 0) - s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - done / unit, expected / unit); - else - s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit); - else - s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit); - s = fmt(itemFmt, s); - - if (failed) - s += fmt(" (" ANSI_RED "%d failed" ANSI_NORMAL ")", failed / unit); - } - - return s; - }; - - auto showActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) { - auto s = renderActivity(type, itemFmt, numberFmt, unit); - if (s.empty()) return; - if (!res.empty()) res += ", "; - res += s; - }; - - showActivity(actBuilds, "%s built"); - - auto s1 = renderActivity(actCopyPaths, "%s copied"); - auto s2 = renderActivity(actCopyPath, "%s MiB", "%.1f", MiB); - - if (!s1.empty() || !s2.empty()) { - if (!res.empty()) res += ", "; - if (s1.empty()) res += "0 copied"; else res += s1; - if (!s2.empty()) { res += " ("; res += s2; res += ')'; } - } - - showActivity(actDownload, "%s MiB DL", "%.1f", MiB); + // FIXME: don't show "done" paths in green. + showActivity(actVerifyPaths, "%s paths verified"); - { - auto s = renderActivity(actOptimiseStore, "%s paths optimised"); - if (s != "") { - s += fmt(", %.1f MiB / %d inodes freed", state.bytesLinked / MiB, state.filesLinked); - if (!res.empty()) res += ", "; - res += s; - } - } - - // FIXME: don't show "done" paths in green. - showActivity(actVerifyPaths, "%s paths verified"); - - if (state.corruptedPaths) { - if (!res.empty()) res += ", "; - res += fmt(ANSI_RED "%d corrupted" ANSI_NORMAL, state.corruptedPaths); - } - - if (state.untrustedPaths) { - if (!res.empty()) res += ", "; - res += fmt(ANSI_RED "%d untrusted" ANSI_NORMAL, state.untrustedPaths); - } + if (state.corruptedPaths) { + if (!res.empty()) res += ", "; + res += fmt(ANSI_RED "%d corrupted" ANSI_NORMAL, state.corruptedPaths); + } - return res; + if (state.untrustedPaths) { + if (!res.empty()) res += ", "; + res += fmt(ANSI_RED "%d untrusted" ANSI_NORMAL, state.untrustedPaths); } + + return res; + } }; -void startProgressBar(bool printBuildLogs) -{ - logger = new ProgressBar( - printBuildLogs, - isatty(STDERR_FILENO) && getEnv("TERM", "dumb") != "dumb"); +void startProgressBar(bool printBuildLogs) { + logger = + new ProgressBar(printBuildLogs, isatty(STDERR_FILENO) && + getEnv("TERM", "dumb") != "dumb"); } -void stopProgressBar() -{ - auto progressBar = dynamic_cast<ProgressBar *>(logger); - if (progressBar) progressBar->stop(); - +void stopProgressBar() { + auto progressBar = dynamic_cast<ProgressBar*>(logger); + if (progressBar) progressBar->stop(); } -} +} // namespace nix diff --git a/third_party/nix/src/nix/progress-bar.hh b/third_party/nix/src/nix/progress-bar.hh index 4d61175c24e4..d9963f9d9456 100644 --- a/third_party/nix/src/nix/progress-bar.hh +++ b/third_party/nix/src/nix/progress-bar.hh @@ -8,4 +8,4 @@ void startProgressBar(bool printBuildLogs = false); void stopProgressBar(); -} +} // namespace nix diff --git a/third_party/nix/src/nix/repl.cc b/third_party/nix/src/nix/repl.cc index f857b2e89c29..72609fb16401 100644 --- a/third_party/nix/src/nix/repl.cc +++ b/third_party/nix/src/nix/repl.cc @@ -1,16 +1,16 @@ -#include <iostream> +#include <setjmp.h> +#include <climits> #include <cstdlib> #include <cstring> -#include <climits> - -#include <setjmp.h> +#include <iostream> #ifdef READLINE #include <readline/history.h> #include <readline/readline.h> #else // editline < 1.15.2 don't wrap their API for C++ usage -// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// (added in +// https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). // This results in linker errors due to to name-mangling of editline C symbols. // For compatibility with these versions, we wrap the API here // (wrapping multiple times on newer versions is no problem). @@ -19,17 +19,17 @@ extern "C" { } #endif -#include "shared.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "store-api.hh" -#include "common-eval-args.hh" -#include "get-drvs.hh" -#include "derivations.hh" #include "affinity.hh" -#include "globals.hh" #include "command.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" #include "finally.hh" +#include "get-drvs.hh" +#include "globals.hh" +#include "shared.hh" +#include "store-api.hh" namespace nix { @@ -41,115 +41,113 @@ namespace nix { #define ESC_CYA "\033[36m" #define ESC_END "\033[0m" -struct NixRepl -{ - string curDir; - EvalState state; - Bindings * autoArgs; - - Strings loadedFiles; - - const static int envSize = 32768; - StaticEnv staticEnv; - Env * env; - int displ; - StringSet varNames; - - const Path historyFile; - - NixRepl(const Strings & searchPath, nix::ref<Store> store); - ~NixRepl(); - void mainLoop(const std::vector<std::string> & files); - StringSet completePrefix(string prefix); - bool getLine(string & input, const std::string &prompt); - Path getDerivationPath(Value & v); - bool processLine(string line); - void loadFile(const Path & path); - void initEnv(); - void reloadFiles(); - void addAttrsToScope(Value & attrs); - void addVarToScope(const Symbol & name, Value & v); - Expr * parseString(string s); - void evalString(string s, Value & v); - - typedef set<Value *> ValuesSeen; - std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth); - std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); +struct NixRepl { + string curDir; + EvalState state; + Bindings* autoArgs; + + Strings loadedFiles; + + const static int envSize = 32768; + StaticEnv staticEnv; + Env* env; + int displ; + StringSet varNames; + + const Path historyFile; + + NixRepl(const Strings& searchPath, nix::ref<Store> store); + ~NixRepl(); + void mainLoop(const std::vector<std::string>& files); + StringSet completePrefix(string prefix); + bool getLine(string& input, const std::string& prompt); + Path getDerivationPath(Value& v); + bool processLine(string line); + void loadFile(const Path& path); + void initEnv(); + void reloadFiles(); + void addAttrsToScope(Value& attrs); + void addVarToScope(const Symbol& name, Value& v); + Expr* parseString(string s); + void evalString(string s, Value& v); + + typedef set<Value*> ValuesSeen; + std::ostream& printValue(std::ostream& str, Value& v, unsigned int maxDepth); + std::ostream& printValue(std::ostream& str, Value& v, unsigned int maxDepth, + ValuesSeen& seen); }; - -void printHelp() -{ - std::cout - << "Usage: nix-repl [--help] [--version] [-I path] paths...\n" - << "\n" - << "nix-repl is a simple read-eval-print loop (REPL) for the Nix package manager.\n" - << "\n" - << "Options:\n" - << " --help\n" - << " Prints out a summary of the command syntax and exits.\n" - << "\n" - << " --version\n" - << " Prints out the Nix version number on standard output and exits.\n" - << "\n" - << " -I path\n" - << " Add a path to the Nix expression search path. This option may be given\n" - << " multiple times. See the NIX_PATH environment variable for information on\n" - << " the semantics of the Nix search path. Paths added through -I take\n" - << " precedence over NIX_PATH.\n" - << "\n" - << " paths...\n" - << " A list of paths to files containing Nix expressions which nix-repl will\n" - << " load and add to its scope.\n" - << "\n" - << " A path surrounded in < and > will be looked up in the Nix expression search\n" - << " path, as in the Nix language itself.\n" - << "\n" - << " If an element of paths starts with http:// or https://, it is interpreted\n" - << " as the URL of a tarball that will be downloaded and unpacked to a temporary\n" - << " location. The tarball must include a single top-level directory containing\n" - << " at least a file named default.nix.\n"; +void printHelp() { + std::cout << "Usage: nix-repl [--help] [--version] [-I path] paths...\n" + << "\n" + << "nix-repl is a simple read-eval-print loop (REPL) for the Nix " + "package manager.\n" + << "\n" + << "Options:\n" + << " --help\n" + << " Prints out a summary of the command syntax and exits.\n" + << "\n" + << " --version\n" + << " Prints out the Nix version number on standard output " + "and exits.\n" + << "\n" + << " -I path\n" + << " Add a path to the Nix expression search path. This " + "option may be given\n" + << " multiple times. See the NIX_PATH environment variable " + "for information on\n" + << " the semantics of the Nix search path. Paths added " + "through -I take\n" + << " precedence over NIX_PATH.\n" + << "\n" + << " paths...\n" + << " A list of paths to files containing Nix expressions " + "which nix-repl will\n" + << " load and add to its scope.\n" + << "\n" + << " A path surrounded in < and > will be looked up in the " + "Nix expression search\n" + << " path, as in the Nix language itself.\n" + << "\n" + << " If an element of paths starts with http:// or " + "https://, it is interpreted\n" + << " as the URL of a tarball that will be downloaded and " + "unpacked to a temporary\n" + << " location. The tarball must include a single top-level " + "directory containing\n" + << " at least a file named default.nix.\n"; } - -string removeWhitespace(string s) -{ - s = chomp(s); - size_t n = s.find_first_not_of(" \n\r\t"); - if (n != string::npos) s = string(s, n); - return s; +string removeWhitespace(string s) { + s = chomp(s); + size_t n = s.find_first_not_of(" \n\r\t"); + if (n != string::npos) s = string(s, n); + return s; } - -NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store) - : state(searchPath, store) - , staticEnv(false, &state.staticBaseEnv) - , historyFile(getDataDir() + "/nix/repl-history") -{ - curDir = absPath("."); +NixRepl::NixRepl(const Strings& searchPath, nix::ref<Store> store) + : state(searchPath, store), + staticEnv(false, &state.staticBaseEnv), + historyFile(getDataDir() + "/nix/repl-history") { + curDir = absPath("."); } +NixRepl::~NixRepl() { write_history(historyFile.c_str()); } -NixRepl::~NixRepl() -{ - write_history(historyFile.c_str()); -} - -static NixRepl * curRepl; // ugly +static NixRepl* curRepl; // ugly -static char * completionCallback(char * s, int *match) { +static char* completionCallback(char* s, int* match) { auto possible = curRepl->completePrefix(s); if (possible.size() == 1) { *match = 1; - auto *res = strdup(possible.begin()->c_str() + strlen(s)); + auto* res = strdup(possible.begin()->c_str() + strlen(s)); if (!res) throw Error("allocation failure"); return res; } else if (possible.size() > 1) { auto checkAllHaveSameAt = [&](size_t pos) { - auto &first = *possible.begin(); - for (auto &p : possible) { - if (p.size() <= pos || p[pos] != first[pos]) - return false; + auto& first = *possible.begin(); + for (auto& p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) return false; } return true; }; @@ -158,7 +156,7 @@ static char * completionCallback(char * s, int *match) { while (checkAllHaveSameAt(start + len)) ++len; if (len > 0) { *match = 1; - auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); + auto* res = strdup(std::string(*possible.begin(), start, len).c_str()); if (!res) throw Error("allocation failure"); return res; } @@ -168,20 +166,19 @@ static char * completionCallback(char * s, int *match) { return nullptr; } -static int listPossibleCallback(char *s, char ***avp) { +static int listPossibleCallback(char* s, char*** avp) { auto possible = curRepl->completePrefix(s); if (possible.size() > (INT_MAX / sizeof(char*))) throw Error("too many completions"); int ac = 0; - char **vp = nullptr; + char** vp = nullptr; - auto check = [&](auto *p) { + auto check = [&](auto* p) { if (!p) { if (vp) { - while (--ac >= 0) - free(vp[ac]); + while (--ac >= 0) free(vp[ac]); free(vp); } throw Error("allocation failure"); @@ -189,10 +186,9 @@ static int listPossibleCallback(char *s, char ***avp) { return p; }; - vp = check((char **)malloc(possible.size() * sizeof(char*))); + vp = check((char**)malloc(possible.size() * sizeof(char*))); - for (auto & p : possible) - vp[ac++] = check(strdup(p.c_str())); + for (auto& p : possible) vp[ac++] = check(strdup(p.c_str())); *avp = vp; @@ -200,592 +196,567 @@ static int listPossibleCallback(char *s, char ***avp) { } namespace { - // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. - volatile sig_atomic_t g_signal_received = 0; +// Used to communicate to NixRepl::getLine whether a signal occurred in +// ::readline. +volatile sig_atomic_t g_signal_received = 0; - void sigintHandler(int signo) { - g_signal_received = signo; - } -} +void sigintHandler(int signo) { g_signal_received = signo; } +} // namespace -void NixRepl::mainLoop(const std::vector<std::string> & files) -{ - string error = ANSI_RED "error:" ANSI_NORMAL " "; - std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." << std::endl << std::endl; +void NixRepl::mainLoop(const std::vector<std::string>& files) { + string error = ANSI_RED "error:" ANSI_NORMAL " "; + std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." + << std::endl + << std::endl; - for (auto & i : files) - loadedFiles.push_back(i); + for (auto& i : files) loadedFiles.push_back(i); - reloadFiles(); - if (!loadedFiles.empty()) std::cout << std::endl; + reloadFiles(); + if (!loadedFiles.empty()) std::cout << std::endl; - // Allow nix-repl specific settings in .inputrc - rl_readline_name = "nix-repl"; - createDirs(dirOf(historyFile)); + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + createDirs(dirOf(historyFile)); #ifndef READLINE - el_hist_size = 1000; + el_hist_size = 1000; #endif - read_history(historyFile.c_str()); - curRepl = this; + read_history(historyFile.c_str()); + curRepl = this; #ifndef READLINE - rl_set_complete_func(completionCallback); - rl_set_list_possib_func(listPossibleCallback); + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); #endif - std::string input; - - while (true) { - // When continuing input from previous lines, don't print a prompt, just align to the same - // number of chars as the prompt. - if (!getLine(input, input.empty() ? "nix-repl> " : " ")) - break; - - try { - if (!removeWhitespace(input).empty() && !processLine(input)) return; - } catch (ParseError & e) { - if (e.msg().find("unexpected $end") != std::string::npos) { - // For parse errors on incomplete input, we continue waiting for the next line of - // input without clearing the input so far. - continue; - } else { - printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); - } - } catch (Error & e) { - printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); - } catch (Interrupted & e) { - printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); - } - - // We handled the current input fully, so we should clear it - // and read brand new input. - input.clear(); - std::cout << std::endl; + std::string input; + + while (true) { + // When continuing input from previous lines, don't print a prompt, just + // align to the same number of chars as the prompt. + if (!getLine(input, input.empty() ? "nix-repl> " : " ")) break; + + try { + if (!removeWhitespace(input).empty() && !processLine(input)) return; + } catch (ParseError& e) { + if (e.msg().find("unexpected $end") != std::string::npos) { + // For parse errors on incomplete input, we continue waiting for the + // next line of input without clearing the input so far. + continue; + } else { + printMsg(lvlError, format(error + "%1%%2%") % + (settings.showTrace ? e.prefix() : "") % + e.msg()); + } + } catch (Error& e) { + printMsg(lvlError, format(error + "%1%%2%") % + (settings.showTrace ? e.prefix() : "") % e.msg()); + } catch (Interrupted& e) { + printMsg(lvlError, format(error + "%1%%2%") % + (settings.showTrace ? e.prefix() : "") % e.msg()); } -} + // We handled the current input fully, so we should clear it + // and read brand new input. + input.clear(); + std::cout << std::endl; + } +} -bool NixRepl::getLine(string & input, const std::string &prompt) -{ - struct sigaction act, old; - sigset_t savedSignalMask, set; - - auto setupSignals = [&]() { - act.sa_handler = sigintHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, &old)) - throw SysError("installing handler for SIGINT"); - - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) - throw SysError("unblocking SIGINT"); - }; - auto restoreSignals = [&]() { - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); - - if (sigaction(SIGINT, &old, 0)) - throw SysError("restoring handler for SIGINT"); - }; +bool NixRepl::getLine(string& input, const std::string& prompt) { + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); - setupSignals(); - char * s = readline(prompt.c_str()); - Finally doFree([&]() { free(s); }); - restoreSignals(); + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; - if (g_signal_received) { - g_signal_received = 0; - input.clear(); - return true; - } + setupSignals(); + char* s = readline(prompt.c_str()); + Finally doFree([&]() { free(s); }); + restoreSignals(); - if (!s) - return false; - input += s; - input += '\n'; + if (g_signal_received) { + g_signal_received = 0; + input.clear(); return true; + } + + if (!s) return false; + input += s; + input += '\n'; + return true; } +StringSet NixRepl::completePrefix(string prefix) { + StringSet completions; + + size_t start = prefix.find_last_of(" \n\r\t(){}[]"); + std::string prev, cur; + if (start == std::string::npos) { + prev = ""; + cur = prefix; + } else { + prev = std::string(prefix, 0, start + 1); + cur = std::string(prefix, start + 1); + } -StringSet NixRepl::completePrefix(string prefix) -{ - StringSet completions; + size_t slash, dot; - size_t start = prefix.find_last_of(" \n\r\t(){}[]"); - std::string prev, cur; - if (start == std::string::npos) { - prev = ""; - cur = prefix; - } else { - prev = std::string(prefix, 0, start + 1); - cur = std::string(prefix, start + 1); + if ((slash = cur.rfind('/')) != string::npos) { + try { + auto dir = std::string(cur, 0, slash); + auto prefix2 = std::string(cur, slash + 1); + for (auto& entry : readDirectory(dir == "" ? "/" : dir)) { + if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) + completions.insert(prev + dir + "/" + entry.name); + } + } catch (Error&) { } - - size_t slash, dot; - - if ((slash = cur.rfind('/')) != string::npos) { - try { - auto dir = std::string(cur, 0, slash); - auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { - if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) - completions.insert(prev + dir + "/" + entry.name); - } - } catch (Error &) { - } - } else if ((dot = cur.rfind('.')) == string::npos) { - /* This is a variable name; look it up in the current scope. */ - StringSet::iterator i = varNames.lower_bound(cur); - while (i != varNames.end()) { - if (string(*i, 0, cur.size()) != cur) break; - completions.insert(prev + *i); - i++; - } - } else { - try { - /* This is an expression that should evaluate to an - attribute set. Evaluate it to get the names of the - attributes. */ - string expr(cur, 0, dot); - string cur2 = string(cur, dot + 1); - - Expr * e = parseString(expr); - Value v; - e->eval(state, *env, v); - state.forceAttrs(v); - - for (auto & i : *v.attrs) { - string name = i.name; - if (string(name, 0, cur2.size()) != cur2) continue; - completions.insert(prev + expr + "." + name); - } - - } catch (ParseError & e) { - // Quietly ignore parse errors. - } catch (EvalError & e) { - // Quietly ignore evaluation errors. - } catch (UndefinedVarError & e) { - // Quietly ignore undefined variable errors. - } + } else if ((dot = cur.rfind('.')) == string::npos) { + /* This is a variable name; look it up in the current scope. */ + StringSet::iterator i = varNames.lower_bound(cur); + while (i != varNames.end()) { + if (string(*i, 0, cur.size()) != cur) break; + completions.insert(prev + *i); + i++; } + } else { + try { + /* This is an expression that should evaluate to an + attribute set. Evaluate it to get the names of the + attributes. */ + string expr(cur, 0, dot); + string cur2 = string(cur, dot + 1); + + Expr* e = parseString(expr); + Value v; + e->eval(state, *env, v); + state.forceAttrs(v); + + for (auto& i : *v.attrs) { + string name = i.name; + if (string(name, 0, cur2.size()) != cur2) continue; + completions.insert(prev + expr + "." + name); + } - return completions; -} - - -static int runProgram(const string & program, const Strings & args) -{ - Strings args2(args); - args2.push_front(program); - - Pid pid; - pid = fork(); - if (pid == -1) throw SysError("forking"); - if (pid == 0) { - restoreAffinity(); - execvp(program.c_str(), stringsToCharPtrs(args2).data()); - _exit(1); + } catch (ParseError& e) { + // Quietly ignore parse errors. + } catch (EvalError& e) { + // Quietly ignore evaluation errors. + } catch (UndefinedVarError& e) { + // Quietly ignore undefined variable errors. } + } - return pid.wait(); + return completions; } +static int runProgram(const string& program, const Strings& args) { + Strings args2(args); + args2.push_front(program); + + Pid pid; + pid = fork(); + if (pid == -1) throw SysError("forking"); + if (pid == 0) { + restoreAffinity(); + execvp(program.c_str(), stringsToCharPtrs(args2).data()); + _exit(1); + } -bool isVarName(const string & s) -{ - if (s.size() == 0) return false; - char c = s[0]; - if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; - for (auto & i : s) - if (!((i >= 'a' && i <= 'z') || - (i >= 'A' && i <= 'Z') || - (i >= '0' && i <= '9') || - i == '_' || i == '-' || i == '\'')) - return false; - return true; + return pid.wait(); } - -Path NixRepl::getDerivationPath(Value & v) { - auto drvInfo = getDerivation(state, v, false); - if (!drvInfo) - throw Error("expression does not evaluate to a derivation, so I can't build it"); - Path drvPath = drvInfo->queryDrvPath(); - if (drvPath == "" || !state.store->isValidPath(drvPath)) - throw Error("expression did not evaluate to a valid derivation"); - return drvPath; +bool isVarName(const string& s) { + if (s.size() == 0) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto& i : s) + if (!((i >= 'a' && i <= 'z') || (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || i == '_' || i == '-' || i == '\'')) + return false; + return true; } +Path NixRepl::getDerivationPath(Value& v) { + auto drvInfo = getDerivation(state, v, false); + if (!drvInfo) + throw Error( + "expression does not evaluate to a derivation, so I can't build it"); + Path drvPath = drvInfo->queryDrvPath(); + if (drvPath == "" || !state.store->isValidPath(drvPath)) + throw Error("expression did not evaluate to a valid derivation"); + return drvPath; +} -bool NixRepl::processLine(string line) -{ - if (line == "") return true; - - string command, arg; - - if (line[0] == ':') { - size_t p = line.find_first_of(" \n\r\t"); - command = string(line, 0, p); - if (p != string::npos) arg = removeWhitespace(string(line, p)); - } else { - arg = line; - } +bool NixRepl::processLine(string line) { + if (line == "") return true; - if (command == ":?" || command == ":help") { - std::cout - << "The following commands are available:\n" - << "\n" - << " <expr> Evaluate and print expression\n" - << " <x> = <expr> Bind expression to variable\n" - << " :a <expr> Add attributes from resulting set to scope\n" - << " :b <expr> Build derivation\n" - << " :i <expr> Build derivation, then install result into current profile\n" - << " :l <path> Load Nix expression and add it to scope\n" - << " :p <expr> Evaluate and print expression recursively\n" - << " :q Exit nix-repl\n" - << " :r Reload all files\n" - << " :s <expr> Build dependencies of derivation, then start nix-shell\n" - << " :t <expr> Describe result of evaluation\n" - << " :u <expr> Build derivation, then start nix-shell\n"; - } + string command, arg; - else if (command == ":a" || command == ":add") { - Value v; - evalString(arg, v); - addAttrsToScope(v); - } + if (line[0] == ':') { + size_t p = line.find_first_of(" \n\r\t"); + command = string(line, 0, p); + if (p != string::npos) arg = removeWhitespace(string(line, p)); + } else { + arg = line; + } - else if (command == ":l" || command == ":load") { - state.resetFileCache(); - loadFile(arg); - } + if (command == ":?" || command == ":help") { + std::cout << "The following commands are available:\n" + << "\n" + << " <expr> Evaluate and print expression\n" + << " <x> = <expr> Bind expression to variable\n" + << " :a <expr> Add attributes from resulting set to scope\n" + << " :b <expr> Build derivation\n" + << " :i <expr> Build derivation, then install result into " + "current profile\n" + << " :l <path> Load Nix expression and add it to scope\n" + << " :p <expr> Evaluate and print expression recursively\n" + << " :q Exit nix-repl\n" + << " :r Reload all files\n" + << " :s <expr> Build dependencies of derivation, then start " + "nix-shell\n" + << " :t <expr> Describe result of evaluation\n" + << " :u <expr> Build derivation, then start nix-shell\n"; + } - else if (command == ":r" || command == ":reload") { - state.resetFileCache(); - reloadFiles(); - } + else if (command == ":a" || command == ":add") { + Value v; + evalString(arg, v); + addAttrsToScope(v); + } - else if (command == ":t") { - Value v; - evalString(arg, v); - std::cout << showType(v) << std::endl; + else if (command == ":l" || command == ":load") { + state.resetFileCache(); + loadFile(arg); + } - } else if (command == ":u") { - Value v, f, result; - evalString(arg, v); - evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f); - state.callFunction(f, v, result, Pos()); + else if (command == ":r" || command == ":reload") { + state.resetFileCache(); + reloadFiles(); + } - Path drvPath = getDerivationPath(result); - runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); - } + else if (command == ":t") { + Value v; + evalString(arg, v); + std::cout << showType(v) << std::endl; + + } else if (command == ":u") { + Value v, f, result; + evalString(arg, v); + evalString( + "drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv " + "]; } \"\"", + f); + state.callFunction(f, v, result, Pos()); + + Path drvPath = getDerivationPath(result); + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + } - else if (command == ":b" || command == ":i" || command == ":s") { - Value v; - evalString(arg, v); - Path drvPath = getDerivationPath(v); - - if (command == ":b") { - /* We could do the build in this process using buildPaths(), - but doing it in a child makes it easier to recover from - problems / SIGINT. */ - if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { - Derivation drv = readDerivation(drvPath); - std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; - for (auto & i : drv.outputs) - std::cout << format(" %1% -> %2%") % i.first % i.second.path << std::endl; - } - } else if (command == ":i") { - runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); - } else { - runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); - } + else if (command == ":b" || command == ":i" || command == ":s") { + Value v; + evalString(arg, v); + Path drvPath = getDerivationPath(v); + + if (command == ":b") { + /* We could do the build in this process using buildPaths(), + but doing it in a child makes it easier to recover from + problems / SIGINT. */ + if (runProgram(settings.nixBinDir + "/nix", + Strings{"build", "--no-link", drvPath}) == 0) { + Derivation drv = readDerivation(drvPath); + std::cout << std::endl + << "this derivation produced the following outputs:" + << std::endl; + for (auto& i : drv.outputs) + std::cout << format(" %1% -> %2%") % i.first % i.second.path + << std::endl; + } + } else if (command == ":i") { + runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); + } else { + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); } + } - else if (command == ":p" || command == ":print") { - Value v; - evalString(arg, v); - printValue(std::cout, v, 1000000000) << std::endl; - } + else if (command == ":p" || command == ":print") { + Value v; + evalString(arg, v); + printValue(std::cout, v, 1000000000) << std::endl; + } - else if (command == ":q" || command == ":quit") - return false; - - else if (command != "") - throw Error(format("unknown command '%1%'") % command); - - else { - size_t p = line.find('='); - string name; - if (p != string::npos && - p < line.size() && - line[p + 1] != '=' && - isVarName(name = removeWhitespace(string(line, 0, p)))) - { - Expr * e = parseString(string(line, p + 1)); - Value & v(*state.allocValue()); - v.type = tThunk; - v.thunk.env = env; - v.thunk.expr = e; - addVarToScope(state.symbols.create(name), v); - } else { - Value v; - evalString(line, v); - printValue(std::cout, v, 1) << std::endl; - } + else if (command == ":q" || command == ":quit") + return false; + + else if (command != "") + throw Error(format("unknown command '%1%'") % command); + + else { + size_t p = line.find('='); + string name; + if (p != string::npos && p < line.size() && line[p + 1] != '=' && + isVarName(name = removeWhitespace(string(line, 0, p)))) { + Expr* e = parseString(string(line, p + 1)); + Value& v(*state.allocValue()); + v.type = tThunk; + v.thunk.env = env; + v.thunk.expr = e; + addVarToScope(state.symbols.create(name), v); + } else { + Value v; + evalString(line, v); + printValue(std::cout, v, 1) << std::endl; } + } - return true; + return true; } - -void NixRepl::loadFile(const Path & path) -{ - loadedFiles.remove(path); - loadedFiles.push_back(path); - Value v, v2; - state.evalFile(lookupFileArg(state, path), v); - state.autoCallFunction(*autoArgs, v, v2); - addAttrsToScope(v2); +void NixRepl::loadFile(const Path& path) { + loadedFiles.remove(path); + loadedFiles.push_back(path); + Value v, v2; + state.evalFile(lookupFileArg(state, path), v); + state.autoCallFunction(*autoArgs, v, v2); + addAttrsToScope(v2); } +void NixRepl::initEnv() { + env = &state.allocEnv(envSize); + env->up = &state.baseEnv; + displ = 0; + staticEnv.vars.clear(); -void NixRepl::initEnv() -{ - env = &state.allocEnv(envSize); - env->up = &state.baseEnv; - displ = 0; - staticEnv.vars.clear(); - - varNames.clear(); - for (auto & i : state.staticBaseEnv.vars) - varNames.insert(i.first); + varNames.clear(); + for (auto& i : state.staticBaseEnv.vars) varNames.insert(i.first); } +void NixRepl::reloadFiles() { + initEnv(); -void NixRepl::reloadFiles() -{ - initEnv(); + Strings old = loadedFiles; + loadedFiles.clear(); - Strings old = loadedFiles; - loadedFiles.clear(); - - bool first = true; - for (auto & i : old) { - if (!first) std::cout << std::endl; - first = false; - std::cout << format("Loading '%1%'...") % i << std::endl; - loadFile(i); - } + bool first = true; + for (auto& i : old) { + if (!first) std::cout << std::endl; + first = false; + std::cout << format("Loading '%1%'...") % i << std::endl; + loadFile(i); + } } - -void NixRepl::addAttrsToScope(Value & attrs) -{ - state.forceAttrs(attrs); - for (auto & i : *attrs.attrs) - addVarToScope(i.name, *i.value); - std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl; +void NixRepl::addAttrsToScope(Value& attrs) { + state.forceAttrs(attrs); + for (auto& i : *attrs.attrs) addVarToScope(i.name, *i.value); + std::cout << format("Added %1% variables.") % attrs.attrs->size() + << std::endl; } - -void NixRepl::addVarToScope(const Symbol & name, Value & v) -{ - if (displ >= envSize) - throw Error("environment full; cannot add more variables"); - staticEnv.vars[name] = displ; - env->values[displ++] = &v; - varNames.insert((string) name); +void NixRepl::addVarToScope(const Symbol& name, Value& v) { + if (displ >= envSize) + throw Error("environment full; cannot add more variables"); + staticEnv.vars[name] = displ; + env->values[displ++] = &v; + varNames.insert((string)name); } - -Expr * NixRepl::parseString(string s) -{ - Expr * e = state.parseExprFromString(s, curDir, staticEnv); - return e; +Expr* NixRepl::parseString(string s) { + Expr* e = state.parseExprFromString(s, curDir, staticEnv); + return e; } - -void NixRepl::evalString(string s, Value & v) -{ - Expr * e = parseString(s); - e->eval(state, *env, v); - state.forceValue(v); +void NixRepl::evalString(string s, Value& v) { + Expr* e = parseString(s); + e->eval(state, *env, v); + state.forceValue(v); } - -std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth) -{ - ValuesSeen seen; - return printValue(str, v, maxDepth, seen); +std::ostream& NixRepl::printValue(std::ostream& str, Value& v, + unsigned int maxDepth) { + ValuesSeen seen; + return printValue(str, v, maxDepth, seen); } - -std::ostream & printStringValue(std::ostream & str, const char * string) { - str << "\""; - for (const char * i = string; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else str << *i; - str << "\""; - return str; +std::ostream& printStringValue(std::ostream& str, const char* string) { + str << "\""; + for (const char* i = string; *i; i++) + if (*i == '\"' || *i == '\\') + str << "\\" << *i; + else if (*i == '\n') + str << "\\n"; + else if (*i == '\r') + str << "\\r"; + else if (*i == '\t') + str << "\\t"; + else + str << *i; + str << "\""; + return str; } - // FIXME: lot of cut&paste from Nix's eval.cc. -std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen) -{ - str.flush(); - checkInterrupt(); +std::ostream& NixRepl::printValue(std::ostream& str, Value& v, + unsigned int maxDepth, ValuesSeen& seen) { + str.flush(); + checkInterrupt(); - state.forceValue(v); - - switch (v.type) { + state.forceValue(v); + switch (v.type) { case tInt: - str << ESC_CYA << v.integer << ESC_END; - break; + str << ESC_CYA << v.integer << ESC_END; + break; case tBool: - str << ESC_CYA << (v.boolean ? "true" : "false") << ESC_END; - break; + str << ESC_CYA << (v.boolean ? "true" : "false") << ESC_END; + break; case tString: - str << ESC_YEL; - printStringValue(str, v.string.s); - str << ESC_END; - break; + str << ESC_YEL; + printStringValue(str, v.string.s); + str << ESC_END; + break; case tPath: - str << ESC_GRE << v.path << ESC_END; // !!! escaping? - break; + str << ESC_GRE << v.path << ESC_END; // !!! escaping? + break; case tNull: - str << ESC_CYA "null" ESC_END; - break; + str << ESC_CYA "null" ESC_END; + break; case tAttrs: { - seen.insert(&v); - - bool isDrv = state.isDerivation(v); - - if (isDrv) { - str << "«derivation "; - Bindings::iterator i = v.attrs->find(state.sDrvPath); - PathSet context; - Path drvPath = i != v.attrs->end() ? state.coerceToPath(*i->pos, *i->value, context) : "???"; - str << drvPath << "»"; - } + seen.insert(&v); + + bool isDrv = state.isDerivation(v); + + if (isDrv) { + str << "«derivation "; + Bindings::iterator i = v.attrs->find(state.sDrvPath); + PathSet context; + Path drvPath = i != v.attrs->end() + ? state.coerceToPath(*i->pos, *i->value, context) + : "???"; + str << drvPath << "»"; + } - else if (maxDepth > 0) { - str << "{ "; - - typedef std::map<string, Value *> Sorted; - Sorted sorted; - for (auto & i : *v.attrs) - sorted[i.name] = i.value; - - for (auto & i : sorted) { - if (isVarName(i.first)) - str << i.first; - else - printStringValue(str, i.first.c_str()); - str << " = "; - if (seen.find(i.second) != seen.end()) - str << "«repeated»"; - else - try { - printValue(str, *i.second, maxDepth - 1, seen); - } catch (AssertionError & e) { - str << ESC_RED "«error: " << e.msg() << "»" ESC_END; - } - str << "; "; + else if (maxDepth > 0) { + str << "{ "; + + typedef std::map<string, Value*> Sorted; + Sorted sorted; + for (auto& i : *v.attrs) sorted[i.name] = i.value; + + for (auto& i : sorted) { + if (isVarName(i.first)) + str << i.first; + else + printStringValue(str, i.first.c_str()); + str << " = "; + if (seen.find(i.second) != seen.end()) + str << "«repeated»"; + else + try { + printValue(str, *i.second, maxDepth - 1, seen); + } catch (AssertionError& e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; } + str << "; "; + } - str << "}"; - } else - str << "{ ... }"; + str << "}"; + } else + str << "{ ... }"; - break; + break; } case tList1: case tList2: case tListN: - seen.insert(&v); - - str << "[ "; - if (maxDepth > 0) - for (unsigned int n = 0; n < v.listSize(); ++n) { - if (seen.find(v.listElems()[n]) != seen.end()) - str << "«repeated»"; - else - try { - printValue(str, *v.listElems()[n], maxDepth - 1, seen); - } catch (AssertionError & e) { - str << ESC_RED "«error: " << e.msg() << "»" ESC_END; - } - str << " "; + seen.insert(&v); + + str << "[ "; + if (maxDepth > 0) + for (unsigned int n = 0; n < v.listSize(); ++n) { + if (seen.find(v.listElems()[n]) != seen.end()) + str << "«repeated»"; + else + try { + printValue(str, *v.listElems()[n], maxDepth - 1, seen); + } catch (AssertionError& e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; } - else - str << "... "; - str << "]"; - break; + str << " "; + } + else + str << "... "; + str << "]"; + break; case tLambda: { - std::ostringstream s; - s << v.lambda.fun->pos; - str << ESC_BLU "«lambda @ " << filterANSIEscapes(s.str()) << "»" ESC_END; - break; + std::ostringstream s; + s << v.lambda.fun->pos; + str << ESC_BLU "«lambda @ " << filterANSIEscapes(s.str()) << "»" ESC_END; + break; } case tPrimOp: - str << ESC_MAG "«primop»" ESC_END; - break; + str << ESC_MAG "«primop»" ESC_END; + break; case tPrimOpApp: - str << ESC_BLU "«primop-app»" ESC_END; - break; + str << ESC_BLU "«primop-app»" ESC_END; + break; case tFloat: - str << v.fpoint; - break; + str << v.fpoint; + break; default: - str << ESC_RED "«unknown»" ESC_END; - break; - } + str << ESC_RED "«unknown»" ESC_END; + break; + } - return str; + return str; } -struct CmdRepl : StoreCommand, MixEvalArgs -{ - std::vector<std::string> files; +struct CmdRepl : StoreCommand, MixEvalArgs { + std::vector<std::string> files; - CmdRepl() - { - expectArgs("files", &files); - } + CmdRepl() { expectArgs("files", &files); } - std::string name() override { return "repl"; } + std::string name() override { return "repl"; } - std::string description() override - { - return "start an interactive environment for evaluating Nix expressions"; - } + std::string description() override { + return "start an interactive environment for evaluating Nix expressions"; + } - void run(ref<Store> store) override - { - auto repl = std::make_unique<NixRepl>(searchPath, openStore()); - repl->autoArgs = getAutoArgs(repl->state); - repl->mainLoop(files); - } + void run(ref<Store> store) override { + auto repl = std::make_unique<NixRepl>(searchPath, openStore()); + repl->autoArgs = getAutoArgs(repl->state); + repl->mainLoop(files); + } }; static RegisterCommand r1(make_ref<CmdRepl>()); -} +} // namespace nix diff --git a/third_party/nix/src/nix/run.cc b/third_party/nix/src/nix/run.cc index 90b76d6663e9..d78589172c1b 100644 --- a/third_party/nix/src/nix/run.cc +++ b/third_party/nix/src/nix/run.cc @@ -1,13 +1,13 @@ +#include "affinity.hh" #include "command.hh" #include "common-args.hh" -#include "shared.hh" -#include "store-api.hh" #include "derivations.hh" -#include "local-store.hh" #include "finally.hh" #include "fs-accessor.hh" +#include "local-store.hh" #include "progress-bar.hh" -#include "affinity.hh" +#include "shared.hh" +#include "store-api.hh" #if __linux__ #include <sys/mount.h> @@ -19,242 +19,229 @@ using namespace nix; std::string chrootHelperName = "__run_in_chroot"; -struct CmdRun : InstallablesCommand -{ - std::vector<std::string> command = { "bash" }; - StringSet keep, unset; - bool ignoreEnvironment = false; - - CmdRun() - { - mkFlag() - .longName("command") - .shortName('c') - .description("command and arguments to be executed; defaults to 'bash'") - .labels({"command", "args"}) - .arity(ArityAny) - .handler([&](std::vector<std::string> ss) { - if (ss.empty()) throw UsageError("--command requires at least one argument"); - command = ss; - }); - - mkFlag() - .longName("ignore-environment") - .shortName('i') - .description("clear the entire environment (except those specified with --keep)") - .set(&ignoreEnvironment, true); - - mkFlag() - .longName("keep") - .shortName('k') - .description("keep specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); }); - - mkFlag() - .longName("unset") - .shortName('u') - .description("unset specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); }); - } - - std::string name() override - { - return "run"; - } - - std::string description() override - { - return "run a shell in which the specified packages are available"; - } - - Examples examples() override - { - return { - Example{ - "To start a shell providing GNU Hello from NixOS 17.03:", - "nix run -f channel:nixos-17.03 hello" - }, - Example{ - "To start a shell providing youtube-dl from your 'nixpkgs' channel:", - "nix run nixpkgs.youtube-dl" - }, - Example{ - "To run GNU Hello:", - "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'" - }, - Example{ - "To run GNU Hello in a chroot store:", - "nix run --store ~/my-nix nixpkgs.hello -c hello" - }, - }; +struct CmdRun : InstallablesCommand { + std::vector<std::string> command = {"bash"}; + StringSet keep, unset; + bool ignoreEnvironment = false; + + CmdRun() { + mkFlag() + .longName("command") + .shortName('c') + .description("command and arguments to be executed; defaults to 'bash'") + .labels({"command", "args"}) + .arity(ArityAny) + .handler([&](std::vector<std::string> ss) { + if (ss.empty()) + throw UsageError("--command requires at least one argument"); + command = ss; + }); + + mkFlag() + .longName("ignore-environment") + .shortName('i') + .description( + "clear the entire environment (except those specified with --keep)") + .set(&ignoreEnvironment, true); + + mkFlag() + .longName("keep") + .shortName('k') + .description("keep specified environment variable") + .arity(1) + .labels({"name"}) + .handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); }); + + mkFlag() + .longName("unset") + .shortName('u') + .description("unset specified environment variable") + .arity(1) + .labels({"name"}) + .handler( + [&](std::vector<std::string> ss) { unset.insert(ss.front()); }); + } + + std::string name() override { return "run"; } + + std::string description() override { + return "run a shell in which the specified packages are available"; + } + + Examples examples() override { + return { + Example{"To start a shell providing GNU Hello from NixOS 17.03:", + "nix run -f channel:nixos-17.03 hello"}, + Example{"To start a shell providing youtube-dl from your 'nixpkgs' " + "channel:", + "nix run nixpkgs.youtube-dl"}, + Example{"To run GNU Hello:", + "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'"}, + Example{"To run GNU Hello in a chroot store:", + "nix run --store ~/my-nix nixpkgs.hello -c hello"}, + }; + } + + void run(ref<Store> store) override { + auto outPaths = toStorePaths(store, Build, installables); + + auto accessor = store->getFSAccessor(); + + if (ignoreEnvironment) { + if (!unset.empty()) + throw UsageError( + "--unset does not make sense with --ignore-environment"); + + std::map<std::string, std::string> kept; + for (auto& var : keep) { + auto s = getenv(var.c_str()); + if (s) kept[var] = s; + } + + clearEnv(); + + for (auto& var : kept) setenv(var.first.c_str(), var.second.c_str(), 1); + + } else { + if (!keep.empty()) + throw UsageError( + "--keep does not make sense without --ignore-environment"); + + for (auto& var : unset) unsetenv(var.c_str()); } - void run(ref<Store> store) override - { - auto outPaths = toStorePaths(store, Build, installables); - - auto accessor = store->getFSAccessor(); - - if (ignoreEnvironment) { - - if (!unset.empty()) - throw UsageError("--unset does not make sense with --ignore-environment"); - - std::map<std::string, std::string> kept; - for (auto & var : keep) { - auto s = getenv(var.c_str()); - if (s) kept[var] = s; - } + std::unordered_set<Path> done; + std::queue<Path> todo; + for (auto& path : outPaths) todo.push(path); - clearEnv(); + auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":"); - for (auto & var : kept) - setenv(var.first.c_str(), var.second.c_str(), 1); + while (!todo.empty()) { + Path path = todo.front(); + todo.pop(); + if (!done.insert(path).second) continue; - } else { + if (true) unixPath.push_front(path + "/bin"); - if (!keep.empty()) - throw UsageError("--keep does not make sense without --ignore-environment"); - - for (auto & var : unset) - unsetenv(var.c_str()); - } - - std::unordered_set<Path> done; - std::queue<Path> todo; - for (auto & path : outPaths) todo.push(path); - - auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":"); + auto propPath = path + "/nix-support/propagated-user-env-packages"; + if (accessor->stat(propPath).type == FSAccessor::tRegular) { + for (auto& p : tokenizeString<Paths>(readFile(propPath))) todo.push(p); + } + } - while (!todo.empty()) { - Path path = todo.front(); - todo.pop(); - if (!done.insert(path).second) continue; + setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); - if (true) - unixPath.push_front(path + "/bin"); + std::string cmd = *command.begin(); + Strings args; + for (auto& arg : command) args.push_back(arg); - auto propPath = path + "/nix-support/propagated-user-env-packages"; - if (accessor->stat(propPath).type == FSAccessor::tRegular) { - for (auto & p : tokenizeString<Paths>(readFile(propPath))) - todo.push(p); - } - } + stopProgressBar(); - setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); + restoreSignals(); - std::string cmd = *command.begin(); - Strings args; - for (auto & arg : command) args.push_back(arg); + restoreAffinity(); - stopProgressBar(); + /* If this is a diverted store (i.e. its "logical" location + (typically /nix/store) differs from its "physical" location + (e.g. /home/eelco/nix/store), then run the command in a + chroot. For non-root users, this requires running it in new + mount and user namespaces. Unfortunately, + unshare(CLONE_NEWUSER) doesn't work in a multithreaded + program (which "nix" is), so we exec() a single-threaded + helper program (chrootHelper() below) to do the work. */ + auto store2 = store.dynamic_pointer_cast<LocalStore>(); - restoreSignals(); + if (store2 && store->storeDir != store2->realStoreDir) { + Strings helperArgs = {chrootHelperName, store->storeDir, + store2->realStoreDir, cmd}; + for (auto& arg : args) helperArgs.push_back(arg); - restoreAffinity(); + execv(readLink("/proc/self/exe").c_str(), + stringsToCharPtrs(helperArgs).data()); - /* If this is a diverted store (i.e. its "logical" location - (typically /nix/store) differs from its "physical" location - (e.g. /home/eelco/nix/store), then run the command in a - chroot. For non-root users, this requires running it in new - mount and user namespaces. Unfortunately, - unshare(CLONE_NEWUSER) doesn't work in a multithreaded - program (which "nix" is), so we exec() a single-threaded - helper program (chrootHelper() below) to do the work. */ - auto store2 = store.dynamic_pointer_cast<LocalStore>(); + throw SysError("could not execute chroot helper"); + } - if (store2 && store->storeDir != store2->realStoreDir) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd }; - for (auto & arg : args) helperArgs.push_back(arg); + execvp(cmd.c_str(), stringsToCharPtrs(args).data()); - execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); + throw SysError("unable to exec '%s'", cmd); + } +}; - throw SysError("could not execute chroot helper"); - } +static RegisterCommand r1(make_ref<CmdRun>()); - execvp(cmd.c_str(), stringsToCharPtrs(args).data()); +void chrootHelper(int argc, char** argv) { + int p = 1; + std::string storeDir = argv[p++]; + std::string realStoreDir = argv[p++]; + std::string cmd = argv[p++]; + Strings args; + while (p < argc) args.push_back(argv[p++]); - throw SysError("unable to exec '%s'", cmd); +#if __linux__ + uid_t uid = getuid(); + uid_t gid = getgid(); + + if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1) + /* Try with just CLONE_NEWNS in case user namespaces are + specifically disabled. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + + /* Bind-mount realStoreDir on /nix/store. If the latter mount + point doesn't already exists, we have to create a chroot + environment containing the mount point and bind mounts for the + children of /. Would be nice if we could use overlayfs here, + but that doesn't work in a user namespace yet (Ubuntu has a + patch for this: + https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */ + if (!pathExists(storeDir)) { + // FIXME: Use overlayfs? + + Path tmpDir = createTempDir(); + + createDirs(tmpDir + storeDir); + + if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, + 0) == -1) + throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); + + for (auto entry : readDirectory("/")) { + auto src = "/" + entry.name; + auto st = lstat(src); + if (!S_ISDIR(st.st_mode)) continue; + Path dst = tmpDir + "/" + entry.name; + if (pathExists(dst)) continue; + if (mkdir(dst.c_str(), 0700) == -1) + throw SysError("creating directory '%s'", dst); + if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("mounting '%s' on '%s'", src, dst); } -}; -static RegisterCommand r1(make_ref<CmdRun>()); + char* cwd = getcwd(0, 0); + if (!cwd) throw SysError("getting current directory"); + Finally freeCwd([&]() { free(cwd); }); -void chrootHelper(int argc, char * * argv) -{ - int p = 1; - std::string storeDir = argv[p++]; - std::string realStoreDir = argv[p++]; - std::string cmd = argv[p++]; - Strings args; - while (p < argc) - args.push_back(argv[p++]); + if (chroot(tmpDir.c_str()) == -1) + throw SysError(format("chrooting into '%s'") % tmpDir); -#if __linux__ - uid_t uid = getuid(); - uid_t gid = getgid(); - - if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1) - /* Try with just CLONE_NEWNS in case user namespaces are - specifically disabled. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - - /* Bind-mount realStoreDir on /nix/store. If the latter mount - point doesn't already exists, we have to create a chroot - environment containing the mount point and bind mounts for the - children of /. Would be nice if we could use overlayfs here, - but that doesn't work in a user namespace yet (Ubuntu has a - patch for this: - https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */ - if (!pathExists(storeDir)) { - // FIXME: Use overlayfs? - - Path tmpDir = createTempDir(); - - createDirs(tmpDir + storeDir); - - if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1) - throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - - for (auto entry : readDirectory("/")) { - auto src = "/" + entry.name; - auto st = lstat(src); - if (!S_ISDIR(st.st_mode)) continue; - Path dst = tmpDir + "/" + entry.name; - if (pathExists(dst)) continue; - if (mkdir(dst.c_str(), 0700) == -1) - throw SysError("creating directory '%s'", dst); - if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("mounting '%s' on '%s'", src, dst); - } - - char * cwd = getcwd(0, 0); - if (!cwd) throw SysError("getting current directory"); - Finally freeCwd([&]() { free(cwd); }); - - if (chroot(tmpDir.c_str()) == -1) - throw SysError(format("chrooting into '%s'") % tmpDir); - - if (chdir(cwd) == -1) - throw SysError(format("chdir to '%s' in chroot") % cwd); - } else - if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1) - throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - - writeFile("/proc/self/setgroups", "deny"); - writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1)); - writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1)); + if (chdir(cwd) == -1) + throw SysError(format("chdir to '%s' in chroot") % cwd); + } else if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == + -1) + throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - execvp(cmd.c_str(), stringsToCharPtrs(args).data()); + writeFile("/proc/self/setgroups", "deny"); + writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1)); + writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1)); - throw SysError("unable to exec '%s'", cmd); + execvp(cmd.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to exec '%s'", cmd); #else - throw Error("mounting the Nix store on '%s' is not supported on this platform", storeDir); + throw Error( + "mounting the Nix store on '%s' is not supported on this platform", + storeDir); #endif } diff --git a/third_party/nix/src/nix/search.cc b/third_party/nix/src/nix/search.cc index eb75493e4759..fab8bba6283a 100644 --- a/third_party/nix/src/nix/search.cc +++ b/third_party/nix/src/nix/search.cc @@ -1,280 +1,262 @@ +#include <fstream> +#include <regex> #include "command.hh" -#include "globals.hh" -#include "eval.hh" +#include "common-args.hh" #include "eval-inline.hh" -#include "names.hh" +#include "eval.hh" #include "get-drvs.hh" -#include "common-args.hh" -#include "json.hh" +#include "globals.hh" #include "json-to-value.hh" +#include "json.hh" +#include "names.hh" #include "shared.hh" -#include <regex> -#include <fstream> - using namespace nix; -std::string wrap(std::string prefix, std::string s) -{ - return prefix + s + ANSI_NORMAL; +std::string wrap(std::string prefix, std::string s) { + return prefix + s + ANSI_NORMAL; } -std::string hilite(const std::string & s, const std::smatch & m, std::string postfix) -{ - return - m.empty() - ? s - : std::string(m.prefix()) - + ANSI_RED + std::string(m.str()) + postfix - + std::string(m.suffix()); +std::string hilite(const std::string& s, const std::smatch& m, + std::string postfix) { + return m.empty() ? s + : std::string(m.prefix()) + ANSI_RED + std::string(m.str()) + + postfix + std::string(m.suffix()); } -struct CmdSearch : SourceExprCommand, MixJSON -{ - std::vector<std::string> res; +struct CmdSearch : SourceExprCommand, MixJSON { + std::vector<std::string> res; + + bool writeCache = true; + bool useCache = true; + + CmdSearch() { + expectArgs("regex", &res); + + mkFlag() + .longName("update-cache") + .shortName('u') + .description("update the package search cache") + .handler([&]() { + writeCache = true; + useCache = false; + }); + + mkFlag() + .longName("no-cache") + .description("do not use or update the package search cache") + .handler([&]() { + writeCache = false; + useCache = false; + }); + } + + std::string name() override { return "search"; } + + std::string description() override { return "query available packages"; } + + Examples examples() override { + return {Example{"To show all available packages:", "nix search"}, + Example{"To show any packages containing 'blender' in its name or " + "description:", + "nix search blender"}, + Example{"To search for Firefox or Chromium:", + "nix search 'firefox|chromium'"}, + Example{"To search for git and frontend or gui:", + "nix search git 'frontend|gui'"}}; + } + + void run(ref<Store> store) override { + settings.readOnlyMode = true; + + // Empty search string should match all packages + // Use "^" here instead of ".*" due to differences in resulting highlighting + // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) + if (res.empty()) { + res.push_back("^"); + } - bool writeCache = true; - bool useCache = true; + std::vector<std::regex> regexes; + regexes.reserve(res.size()); - CmdSearch() - { - expectArgs("regex", &res); + for (auto& re : res) { + regexes.push_back( + std::regex(re, std::regex::extended | std::regex::icase)); + } - mkFlag() - .longName("update-cache") - .shortName('u') - .description("update the package search cache") - .handler([&]() { writeCache = true; useCache = false; }); + auto state = getEvalState(); - mkFlag() - .longName("no-cache") - .description("do not use or update the package search cache") - .handler([&]() { writeCache = false; useCache = false; }); - } + auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr; - std::string name() override - { - return "search"; - } + auto sToplevel = state->symbols.create("_toplevel"); + auto sRecurse = state->symbols.create("recurseForDerivations"); - std::string description() override - { - return "query available packages"; - } + bool fromCache = false; - Examples examples() override - { - return { - Example{ - "To show all available packages:", - "nix search" - }, - Example{ - "To show any packages containing 'blender' in its name or description:", - "nix search blender" - }, - Example{ - "To search for Firefox or Chromium:", - "nix search 'firefox|chromium'" - }, - Example{ - "To search for git and frontend or gui:", - "nix search git 'frontend|gui'" - } - }; - } + std::map<std::string, std::string> results; - void run(ref<Store> store) override - { - settings.readOnlyMode = true; + std::function<void(Value*, std::string, bool, JSONObject*)> doExpr; - // Empty search string should match all packages - // Use "^" here instead of ".*" due to differences in resulting highlighting - // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) - if (res.empty()) { - res.push_back("^"); - } + doExpr = [&](Value* v, std::string attrPath, bool toplevel, + JSONObject* cache) { + debug("at attribute '%s'", attrPath); + + try { + uint found = 0; - std::vector<std::regex> regexes; - regexes.reserve(res.size()); + state->forceValue(*v); - for (auto &re : res) { - regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase)); + if (v->type == tLambda && toplevel) { + Value* v2 = state->allocValue(); + state->autoCallFunction(*state->allocBindings(1), *v, *v2); + v = v2; + state->forceValue(*v); } - auto state = getEvalState(); + if (state->isDerivation(*v)) { + DrvInfo drv(*state, attrPath, v->attrs); + std::string description; + std::smatch attrPathMatch; + std::smatch descriptionMatch; + std::smatch nameMatch; + std::string name; - auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr; - - auto sToplevel = state->symbols.create("_toplevel"); - auto sRecurse = state->symbols.create("recurseForDerivations"); - - bool fromCache = false; - - std::map<std::string, std::string> results; - - std::function<void(Value *, std::string, bool, JSONObject *)> doExpr; - - doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) { - debug("at attribute '%s'", attrPath); - - try { - uint found = 0; - - state->forceValue(*v); - - if (v->type == tLambda && toplevel) { - Value * v2 = state->allocValue(); - state->autoCallFunction(*state->allocBindings(1), *v, *v2); - v = v2; - state->forceValue(*v); - } - - if (state->isDerivation(*v)) { - - DrvInfo drv(*state, attrPath, v->attrs); - std::string description; - std::smatch attrPathMatch; - std::smatch descriptionMatch; - std::smatch nameMatch; - std::string name; - - DrvName parsed(drv.queryName()); - - for (auto ®ex : regexes) { - std::regex_search(attrPath, attrPathMatch, regex); - - name = parsed.name; - std::regex_search(name, nameMatch, regex); - - description = drv.queryMetaString("description"); - std::replace(description.begin(), description.end(), '\n', ' '); - std::regex_search(description, descriptionMatch, regex); - - if (!attrPathMatch.empty() - || !nameMatch.empty() - || !descriptionMatch.empty()) - { - found++; - } - } - - if (found == res.size()) { - if (json) { - - auto jsonElem = jsonOut->object(attrPath); - - jsonElem.attr("pkgName", parsed.name); - jsonElem.attr("version", parsed.version); - jsonElem.attr("description", description); - - } else { - auto name = hilite(parsed.name, nameMatch, "\e[0;2m") - + std::string(parsed.fullName, parsed.name.length()); - results[attrPath] = fmt( - "* %s (%s)\n %s\n", - wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")), - wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")), - hilite(description, descriptionMatch, ANSI_NORMAL)); - } - } - - if (cache) { - cache->attr("type", "derivation"); - cache->attr("name", drv.queryName()); - cache->attr("system", drv.querySystem()); - if (description != "") { - auto meta(cache->object("meta")); - meta.attr("description", description); - } - } - } - - else if (v->type == tAttrs) { - - if (!toplevel) { - auto attrs = v->attrs; - Bindings::iterator j = attrs->find(sRecurse); - if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) { - debug("skip attribute '%s'", attrPath); - return; - } - } - - bool toplevel2 = false; - if (!fromCache) { - Bindings::iterator j = v->attrs->find(sToplevel); - toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos); - } - - for (auto & i : *v->attrs) { - auto cache2 = - cache ? std::make_unique<JSONObject>(cache->object(i.name)) : nullptr; - doExpr(i.value, - attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name, - toplevel2 || fromCache, cache2 ? cache2.get() : nullptr); - } - } - - } catch (AssertionError & e) { - } catch (Error & e) { - if (!toplevel) { - e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath)); - throw; - } - } - }; + DrvName parsed(drv.queryName()); - Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json"; + for (auto& regex : regexes) { + std::regex_search(attrPath, attrPathMatch, regex); - if (useCache && pathExists(jsonCacheFileName)) { + name = parsed.name; + std::regex_search(name, nameMatch, regex); - warn("using cached results; pass '-u' to update the cache"); + description = drv.queryMetaString("description"); + std::replace(description.begin(), description.end(), '\n', ' '); + std::regex_search(description, descriptionMatch, regex); - Value vRoot; - parseJSON(*state, readFile(jsonCacheFileName), vRoot); + if (!attrPathMatch.empty() || !nameMatch.empty() || + !descriptionMatch.empty()) { + found++; + } + } + + if (found == res.size()) { + if (json) { + auto jsonElem = jsonOut->object(attrPath); + + jsonElem.attr("pkgName", parsed.name); + jsonElem.attr("version", parsed.version); + jsonElem.attr("description", description); + + } else { + auto name = hilite(parsed.name, nameMatch, "\e[0;2m") + + std::string(parsed.fullName, parsed.name.length()); + results[attrPath] = fmt( + "* %s (%s)\n %s\n", + wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")), + wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")), + hilite(description, descriptionMatch, ANSI_NORMAL)); + } + } + + if (cache) { + cache->attr("type", "derivation"); + cache->attr("name", drv.queryName()); + cache->attr("system", drv.querySystem()); + if (description != "") { + auto meta(cache->object("meta")); + meta.attr("description", description); + } + } + } - fromCache = true; + else if (v->type == tAttrs) { + if (!toplevel) { + auto attrs = v->attrs; + Bindings::iterator j = attrs->find(sRecurse); + if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) { + debug("skip attribute '%s'", attrPath); + return; + } + } + + bool toplevel2 = false; + if (!fromCache) { + Bindings::iterator j = v->attrs->find(sToplevel); + toplevel2 = + j != v->attrs->end() && state->forceBool(*j->value, *j->pos); + } + + for (auto& i : *v->attrs) { + auto cache2 = + cache ? std::make_unique<JSONObject>(cache->object(i.name)) + : nullptr; + doExpr(i.value, + attrPath == "" ? (std::string)i.name + : attrPath + "." + (std::string)i.name, + toplevel2 || fromCache, cache2 ? cache2.get() : nullptr); + } + } - doExpr(&vRoot, "", true, nullptr); + } catch (AssertionError& e) { + } catch (Error& e) { + if (!toplevel) { + e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath)); + throw; } + } + }; - else { - createDirs(dirOf(jsonCacheFileName)); + Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json"; - Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid()); + if (useCache && pathExists(jsonCacheFileName)) { + warn("using cached results; pass '-u' to update the cache"); - std::ofstream jsonCacheFile; + Value vRoot; + parseJSON(*state, readFile(jsonCacheFileName), vRoot); - try { - // iostream considered harmful - jsonCacheFile.exceptions(std::ofstream::failbit); - jsonCacheFile.open(tmpFile); + fromCache = true; - auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr; + doExpr(&vRoot, "", true, nullptr); + } - doExpr(getSourceExpr(*state), "", true, cache.get()); + else { + createDirs(dirOf(jsonCacheFileName)); - } catch (std::exception &) { - /* Fun fact: catching std::ios::failure does not work - due to C++11 ABI shenanigans. - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */ - if (!jsonCacheFile) - throw Error("error writing to %s", tmpFile); - throw; - } + Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid()); - if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) - throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName); - } + std::ofstream jsonCacheFile; - if (results.size() == 0) - throw Error("no results for the given search term(s)!"); + try { + // iostream considered harmful + jsonCacheFile.exceptions(std::ofstream::failbit); + jsonCacheFile.open(tmpFile); - RunPager pager; - for (auto el : results) std::cout << el.second << "\n"; + auto cache = writeCache + ? std::make_unique<JSONObject>(jsonCacheFile, false) + : nullptr; + doExpr(getSourceExpr(*state), "", true, cache.get()); + + } catch (std::exception&) { + /* Fun fact: catching std::ios::failure does not work + due to C++11 ABI shenanigans. + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */ + if (!jsonCacheFile) throw Error("error writing to %s", tmpFile); + throw; + } + + if (writeCache && + rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) + throw SysError("cannot rename '%s' to '%s'", tmpFile, + jsonCacheFileName); } + + if (results.size() == 0) + throw Error("no results for the given search term(s)!"); + + RunPager pager; + for (auto el : results) std::cout << el.second << "\n"; + } }; static RegisterCommand r1(make_ref<CmdSearch>()); diff --git a/third_party/nix/src/nix/show-config.cc b/third_party/nix/src/nix/show-config.cc index 86638b50d2c6..dba94732a9c3 100644 --- a/third_party/nix/src/nix/show-config.cc +++ b/third_party/nix/src/nix/show-config.cc @@ -1,40 +1,30 @@ #include "command.hh" #include "common-args.hh" +#include "json.hh" #include "shared.hh" #include "store-api.hh" -#include "json.hh" using namespace nix; -struct CmdShowConfig : Command, MixJSON -{ - CmdShowConfig() - { - } +struct CmdShowConfig : Command, MixJSON { + CmdShowConfig() {} - std::string name() override - { - return "show-config"; - } + std::string name() override { return "show-config"; } - std::string description() override - { - return "show the Nix configuration"; - } + std::string description() override { return "show the Nix configuration"; } - void run() override - { - if (json) { - // FIXME: use appropriate JSON types (bool, ints, etc). - JSONObject jsonObj(std::cout); - globalConfig.toJSON(jsonObj); - } else { - std::map<std::string, Config::SettingInfo> settings; - globalConfig.getSettings(settings); - for (auto & s : settings) - std::cout << s.first + " = " + s.second.value + "\n"; - } + void run() override { + if (json) { + // FIXME: use appropriate JSON types (bool, ints, etc). + JSONObject jsonObj(std::cout); + globalConfig.toJSON(jsonObj); + } else { + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (auto& s : settings) + std::cout << s.first + " = " + s.second.value + "\n"; } + } }; static RegisterCommand r1(make_ref<CmdShowConfig>()); diff --git a/third_party/nix/src/nix/show-derivation.cc b/third_party/nix/src/nix/show-derivation.cc index ee94fded364f..44e33cef4c98 100644 --- a/third_party/nix/src/nix/show-derivation.cc +++ b/third_party/nix/src/nix/show-derivation.cc @@ -1,119 +1,103 @@ // FIXME: integrate this with nix path-info? +#include "archive.hh" #include "command.hh" #include "common-args.hh" -#include "store-api.hh" -#include "archive.hh" -#include "json.hh" #include "derivations.hh" +#include "json.hh" +#include "store-api.hh" using namespace nix; -struct CmdShowDerivation : InstallablesCommand -{ - bool recursive = false; - - CmdShowDerivation() - { - mkFlag() - .longName("recursive") - .shortName('r') - .description("include the dependencies of the specified derivations") - .set(&recursive, true); - } - - std::string name() override - { - return "show-derivation"; +struct CmdShowDerivation : InstallablesCommand { + bool recursive = false; + + CmdShowDerivation() { + mkFlag() + .longName("recursive") + .shortName('r') + .description("include the dependencies of the specified derivations") + .set(&recursive, true); + } + + std::string name() override { return "show-derivation"; } + + std::string description() override { + return "show the contents of a store derivation"; + } + + Examples examples() override { + return { + Example{"To show the store derivation that results from evaluating the " + "Hello package:", + "nix show-derivation nixpkgs.hello"}, + Example{"To show the full derivation graph (if available) that " + "produced your NixOS system:", + "nix show-derivation -r /run/current-system"}, + }; + } + + void run(ref<Store> store) override { + auto drvPaths = toDerivations(store, installables, true); + + if (recursive) { + PathSet closure; + store->computeFSClosure(drvPaths, closure); + drvPaths = closure; } - std::string description() override { - return "show the contents of a store derivation"; - } + JSONObject jsonRoot(std::cout, true); - Examples examples() override - { - return { - Example{ - "To show the store derivation that results from evaluating the Hello package:", - "nix show-derivation nixpkgs.hello" - }, - Example{ - "To show the full derivation graph (if available) that produced your NixOS system:", - "nix show-derivation -r /run/current-system" - }, - }; - } + for (auto& drvPath : drvPaths) { + if (!isDerivation(drvPath)) continue; - void run(ref<Store> store) override - { - auto drvPaths = toDerivations(store, installables, true); + auto drvObj(jsonRoot.object(drvPath)); - if (recursive) { - PathSet closure; - store->computeFSClosure(drvPaths, closure); - drvPaths = closure; - } + auto drv = readDerivation(drvPath); { - - JSONObject jsonRoot(std::cout, true); - - for (auto & drvPath : drvPaths) { - if (!isDerivation(drvPath)) continue; - - auto drvObj(jsonRoot.object(drvPath)); - - auto drv = readDerivation(drvPath); - - { - auto outputsObj(drvObj.object("outputs")); - for (auto & output : drv.outputs) { - auto outputObj(outputsObj.object(output.first)); - outputObj.attr("path", output.second.path); - if (output.second.hash != "") { - outputObj.attr("hashAlgo", output.second.hashAlgo); - outputObj.attr("hash", output.second.hash); - } - } - } - - { - auto inputsList(drvObj.list("inputSrcs")); - for (auto & input : drv.inputSrcs) - inputsList.elem(input); + auto outputsObj(drvObj.object("outputs")); + for (auto& output : drv.outputs) { + auto outputObj(outputsObj.object(output.first)); + outputObj.attr("path", output.second.path); + if (output.second.hash != "") { + outputObj.attr("hashAlgo", output.second.hashAlgo); + outputObj.attr("hash", output.second.hash); } + } + } - { - auto inputDrvsObj(drvObj.object("inputDrvs")); - for (auto & input : drv.inputDrvs) { - auto inputList(inputDrvsObj.list(input.first)); - for (auto & outputId : input.second) - inputList.elem(outputId); - } - } + { + auto inputsList(drvObj.list("inputSrcs")); + for (auto& input : drv.inputSrcs) inputsList.elem(input); + } - drvObj.attr("platform", drv.platform); - drvObj.attr("builder", drv.builder); + { + auto inputDrvsObj(drvObj.object("inputDrvs")); + for (auto& input : drv.inputDrvs) { + auto inputList(inputDrvsObj.list(input.first)); + for (auto& outputId : input.second) inputList.elem(outputId); + } + } - { - auto argsList(drvObj.list("args")); - for (auto & arg : drv.args) - argsList.elem(arg); - } + drvObj.attr("platform", drv.platform); + drvObj.attr("builder", drv.builder); - { - auto envObj(drvObj.object("env")); - for (auto & var : drv.env) - envObj.attr(var.first, var.second); - } + { + auto argsList(drvObj.list("args")); + for (auto& arg : drv.args) argsList.elem(arg); } + { + auto envObj(drvObj.object("env")); + for (auto& var : drv.env) envObj.attr(var.first, var.second); } - - std::cout << "\n"; + } } + + std::cout << "\n"; + } }; static RegisterCommand r1(make_ref<CmdShowDerivation>()); diff --git a/third_party/nix/src/nix/sigs.cc b/third_party/nix/src/nix/sigs.cc index b1825c412c2d..da453d1486fe 100644 --- a/third_party/nix/src/nix/sigs.cc +++ b/third_party/nix/src/nix/sigs.cc @@ -1,149 +1,133 @@ +#include <atomic> #include "command.hh" #include "shared.hh" #include "store-api.hh" #include "thread-pool.hh" -#include <atomic> - using namespace nix; -struct CmdCopySigs : StorePathsCommand -{ - Strings substituterUris; - - CmdCopySigs() - { - mkFlag() - .longName("substituter") - .shortName('s') - .labels({"store-uri"}) - .description("use signatures from specified store") - .arity(1) - .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); }); - } +struct CmdCopySigs : StorePathsCommand { + Strings substituterUris; - std::string name() override - { - return "copy-sigs"; - } + CmdCopySigs() { + mkFlag() + .longName("substituter") + .shortName('s') + .labels({"store-uri"}) + .description("use signatures from specified store") + .arity(1) + .handler([&](std::vector<std::string> ss) { + substituterUris.push_back(ss[0]); + }); + } - std::string description() override - { - return "copy path signatures from substituters (like binary caches)"; - } + std::string name() override { return "copy-sigs"; } + + std::string description() override { + return "copy path signatures from substituters (like binary caches)"; + } - void run(ref<Store> store, Paths storePaths) override - { - if (substituterUris.empty()) - throw UsageError("you must specify at least one substituter using '-s'"); + void run(ref<Store> store, Paths storePaths) override { + if (substituterUris.empty()) + throw UsageError("you must specify at least one substituter using '-s'"); - // FIXME: factor out commonality with MixVerify. - std::vector<ref<Store>> substituters; - for (auto & s : substituterUris) - substituters.push_back(openStore(s)); + // FIXME: factor out commonality with MixVerify. + std::vector<ref<Store>> substituters; + for (auto& s : substituterUris) substituters.push_back(openStore(s)); - ThreadPool pool; + ThreadPool pool; - std::string doneLabel = "done"; - std::atomic<size_t> added{0}; + std::string doneLabel = "done"; + std::atomic<size_t> added{0}; - //logger->setExpected(doneLabel, storePaths.size()); + // logger->setExpected(doneLabel, storePaths.size()); - auto doPath = [&](const Path & storePath) { - //Activity act(*logger, lvlInfo, format("getting signatures for '%s'") % storePath); + auto doPath = [&](const Path& storePath) { + // Activity act(*logger, lvlInfo, format("getting signatures for '%s'") % + // storePath); - checkInterrupt(); + checkInterrupt(); - auto info = store->queryPathInfo(storePath); + auto info = store->queryPathInfo(storePath); - StringSet newSigs; + StringSet newSigs; - for (auto & store2 : substituters) { - try { - auto info2 = store2->queryPathInfo(storePath); + for (auto& store2 : substituters) { + try { + auto info2 = store2->queryPathInfo(storePath); - /* Don't import signatures that don't match this - binary. */ - if (info->narHash != info2->narHash || - info->narSize != info2->narSize || - info->references != info2->references) - continue; + /* Don't import signatures that don't match this + binary. */ + if (info->narHash != info2->narHash || + info->narSize != info2->narSize || + info->references != info2->references) + continue; - for (auto & sig : info2->sigs) - if (!info->sigs.count(sig)) - newSigs.insert(sig); - } catch (InvalidPath &) { - } - } + for (auto& sig : info2->sigs) + if (!info->sigs.count(sig)) newSigs.insert(sig); + } catch (InvalidPath&) { + } + } - if (!newSigs.empty()) { - store->addSignatures(storePath, newSigs); - added += newSigs.size(); - } + if (!newSigs.empty()) { + store->addSignatures(storePath, newSigs); + added += newSigs.size(); + } - //logger->incProgress(doneLabel); - }; + // logger->incProgress(doneLabel); + }; - for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, storePath)); + for (auto& storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); - pool.process(); + pool.process(); - printInfo(format("imported %d signatures") % added); - } + printInfo(format("imported %d signatures") % added); + } }; static RegisterCommand r1(make_ref<CmdCopySigs>()); -struct CmdSignPaths : StorePathsCommand -{ - Path secretKeyFile; - - CmdSignPaths() - { - mkFlag() - .shortName('k') - .longName("key-file") - .label("file") - .description("file containing the secret signing key") - .dest(&secretKeyFile); - } +struct CmdSignPaths : StorePathsCommand { + Path secretKeyFile; - std::string name() override - { - return "sign-paths"; - } + CmdSignPaths() { + mkFlag() + .shortName('k') + .longName("key-file") + .label("file") + .description("file containing the secret signing key") + .dest(&secretKeyFile); + } - std::string description() override - { - return "sign the specified paths"; - } + std::string name() override { return "sign-paths"; } - void run(ref<Store> store, Paths storePaths) override - { - if (secretKeyFile.empty()) - throw UsageError("you must specify a secret key file using '-k'"); + std::string description() override { return "sign the specified paths"; } - SecretKey secretKey(readFile(secretKeyFile)); + void run(ref<Store> store, Paths storePaths) override { + if (secretKeyFile.empty()) + throw UsageError("you must specify a secret key file using '-k'"); - size_t added{0}; + SecretKey secretKey(readFile(secretKeyFile)); - for (auto & storePath : storePaths) { - auto info = store->queryPathInfo(storePath); + size_t added{0}; - auto info2(*info); - info2.sigs.clear(); - info2.sign(secretKey); - assert(!info2.sigs.empty()); + for (auto& storePath : storePaths) { + auto info = store->queryPathInfo(storePath); - if (!info->sigs.count(*info2.sigs.begin())) { - store->addSignatures(storePath, info2.sigs); - added++; - } - } + auto info2(*info); + info2.sigs.clear(); + info2.sign(secretKey); + assert(!info2.sigs.empty()); - printInfo(format("added %d signatures") % added); + if (!info->sigs.count(*info2.sigs.begin())) { + store->addSignatures(storePath, info2.sigs); + added++; + } } + + printInfo(format("added %d signatures") % added); + } }; static RegisterCommand r3(make_ref<CmdSignPaths>()); diff --git a/third_party/nix/src/nix/upgrade-nix.cc b/third_party/nix/src/nix/upgrade-nix.cc index 35c44a70cf52..66230a9ddf4f 100644 --- a/third_party/nix/src/nix/upgrade-nix.cc +++ b/third_party/nix/src/nix/upgrade-nix.cc @@ -1,160 +1,156 @@ +#include "attr-path.hh" #include "command.hh" #include "common-args.hh" -#include "store-api.hh" #include "download.hh" #include "eval.hh" -#include "attr-path.hh" #include "names.hh" #include "progress-bar.hh" +#include "store-api.hh" using namespace nix; -struct CmdUpgradeNix : MixDryRun, StoreCommand -{ - Path profileDir; - std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; - - CmdUpgradeNix() +struct CmdUpgradeNix : MixDryRun, StoreCommand { + Path profileDir; + std::string storePathsUrl = + "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/" + "tools/nix-fallback-paths.nix"; + + CmdUpgradeNix() { + mkFlag() + .longName("profile") + .shortName('p') + .labels({"profile-dir"}) + .description("the Nix profile to upgrade") + .dest(&profileDir); + + mkFlag() + .longName("nix-store-paths-url") + .labels({"url"}) + .description( + "URL of the file that contains the store paths of the latest Nix " + "release") + .dest(&storePathsUrl); + } + + std::string name() override { return "upgrade-nix"; } + + std::string description() override { + return "upgrade Nix to the latest stable version"; + } + + Examples examples() override { + return { + Example{"To upgrade Nix to the latest stable version:", + "nix upgrade-nix"}, + Example{ + "To upgrade Nix in a specific profile:", + "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile"}, + }; + } + + void run(ref<Store> store) override { + evalSettings.pureEval = true; + + if (profileDir == "") profileDir = getProfileDir(store); + + printInfo("upgrading Nix in profile '%s'", profileDir); + + Path storePath; { - mkFlag() - .longName("profile") - .shortName('p') - .labels({"profile-dir"}) - .description("the Nix profile to upgrade") - .dest(&profileDir); - - mkFlag() - .longName("nix-store-paths-url") - .labels({"url"}) - .description("URL of the file that contains the store paths of the latest Nix release") - .dest(&storePathsUrl); + Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); + storePath = getLatestNix(store); } - std::string name() override - { - return "upgrade-nix"; - } + auto version = DrvName(storePathToName(storePath)).version; - std::string description() override - { - return "upgrade Nix to the latest stable version"; + if (dryRun) { + stopProgressBar(); + printError("would upgrade to version %s", version); + return; } - Examples examples() override { - return { - Example{ - "To upgrade Nix to the latest stable version:", - "nix upgrade-nix" - }, - Example{ - "To upgrade Nix in a specific profile:", - "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile" - }, - }; + Activity act(*logger, lvlInfo, actUnknown, + fmt("downloading '%s'...", storePath)); + store->ensurePath(storePath); } - void run(ref<Store> store) override { - evalSettings.pureEval = true; - - if (profileDir == "") - profileDir = getProfileDir(store); - - printInfo("upgrading Nix in profile '%s'", profileDir); - - Path storePath; - { - Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); - storePath = getLatestNix(store); - } - - auto version = DrvName(storePathToName(storePath)).version; - - if (dryRun) { - stopProgressBar(); - printError("would upgrade to version %s", version); - return; - } - - { - Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath)); - store->ensurePath(storePath); - } - - { - Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", storePath)); - auto program = storePath + "/bin/nix-env"; - auto s = runProgram(program, false, {"--version"}); - if (s.find("Nix") == std::string::npos) - throw Error("could not verify that '%s' works", program); - } - - stopProgressBar(); + Activity act(*logger, lvlInfo, actUnknown, + fmt("verifying that '%s' works...", storePath)); + auto program = storePath + "/bin/nix-env"; + auto s = runProgram(program, false, {"--version"}); + if (s.find("Nix") == std::string::npos) + throw Error("could not verify that '%s' works", program); + } - { - Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir)); - runProgram(settings.nixBinDir + "/nix-env", false, - {"--profile", profileDir, "-i", storePath, "--no-sandbox"}); - } + stopProgressBar(); - printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); + { + Activity act( + *logger, lvlInfo, actUnknown, + fmt("installing '%s' into profile '%s'...", storePath, profileDir)); + runProgram(settings.nixBinDir + "/nix-env", false, + {"--profile", profileDir, "-i", storePath, "--no-sandbox"}); } - /* Return the profile in which Nix is installed. */ - Path getProfileDir(ref<Store> store) - { - Path where; + printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); + } - for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) - if (pathExists(dir + "/nix-env")) { - where = dir; - break; - } + /* Return the profile in which Nix is installed. */ + Path getProfileDir(ref<Store> store) { + Path where; - if (where == "") - throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); + for (auto& dir : tokenizeString<Strings>(getEnv("PATH"), ":")) + if (pathExists(dir + "/nix-env")) { + where = dir; + break; + } - printInfo("found Nix in '%s'", where); + if (where == "") + throw Error( + "couldn't figure out how Nix is installed, so I can't upgrade it"); - if (hasPrefix(where, "/run/current-system")) - throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); + printInfo("found Nix in '%s'", where); - Path profileDir = dirOf(where); + if (hasPrefix(where, "/run/current-system")) + throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - // Resolve profile to /nix/var/nix/profiles/<name> link. - while (canonPath(profileDir).find("/profiles/") == std::string::npos && isLink(profileDir)) - profileDir = readLink(profileDir); + Path profileDir = dirOf(where); - printInfo("found profile '%s'", profileDir); + // Resolve profile to /nix/var/nix/profiles/<name> link. + while (canonPath(profileDir).find("/profiles/") == std::string::npos && + isLink(profileDir)) + profileDir = readLink(profileDir); - Path userEnv = canonPath(profileDir, true); + printInfo("found profile '%s'", profileDir); - if (baseNameOf(where) != "bin" || - !hasSuffix(userEnv, "user-environment")) - throw Error("directory '%s' does not appear to be part of a Nix profile", where); + Path userEnv = canonPath(profileDir, true); - if (!store->isValidPath(userEnv)) - throw Error("directory '%s' is not in the Nix store", userEnv); + if (baseNameOf(where) != "bin" || !hasSuffix(userEnv, "user-environment")) + throw Error("directory '%s' does not appear to be part of a Nix profile", + where); - return profileDir; - } + if (!store->isValidPath(userEnv)) + throw Error("directory '%s' is not in the Nix store", userEnv); - /* Return the store path of the latest stable Nix. */ - Path getLatestNix(ref<Store> store) - { - // FIXME: use nixos.org? - auto req = DownloadRequest(storePathsUrl); - auto res = getDownloader()->download(req); + return profileDir; + } - auto state = std::make_unique<EvalState>(Strings(), store); - auto v = state->allocValue(); - state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v); - Bindings & bindings(*state->allocBindings(0)); - auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v); + /* Return the store path of the latest stable Nix. */ + Path getLatestNix(ref<Store> store) { + // FIXME: use nixos.org? + auto req = DownloadRequest(storePathsUrl); + auto res = getDownloader()->download(req); - return state->forceString(*v2); - } + auto state = std::make_unique<EvalState>(Strings(), store); + auto v = state->allocValue(); + state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v); + Bindings& bindings(*state->allocBindings(0)); + auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v); + + return state->forceString(*v2); + } }; static RegisterCommand r1(make_ref<CmdUpgradeNix>()); diff --git a/third_party/nix/src/nix/verify.cc b/third_party/nix/src/nix/verify.cc index 8893fded5ed1..87405b7b37d1 100644 --- a/third_party/nix/src/nix/verify.cc +++ b/third_party/nix/src/nix/verify.cc @@ -1,178 +1,167 @@ +#include <atomic> #include "command.hh" #include "shared.hh" #include "store-api.hh" #include "sync.hh" #include "thread-pool.hh" -#include <atomic> - using namespace nix; -struct CmdVerify : StorePathsCommand -{ - bool noContents = false; - bool noTrust = false; - Strings substituterUris; - size_t sigsNeeded = 0; - - CmdVerify() - { - mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); - mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); - mkFlag() - .longName("substituter") - .shortName('s') - .labels({"store-uri"}) - .description("use signatures from specified store") - .arity(1) - .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); }); - mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); - } - - std::string name() override - { - return "verify"; - } - - std::string description() override - { - return "verify the integrity of store paths"; - } - - Examples examples() override - { - return { - Example{ - "To verify the entire Nix store:", - "nix verify --all" - }, - Example{ - "To check whether each path in the closure of Firefox has at least 2 signatures:", - "nix verify -r -n2 --no-contents $(type -p firefox)" - }, - }; - } - - void run(ref<Store> store, Paths storePaths) override - { - std::vector<ref<Store>> substituters; - for (auto & s : substituterUris) - substituters.push_back(openStore(s)); - - auto publicKeys = getDefaultPublicKeys(); - - Activity act(*logger, actVerifyPaths); - - std::atomic<size_t> done{0}; - std::atomic<size_t> untrusted{0}; - std::atomic<size_t> corrupted{0}; - std::atomic<size_t> failed{0}; - std::atomic<size_t> active{0}; - - auto update = [&]() { - act.progress(done, storePaths.size(), active, failed); - }; - - ThreadPool pool; - - auto doPath = [&](const Path & storePath) { - try { - checkInterrupt(); - - Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath)); - - MaintainCount<std::atomic<size_t>> mcActive(active); - update(); - - auto info = store->queryPathInfo(storePath); - - if (!noContents) { - - HashSink sink(info->narHash.type); - store->narFromPath(info->path, sink); - - auto hash = sink.finish(); - - if (hash.first != info->narHash) { - corrupted++; - act2.result(resCorruptedPath, info->path); - printError( - format("path '%s' was modified! expected hash '%s', got '%s'") - % info->path % info->narHash.to_string() % hash.first.to_string()); - } - - } - - if (!noTrust) { - - bool good = false; - - if (info->ultimate && !sigsNeeded) - good = true; - - else { - - StringSet sigsSeen; - size_t actualSigsNeeded = std::max(sigsNeeded, (size_t) 1); - size_t validSigs = 0; - - auto doSigs = [&](StringSet sigs) { - for (auto sig : sigs) { - if (sigsSeen.count(sig)) continue; - sigsSeen.insert(sig); - if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig)) - validSigs++; - } - }; - - if (info->isContentAddressed(*store)) validSigs = ValidPathInfo::maxSigs; - - doSigs(info->sigs); - - for (auto & store2 : substituters) { - if (validSigs >= actualSigsNeeded) break; - try { - auto info2 = store2->queryPathInfo(info->path); - if (info2->isContentAddressed(*store)) validSigs = ValidPathInfo::maxSigs; - doSigs(info2->sigs); - } catch (InvalidPath &) { - } catch (Error & e) { - printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); - } - } - - if (validSigs >= actualSigsNeeded) - good = true; - } +struct CmdVerify : StorePathsCommand { + bool noContents = false; + bool noTrust = false; + Strings substituterUris; + size_t sigsNeeded = 0; + + CmdVerify() { + mkFlag(0, "no-contents", "do not verify the contents of each store path", + &noContents); + mkFlag(0, "no-trust", "do not verify whether each store path is trusted", + &noTrust); + mkFlag() + .longName("substituter") + .shortName('s') + .labels({"store-uri"}) + .description("use signatures from specified store") + .arity(1) + .handler([&](std::vector<std::string> ss) { + substituterUris.push_back(ss[0]); + }); + mkIntFlag('n', "sigs-needed", + "require that each path has at least N valid signatures", + &sigsNeeded); + } + + std::string name() override { return "verify"; } + + std::string description() override { + return "verify the integrity of store paths"; + } + + Examples examples() override { + return { + Example{"To verify the entire Nix store:", "nix verify --all"}, + Example{"To check whether each path in the closure of Firefox has at " + "least 2 signatures:", + "nix verify -r -n2 --no-contents $(type -p firefox)"}, + }; + } + + void run(ref<Store> store, Paths storePaths) override { + std::vector<ref<Store>> substituters; + for (auto& s : substituterUris) substituters.push_back(openStore(s)); + + auto publicKeys = getDefaultPublicKeys(); + + Activity act(*logger, actVerifyPaths); + + std::atomic<size_t> done{0}; + std::atomic<size_t> untrusted{0}; + std::atomic<size_t> corrupted{0}; + std::atomic<size_t> failed{0}; + std::atomic<size_t> active{0}; + + auto update = [&]() { + act.progress(done, storePaths.size(), active, failed); + }; + + ThreadPool pool; + + auto doPath = [&](const Path& storePath) { + try { + checkInterrupt(); + + Activity act2(*logger, lvlInfo, actUnknown, + fmt("checking '%s'", storePath)); + + MaintainCount<std::atomic<size_t>> mcActive(active); + update(); + + auto info = store->queryPathInfo(storePath); + + if (!noContents) { + HashSink sink(info->narHash.type); + store->narFromPath(info->path, sink); + + auto hash = sink.finish(); + + if (hash.first != info->narHash) { + corrupted++; + act2.result(resCorruptedPath, info->path); + printError( + format("path '%s' was modified! expected hash '%s', got '%s'") % + info->path % info->narHash.to_string() % + hash.first.to_string()); + } + } + + if (!noTrust) { + bool good = false; + + if (info->ultimate && !sigsNeeded) + good = true; + + else { + StringSet sigsSeen; + size_t actualSigsNeeded = std::max(sigsNeeded, (size_t)1); + size_t validSigs = 0; + + auto doSigs = [&](StringSet sigs) { + for (auto sig : sigs) { + if (sigsSeen.count(sig)) continue; + sigsSeen.insert(sig); + if (validSigs < ValidPathInfo::maxSigs && + info->checkSignature(publicKeys, sig)) + validSigs++; + } + }; + + if (info->isContentAddressed(*store)) + validSigs = ValidPathInfo::maxSigs; + + doSigs(info->sigs); + + for (auto& store2 : substituters) { + if (validSigs >= actualSigsNeeded) break; + try { + auto info2 = store2->queryPathInfo(info->path); + if (info2->isContentAddressed(*store)) + validSigs = ValidPathInfo::maxSigs; + doSigs(info2->sigs); + } catch (InvalidPath&) { + } catch (Error& e) { + printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % + e.what()); + } + } - if (!good) { - untrusted++; - act2.result(resUntrustedPath, info->path); - printError(format("path '%s' is untrusted") % info->path); - } + if (validSigs >= actualSigsNeeded) good = true; + } - } + if (!good) { + untrusted++; + act2.result(resUntrustedPath, info->path); + printError(format("path '%s' is untrusted") % info->path); + } + } - done++; + done++; - } catch (Error & e) { - printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); - failed++; - } + } catch (Error& e) { + printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + failed++; + } - update(); - }; + update(); + }; - for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, storePath)); + for (auto& storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); - pool.process(); + pool.process(); - throw Exit( - (corrupted ? 1 : 0) | - (untrusted ? 2 : 0) | - (failed ? 4 : 0)); - } + throw Exit((corrupted ? 1 : 0) | (untrusted ? 2 : 0) | (failed ? 4 : 0)); + } }; static RegisterCommand r1(make_ref<CmdVerify>()); diff --git a/third_party/nix/src/nix/why-depends.cc b/third_party/nix/src/nix/why-depends.cc index 325a2be0a793..7b8650a4e8eb 100644 --- a/third_party/nix/src/nix/why-depends.cc +++ b/third_party/nix/src/nix/why-depends.cc @@ -1,267 +1,246 @@ +#include <queue> #include "command.hh" -#include "store-api.hh" -#include "progress-bar.hh" #include "fs-accessor.hh" +#include "progress-bar.hh" #include "shared.hh" - -#include <queue> +#include "store-api.hh" using namespace nix; -static std::string hilite(const std::string & s, size_t pos, size_t len, - const std::string & colour = ANSI_RED) -{ - return - std::string(s, 0, pos) - + colour - + std::string(s, pos, len) - + ANSI_NORMAL - + std::string(s, pos + len); +static std::string hilite(const std::string& s, size_t pos, size_t len, + const std::string& colour = ANSI_RED) { + return std::string(s, 0, pos) + colour + std::string(s, pos, len) + + ANSI_NORMAL + std::string(s, pos + len); } -static std::string filterPrintable(const std::string & s) -{ - std::string res; - for (char c : s) - res += isprint(c) ? c : '.'; - return res; +static std::string filterPrintable(const std::string& s) { + std::string res; + for (char c : s) res += isprint(c) ? c : '.'; + return res; } -struct CmdWhyDepends : SourceExprCommand -{ - std::string _package, _dependency; - bool all = false; - - CmdWhyDepends() - { - expectArg("package", &_package); - expectArg("dependency", &_dependency); - - mkFlag() - .longName("all") - .shortName('a') - .description("show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path") - .set(&all, true); +struct CmdWhyDepends : SourceExprCommand { + std::string _package, _dependency; + bool all = false; + + CmdWhyDepends() { + expectArg("package", &_package); + expectArg("dependency", &_dependency); + + mkFlag() + .longName("all") + .shortName('a') + .description( + "show all edges in the dependency graph leading from 'package' to " + "'dependency', rather than just a shortest path") + .set(&all, true); + } + + std::string name() override { return "why-depends"; } + + std::string description() override { + return "show why a package has another package in its closure"; + } + + Examples examples() override { + return { + Example{"To show one path through the dependency graph leading from " + "Hello to Glibc:", + "nix why-depends nixpkgs.hello nixpkgs.glibc"}, + Example{ + "To show all files and paths in the dependency graph leading from " + "Thunderbird to libX11:", + "nix why-depends --all nixpkgs.thunderbird nixpkgs.xorg.libX11"}, + Example{"To show why Glibc depends on itself:", + "nix why-depends nixpkgs.glibc nixpkgs.glibc"}, + }; + } + + void run(ref<Store> store) override { + auto package = parseInstallable(*this, store, _package, false); + auto packagePath = toStorePath(store, Build, package); + auto dependency = parseInstallable(*this, store, _dependency, false); + auto dependencyPath = toStorePath(store, NoBuild, dependency); + auto dependencyPathHash = storePathToHash(dependencyPath); + + PathSet closure; + store->computeFSClosure({packagePath}, closure, false, false); + + if (!closure.count(dependencyPath)) { + printError("'%s' does not depend on '%s'", package->what(), + dependency->what()); + return; } - std::string name() override - { - return "why-depends"; - } + stopProgressBar(); // FIXME - std::string description() override - { - return "show why a package has another package in its closure"; - } + auto accessor = store->getFSAccessor(); - Examples examples() override - { - return { - Example{ - "To show one path through the dependency graph leading from Hello to Glibc:", - "nix why-depends nixpkgs.hello nixpkgs.glibc" - }, - Example{ - "To show all files and paths in the dependency graph leading from Thunderbird to libX11:", - "nix why-depends --all nixpkgs.thunderbird nixpkgs.xorg.libX11" - }, - Example{ - "To show why Glibc depends on itself:", - "nix why-depends nixpkgs.glibc nixpkgs.glibc" - }, - }; - } + auto const inf = std::numeric_limits<size_t>::max(); + + struct Node { + Path path; + PathSet refs; + PathSet rrefs; + size_t dist = inf; + Node* prev = nullptr; + bool queued = false; + bool visited = false; + }; + + std::map<Path, Node> graph; + + for (auto& path : closure) + graph.emplace(path, Node{path, store->queryPathInfo(path)->references}); + + // Transpose the graph. + for (auto& node : graph) + for (auto& ref : node.second.refs) graph[ref].rrefs.insert(node.first); + + /* Run Dijkstra's shortest path algorithm to get the distance + of every path in the closure to 'dependency'. */ + graph[dependencyPath].dist = 0; + + std::priority_queue<Node*> queue; - void run(ref<Store> store) override - { - auto package = parseInstallable(*this, store, _package, false); - auto packagePath = toStorePath(store, Build, package); - auto dependency = parseInstallable(*this, store, _dependency, false); - auto dependencyPath = toStorePath(store, NoBuild, dependency); - auto dependencyPathHash = storePathToHash(dependencyPath); + queue.push(&graph.at(dependencyPath)); - PathSet closure; - store->computeFSClosure({packagePath}, closure, false, false); + while (!queue.empty()) { + auto& node = *queue.top(); + queue.pop(); - if (!closure.count(dependencyPath)) { - printError("'%s' does not depend on '%s'", package->what(), dependency->what()); - return; + for (auto& rref : node.rrefs) { + auto& node2 = graph.at(rref); + auto dist = node.dist + 1; + if (dist < node2.dist) { + node2.dist = dist; + node2.prev = &node; + if (!node2.queued) { + node2.queued = true; + queue.push(&node2); + } } + } + } - stopProgressBar(); // FIXME + /* Print the subgraph of nodes that have 'dependency' in their + closure (i.e., that have a non-infinite distance to + 'dependency'). Print every edge on a path between `package` + and `dependency`. */ + std::function<void(Node&, const string&, const string&)> printNode; - auto accessor = store->getFSAccessor(); + const string treeConn = "╠═══"; + const string treeLast = "╚═══"; + const string treeLine = "║ "; + const string treeNull = " "; - auto const inf = std::numeric_limits<size_t>::max(); + struct BailOut {}; - struct Node - { - Path path; - PathSet refs; - PathSet rrefs; - size_t dist = inf; - Node * prev = nullptr; - bool queued = false; - bool visited = false; - }; + printNode = [&](Node& node, const string& firstPad, const string& tailPad) { + assert(node.dist != inf); + std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n", firstPad, + node.visited ? "\e[38;5;244m" : "", + firstPad != "" ? "=> " : "", node.path); - std::map<Path, Node> graph; + if (node.path == dependencyPath && !all && packagePath != dependencyPath) + throw BailOut(); - for (auto & path : closure) - graph.emplace(path, Node{path, store->queryPathInfo(path)->references}); + if (node.visited) return; + node.visited = true; - // Transpose the graph. - for (auto & node : graph) - for (auto & ref : node.second.refs) - graph[ref].rrefs.insert(node.first); + /* Sort the references by distance to `dependency` to + ensure that the shortest path is printed first. */ + std::multimap<size_t, Node*> refs; + std::set<std::string> hashes; - /* Run Dijkstra's shortest path algorithm to get the distance - of every path in the closure to 'dependency'. */ - graph[dependencyPath].dist = 0; + for (auto& ref : node.refs) { + if (ref == node.path && packagePath != dependencyPath) continue; + auto& node2 = graph.at(ref); + if (node2.dist == inf) continue; + refs.emplace(node2.dist, &node2); + hashes.insert(storePathToHash(node2.path)); + } - std::priority_queue<Node *> queue; + /* For each reference, find the files and symlinks that + contain the reference. */ + std::map<std::string, Strings> hits; - queue.push(&graph.at(dependencyPath)); + std::function<void(const Path&)> visitPath; - while (!queue.empty()) { - auto & node = *queue.top(); - queue.pop(); + visitPath = [&](const Path& p) { + auto st = accessor->stat(p); - for (auto & rref : node.rrefs) { - auto & node2 = graph.at(rref); - auto dist = node.dist + 1; - if (dist < node2.dist) { - node2.dist = dist; - node2.prev = &node; - if (!node2.queued) { - node2.queued = true; - queue.push(&node2); - } - } + auto p2 = p == node.path ? "/" : std::string(p, node.path.size() + 1); - } + auto getColour = [&](const std::string& hash) { + return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; + }; + + if (st.type == FSAccessor::Type::tDirectory) { + auto names = accessor->readDirectory(p); + for (auto& name : names) visitPath(p + "/" + name); } - /* Print the subgraph of nodes that have 'dependency' in their - closure (i.e., that have a non-infinite distance to - 'dependency'). Print every edge on a path between `package` - and `dependency`. */ - std::function<void(Node &, const string &, const string &)> printNode; - - const string treeConn = "╠═══"; - const string treeLast = "╚═══"; - const string treeLine = "║ "; - const string treeNull = " "; - - struct BailOut { }; - - printNode = [&](Node & node, const string & firstPad, const string & tailPad) { - assert(node.dist != inf); - std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n", - firstPad, - node.visited ? "\e[38;5;244m" : "", - firstPad != "" ? "=> " : "", - node.path); - - if (node.path == dependencyPath && !all - && packagePath != dependencyPath) - throw BailOut(); - - if (node.visited) return; - node.visited = true; - - /* Sort the references by distance to `dependency` to - ensure that the shortest path is printed first. */ - std::multimap<size_t, Node *> refs; - std::set<std::string> hashes; - - for (auto & ref : node.refs) { - if (ref == node.path && packagePath != dependencyPath) continue; - auto & node2 = graph.at(ref); - if (node2.dist == inf) continue; - refs.emplace(node2.dist, &node2); - hashes.insert(storePathToHash(node2.path)); + else if (st.type == FSAccessor::Type::tRegular) { + auto contents = accessor->readFile(p); + + for (auto& hash : hashes) { + auto pos = contents.find(hash); + if (pos != std::string::npos) { + size_t margin = 32; + auto pos2 = pos >= margin ? pos - margin : 0; + hits[hash].emplace_back(fmt( + "%s: …%s…\n", p2, + hilite( + filterPrintable(std::string( + contents, pos2, pos - pos2 + hash.size() + margin)), + pos - pos2, storePathHashLen, getColour(hash)))); } + } + } - /* For each reference, find the files and symlinks that - contain the reference. */ - std::map<std::string, Strings> hits; - - std::function<void(const Path &)> visitPath; - - visitPath = [&](const Path & p) { - auto st = accessor->stat(p); - - auto p2 = p == node.path ? "/" : std::string(p, node.path.size() + 1); - - auto getColour = [&](const std::string & hash) { - return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; - }; - - if (st.type == FSAccessor::Type::tDirectory) { - auto names = accessor->readDirectory(p); - for (auto & name : names) - visitPath(p + "/" + name); - } - - else if (st.type == FSAccessor::Type::tRegular) { - auto contents = accessor->readFile(p); - - for (auto & hash : hashes) { - auto pos = contents.find(hash); - if (pos != std::string::npos) { - size_t margin = 32; - auto pos2 = pos >= margin ? pos - margin : 0; - hits[hash].emplace_back(fmt("%s: …%s…\n", - p2, - hilite(filterPrintable( - std::string(contents, pos2, pos - pos2 + hash.size() + margin)), - pos - pos2, storePathHashLen, - getColour(hash)))); - } - } - } - - else if (st.type == FSAccessor::Type::tSymlink) { - auto target = accessor->readLink(p); - - for (auto & hash : hashes) { - auto pos = target.find(hash); - if (pos != std::string::npos) - hits[hash].emplace_back(fmt("%s -> %s\n", p2, - hilite(target, pos, storePathHashLen, getColour(hash)))); - } - } - }; - - // FIXME: should use scanForReferences(). - - visitPath(node.path); - - RunPager pager; - for (auto & ref : refs) { - auto hash = storePathToHash(ref.second->path); - - bool last = all ? ref == *refs.rbegin() : true; - - for (auto & hit : hits[hash]) { - bool first = hit == *hits[hash].begin(); - std::cout << tailPad - << (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)) - << hit; - if (!all) break; - } - - printNode(*ref.second, - tailPad + (last ? treeNull : treeLine), - tailPad + (last ? treeNull : treeLine)); - } - }; + else if (st.type == FSAccessor::Type::tSymlink) { + auto target = accessor->readLink(p); + + for (auto& hash : hashes) { + auto pos = target.find(hash); + if (pos != std::string::npos) + hits[hash].emplace_back( + fmt("%s -> %s\n", p2, + hilite(target, pos, storePathHashLen, getColour(hash)))); + } + } + }; + + // FIXME: should use scanForReferences(). + + visitPath(node.path); + + RunPager pager; + for (auto& ref : refs) { + auto hash = storePathToHash(ref.second->path); + + bool last = all ? ref == *refs.rbegin() : true; + + for (auto& hit : hits[hash]) { + bool first = hit == *hits[hash].begin(); + std::cout << tailPad + << (first ? (last ? treeLast : treeConn) + : (last ? treeNull : treeLine)) + << hit; + if (!all) break; + } + + printNode(*ref.second, tailPad + (last ? treeNull : treeLine), + tailPad + (last ? treeNull : treeLine)); + } + }; - try { - printNode(graph.at(packagePath), "", ""); - } catch (BailOut & ) { } + try { + printNode(graph.at(packagePath), "", ""); + } catch (BailOut&) { } + } }; static RegisterCommand r1(make_ref<CmdWhyDepends>()); diff --git a/third_party/nix/src/resolve-system-dependencies/resolve-system-dependencies.cc b/third_party/nix/src/resolve-system-dependencies/resolve-system-dependencies.cc index e02cfc03ec97..1b20d65e0b84 100644 --- a/third_party/nix/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/third_party/nix/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -1,15 +1,15 @@ +#include <fcntl.h> +#include <mach-o/loader.h> +#include <mach-o/swap.h> +#include <sys/mman.h> +#include <sys/utsname.h> +#include <algorithm> +#include <fstream> +#include <iostream> #include "derivations.hh" #include "globals.hh" #include "shared.hh" #include "store-api.hh" -#include <sys/utsname.h> -#include <algorithm> -#include <iostream> -#include <fstream> -#include <sys/mman.h> -#include <fcntl.h> -#include <mach-o/loader.h> -#include <mach-o/swap.h> #define DO_SWAP(x, y) ((x) ? OSSwapInt32(y) : (y)) @@ -17,184 +17,172 @@ using namespace nix; static auto cacheDir = Path{}; -Path resolveCacheFile(Path lib) -{ - std::replace(lib.begin(), lib.end(), '/', '%'); - return cacheDir + "/" + lib; +Path resolveCacheFile(Path lib) { + std::replace(lib.begin(), lib.end(), '/', '%'); + return cacheDir + "/" + lib; } -std::set<string> readCacheFile(const Path & file) -{ - return tokenizeString<set<string>>(readFile(file), "\n"); +std::set<string> readCacheFile(const Path& file) { + return tokenizeString<set<string>>(readFile(file), "\n"); } -std::set<std::string> runResolver(const Path & filename) -{ - AutoCloseFD fd = open(filename.c_str(), O_RDONLY); - if (!fd) - throw SysError("opening '%s'", filename); - - struct stat st; - if (fstat(fd.get(), &st)) - throw SysError("statting '%s'", filename); - - if (!S_ISREG(st.st_mode)) { - printError("file '%s' is not a regular file", filename); - return {}; +std::set<std::string> runResolver(const Path& filename) { + AutoCloseFD fd = open(filename.c_str(), O_RDONLY); + if (!fd) throw SysError("opening '%s'", filename); + + struct stat st; + if (fstat(fd.get(), &st)) throw SysError("statting '%s'", filename); + + if (!S_ISREG(st.st_mode)) { + printError("file '%s' is not a regular file", filename); + return {}; + } + + if (st.st_size < sizeof(mach_header_64)) { + printError("file '%s' is too short for a MACH binary", filename); + return {}; + } + + char* obj = (char*)mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd.get(), 0); + if (!obj) throw SysError("mmapping '%s'", filename); + + ptrdiff_t mach64_offset = 0; + + uint32_t magic = ((mach_header_64*)obj)->magic; + if (magic == FAT_CIGAM || magic == FAT_MAGIC) { + bool should_swap = magic == FAT_CIGAM; + uint32_t narches = DO_SWAP(should_swap, ((fat_header*)obj)->nfat_arch); + for (uint32_t i = 0; i < narches; i++) { + fat_arch* arch = + (fat_arch*)(obj + sizeof(fat_header) + sizeof(fat_arch) * i); + if (DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) { + mach64_offset = (ptrdiff_t)DO_SWAP(should_swap, arch->offset); + break; + } } - - if (st.st_size < sizeof(mach_header_64)) { - printError("file '%s' is too short for a MACH binary", filename); - return {}; - } - - char* obj = (char*) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd.get(), 0); - if (!obj) - throw SysError("mmapping '%s'", filename); - - ptrdiff_t mach64_offset = 0; - - uint32_t magic = ((mach_header_64*) obj)->magic; - if (magic == FAT_CIGAM || magic == FAT_MAGIC) { - bool should_swap = magic == FAT_CIGAM; - uint32_t narches = DO_SWAP(should_swap, ((fat_header *) obj)->nfat_arch); - for (uint32_t i = 0; i < narches; i++) { - fat_arch* arch = (fat_arch*) (obj + sizeof(fat_header) + sizeof(fat_arch) * i); - if (DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) { - mach64_offset = (ptrdiff_t) DO_SWAP(should_swap, arch->offset); - break; - } - } - if (mach64_offset == 0) { - printError(format("Could not find any mach64 blobs in file '%1%', continuing...") % filename); - return {}; - } - } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { - mach64_offset = 0; - } else { - printError(format("Object file has unknown magic number '%1%', skipping it...") % magic); - return {}; + if (mach64_offset == 0) { + printError( + format( + "Could not find any mach64 blobs in file '%1%', continuing...") % + filename); + return {}; } - - mach_header_64 * m_header = (mach_header_64 *) (obj + mach64_offset); - - bool should_swap = magic == MH_CIGAM_64; - ptrdiff_t cmd_offset = mach64_offset + sizeof(mach_header_64); - - std::set<string> libs; - for (uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) { - load_command * cmd = (load_command *) (obj + cmd_offset); - switch(DO_SWAP(should_swap, cmd->cmd)) { - case LC_LOAD_UPWARD_DYLIB: - case LC_LOAD_DYLIB: - case LC_REEXPORT_DYLIB: - libs.insert(std::string((char *) cmd + ((dylib_command*) cmd)->dylib.name.offset)); - break; - } - cmd_offset += DO_SWAP(should_swap, cmd->cmdsize); + } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { + mach64_offset = 0; + } else { + printError( + format("Object file has unknown magic number '%1%', skipping it...") % + magic); + return {}; + } + + mach_header_64* m_header = (mach_header_64*)(obj + mach64_offset); + + bool should_swap = magic == MH_CIGAM_64; + ptrdiff_t cmd_offset = mach64_offset + sizeof(mach_header_64); + + std::set<string> libs; + for (uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) { + load_command* cmd = (load_command*)(obj + cmd_offset); + switch (DO_SWAP(should_swap, cmd->cmd)) { + case LC_LOAD_UPWARD_DYLIB: + case LC_LOAD_DYLIB: + case LC_REEXPORT_DYLIB: + libs.insert( + std::string((char*)cmd + ((dylib_command*)cmd)->dylib.name.offset)); + break; } + cmd_offset += DO_SWAP(should_swap, cmd->cmdsize); + } - return libs; + return libs; } -bool isSymlink(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st) == -1) - throw SysError("getting attributes of path '%1%'", path); +bool isSymlink(const Path& path) { + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError("getting attributes of path '%1%'", path); - return S_ISLNK(st.st_mode); + return S_ISLNK(st.st_mode); } -Path resolveSymlink(const Path & path) -{ - auto target = readLink(path); - return hasPrefix(target, "/") - ? target - : dirOf(path) + "/" + target; +Path resolveSymlink(const Path& path) { + auto target = readLink(path); + return hasPrefix(target, "/") ? target : dirOf(path) + "/" + target; } -std::set<string> resolveTree(const Path & path, PathSet & deps) -{ - std::set<string> results; - if (deps.count(path)) - return {}; - deps.insert(path); - for (auto & lib : runResolver(path)) { - results.insert(lib); - for (auto & p : resolveTree(lib, deps)) { - results.insert(p); - } +std::set<string> resolveTree(const Path& path, PathSet& deps) { + std::set<string> results; + if (deps.count(path)) return {}; + deps.insert(path); + for (auto& lib : runResolver(path)) { + results.insert(lib); + for (auto& p : resolveTree(lib, deps)) { + results.insert(p); } - return results; + } + return results; } -std::set<string> getPath(const Path & path) -{ - if (hasPrefix(path, "/dev")) return {}; +std::set<string> getPath(const Path& path) { + if (hasPrefix(path, "/dev")) return {}; - Path cacheFile = resolveCacheFile(path); - if (pathExists(cacheFile)) - return readCacheFile(cacheFile); + Path cacheFile = resolveCacheFile(path); + if (pathExists(cacheFile)) return readCacheFile(cacheFile); - std::set<string> deps, paths; - paths.insert(path); + std::set<string> deps, paths; + paths.insert(path); - Path nextPath(path); - while (isSymlink(nextPath)) { - nextPath = resolveSymlink(nextPath); - paths.insert(nextPath); - } + Path nextPath(path); + while (isSymlink(nextPath)) { + nextPath = resolveSymlink(nextPath); + paths.insert(nextPath); + } - for (auto & t : resolveTree(nextPath, deps)) - paths.insert(t); + for (auto& t : resolveTree(nextPath, deps)) paths.insert(t); - writeFile(cacheFile, concatStringsSep("\n", paths)); + writeFile(cacheFile, concatStringsSep("\n", paths)); - return paths; + return paths; } -int main(int argc, char ** argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); +int main(int argc, char** argv) { + return handleExceptions(argv[0], [&]() { + initNix(); - struct utsname _uname; + struct utsname _uname; - uname(&_uname); + uname(&_uname); - auto cacheParentDir = (format("%1%/dependency-maps") % settings.nixStateDir).str(); + auto cacheParentDir = + (format("%1%/dependency-maps") % settings.nixStateDir).str(); - cacheDir = (format("%1%/%2%-%3%-%4%") - % cacheParentDir - % _uname.machine - % _uname.sysname - % _uname.release).str(); + cacheDir = (format("%1%/%2%-%3%-%4%") % cacheParentDir % _uname.machine % + _uname.sysname % _uname.release) + .str(); - mkdir(cacheParentDir.c_str(), 0755); - mkdir(cacheDir.c_str(), 0755); + mkdir(cacheParentDir.c_str(), 0755); + mkdir(cacheDir.c_str(), 0755); - auto store = openStore(); + auto store = openStore(); - StringSet impurePaths; + StringSet impurePaths; - if (std::string(argv[1]) == "--test") - impurePaths.insert(argv[2]); - else { - auto drv = store->derivationFromPath(Path(argv[1])); - impurePaths = tokenizeString<StringSet>(get(drv.env, "__impureHostDeps")); - impurePaths.insert("/usr/lib/libSystem.dylib"); - } + if (std::string(argv[1]) == "--test") + impurePaths.insert(argv[2]); + else { + auto drv = store->derivationFromPath(Path(argv[1])); + impurePaths = tokenizeString<StringSet>(get(drv.env, "__impureHostDeps")); + impurePaths.insert("/usr/lib/libSystem.dylib"); + } - std::set<string> allPaths; + std::set<string> allPaths; - for (auto & path : impurePaths) - for (auto & p : getPath(path)) - allPaths.insert(p); + for (auto& path : impurePaths) + for (auto& p : getPath(path)) allPaths.insert(p); - std::cout << "extra-chroot-dirs" << std::endl; - for (auto & path : allPaths) - std::cout << path << std::endl; - std::cout << std::endl; - }); + std::cout << "extra-chroot-dirs" << std::endl; + for (auto& path : allPaths) std::cout << path << std::endl; + std::cout << std::endl; + }); } |