From 3c92ea399d717dc45b3fa91424c0dadc0239ebf2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 2 Aug 2008 12:54:35 +0000 Subject: * Make nix-env --dry-run print the paths to be substituted correctly again. (After the previous substituter mechanism refactoring I didn't update the code that obtains the references of substitutable paths.) This required some refactoring: the substituter programs are now kept running and receive/respond to info requests via stdin/stdout. --- src/libstore/build.cc | 171 +++++++++++--------------------------------------- 1 file changed, 36 insertions(+), 135 deletions(-) (limited to 'src/libstore/build.cc') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1d624723f92b..22f1c64ae3a8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -184,11 +184,6 @@ private: /* Goals waiting for a build slot. */ WeakGoals wantingToBuild; - /* Goals waiting for info from substituters (using --query-info), - and the info they're (collectively) waiting for. */ - WeakGoals waitingForInfo; - map requestedInfo; - /* Child processes currently running. */ Children children; @@ -243,11 +238,6 @@ public: call is made to childTerminate(..., true). */ void waitForChildTermination(GoalPtr goal); - /* Put `goal' to sleep until the top-level loop has run `sub' to - get info about `storePath' (with --query-info). We combine - substituter invocations to reduce overhead. */ - void waitForInfo(GoalPtr goal, Path sub, Path storePath); - /* Wait for any goal to finish. Pretty indiscriminate way to wait for some resource that some other goal is holding. */ void waitForAnyGoal(GoalPtr goal); @@ -257,12 +247,6 @@ public: /* Wait for input to become available. */ void waitForInput(); - -private: - - /* Process the pending paths in requestedInfo and wake up the - goals in waitingForInfo. */ - void getInfo(); }; @@ -2042,12 +2026,12 @@ void DerivationGoal::initChild() } /* Close all other file descriptors. */ - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO - && (!inHook || (fd != 3 && fd != 4))) - close(fd); /* ignore result */ + set exceptions; + if (inHook) { + exceptions.insert(3); + exceptions.insert(4); + } + closeMostFDs(exceptions); } @@ -2118,10 +2102,8 @@ private: /* The current substituter. */ Path sub; - /* Path info returned by the substituter's --query-info operation. */ - bool infoOkay; - PathSet references; - Path deriver; + /* Path info returned by the substituter's query info operation. */ + SubstitutablePathInfo info; /* Pipe for the substitute's standard output/error. */ Pipe logPipe; @@ -2205,48 +2187,17 @@ void SubstitutionGoal::init() return; } - subs = substituters; - - tryNext(); -} - - -void SubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ + if (!worker.store.querySubstitutablePathInfo(storePath, info)) { printMsg(lvlError, - format("path `%1%' is required, but there is no substituter that can build it") + format("path `%1%' is required, but there is no substituter that knows anything about it") % storePath); amDone(ecFailed); return; } - sub = subs.front(); - subs.pop_front(); - - infoOkay = false; - state = &SubstitutionGoal::gotInfo; - worker.waitForInfo(shared_from_this(), sub, storePath); -} - - -void SubstitutionGoal::gotInfo() -{ - trace("got info"); - - if (!infoOkay) { - tryNext(); - return; - } - /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - for (PathSet::iterator i = references.begin(); - i != references.end(); ++i) + foreach (PathSet::iterator, i, info.references) if (*i != storePath) /* ignore self-references */ addWaitee(worker.makeSubstitutionGoal(*i)); @@ -2268,11 +2219,33 @@ void SubstitutionGoal::referencesValid() return; } - for (PathSet::iterator i = references.begin(); - i != references.end(); ++i) + foreach (PathSet::iterator, i, info.references) if (*i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(*i)); + subs = substituters; + + tryNext(); +} + + +void SubstitutionGoal::tryNext() +{ + trace("trying next substituter"); + + if (subs.size() == 0) { + /* None left. Terminate this goal and let someone else deal + with it. */ + printMsg(lvlError, + format("path `%1%' is required, but there is no substituter that can build it") + % storePath); + amDone(ecFailed); + return; + } + + sub = subs.front(); + subs.pop_front(); + state = &SubstitutionGoal::tryToRun; worker.waitForBuildSlot(shared_from_this()); } @@ -2413,7 +2386,7 @@ void SubstitutionGoal::finished() Hash contentHash = hashPath(htSHA256, storePath); worker.store.registerValidPath(storePath, contentHash, - references, deriver); + info.references, info.deriver); outputLock->setDeletion(true); @@ -2608,14 +2581,6 @@ void Worker::waitForChildTermination(GoalPtr goal) } -void Worker::waitForInfo(GoalPtr goal, Path sub, Path storePath) -{ - debug("wait for info"); - requestedInfo[sub].insert(storePath); - waitingForInfo.insert(goal); -} - - void Worker::waitForAnyGoal(GoalPtr goal) { debug("wait for any goal"); @@ -2623,68 +2588,6 @@ void Worker::waitForAnyGoal(GoalPtr goal) } -void Worker::getInfo() -{ - for (map::iterator i = requestedInfo.begin(); - i != requestedInfo.end(); ++i) - { - Path sub = i->first; - PathSet paths = i->second; - - while (!paths.empty()) { - - /* Run the substituter for at most 100 paths at a time to - prevent command line overflows. */ - PathSet paths2; - while (!paths.empty() && paths2.size() < 100) { - paths2.insert(*paths.begin()); - paths.erase(paths.begin()); - } - - /* Ask the substituter for the references and deriver of - the paths. */ - debug(format("running `%1%' to get info about `%2%'") % sub % showPaths(paths2)); - Strings args; - args.push_back("--query-info"); - args.insert(args.end(), paths2.begin(), paths2.end()); - string res = runProgram(sub, false, args); - std::istringstream str(res); - - while (true) { - ValidPathInfo info = decodeValidPathInfo(str); - if (info.path == "") break; - - /* !!! inefficient */ - for (WeakGoals::iterator k = waitingForInfo.begin(); - k != waitingForInfo.end(); ++k) - { - GoalPtr goal = k->lock(); - if (goal) { - SubstitutionGoal * goal2 = dynamic_cast(goal.get()); - if (goal2->storePath == info.path) { - goal2->references = info.references; - goal2->deriver = info.deriver; - goal2->infoOkay = true; - wakeUp(goal); - } - } - } - } - } - } - - for (WeakGoals::iterator k = waitingForInfo.begin(); - k != waitingForInfo.end(); ++k) - { - GoalPtr goal = k->lock(); - if (goal) wakeUp(goal); - } - - requestedInfo.clear(); - waitingForInfo.clear(); // !!! have we done them all? -} - - void Worker::run(const Goals & _topGoals) { for (Goals::iterator i = _topGoals.begin(); @@ -2711,8 +2614,6 @@ void Worker::run(const Goals & _topGoals) if (topGoals.empty()) break; - getInfo(); - /* Wait for input. */ if (!children.empty()) waitForInput(); -- cgit 1.4.1