From 05f0430de1d8eeae222a1306d4d0f7f407c8ce7d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Jan 2005 14:10:19 +0000 Subject: * Another change to low-level derivations. The last one this year, I promise :-) This allows derivations to specify on *what* output paths of input derivations they are dependent. This helps to prevent unnecessary downloads. For instance, a build might be dependent on the `devel' and `lib' outputs of some library component, but not the `docs' output. --- src/libexpr/primops.cc | 16 +++++++++------- src/libstore/build.cc | 28 ++++++++++++++++----------- src/libstore/derivations-ast.def | 1 + src/libstore/derivations.cc | 41 +++++++++++++++++++++++++++++----------- src/libstore/derivations.hh | 7 ++++++- src/libstore/misc.cc | 7 +++++-- src/libutil/util.cc | 2 ++ 7 files changed, 70 insertions(+), 32 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9335ee3138a4..f10597557203 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -68,17 +68,17 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv) /* For other derivations, replace the inputs paths with recursive calls to this function.*/ - PathSet inputs2; - for (PathSet::iterator i = drv.inputDrvs.begin(); + DerivationInputs inputs2; + for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i) { - Hash h = state.drvHashes[*i]; + Hash h = state.drvHashes[i->first]; if (h.type == htUnknown) { - Derivation drv2 = derivationFromPath(*i); + Derivation drv2 = derivationFromPath(i->first); h = hashDerivationModulo(state, drv2); - state.drvHashes[*i] = h; + state.drvHashes[i->first] = h; } - inputs2.insert(printHash(h)); + inputs2[printHash(h)] = i->second; } drv.inputDrvs = inputs2; @@ -119,7 +119,9 @@ static void processBinding(EvalState & state, Expr e, Derivation & drv, /* !!! supports only single output path */ Path outPath = evalPath(state, a); - drv.inputDrvs.insert(drvPath); + StringSet ids; + ids.insert("out"); + drv.inputDrvs[drvPath] = ids; ss.push_back(outPath); } else throw Error("invalid derivation attribute"); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 60e72c9dc195..9e984a5b3fd9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -448,9 +448,10 @@ void DerivationGoal::haveStoreExpr() } /* Inputs must be built before we can build this goal. */ - for (PathSet::iterator i = drv.inputDrvs.begin(); + /* !!! but if possible, only install the paths that we need */ + for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i) - addWaitee(worker.makeDerivationGoal(*i)); + addWaitee(worker.makeDerivationGoal(i->first)); for (PathSet::iterator i = drv.inputSrcs.begin(); i != drv.inputSrcs.end(); ++i) @@ -812,18 +813,23 @@ bool DerivationGoal::prepareBuild() /* Determine the full set of input paths. */ /* First, the input derivations. */ - for (PathSet::iterator i = drv.inputDrvs.begin(); + for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i) { - /* Add all the output closures of the input derivation `*i' as - input paths. !!! there should be a way to indicate - specific outputs. */ + /* Add the relevant output closures of the input derivation + `*i' as input paths. Only add the closures of output paths + that are specified as inputs. */ /* !!! is `*i' present? */ - assert(isValidPath(*i)); - Derivation inDrv = derivationFromPath(*i); - for (DerivationOutputs::iterator j = inDrv.outputs.begin(); - j != inDrv.outputs.end(); ++j) - computeFSClosure(j->second.path, inputPaths); + assert(isValidPath(i->first)); + Derivation inDrv = derivationFromPath(i->first); + for (StringSet::iterator j = i->second.begin(); + j != i->second.begin(); ++j) + if (inDrv.outputs.find(*j) != inDrv.outputs.end()) + computeFSClosure(inDrv.outputs[*j].path, inputPaths); + else + throw Error( + format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'") + % drvPath % *j % i->first); } for (PathSet::iterator i = inputPaths.begin(); i != inputPaths.end(); ++i) diff --git a/src/libstore/derivations-ast.def b/src/libstore/derivations-ast.def index 2dce8de12c01..38689845883b 100644 --- a/src/libstore/derivations-ast.def +++ b/src/libstore/derivations-ast.def @@ -3,4 +3,5 @@ init initDerivationsHelpers Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm | | string string | ATerm | EnvBinding | +| string ATermList | ATerm | DerivationInput | | string string string string | ATerm | DerivationOutput | diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 2f37c66fb60d..9008be43981c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -26,13 +26,13 @@ static void checkPath(const string & s) } -static void parsePaths(ATermList paths, PathSet & out) +static void parseStrings(ATermList paths, StringSet & out, bool arePaths) { for (ATermIterator i(paths); i; ++i) { if (ATgetType(*i) != AT_APPL) throw badTerm("not a path", *i); string s = aterm2String(*i); - checkPath(s); + if (arePaths) checkPath(s); out.insert(s); } } @@ -65,8 +65,19 @@ Derivation parseDerivation(ATerm t) drv.outputs[aterm2String(id)] = out; } - parsePaths(inDrvs, drv.inputDrvs); - parsePaths(inSrcs, drv.inputSrcs); + for (ATermIterator i(inDrvs); i; ++i) { + ATerm drvPath; + ATermList ids; + if (!matchDerivationInput(*i, drvPath, ids)) + throwBadDrv(t); + Path drvPath2 = aterm2String(drvPath); + checkPath(drvPath2); + StringSet ids2; + parseStrings(ids, ids2, false); + drv.inputDrvs[drvPath2] = ids2; + } + + parseStrings(inSrcs, drv.inputSrcs, true); drv.builder = aterm2String(builder); drv.platform = aterm2String(platform); @@ -88,11 +99,11 @@ Derivation parseDerivation(ATerm t) } -static ATermList unparsePaths(const PathSet & paths) +static ATermList unparseStrings(const StringSet & paths) { ATermList l = ATempty; for (PathSet::const_iterator i = paths.begin(); - i != paths.end(); i++) + i != paths.end(); ++i) l = ATinsert(l, toATerm(*i)); return ATreverse(l); } @@ -102,7 +113,7 @@ ATerm unparseDerivation(const Derivation & drv) { ATermList outputs = ATempty; for (DerivationOutputs::const_iterator i = drv.outputs.begin(); - i != drv.outputs.end(); i++) + i != drv.outputs.end(); ++i) outputs = ATinsert(outputs, makeDerivationOutput( toATerm(i->first), @@ -110,14 +121,22 @@ ATerm unparseDerivation(const Derivation & drv) toATerm(i->second.hashAlgo), toATerm(i->second.hash))); + ATermList inDrvs = ATempty; + for (DerivationInputs::const_iterator i = drv.inputDrvs.begin(); + i != drv.inputDrvs.end(); ++i) + inDrvs = ATinsert(inDrvs, + makeDerivationInput( + toATerm(i->first), + unparseStrings(i->second))); + ATermList args = ATempty; for (Strings::const_iterator i = drv.args.begin(); - i != drv.args.end(); i++) + i != drv.args.end(); ++i) args = ATinsert(args, toATerm(*i)); ATermList env = ATempty; for (StringPairs::const_iterator i = drv.env.begin(); - i != drv.env.end(); i++) + i != drv.env.end(); ++i) env = ATinsert(env, makeEnvBinding( toATerm(i->first), @@ -125,8 +144,8 @@ ATerm unparseDerivation(const Derivation & drv) return makeDerive( ATreverse(outputs), - unparsePaths(drv.inputDrvs), - unparsePaths(drv.inputSrcs), + ATreverse(inDrvs), + unparseStrings(drv.inputSrcs), toATerm(drv.platform), toATerm(drv.builder), ATreverse(args), diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index c264981d265c..9358db2edae0 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -28,12 +28,17 @@ struct DerivationOutput }; typedef map DerivationOutputs; + +/* For inputs that are sub-derivations, we specify exactly which + output IDs we are interested in. */ +typedef map DerivationInputs; + typedef map StringPairs; struct Derivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ - PathSet inputDrvs; /* inputs that are sub-derivations */ + DerivationInputs inputDrvs; /* inputs that are sub-derivations */ PathSet inputSrcs; /* inputs that are sources */ string platform; Path builder; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c585a9aeac5c..dbb9273f7955 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -39,9 +39,12 @@ void storePathRequisites(const Path & storePath, Derivation drv = derivationFromPath(storePath); - for (PathSet::iterator i = drv.inputDrvs.begin(); + for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i) - storePathRequisites(*i, includeOutputs, paths); + /* !!! Maybe this is too strict, since it will include + *all* output paths of the input derivation, not just + the ones needed by this derivation. */ + storePathRequisites(i->first, includeOutputs, paths); for (PathSet::iterator i = drv.inputSrcs.begin(); i != drv.inputSrcs.end(); ++i) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 266ab67f04ee..e77009321d8d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -587,6 +587,8 @@ Strings unpackStrings(const string & s) len |= ((unsigned char) *i++) << 8; len |= ((unsigned char) *i++) << 16; len |= ((unsigned char) *i++) << 24; + + if (len == 0xffffffff) return strings; /* explicit end-of-list */ if (i + len > s.end()) throw Error(format("short db entry: `%1%'") % s); -- cgit 1.4.1