diff options
| -rw-r--r-- | configure.ac | 2 | ||||
| -rw-r--r-- | corepkgs/nar/unnar.fix | 6 | ||||
| -rw-r--r-- | scripts/nix-pull.in | 151 | ||||
| -rw-r--r-- | src/dotgraph.cc | 5 | ||||
| -rw-r--r-- | src/expr.cc | 8 | ||||
| -rw-r--r-- | src/expr.hh | 3 | ||||
| -rw-r--r-- | src/fix.cc | 12 | ||||
| -rw-r--r-- | src/normalise.cc | 53 | ||||
| -rw-r--r-- | src/normalise.hh | 8 | ||||
| -rw-r--r-- | src/shared.cc | 3 | ||||
| -rw-r--r-- | src/store.cc | 14 | ||||
| -rw-r--r-- | src/store.hh | 3 |
12 files changed, 174 insertions, 94 deletions
diff --git a/configure.ac b/configure.ac index 8e4a9d12b2ed..8c878f2e30a5 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(nix, "0.3") +AC_INIT(nix, "0.4") AC_CONFIG_SRCDIR(src/nix.cc) AC_CONFIG_AUX_DIR(config) AM_INIT_AUTOMAKE diff --git a/corepkgs/nar/unnar.fix b/corepkgs/nar/unnar.fix index 315be5873552..cd5079e50a05 100644 --- a/corepkgs/nar/unnar.fix +++ b/corepkgs/nar/unnar.fix @@ -1,9 +1,9 @@ -Function(["nar", "name"], +Function(["nar", "outPath"], Package( - [ ("name", Var("name")) + [ ("name", "unnar") + , ("outPath", Var("outPath")) , ("build", Relative("nar/unnar.sh")) , ("nar", Var("nar")) - , ("id", Var("id")) ] ) ) \ No newline at end of file diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in index a3d23ea167af..8cd276801ee6 100644 --- a/scripts/nix-pull.in +++ b/scripts/nix-pull.in @@ -2,11 +2,18 @@ use strict; use IPC::Open2; +use POSIX qw(tmpnam); -my $tmpfile = "@localstatedir@/nix/pull.tmp"; +my $tmpdir; +do { $tmpdir = tmpnam(); } +until mkdir $tmpdir, 0777; + +my $manifest = "$tmpdir/manifest"; my $conffile = "@sysconfdir@/nix/prebuilts.conf"; -my @ids; +#END { unlink $manifest; rmdir $tmpdir; } + +my @srcpaths; my @subs; my @sucs; @@ -20,70 +27,89 @@ while (<CONFFILE>) { chomp; if (/^\s*(\S+)\s*(\#.*)?$/) { my $url = $1; + $url =~ s/\/$//; print "obtaining list of Nix archives at $url...\n"; - system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape + system "wget '$url'/MANIFEST -O '$manifest' 2> /dev/null"; # !!! escape if ($?) { die "`wget' failed"; } - open INDEX, "<$tmpfile"; - - while (<INDEX>) { - # Get all links to prebuilts, that is, file names of the - # form foo-HASH-HASH.tar.bz2. - next unless (/HREF=\"([^\"]*)\"/); - my $fn = $1; - next if $fn =~ /\.\./; - next if $fn =~ /\//; - next unless $fn =~ /^([0-9a-z]{32})-([0-9a-z]{32})(.*)\.nar\.bz2$/; - my $hash = $1; - my $id = $2; - my $outname = $3; - my $fsid; - if ($outname =~ /^-/) { - next unless $outname =~ /^-((s-([0-9a-z]{32}))?.*)$/; - $outname = $1; - $fsid = $3; - } else { - $outname = "unnamed"; - } - - print STDERR "$id ($outname)\n"; - - # Construct a Fix expression that fetches and unpacks a - # Nix archive from the network. - my $fetch = - "App(IncludeFix(\"fetchurl/fetchurl.fix\"), " . - "[(\"url\", \"$url/$fn\"), (\"md5\", \"$hash\")])"; - my $fixexpr = - "App(IncludeFix(\"nar/unnar.fix\"), " . - "[ (\"nar\", $fetch)" . - ", (\"name\", \"$outname\")" . - ", (\"id\", \"$id\")" . - "])"; - - if (!$first) { $fullexpr .= "," }; - $first = 0; - $fullexpr .= $fixexpr; # !!! O(n^2)? - - push @ids, $id; - - # Does the name encode a successor relation? - if (defined $fsid) { - push @sucs, $fsid; - push @sucs, $id; - } + open MANIFEST, "<$manifest"; + + my $inside = 0; + + my $storepath; + my $narname; + my $hash; + my @preds; + + while (<MANIFEST>) { + chomp; + s/\#.*$//g; + next if (/^$/); + + if (!$inside) { + if (/^\{$/) { + $inside = 1; + undef $storepath; + undef $narname; + undef $hash; + @preds = (); + } + else { die "bad line: $_"; } + } else { + if (/^\}$/) { + $inside = 0; + my $fullurl = "$url/$narname"; + print "$storepath\n"; + + # Construct a Fix expression that fetches and unpacks a + # Nix archive from the network. + my $fetch = + "App(IncludeFix(\"fetchurl/fetchurl.fix\"), " . + "[(\"url\", \"$fullurl\"), (\"md5\", \"$hash\")])"; + my $fixexpr = + "App(IncludeFix(\"nar/unnar.fix\"), " . + "[ (\"nar\", $fetch)" . + ", (\"outPath\", \"$storepath\")" . + "])"; + + if (!$first) { $fullexpr .= "," }; + $first = 0; + $fullexpr .= $fixexpr; # !!! O(n^2)? + + push @srcpaths, $storepath; + + foreach my $p (@preds) { + push @sucs, $p; + push @sucs, $storepath; + } + + } + elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { + $storepath = $1; + } + elsif (/^\s*NarName:\s*(\S+)\s*$/) { + $narname = $1; + } + elsif (/^\s*MD5:\s*(\S+)\s*$/) { + $hash = $1; + } + elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) { + push @preds, $1; + } + else { die "bad line: $_"; } + } } - close INDEX; - - unlink $tmpfile; + close MANIFEST; } } $fullexpr .= "]"; + # Instantiate Nix expressions from the Fix expressions we created above. print STDERR "running fix...\n"; my $pid = open2(\*READ, \*WRITE, "fix -") or die "cannot run fix"; @@ -93,23 +119,28 @@ close WRITE; my $i = 0; while (<READ>) { chomp; - die unless /^([0-9a-z]{32})$/; - my $nid = $1; - die unless ($i < scalar @ids); - my $id = $ids[$i++]; - push @subs, $id; - push @subs, $nid; + die unless /^\//; + my $subpath = $_; + die unless ($i < scalar @srcpaths); + my $srcpath = $srcpaths[$i++]; + push @subs, $srcpath; + push @subs, $subpath; + print "$srcpath $subpath\n"; } waitpid $pid, 0; $? == 0 or die "fix failed"; + # Register all substitutes. print STDERR "registering substitutes...\n"; +print "@subs\n"; system "nix --substitute @subs"; if ($?) { die "`nix --substitute' failed"; } + # Register all successors. print STDERR "registering successors...\n"; -system "nix --successor @sucs"; +print "@sucs\n"; +system "nix --successor -vvvv @sucs"; if ($?) { die "`nix --successor' failed"; } diff --git a/src/dotgraph.cc b/src/dotgraph.cc index 514fda325914..36daf7f9966d 100644 --- a/src/dotgraph.cc +++ b/src/dotgraph.cc @@ -1,4 +1,5 @@ #include "dotgraph.hh" +#include "normalise.hh" static string dotQuote(const string & s) @@ -98,8 +99,8 @@ void printDotGraph(const PathSet & roots) if (doneSet.find(nePath) == doneSet.end()) { doneSet.insert(nePath); - - NixExpr ne = parseNixExpr(termFromPath(nePath)); + + NixExpr ne = exprFromPath(nePath); string label, colour; diff --git a/src/expr.cc b/src/expr.cc index cfc4af1f39a8..cead803425ba 100644 --- a/src/expr.cc +++ b/src/expr.cc @@ -22,14 +22,6 @@ Hash hashTerm(ATerm t) } -ATerm termFromPath(const Path & path) -{ - ATerm t = ATreadFromNamedFile(path.c_str()); - if (!t) throw Error(format("cannot read aterm from `%1%'") % path); - return t; -} - - Path writeTerm(ATerm t, const string & suffix) { /* The id of a term is its hash. */ diff --git a/src/expr.hh b/src/expr.hh index b345643223cc..7d0420935f9d 100644 --- a/src/expr.hh +++ b/src/expr.hh @@ -53,9 +53,6 @@ Error badTerm(const format & f, ATerm t); /* Hash an aterm. */ Hash hashTerm(ATerm t); -/* Read an aterm from disk. */ -ATerm termFromPath(const Path & path); - /* Write an aterm to the Nix store directory, and return its path. */ Path writeTerm(ATerm t, const string & suffix); diff --git a/src/fix.cc b/src/fix.cc index cbf759b31763..c1f9c1ad68d0 100644 --- a/src/fix.cc +++ b/src/fix.cc @@ -299,6 +299,7 @@ static Expr evalExpr2(EvalState & state, Expr e) ne.type = NixExpr::neDerivation; ne.derivation.platform = SYSTEM; string name; + Path outPath; Hash outHash; bool outHashGiven = false; bnds = ATempty; @@ -327,6 +328,7 @@ static Expr evalExpr2(EvalState & state, Expr e) if (key == "build") ne.derivation.builder = s; if (key == "name") name = s; + if (key == "outPath") outPath = s; if (key == "id") { outHash = parseHash(s); outHashGiven = true; @@ -343,11 +345,13 @@ static Expr evalExpr2(EvalState & state, Expr e) if (name == "") throw badTerm("no package name specified", e); - /* Hash the Nix expression with no outputs to produce a - unique but deterministic path name for this package. */ + /* Determine the output path. */ if (!outHashGiven) outHash = hashPackage(state, ne); - Path outPath = - canonPath(nixStore + "/" + ((string) outHash).c_str() + "-" + name); + if (outPath == "") + /* Hash the Nix expression with no outputs to produce a + unique but deterministic path name for this package. */ + outPath = + canonPath(nixStore + "/" + ((string) outHash).c_str() + "-" + name); ne.derivation.env["out"] = outPath; ne.derivation.outputs.insert(outPath); diff --git a/src/normalise.cc b/src/normalise.cc index 160130d96634..be71081ffb25 100644 --- a/src/normalise.cc +++ b/src/normalise.cc @@ -122,7 +122,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending) Path nePath = useSuccessor(_nePath); /* Get the Nix expression. */ - NixExpr ne = parseNixExpr(termFromPath(nePath)); + NixExpr ne = exprFromPath(nePath, pending); /* If this is a normal form (i.e., a closure) we are done. */ if (ne.type == NixExpr::neClosure) return nePath; @@ -172,7 +172,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending) { Path nePath2 = useSuccessor(nePath); if (nePath != nePath2) { - NixExpr ne = parseNixExpr(termFromPath(nePath2)); + NixExpr ne = exprFromPath(nePath2, pending); debug(format("skipping build of expression `%1%', someone beat us to it") % (string) nePath); if (ne.type != NixExpr::neClosure) abort(); @@ -193,7 +193,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending) realiseClosure(nfPath, pending); /* !!! nfPath should be a root of the garbage collector while we are building */ - NixExpr ne = parseNixExpr(termFromPath(nfPath)); + NixExpr ne = exprFromPath(nfPath, pending); if (ne.type != NixExpr::neClosure) abort(); for (ClosureElems::iterator j = ne.closure.elems.begin(); j != ne.closure.elems.end(); j++) @@ -364,16 +364,49 @@ void realiseClosure(const Path & nePath, PathSet pending) { Nest nest(lvlDebug, format("realising closure `%1%'") % nePath); - NixExpr ne = parseNixExpr(termFromPath(nePath)); + NixExpr ne = exprFromPath(nePath, pending); if (ne.type != NixExpr::neClosure) throw Error(format("expected closure in `%1%'") % nePath); for (ClosureElems::const_iterator i = ne.closure.elems.begin(); i != ne.closure.elems.end(); i++) - assert(isValidPath(i->first)); -#if 0 - expandId(i->second.id, i->first, "/", pending); -#endif + ensurePath(i->first, pending); +} + + +void ensurePath(const Path & path, PathSet pending) +{ + /* If the path is already valid, we're done. */ + if (isValidPath(path)) return; + + /* Otherwise, try the substitutes. */ + Paths subPaths = querySubstitutes(path); + + for (Paths::iterator i = subPaths.begin(); + i != subPaths.end(); i++) + { + try { + normaliseNixExpr(*i, pending); + if (isValidPath(path)) return; + throw Error(format("substitute failed to produce expected output path")); + } catch (Error & e) { + msg(lvlTalkative, + format("building of substitute `%1%' for `%2%' failed: %3%") + % *i % path % e.what()); + } + } + + throw Error(format("path `%1%' is required, " + "but there are no (successful) substitutes") % path); +} + + +NixExpr exprFromPath(const Path & path, PathSet pending) +{ + ensurePath(path, pending); + ATerm t = ATreadFromNamedFile(path.c_str()); + if (!t) throw Error(format("cannot read aterm from `%1%'") % path); + return parseNixExpr(t); } @@ -381,7 +414,7 @@ PathSet nixExprRoots(const Path & nePath) { PathSet paths; - NixExpr ne = parseNixExpr(termFromPath(nePath)); + NixExpr ne = exprFromPath(nePath); if (ne.type == NixExpr::neClosure) paths.insert(ne.closure.roots.begin(), ne.closure.roots.end()); @@ -401,7 +434,7 @@ static void requisitesWorker(const Path & nePath, if (doneSet.find(nePath) != doneSet.end()) return; doneSet.insert(nePath); - NixExpr ne = parseNixExpr(termFromPath(nePath)); + NixExpr ne = exprFromPath(nePath); if (ne.type == NixExpr::neClosure) for (ClosureElems::iterator i = ne.closure.elems.begin(); diff --git a/src/normalise.hh b/src/normalise.hh index e8e72f5bc857..bbe846404cc0 100644 --- a/src/normalise.hh +++ b/src/normalise.hh @@ -18,6 +18,14 @@ Path normaliseNixExpr(const Path & nePath, PathSet pending = PathSet()); its output paths through substitutes... kaboom!). */ void realiseClosure(const Path & nePath, PathSet pending = PathSet()); +/* Ensure that a path exists, possibly by instantiating it by + realising a substitute. */ +void ensurePath(const Path & path, PathSet pending = PathSet()); + +/* Read a Nix expression, after ensuring its existence through + ensurePath(). */ +NixExpr exprFromPath(const Path & path, PathSet pending = PathSet()); + /* Get the list of root (output) paths of the given Nix expression. */ PathSet nixExprRoots(const Path & nePath); diff --git a/src/shared.cc b/src/shared.cc index dcda0b50abe6..80463308a3f0 100644 --- a/src/shared.cc +++ b/src/shared.cc @@ -47,6 +47,8 @@ static void initAndRun(int argc, char * * argv) } +static char buf[1024]; + int main(int argc, char * * argv) { /* ATerm setup. */ @@ -54,7 +56,6 @@ int main(int argc, char * * argv) ATinit(argc, argv, &bottomOfStack); /* Turn on buffering for cerr. */ - char buf[1024]; cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); try { diff --git a/src/store.cc b/src/store.cc index 7f10c6377dc6..2d223313b612 100644 --- a/src/store.cc +++ b/src/store.cc @@ -175,7 +175,8 @@ void registerSuccessor(const Transaction & txn, Paths revs; nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs); - revs.push_back(srcPath); + if (find(revs.begin(), revs.end(), srcPath) == revs.end()) + revs.push_back(srcPath); nixDB.setString(txn, dbSuccessors, srcPath, sucPath); nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs); @@ -212,7 +213,8 @@ void registerSubstitute(const Path & srcPath, const Path & subPath) Paths revs; nixDB.queryStrings(txn, dbSubstitutesRev, subPath, revs); - revs.push_back(srcPath); + if (find(revs.begin(), revs.end(), srcPath) == revs.end()) + revs.push_back(srcPath); nixDB.setStrings(txn, dbSubstitutes, srcPath, subs); nixDB.setStrings(txn, dbSubstitutesRev, subPath, revs); @@ -221,6 +223,14 @@ void registerSubstitute(const Path & srcPath, const Path & subPath) } +Paths querySubstitutes(const Path & srcPath) +{ + Paths subPaths; + nixDB.queryStrings(noTxn, dbSubstitutes, srcPath, subPaths); + return subPaths; +} + + void registerValidPath(const Transaction & txn, const Path & _path) { Path path(canonPath(_path)); diff --git a/src/store.hh b/src/store.hh index 3d7575c3e86c..dab3d603f802 100644 --- a/src/store.hh +++ b/src/store.hh @@ -42,6 +42,9 @@ Paths queryPredecessors(const Path & sucPath); /* Register a substitute. */ void registerSubstitute(const Path & srcPath, const Path & subPath); +/* Return the substitutes expression for the given path. */ +Paths querySubstitutes(const Path & srcPath); + /* Register the validity of a path. */ void registerValidPath(const Transaction & txn, const Path & path); |