diff options
| -rw-r--r-- | scripts/Makefile.am | 6 | ||||
| -rwxr-xr-x | scripts/nix-generate-regscript | 20 | ||||
| -rwxr-xr-x | scripts/nix-pull-prebuilts | 69 | ||||
| -rwxr-xr-x | scripts/nix-push-prebuilts | 40 | ||||
| -rw-r--r-- | scripts/prebuilts.conf | 4 | ||||
| -rw-r--r-- | src/Makefile.am | 3 | ||||
| -rw-r--r-- | src/fix.cc | 69 | ||||
| -rw-r--r-- | src/nix.cc | 62 | ||||
| -rw-r--r-- | src/util.hh | 36 |
9 files changed, 230 insertions, 79 deletions
diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 4140cdf5b311..f1d008f1e76d 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -1,5 +1,9 @@ -bin_SCRIPTS = nix-generate-regscript nix-switch nix-collect-garbage +bin_SCRIPTS = nix-generate-regscript nix-switch nix-collect-garbage \ + nix-pull-prebuilts nix-push-prebuilts install-exec-local: $(INSTALL) -d $(sysconfdir)/profile.d $(INSTALL_PROGRAM) nix-profile.sh $(sysconfdir)/profile.d/nix.sh + $(INSTALL) -d $(sysconfdir)/nix + # !!! don't overwrite local modifications + $(INSTALL_PROGRAM) prebuilts.conf $(sysconfdir)/nix/prebuilts.conf diff --git a/scripts/nix-generate-regscript b/scripts/nix-generate-regscript deleted file mode 100755 index bf370f8d789e..000000000000 --- a/scripts/nix-generate-regscript +++ /dev/null @@ -1,20 +0,0 @@ -#! /usr/bin/perl -w - -my $dir = shift @ARGV; -$dir || die "missing directory"; -my $url = shift @ARGV; -$url || die "missing base url"; - -chdir $dir || die "cannot chdir to $dir"; - -foreach my $prebuilt (glob("*.tar.bz2")) { - - $prebuilt =~ /-([a-z0-9]+)-([a-z0-9]+).tar.bz2$/ - || die "invalid file name: $prebuilt"; - - my $pkgHash = $1; - my $prebuiltHash = $2; - - print "regprebuilt $pkgHash $prebuiltHash\n"; - print "regurl $prebuiltHash $url/$prebuilt\n"; -} diff --git a/scripts/nix-pull-prebuilts b/scripts/nix-pull-prebuilts new file mode 100755 index 000000000000..91bbf8082b35 --- /dev/null +++ b/scripts/nix-pull-prebuilts @@ -0,0 +1,69 @@ +#! /usr/bin/perl -w + +my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix +my $etcdir = "$prefix/etc/nix"; +my $knowns = "$prefix/var/nix/known-prebuilts"; +my $tmpfile = "$prefix/var/nix/prebuilts.tmp"; + +my $conffile = "$etcdir/prebuilts.conf"; + +sub register { + my $fn = shift; + return unless $fn =~ /([^\/]*)-([0-9a-z]{32})-([0-9a-z]{32})\.tar\.bz2/; + my $id = $1; + my $pkghash = $2; + my $prebuilthash = $3; + print "$pkghash => $prebuilthash ($id)\n"; + system "nix regprebuilt $pkghash $prebuilthash"; + if ($?) { die "`nix regprebuilt' failed"; } + print KNOWNS "$pkghash\n"; +} + +open KNOWNS, ">$knowns"; + +open CONFFILE, "<$conffile"; + +while (<CONFFILE>) { + chomp; + if (/^\s*(\S+)\s*(\#.*)?$/) { + my $url = $1; + + print "obtaining prebuilt list from $url...\n"; + + if ($url =~ /^\//) { + + # It's a local path. + + foreach my $fn (glob "$url/*") { + register $fn; + } + + } else { + + # It's a URL. + + system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape + if ($?) { die "`wget' failed"; } + + open INDEX, "<$tmpfile"; + + while (<INDEX>) { + # Get all links to prebuilts, that is, file names of the + # form foo-HASH-HASH.tar.bz2. + next unless (/HREF=\"([^\"]*)\"/); + my $fn = $1; + next if $fn =~ /\.\./; + next if $fn =~ /\//; + register $fn; + } + + close INDEX; + + unlink $tmpfile; + } + } +} + +close CONFFILE; + +close KNOWNS; diff --git a/scripts/nix-push-prebuilts b/scripts/nix-push-prebuilts new file mode 100755 index 000000000000..2e3029b1699f --- /dev/null +++ b/scripts/nix-push-prebuilts @@ -0,0 +1,40 @@ +#! /usr/bin/perl -w + +my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix +my $etcdir = "$prefix/etc/nix"; +my $exportdir = "$prefix/var/nix/prebuilts/exports"; +my $knowns = "$prefix/var/nix/known-prebuilts"; + +# For performance, put the known hashes in an associative array. +my %knowns = (); +open KNOWNS, "<$knowns"; +while (<KNOWNS>) { + next unless /([0-9a-z]{32})/; + $knowns{$1} = 1; +} +close KNOWNS; + +# For each installed package, check whether a prebuilt is known. + +open PKGS, "nix listinst|"; +open KNOWNS, ">>$knowns"; + +while (<PKGS>) { + chomp; + next unless /([0-9a-z]{32})/; + my $pkghash = $1; + if (!defined $knowns{$1}) { + # No known prebuilt exists for this package; so export it. + print "exporting $pkghash...\n"; + system "nix export '$exportdir' $pkghash"; + if ($?) { die "`nix export' failed"; } + print KNOWNS "$pkghash\n"; + } +} + +close KNOWNS; +close PKGS; + +# Push the prebuilts to the server. !!! FIXME + +system "rsync -av -e ssh '$exportdir' losser:/home/eelco/public_html/nix-prebuilts/"; diff --git a/scripts/prebuilts.conf b/scripts/prebuilts.conf new file mode 100644 index 000000000000..9b950cad4dab --- /dev/null +++ b/scripts/prebuilts.conf @@ -0,0 +1,4 @@ +# A list of URLs or local paths from where we obtain prebuilts. +/nix/var/nix/prebuilts/imports +/nix/var/nix/prebuilts/exports +http://losser.st-lab.cs.uu.nl/~eelco/nix-prebuilts/ diff --git a/src/Makefile.am b/src/Makefile.am index d6dbdcb734e0..31fad892583b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -13,5 +13,8 @@ install-data-local: $(INSTALL) -d $(localstatedir)/nix/descriptors $(INSTALL) -d $(localstatedir)/nix/sources $(INSTALL) -d $(localstatedir)/nix/links + $(INSTALL) -d $(localstatedir)/nix/prebuilts + $(INSTALL) -d $(localstatedir)/nix/prebuilts/imports + $(INSTALL) -d $(localstatedir)/nix/prebuilts/exports $(INSTALL) -d $(prefix)/pkg $(bindir)/nix init diff --git a/src/fix.cc b/src/fix.cc index 052c1d4c9103..286b552c381d 100644 --- a/src/fix.cc +++ b/src/fix.cc @@ -13,7 +13,6 @@ extern "C" { static string nixDescriptorDir; -static string nixSourcesDir; static bool verbose = false; @@ -33,46 +32,6 @@ void registerFile(string filename) throw Error("cannot register " + filename + " with Nix"); } - -/* Return the directory part of the given path, i.e., everything - before the final `/'. */ -string dirOf(string s) -{ - unsigned int pos = s.rfind('/'); - if (pos == string::npos) throw Error("invalid file name"); - return string(s, 0, pos); -} - - -/* Return the base name of the given path, i.e., everything following - the final `/'. */ -string baseNameOf(string s) -{ - unsigned int pos = s.rfind('/'); - if (pos == string::npos) throw Error("invalid file name"); - return string(s, pos + 1); -} - - -/* Download object referenced by the given URL into the sources - directory. Return the file name it was downloaded to. */ -string fetchURL(string url) -{ - string filename = baseNameOf(url); - string fullname = nixSourcesDir + "/" + filename; - struct stat st; - if (stat(fullname.c_str(), &st)) { - /* !!! quoting */ - string shellCmd = - "cd " + nixSourcesDir + " && wget --quiet -N \"" + url + "\""; - int res = system(shellCmd.c_str()); - if (WEXITSTATUS(res) != 0) - throw Error("cannot fetch " + url); - } - return fullname; -} - - Error badTerm(const string & msg, ATerm e) { char * s = ATwriteToString(e); @@ -120,7 +79,7 @@ bool evaluateBool(ATerm e, EvalContext ctx) ATerm evaluate(ATerm e, EvalContext ctx) { char * s; - ATerm e2; + ATerm e2, e3; ATerm eCond, eTrue, eFalse; /* Check for normal forms first. */ @@ -166,6 +125,7 @@ ATerm evaluate(ATerm e, EvalContext ctx) instantiateDescriptor(filename, ctx).c_str()); } +#if 0 /* `Source' copies the specified file to nixSourcesDir, registers it with Nix, and returns the hash of the file. */ else if (ATmatch(e, "Source(<term>)", &e2)) { @@ -185,15 +145,29 @@ ATerm evaluate(ATerm e, EvalContext ctx) registerFile(target); return ATmake("File(<str>)", hashFile(target).c_str()); } +#endif + + /* `Local' registers a file with Nix, and returns the file's + hash. */ + else if (ATmatch(e, "Local(<term>)", &e2)) { + string filename = absPath(evaluateStr(e2, ctx), ctx.dir); /* !!! */ + string hash = hashFile(filename); + return ATmake("File(<str>)", hash.c_str()); + } - /* `Url' fetches a file from the network, caching it in - nixSourcesDir and returning the file name. */ - else if (ATmatch(e, "Url(<term>)", &e2)) { - string url = evaluateStr(e2, ctx); + /* `Url' registers a mapping from a hash to an url with Nix, and + returns the hash. */ + else if (ATmatch(e, "Url(<term>, <term>)", &e2, &e3)) { + string hash = evaluateStr(e2, ctx); + checkHash(hash); + string url = evaluateStr(e3, ctx); +#if 0 if (verbose) cerr << "fetching " << url << endl; string filename = fetchURL(url); - return ATmake("Str(<str>)", filename.c_str()); +#endif + /* !!! register */ + return ATmake("File(<str>)", hash.c_str()); } /* `If' provides conditional evaluation. */ @@ -329,7 +303,6 @@ void run(Strings::iterator argCur, Strings::iterator argEnd) if (homeDir) nixHomeDir = homeDir; nixDescriptorDir = nixHomeDir + "/var/nix/descriptors"; - nixSourcesDir = nixHomeDir + "/var/nix/sources"; for ( ; argCur != argEnd; argCur++) { string arg(*argCur); diff --git a/src/nix.cc b/src/nix.cc index 973c36727ac2..cfe879952c27 100644 --- a/src/nix.cc +++ b/src/nix.cc @@ -28,6 +28,9 @@ static string dbInstPkgs = "pkginst"; static string dbPrebuilts = "prebuilts"; +static string nixSourcesDir; + + /* Wrapper classes that ensures that the database is closed upon object destruction. */ class Db2 : public Db @@ -435,9 +438,9 @@ void exportPkgs(string outDir, string pkgDir = getPkg(hash); string tmpFile = outDir + "/export_tmp"; - string cmd = "cd " + pkgDir + " && tar cvfj " + tmpFile + " ."; + string cmd = "cd " + pkgDir + " && tar cfj " + tmpFile + " ."; int res = system(cmd.c_str()); // !!! escaping - if (WEXITSTATUS(res) != 0) + if (!WIFEXITED(res) || WEXITSTATUS(res) != 0) throw Error("cannot tar " + pkgDir); string prebuiltHash = hashFile(tmpFile); @@ -458,10 +461,12 @@ void regPrebuilt(string pkgHash, string prebuiltHash) } -void registerFile(string filename) +string registerFile(string filename) { filename = absPath(filename); - setDB(dbRefs, hashFile(filename), filename); + string hash = hashFile(filename); + setDB(dbRefs, hash, filename); + return hash; } @@ -618,8 +623,46 @@ void printGraph(Strings::iterator first, Strings::iterator last) } -void run(Strings args) +/* Download object referenced by the given URL into the sources + directory. Return the file name it was downloaded to. */ +string fetchURL(string url) +{ + string filename = baseNameOf(url); + string fullname = nixSourcesDir + "/" + filename; + struct stat st; + if (stat(fullname.c_str(), &st)) { + /* !!! quoting */ + string shellCmd = + "cd " + nixSourcesDir + " && wget --quiet -N \"" + url + "\""; + int res = system(shellCmd.c_str()); + if (WEXITSTATUS(res) != 0) + throw Error("cannot fetch " + url); + } + return fullname; +} + + +void fetch(string id) { + string fn; + + /* Fetch the object referenced by id. */ + if (isHash(id)) { + throw Error("not implemented"); + } else { + fn = fetchURL(id); + } + + /* Register it by hash. */ + string hash = registerFile(fn); + cout << hash << endl; +} + + +void fetch(Strings::iterator first, Strings::iterator last) +{ + for (Strings::iterator it = first; it != last; it++) + fetch(*it); } @@ -675,6 +718,11 @@ Subcommands: graph HASH... Like closure, but print a dot graph specification. + + fetch ID... + Fetch the objects identified by ID and place them in the Nix + sources directory. ID can be a hash or URL. Print out the hash + of the object. "; } @@ -686,6 +734,8 @@ void run(Strings::iterator argCur, Strings::iterator argEnd) char * homeDir = getenv(nixHomeDirEnvVar.c_str()); if (homeDir) nixHomeDir = homeDir; + nixSourcesDir = nixHomeDir + "/var/nix/sources"; + /* Parse the global flags. */ for ( ; argCur != argEnd; argCur++) { string arg(*argCur); @@ -742,6 +792,8 @@ void run(Strings::iterator argCur, Strings::iterator argEnd) printClosure(argCur, argEnd); } else if (cmd == "graph") { printGraph(argCur, argEnd); + } else if (cmd == "fetch") { + fetch(argCur, argEnd); } else throw UsageError("unknown command: " + string(cmd)); } diff --git a/src/util.hh b/src/util.hh index fb405b0f1200..9b3f212de359 100644 --- a/src/util.hh +++ b/src/util.hh @@ -85,17 +85,22 @@ string printHash(unsigned char * buf) /* Verify that a reference is valid (that is, is a MD5 hash code). */ -void checkHash(const string & s) +bool isHash(const string & s) { - string err = "invalid reference: " + s; - if (s.length() != 32) - throw BadRefError(err); + if (s.length() != 32) return false; for (int i = 0; i < 32; i++) { char c = s[i]; if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) - throw BadRefError(err); + return false; } + return true; +} + + +void checkHash(const string & s) +{ + if (!isHash(s)) throw BadRefError("invalid reference: " + s); } @@ -113,4 +118,25 @@ string hashFile(string filename) } + +/* Return the directory part of the given path, i.e., everything + before the final `/'. */ +string dirOf(string s) +{ + unsigned int pos = s.rfind('/'); + if (pos == string::npos) throw Error("invalid file name"); + return string(s, 0, pos); +} + + +/* Return the base name of the given path, i.e., everything following + the final `/'. */ +string baseNameOf(string s) +{ + unsigned int pos = s.rfind('/'); + if (pos == string::npos) throw Error("invalid file name"); + return string(s, pos + 1); +} + + #endif /* !__UTIL_H */ |