diff options
30 files changed, 648 insertions, 259 deletions
diff --git a/Makefile b/Makefile index d8d4a7cc5768..fe2e88a995aa 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ makefiles = \ src/nix-instantiate/local.mk \ src/nix-env/local.mk \ src/nix-daemon/local.mk \ + src/nix-collect-garbage/local.mk \ src/download-via-ssh/local.mk \ src/nix-log2xml/local.mk \ src/bsdiff-4.3/local.mk \ diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 89b8aac7834f..ec96f750ea8c 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -562,6 +562,40 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> + <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term> + + <listitem> + + + <para>If set, the path to a program that can set extra + derivation-specific settings for this system. This is used for settings + that can't be captured by the derivation model itself and are too variable + between different versions of the same system to be hard-coded into nix. + </para> + + <para>The hook is passed the derivation path and, if chroots are enabled, + the chroot directory. It can then modify the chroot and send a series of + commands to modify various settings to stdout. The currently recognized + commands are:</para> + + <variablelist> + <varlistentry xml:id="extra-chroot-dirs"><term><literal>extra-chroot-dirs</literal></term> + + <listitem> + + <para>Pass a list of files and directories to be included in the + chroot for this build. One entry per line, terminated by an empty + line. Entries have the same format as build-chroot-dirs.</para> + + </listitem> + + </varlistentry> + </variablelist> + </listitem> + + </varlistentry> + + </variablelist> </para> diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index d37f4bcc1418..45a99b27d7bd 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -703,6 +703,44 @@ $ nix-env -e '.*' <lineannotation>(remove everything)</lineannotation></screen> <!--######################################################################--> +<refsection xml:id="rsec-nix-env-set"><title>Operation <option>--set</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <arg choice='plain'><option>--set</option></arg> + <arg choice='plain'><replaceable>drvname</replaceable></arg> +</cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The <option>--set</option> operation modifies the current generation of a +profile so that it contains exactly the specified derivation, and nothing else. +</para> + +</refsection> + +<refsection><title>Examples</title> + +<para> +The following updates a profile such that its current generation will contain +just Firefox: + +<screen> +$ nix-env -p /nix/var/nix/profiles/browser --set firefox</screen> + +</para> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + <refsection xml:id="rsec-nix-env-set-flag"><title>Operation <option>--set-flag</option></title> <refsection><title>Synopsis</title> diff --git a/misc/vim/syntax/nix.vim b/misc/vim/syntax/nix.vim deleted file mode 100644 index ddddea5f0596..000000000000 --- a/misc/vim/syntax/nix.vim +++ /dev/null @@ -1,37 +0,0 @@ -" Vim syntax file -" Language: nix -" Maintainer: Marc Weber <marco-oweber@gmx.de> -" Modify and commit if you feel that way -" Last Change: 2007 Dec - -" Quit when a (custom) syntax file was already loaded -if exists("b:current_syntax") - finish -endif - -syn keyword nixKeyword let throw inherit import true false null with -syn keyword nixConditional if else then -syn keyword nixBrace ( ) { } = -syn keyword nixBuiltin __currentSystem __currentTime __isFunction __getEnv __trace __toPath __pathExists - \ __readFile __toXML __toFile __filterSource __attrNames __getAttr __hasAttr __isAttrs __listToAttrs __isList - \ __head __tail __add __sub __lessThan __substring __stringLength - -syn match nixAttr "\w\+\ze\s*=" -syn match nixFuncArg "\zs\w\+\ze\s*:" -syn region nixStringParam start=+\${+ end=+}+ -syn region nixMultiLineComment start=+/\*+ skip=+\\"+ end=+\*/+ -syn match nixEndOfLineComment "#.*$" -syn region nixStringIndented start=+''+ skip=+'''\|''${\|"+ end=+''+ contains=nixStringParam -syn region nixString start=+"+ skip=+\\"+ end=+"+ contains=nixStringParam - -hi def link nixKeyword Keyword -hi def link nixConditional Conditional -hi def link nixBrace Special -hi def link nixString String -hi def link nixStringIndented String -hi def link nixBuiltin Special -hi def link nixStringParam Macro -hi def link nixMultiLineComment Comment -hi def link nixEndOfLineComment Comment -hi def link nixAttr Identifier -hi def link nixFuncArg Identifier diff --git a/scripts/local.mk b/scripts/local.mk index f4c5e8097de4..39e1df611c5c 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -1,7 +1,6 @@ nix_bin_scripts := \ $(d)/nix-build \ $(d)/nix-channel \ - $(d)/nix-collect-garbage \ $(d)/nix-copy-closure \ $(d)/nix-generate-patches \ $(d)/nix-install-package \ diff --git a/scripts/nix-build.in b/scripts/nix-build.in index 19de6feb6080..5745a1e06e48 100755 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -44,7 +44,7 @@ $SIG{'INT'} = sub { exit 1 }; # Heuristic to see if we're invoked as a shebang script, namely, if we # have a single argument, it's the name of an executable file, and it # starts with "#!". -if ($runEnv && $ARGV[0] !~ /nix-shell/) { +if ($runEnv && defined $ARGV[0] && $ARGV[0] !~ /nix-shell/) { $script = $ARGV[0]; if (-f $script && -x $script) { open SCRIPT, "<$script" or die "$0: cannot open ‘$script’: $!\n"; diff --git a/scripts/nix-channel.in b/scripts/nix-channel.in index b8c93df18c32..0b499c3f7927 100755 --- a/scripts/nix-channel.in +++ b/scripts/nix-channel.in @@ -6,6 +6,7 @@ use File::Basename; use File::Path qw(mkpath); use Nix::Config; use Nix::Manifest; +use File::Temp qw(tempdir); binmode STDERR, ":encoding(utf8)"; @@ -98,42 +99,14 @@ sub update { my $url = $channels{$name}; my $origUrl = "$url/MANIFEST"; - # Check if $url is a redirect. If so, follow it now to ensure - # consistency if the redirection is changed between - # downloading the manifest and the tarball. - my $headers = `$Nix::Config::curl --silent --head '$url'`; + # We want to download the url to a file to see if it's a tarball while also checking if we + # got redirected in the process, so that we can grab the various parts of a nix channel + # definition from a consistent location if the redirect changes mid-download. + my $tmpdir = tempdir( CLEANUP => 1 ); + my $filename; + ($url, $filename) = `cd $tmpdir && $Nix::Config::curl --silent --write-out '%{url_effective}\n%{filename_effective}' -L '$url' -O`; die "$0: unable to check ‘$url’\n" if $? != 0; - $headers =~ s/\r//g; - $url = $1 if $headers =~ /^Location:\s*(.*)\s*$/m; - - # Check if the channel advertises a binary cache. - my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`; - my $extraAttrs = ""; - my $getManifest = ($Nix::Config::config{"force-manifest"} // "false") eq "true"; - if ($? == 0 && $binaryCacheURL ne "") { - $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; "; - deleteOldManifests($origUrl, undef); - } else { - $getManifest = 1; - } - - if ($getManifest) { - # No binary cache, so pull the channel manifest. - mkdir $manifestDir, 0755 unless -e $manifestDir; - die "$0: you do not have write permission to ‘$manifestDir’!\n" unless -W $manifestDir; - $ENV{'NIX_ORIG_URL'} = $origUrl; - system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0 - or die "cannot pull manifest from ‘$url’\n"; - } - - # Download the channel tarball. - my $fullURL = "$url/nixexprs.tar.xz"; - system("$Nix::Config::curl --fail --silent --head '$fullURL' > /dev/null") == 0 or - $fullURL = "$url/nixexprs.tar.bz2"; - print STDERR "downloading Nix expressions from ‘$fullURL’...\n"; - my ($hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url '$fullURL'`; - die "cannot fetch ‘$fullURL’\n" if $? != 0; - chomp $path; + chomp $url; # If the URL contains a version number, append it to the name # attribute (so that "nix-env -q" on the channels profile @@ -141,6 +114,52 @@ sub update { my $cname = $name; $cname .= $1 if basename($url) =~ /(-\d.*)$/; + my $path; + my $ret = -1; + if (-e "$tmpdir/$filename") { + # Get our temporary download into the store + (my $hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url 'file://$tmpdir/$filename'`; + chomp $path; + + # Try unpacking the expressions to see if they'll be valid for us to process later. + # Like anything in nix, this will cache the result so we don't do it again outside of the loop below + $ret = system("$Nix::Config::binDir/nix-build --no-out-link -E 'import <nix/unpack-channel.nix> " . + "{ name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; }'"); + } + + # The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it + my $extraAttrs = ""; + if ($ret != 0) { + # Check if the channel advertises a binary cache. + my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`; + my $getManifest = ($Nix::Config::config{"force-manifest"} // "false") eq "true"; + if ($? == 0 && $binaryCacheURL ne "") { + $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; "; + deleteOldManifests($origUrl, undef); + } else { + $getManifest = 1; + } + + if ($getManifest) { + # No binary cache, so pull the channel manifest. + mkdir $manifestDir, 0755 unless -e $manifestDir; + die "$0: you do not have write permission to ‘$manifestDir’!\n" unless -W $manifestDir; + $ENV{'NIX_ORIG_URL'} = $origUrl; + system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0 + or die "cannot pull manifest from ‘$url’\n"; + } + + # Download the channel tarball. + my $fullURL = "$url/nixexprs.tar.xz"; + system("$Nix::Config::curl --fail --silent --head '$fullURL' > /dev/null") == 0 or + $fullURL = "$url/nixexprs.tar.bz2"; + print STDERR "downloading Nix expressions from ‘$fullURL’...\n"; + (my $hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url '$fullURL'`; + die "cannot fetch ‘$fullURL’\n" if $? != 0; + chomp $path; + } + + # Regardless of where it came from, add the expression representing this channel to accumulated expression $exprs .= "'f: f { name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; $extraAttrs }' "; } diff --git a/scripts/nix-collect-garbage.in b/scripts/nix-collect-garbage.in deleted file mode 100755 index 55e0ba7a6fab..000000000000 --- a/scripts/nix-collect-garbage.in +++ /dev/null @@ -1,65 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use strict; -use Nix::Config; - -my $profilesDir = "@localstatedir@/nix/profiles"; - - -# Process the command line arguments. -my @args = (); -my $arg; - -my $removeOld = 0; -my $gen; -my $dryRun = 0; - -while ($arg = shift) { - if ($arg eq "--delete-old" || $arg eq "-d") { - $removeOld = 1; - $gen = "old"; - } elsif ($arg eq "--delete-older-than") { - $removeOld = 1; - $gen = shift; - } elsif ($arg eq "--dry-run") { - $dryRun = 1; - } elsif ($arg eq "--help") { - exec "man nix-collect-garbage" or die; - } else { - push @args, $arg; - } -} - - -# If `-d' was specified, remove all old generations of all profiles. -# Of course, this makes rollbacks to before this point in time -# impossible. - -sub removeOldGenerations; -sub removeOldGenerations { - my $dir = shift; - - my $dh; - opendir $dh, $dir or die; - - foreach my $name (sort (readdir $dh)) { - next if $name eq "." || $name eq ".."; - $name = $dir . "/" . $name; - if (-l $name && (readlink($name) =~ /link/)) { - print STDERR "removing old generations of profile $name\n"; - - system("$Nix::Config::binDir/nix-env", "-p", $name, "--delete-generations", $gen, $dryRun ? "--dry-run" : ()); - } - elsif (! -l $name && -d $name) { - removeOldGenerations $name; - } - } - - closedir $dh or die; -} - -removeOldGenerations $profilesDir if $removeOld; - - -# Run the actual garbage collector. -exec "$Nix::Config::binDir/nix-store", "--gc", @args unless $dryRun; diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index a91b54bd589f..eb34fcd75129 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -9,6 +9,7 @@ if [ -n "$HOME" ]; then fi export PATH=$NIX_LINK/bin:$NIX_LINK/sbin:$PATH + export MANPATH=$NIX_LINK/share/man:$MANPATH # Subscribe the user to the Nixpkgs channel by default. if [ ! -e $HOME/.nix-channels ]; then diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc index c03d720bde82..13760490d9c4 100644 --- a/src/libexpr/common-opts.cc +++ b/src/libexpr/common-opts.cc @@ -1,5 +1,6 @@ #include "common-opts.hh" -#include "../libmain/shared.hh" +#include "shared.hh" +#include "download.hh" #include "util.hh" @@ -53,7 +54,9 @@ bool parseSearchPathArg(Strings::iterator & i, Path lookupFileArg(EvalState & state, string s) { - if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + if (isUri(s)) + return downloadFileCached(s, true); + else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); } else diff --git a/src/libexpr/download.cc b/src/libexpr/download.cc new file mode 100644 index 000000000000..18ab6fbcda73 --- /dev/null +++ b/src/libexpr/download.cc @@ -0,0 +1,235 @@ +#include "download.hh" +#include "util.hh" +#include "globals.hh" +#include "hash.hh" +#include "store-api.hh" + +#include <curl/curl.h> + +namespace nix { + +struct Curl +{ + CURL * curl; + string data; + string etag, status, expectedETag; + + struct curl_slist * requestHeaders; + + static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp) + { + Curl & c(* (Curl *) userp); + size_t realSize = size * nmemb; + c.data.append((char *) contents, realSize); + return realSize; + } + + static size_t headerCallback(void * contents, size_t size, size_t nmemb, void * userp) + { + Curl & c(* (Curl *) userp); + size_t realSize = size * nmemb; + string line = string((char *) contents, realSize); + printMsg(lvlVomit, format("got header: %1%") % trim(line)); + if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + c.etag = ""; + auto ss = tokenizeString<vector<string>>(line, " "); + c.status = ss.size() >= 2 ? ss[1] : ""; + } else { + auto i = line.find(':'); + if (i != string::npos) { + string name = trim(string(line, 0, i)); + if (name == "ETag") { // FIXME: case + c.etag = trim(string(line, i + 1)); + /* Hack to work around a GitHub bug: it sends + ETags, but ignores If-None-Match. So if we get + the expected ETag on a 200 response, then shut + down the connection because we already have the + data. */ + printMsg(lvlDebug, format("got ETag: %1%") % c.etag); + if (c.etag == c.expectedETag && c.status == "200") { + printMsg(lvlDebug, format("shutting down on 200 HTTP response with expected ETag")); + return 0; + } + } + } + } + return realSize; + } + + static int progressCallback(void * clientp, double dltotal, double dlnow, double ultotal, double ulnow) + { + return _isInterrupted; + } + + Curl() + { + requestHeaders = 0; + + curl = curl_easy_init(); + if (!curl) throw Error("unable to initialize curl"); + + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str()); + curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl); + + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &curl); + + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); + } + + ~Curl() + { + if (curl) curl_easy_cleanup(curl); + if (requestHeaders) curl_slist_free_all(requestHeaders); + } + + bool fetch(const string & url, const string & expectedETag = "") + { + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + + data.clear(); + + if (requestHeaders) { + curl_slist_free_all(requestHeaders); + requestHeaders = 0; + } + + if (!expectedETag.empty()) { + this->expectedETag = expectedETag; + requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + expectedETag).c_str()); + } + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders); + + CURLcode res = curl_easy_perform(curl); + checkInterrupt(); + if (res == CURLE_WRITE_ERROR && etag == expectedETag) return false; + if (res != CURLE_OK) + throw DownloadError(format("unable to download ‘%1%’: %2% (%3%)") + % url % curl_easy_strerror(res) % res); + + long httpStatus = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); + if (httpStatus == 304) return false; + + return true; + } +}; + + +DownloadResult downloadFile(string url, string expectedETag) +{ + DownloadResult res; + Curl curl; + if (curl.fetch(url, expectedETag)) { + res.cached = false; + res.data = curl.data; + } else + res.cached = true; + res.etag = curl.etag; + return res; +} + + +Path downloadFileCached(const string & url, bool unpack) +{ + Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs"; + createDirs(cacheDir); + + string urlHash = printHash32(hashString(htSHA256, url)); + + Path dataFile = cacheDir + "/" + urlHash + ".info"; + Path fileLink = cacheDir + "/" + urlHash + "-file"; + + Path storePath; + + string expectedETag; + + int ttl = settings.get("tarball-ttl", 60 * 60); + bool skip = false; + + if (pathExists(fileLink) && pathExists(dataFile)) { + storePath = readLink(fileLink); + store->addTempRoot(storePath); + if (store->isValidPath(storePath)) { + auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); + if (ss.size() >= 3 && ss[0] == url) { + time_t lastChecked; + if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) + skip = true; + else if (!ss[1].empty()) { + printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]); + expectedETag = ss[1]; + } + } + } else + storePath = ""; + } + + string name; + auto p = url.rfind('/'); + if (p != string::npos) name = string(url, p + 1); + + if (!skip) { + + if (storePath.empty()) + printMsg(lvlInfo, format("downloading ‘%1%’...") % url); + else + printMsg(lvlInfo, format("checking ‘%1%’...") % url); + + try { + auto res = downloadFile(url, expectedETag); + + if (!res.cached) + storePath = store->addTextToStore(name, res.data, PathSet(), false); + + assert(!storePath.empty()); + replaceSymlink(storePath, fileLink); + + writeFile(dataFile, url + "\n" + res.etag + "\n" + int2String(time(0)) + "\n"); + } catch (DownloadError & e) { + if (storePath.empty()) throw; + printMsg(lvlError, format("warning: %1%; using cached result") % e.msg()); + } + } + + if (unpack) { + Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; + Path unpackedStorePath; + if (pathExists(unpackedLink)) { + unpackedStorePath = readLink(unpackedLink); + store->addTempRoot(unpackedStorePath); + if (!store->isValidPath(unpackedStorePath)) + unpackedStorePath = ""; + } + if (unpackedStorePath.empty()) { + printMsg(lvlInfo, format("unpacking ‘%1%’...") % url); + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, ""); + unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, false); + } + replaceSymlink(unpackedStorePath, unpackedLink); + return unpackedStorePath; + } + + return storePath; +} + + +bool isUri(const string & s) +{ + size_t pos = s.find("://"); + if (pos == string::npos) return false; + string scheme(s, 0, pos); + return scheme == "http" || scheme == "https"; +} + + +} diff --git a/src/libexpr/download.hh b/src/libexpr/download.hh new file mode 100644 index 000000000000..28c9117e4227 --- /dev/null +++ b/src/libexpr/download.hh @@ -0,0 +1,22 @@ +#pragma once + +#include "types.hh" +#include <string> + +namespace nix { + +struct DownloadResult +{ + bool cached; + string data, etag; +}; + +DownloadResult downloadFile(string url, string expectedETag = ""); + +Path downloadFileCached(const string & url, bool unpack); + +MakeError(DownloadError, Error) + +bool isUri(const string & s); + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d8a4193a8e64..301f991b7ab9 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -292,6 +292,11 @@ Path EvalState::checkSourcePath(const Path & path_) if (path == i.second || isInDir(path, i.second)) return path; + /* Hack to support the chroot dependencies of corepkgs (see + corepkgs/config.nix.in). */ + if (path == settings.nixPrefix && isStorePath(settings.nixPrefix)) + return path; + throw RestrictedPathError(format("access to path ‘%1%’ is forbidden in restricted mode") % path_); } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 664d6692f51e..26168b2ed420 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -527,6 +527,8 @@ formal #include <unistd.h> #include <eval.hh> +#include <download.hh> +#include <store-api.hh> namespace nix { @@ -611,6 +613,9 @@ void EvalState::addToSearchPath(const string & s, bool warn) path = string(s, pos + 1); } + if (isUri(path)) + path = downloadFileCached(path, true); + path = absPath(path); if (pathExists(path)) { debug(format("adding path ‘%1%’ to the search path") % path); @@ -629,16 +634,17 @@ Path EvalState::findFile(const string & path) Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos) { - foreach (SearchPath::iterator, i, searchPath) { + for (auto & i : searchPath) { + assert(!isUri(i.second)); Path res; - if (i->first.empty()) - res = i->second + "/" + path; + if (i.first.empty()) + res = i.second + "/" + path; else { - if (path.compare(0, i->first.size(), i->first) != 0 || - (path.size() > i->first.size() && path[i->first.size()] != '/')) + if (path.compare(0, i.first.size(), i.first) != 0 || + (path.size() > i.first.size() && path[i.first.size()] != '/')) continue; - res = i->second + - (path.size() == i->first.size() ? "" : "/" + string(path, i->first.size())); + res = i.second + + (path.size() == i.first.size() ? "" : "/" + string(path, i.first.size())); } if (pathExists(res)) return canonPath(res); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e818496460ea..fe2f1b1e0ae1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -9,6 +9,7 @@ #include "json-to-value.hh" #include "names.hh" #include "eval-inline.hh" +#include "download.hh" #include <sys/types.h> #include <sys/stat.h> @@ -18,8 +19,6 @@ #include <cstring> #include <dlfcn.h> -#include <curl/curl.h> - namespace nix { @@ -104,7 +103,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args } w.attrs->sort(); Value fun; - state.evalFile(state.findFile("nix/imported-drv-to-derivation.nix"), fun); + state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun); state.forceFunction(fun, pos); mkApp(v, fun, w); state.forceAttrs(v, pos); @@ -1486,53 +1485,6 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a *************************************************************/ -struct Curl -{ - CURL * curl; - string data; - - static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp) - { - Curl & c(* (Curl *) userp); - size_t realSize = size * nmemb; - c.data.append((char *) contents, realSize); - return realSize; - } - - Curl() - { - curl = curl_easy_init(); - if (!curl) throw Error("unable to initialize curl"); - - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str()); - curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); - - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl); - } - - ~Curl() - { - if (curl) curl_easy_cleanup(curl); - } - - string fetch(const string & url) - { - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - - data.clear(); - - CURLcode res = curl_easy_perform(curl); - if (res != CURLE_OK) - throw Error(format("unable to download ‘%1%’: %2%") - % url % curl_easy_strerror(res)); - - return data; - } -}; - - void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack) { @@ -1560,25 +1512,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, } else url = state.forceStringNoCtx(*args[0], pos); - // TODO: cache downloads. - - Curl curl; - string data = curl.fetch(url); - - string name; - string::size_type p = url.rfind('/'); - if (p != string::npos) name = string(url, p + 1); - - Path storePath = store->addTextToStore(name, data, PathSet(), state.repair); - - if (unpack) { - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, ""); - storePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, state.repair); - } - - mkString(v, storePath, singleton<PathSet>(storePath)); + mkString(v, downloadFileCached(url, unpack), PathSet({url})); } @@ -1738,8 +1672,7 @@ void EvalState::createBaseEnv() /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ - string path = findFile("nix/derivation.nix"); - assert(!path.empty()); + string path = settings.nixDataDir + "/nix/corepkgs/derivation.nix"; sDerivationNix = symbols.create(path); evalFile(path, v); addConstant("derivation", v); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 90bcccd243a7..50c59c1314d9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -20,6 +20,7 @@ #include <sys/types.h> #include <sys/stat.h> #include <sys/utsname.h> +#include <sys/select.h> #include <fcntl.h> #include <unistd.h> #include <errno.h> @@ -1786,8 +1787,10 @@ void DerivationGoal::startBuilder() if (useChroot) { string defaultChrootDirs; +#if CHROOT_ENABLED if (isInStore(BASH_PATH)) defaultChrootDirs = "/bin/sh=" BASH_PATH; +#endif /* Allow a user-configurable set of directories from the host file system. */ @@ -1895,7 +1898,7 @@ void DerivationGoal::startBuilder() build user. */ Path chrootStoreDir = chrootRootDir + settings.nixStore; createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 0730); + chmod_(chrootStoreDir, 01775); if (chown(chrootStoreDir.c_str(), 0, buildUser.getGID()) == -1) throw SysError(format("cannot change ownership of ‘%1%’") % chrootStoreDir); @@ -1969,6 +1972,42 @@ void DerivationGoal::startBuilder() } } + if (settings.preBuildHook != "") { + printMsg(lvlChatty, format("executing pre-build hook ‘%1%’") + % settings.preBuildHook); + auto args = useChroot ? Strings({drvPath, chrootRootDir}) : + Strings({ drvPath }); + enum BuildHookState { + stBegin, + stExtraChrootDirs + }; + auto state = stBegin; + auto lines = runProgram(settings.preBuildHook, false, args); + auto lastPos = std::string::size_type{0}; + for (auto nlPos = lines.find('\n'); nlPos != string::npos; + nlPos = lines.find('\n', lastPos)) { + auto line = std::string{lines, lastPos, nlPos}; + lastPos = nlPos + 1; + if (state == stBegin) { + if (line == "extra-chroot-dirs") { + state = stExtraChrootDirs; + } else { + throw Error(format("unknown pre-build hook command ‘%1%’") + % line); + } + } else if (state == stExtraChrootDirs) { + if (line == "") { + state = stBegin; + } else { + auto p = line.find('='); + if (p == string::npos) + dirsInChroot[line] = line; + else + dirsInChroot[string(line, 0, p)] = string(line, p + 1); + } + } + } + } /* Run the builder. */ printMsg(lvlChatty, format("executing builder ‘%1%’") % drv.builder); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index e382b3aac03a..d5615d93c7d0 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -67,6 +67,7 @@ Settings::Settings() void Settings::processEnvironment() { + nixPrefix = NIX_PREFIX; nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))); nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)); nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)); @@ -143,6 +144,14 @@ bool Settings::get(const string & name, bool def) } +int Settings::get(const string & name, int def) +{ + int res = def; + _get(res, name); + return res; +} + + void Settings::update() { _get(tryFallback, "build-fallback"); @@ -173,6 +182,7 @@ void Settings::update() _get(logServers, "log-servers"); _get(enableImportNative, "allow-unsafe-native-code-during-evaluation"); _get(useCaseHack, "use-case-hack"); + _get(preBuildHook, "pre-build-hook"); string subs = getEnv("NIX_SUBSTITUTERS", "default"); if (subs == "default") { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0230a540e655..60b11afe6088 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -27,6 +27,8 @@ struct Settings { bool get(const string & name, bool def); + int get(const string & name, int def); + void update(); string pack(); @@ -40,6 +42,8 @@ struct Settings { Path nixDataDir; /* !!! fix */ + Path nixPrefix; + /* The directory where we log various operations. */ Path nixLogDir; @@ -202,6 +206,10 @@ struct Settings { /* Whether the importNative primop should be enabled */ bool enableImportNative; + /* The hook to run just before a build to set derivation-specific + build settings */ + Path preBuildHook; + private: SettingsMap settings, overrides; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3ec467649d3b..1dcd7228f347 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -13,6 +13,7 @@ #include <sys/types.h> #include <sys/stat.h> +#include <sys/select.h> #include <sys/time.h> #include <unistd.h> #include <utime.h> @@ -256,7 +257,7 @@ LocalStore::LocalStore(bool reserveSpace) if (chmod(perUserDir.c_str(), 01777) == -1) throw SysError(format("could not set permissions on ‘%1%’ to 1777") % perUserDir); - mode_t perm = 01735; + mode_t perm = 01775; struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); if (!gr) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 78b4d0fd4b94..771c06753a65 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -15,6 +15,7 @@ ifeq ($(OS), SunOS) endif libstore_CXXFLAGS = \ + -DNIX_PREFIX=\"$(prefix)\" \ -DNIX_STORE_DIR=\"$(storedir)\" \ -DNIX_DATA_DIR=\"$(datadir)\" \ -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index be0a9bf317d1..5cda9a0677b5 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -413,6 +413,17 @@ void createSymlink(const Path & target, const Path & link) } +void replaceSymlink(const Path & target, const Path & link) +{ + Path tmp = canonPath(dirOf(link) + "/.new_" + baseNameOf(link)); + + createSymlink(target, tmp); + + if (rename(tmp.c_str(), link.c_str()) != 0) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % link); +} + + LogType logType = ltPretty; Verbosity verbosity = lvlInfo; @@ -931,16 +942,16 @@ string runProgram(Path program, bool searchPath, const Strings & args, checkInterrupt(); /* Create a pipe. */ - Pipe stdout, stdin; - stdout.create(); - if (!input.empty()) stdin.create(); + Pipe out, in; + out.create(); + if (!input.empty()) in.create(); /* Fork. */ Pid pid = startProcess([&]() { - if (dup2(stdout.writeSide, STDOUT_FILENO) == -1) + if (dup2(out.writeSide, STDOUT_FILENO) == -1) throw SysError("dupping stdout"); if (!input.empty()) { - if (dup2(stdin.readSide, STDIN_FILENO) == -1) + if (dup2(in.readSide, STDIN_FILENO) == -1) throw SysError("dupping stdin"); } @@ -956,16 +967,16 @@ string runProgram(Path program, bool searchPath, const Strings & args, throw SysError(format("executing ‘%1%’") % program); }); - stdout.writeSide.close(); + out.writeSide.close(); /* FIXME: This can deadlock if the input is too long. */ if (!input.empty()) { - stdin.readSide.close(); - writeFull(stdin.writeSide, input); - stdin.writeSide.close(); + in.readSide.close(); + writeFull(in.writeSide, input); + in.writeSide.close(); } - string result = drainFD(stdout.readSide); + string result = drainFD(out.readSide); /* Wait for the child to finish. */ int status = pid.wait(true); @@ -1076,6 +1087,15 @@ string chomp(const string & s) } +string trim(const string & s, const string & whitespace) +{ + auto i = s.find_first_not_of(whitespace); + if (i == string::npos) return ""; + auto j = s.find_last_not_of(whitespace); + return string(s, i, j == string::npos ? j : j - i + 1); +} + + string statusToString(int status) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 20330fb7699e..7d20351eec2e 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -110,6 +110,9 @@ Paths createDirs(const Path & path); /* Create a symlink. */ void createSymlink(const Path & target, const Path & link); +/* Atomically create or replace a symlink. */ +void replaceSymlink(const Path & target, const Path & link); + template<class T, class A> T singleton(const A & a) @@ -334,6 +337,10 @@ string concatStringsSep(const string & sep, const StringSet & ss); string chomp(const string & s); +/* Remove whitespace from the start and end of a string. */ +string trim(const string & s, const string & whitespace = " \n\r\t"); + + /* Convert the exit status of a child as returned by wait() into an error string. */ string statusToString(int status); diff --git a/src/nix-collect-garbage/local.mk b/src/nix-collect-garbage/local.mk new file mode 100644 index 000000000000..02d14cf62199 --- /dev/null +++ b/src/nix-collect-garbage/local.mk @@ -0,0 +1,7 @@ +programs += nix-collect-garbage + +nix-collect-garbage_DIR := $(d) + +nix-collect-garbage_SOURCES := $(d)/nix-collect-garbage.cc + +nix-collect-garbage_LIBS = libmain libstore libutil libformat diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc new file mode 100644 index 000000000000..ae75fd62103c --- /dev/null +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -0,0 +1,91 @@ +#include "hash.hh" +#include "shared.hh" +#include "globals.hh" + +#include <iostream> + +using namespace nix; + +std::string gen = "old"; +bool dryRun = false; + +void runProgramSimple(Path program, const Strings & args) +{ + checkInterrupt(); + + /* Fork. */ + Pid pid = startProcess([&]() { + Strings args_(args); + args_.push_front(program); + auto cargs = stringsToCharPtrs(args_); + + execv(program.c_str(), (char * *) &cargs[0]); + + throw SysError(format("executing ‘%1%’") % program); + }); + + pid.wait(true); +} + + +/* If `-d' was specified, remove all old generations of all profiles. + * Of course, this makes rollbacks to before this point in time + * impossible. */ + +void removeOldGenerations(std::string dir) +{ + for (auto & i : readDirectory(dir)) { + checkInterrupt(); + + auto path = dir + "/" + i.name; + auto type = getFileType(path); + + if (type == DT_LNK) { + auto link = readLink(path); + if (link.find("link") != string::npos) { + printMsg(lvlInfo, format("removing old generations of profile %1%") % path); + + auto args = Strings{"-p", path, "--delete-generations", gen}; + if (dryRun) { + args.push_back("--dry-run"); + } + runProgramSimple(settings.nixBinDir + "/nix-env", args); + } + } else if (type == DT_DIR) { + removeOldGenerations(path); + } + } +} + +int main(int argc, char * * argv) +{ + bool removeOld = false; + Strings extraArgs; + + return handleExceptions(argv[0], [&]() { + initNix(); + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-collect-garbage"); + else if (*arg == "--version") + printVersion("nix-collect-garbage"); + else if (*arg == "--delete-old" || *arg == "-d") removeOld = true; + else if (*arg == "--delete-older-than") { + removeOld = true; + gen = getArg(*arg, arg, end); + } + else if (*arg == "--dry-run") dryRun = true; + else + extraArgs.push_back(*arg); + return true; + }); + + auto profilesDir = settings.nixStateDir + "/profiles"; + if (removeOld) removeOldGenerations(profilesDir); + + // Run the actual garbage collector. + if (!dryRun) runProgramSimple(settings.nixBinDir + "/nix-store", Strings{"--gc"}); + }); +} + diff --git a/src/nix-daemon/local.mk b/src/nix-daemon/local.mk index e5538bada0b2..5a4474465b3c 100644 --- a/src/nix-daemon/local.mk +++ b/src/nix-daemon/local.mk @@ -11,5 +11,3 @@ nix-daemon_LDFLAGS = -pthread ifeq ($(OS), SunOS) nix-daemon_LDFLAGS += -lsocket endif - -$(eval $(call install-symlink, nix-daemon, $(bindir)/nix-worker)) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 10b95dad168c..5cf41e844e83 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1423,6 +1423,8 @@ int main(int argc, char * * argv) if (!op) throw UsageError("no operation specified"); + store = openStore(); + globals.state = std::shared_ptr<EvalState>(new EvalState(searchPath)); globals.state->repair = repair; @@ -1441,8 +1443,6 @@ int main(int argc, char * * argv) : canonPath(settings.nixStateDir + "/profiles/default"); } - store = openStore(); - op(globals, opFlags, opArgs); globals.state->printStats(); diff --git a/src/nix-env/profiles.cc b/src/nix-env/profiles.cc index cedefb157a4b..5b7a533df290 100644 --- a/src/nix-env/profiles.cc +++ b/src/nix-env/profiles.cc @@ -134,16 +134,7 @@ void switchLink(Path link, Path target) /* Hacky. */ if (dirOf(target) == dirOf(link)) target = baseNameOf(target); - Path tmp = canonPath(dirOf(link) + "/.new_" + baseNameOf(link)); - createSymlink(target, tmp); - /* The rename() system call is supposed to be essentially atomic - on Unix. That is, if we have links `current -> X' and - `new_current -> Y', and we rename new_current to current, a - process accessing current will see X or Y, but never a - file-not-found or other error condition. This is sufficient to - atomically switch user environments. */ - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % link); + replaceSymlink(target, link); } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 5abaa617d245..bea04180e513 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -155,14 +155,16 @@ int main(int argc, char * * argv) return true; }); + if (evalOnly && !wantsReadWrite) + settings.readOnlyMode = true; + + store = openStore(); + EvalState state(searchPath); state.repair = repair; Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); - if (evalOnly && !wantsReadWrite) - settings.readOnlyMode = true; - if (attrPaths.empty()) attrPaths.push_back(""); if (findFile) { @@ -174,8 +176,6 @@ int main(int argc, char * * argv) return; } - store = openStore(); - if (readStdin) { Expr * e = parseStdin(state); processExpr(state, attrPaths, parseOnly, strict, autoArgs, @@ -183,10 +183,10 @@ int main(int argc, char * * argv) } else if (files.empty() && !fromArgs) files.push_back("./default.nix"); - foreach (Strings::iterator, i, files) { + for (auto & i : files) { Expr * e = fromArgs - ? state.parseExprFromString(*i, absPath(".")) - : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, *i))); + ? state.parseExprFromString(i, absPath(".")) + : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, i))); processExpr(state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 60080f253a22..506c1a397b55 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -49,12 +49,11 @@ LocalStore & ensureLocalStore() static Path useDeriver(Path path) { - if (!isDerivation(path)) { - path = store->queryDeriver(path); - if (path == "") - throw Error(format("deriver of path ‘%1%’ is not known") % path); - } - return path; + if (isDerivation(path)) return path; + Path drvPath = store->queryDeriver(path); + if (drvPath == "") + throw Error(format("deriver of path ‘%1%’ is not known") % path); + return drvPath; } diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index a25d56bec11e..b3442f6a8471 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -41,3 +41,26 @@ grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml # Do an install. nix-env -i dependencies [ -e $TEST_ROOT/var/nix/profiles/default/foobar ] + + + +clearProfiles +clearManifests +rm -f $TEST_ROOT/.nix-channels + +# Test updating from a tarball +nix-channel --add file://$TEST_ROOT/foo/nixexprs.tar.bz2 foo +nix-channel --update + +# Do a query. +nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml +if [ "$xmllint" != false ]; then + $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML" +fi +grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml +grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml + +# Do an install. +nix-env -i dependencies +[ -e $TEST_ROOT/var/nix/profiles/default/foobar ] + |