diff options
| -rw-r--r-- | .gitignore | 4 | ||||
| -rw-r--r-- | Makefile | 1 | ||||
| -rw-r--r-- | scripts/local.mk | 1 | ||||
| -rwxr-xr-x | scripts/nix-channel.in | 228 | ||||
| -rw-r--r-- | src/libstore/download.cc | 20 | ||||
| -rw-r--r-- | src/libstore/download.hh | 6 | ||||
| -rw-r--r-- | src/nix-channel/local.mk | 7 | ||||
| -rwxr-xr-x | src/nix-channel/nix-channel.cc | 270 |
8 files changed, 304 insertions, 233 deletions
diff --git a/.gitignore b/.gitignore index 245b89f3cb..8e0e59ed12 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,6 @@ Makefile.config /scripts/nix-switch /scripts/nix-collect-garbage /scripts/nix-prefetch-url -/scripts/nix-channel /scripts/nix-build /scripts/nix-copy-closure /scripts/NixConfig.pm @@ -73,6 +72,9 @@ Makefile.config # /src/nix-daemon/ /src/nix-daemon/nix-daemon +# /src/nix-channel/ +/src/nix-channel/nix-channel + # /src/download-via-ssh/ /src/download-via-ssh/download-via-ssh diff --git a/Makefile b/Makefile index 90dca473f3..eb2c01ad95 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ makefiles = \ src/nix-daemon/local.mk \ src/nix-collect-garbage/local.mk \ src/nix-prefetch-url/local.mk \ + src/nix-channel/local.mk \ perl/local.mk \ scripts/local.mk \ corepkgs/local.mk \ diff --git a/scripts/local.mk b/scripts/local.mk index 46b3fe3cf7..db4bea6d44 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -1,6 +1,5 @@ nix_bin_scripts := \ $(d)/nix-build \ - $(d)/nix-channel \ $(d)/nix-copy-closure \ $(d)/nix-push diff --git a/scripts/nix-channel.in b/scripts/nix-channel.in deleted file mode 100755 index 65084ff1f3..0000000000 --- a/scripts/nix-channel.in +++ /dev/null @@ -1,228 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use File::Basename; -use File::Path qw(mkpath); -use Nix::Config; -use Nix::Manifest; -use File::Temp qw(tempdir); - -binmode STDERR, ":encoding(utf8)"; - -Nix::Config::readConfig; - - -# Turn on caching in nix-prefetch-url. -my $channelCache = "$Nix::Config::stateDir/channel-cache"; -mkdir $channelCache, 0755 unless -e $channelCache; -$ENV{'NIX_DOWNLOAD_CACHE'} = $channelCache if -W $channelCache; - -# Figure out the name of the `.nix-channels' file to use. -my $home = $ENV{"HOME"} or die '$HOME not set\n'; -my $channelsList = "$home/.nix-channels"; -my $nixDefExpr = "$home/.nix-defexpr"; - -# Figure out the name of the channels profile. -my $userName = getpwuid($<) || $ENV{"USER"} or die "cannot figure out user name"; -my $profile = "$Nix::Config::stateDir/profiles/per-user/$userName/channels"; -mkpath(dirname $profile, 0, 0755); - -my %channels; - - -# Reads the list of channels. -sub readChannels { - return if (!-f $channelsList); - open CHANNELS, "<$channelsList" or die "cannot open ‘$channelsList’: $!"; - while (<CHANNELS>) { - chomp; - next if /^\s*\#/; - my ($url, $name) = split ' ', $_; - $url =~ s/\/*$//; # remove trailing slashes - $name = basename $url unless defined $name; - $channels{$name} = $url; - } - close CHANNELS; -} - - -# Writes the list of channels. -sub writeChannels { - open CHANNELS, ">$channelsList" or die "cannot open ‘$channelsList’: $!"; - foreach my $name (keys %channels) { - print CHANNELS "$channels{$name} $name\n"; - } - close CHANNELS; -} - - -# Adds a channel. -sub addChannel { - my ($url, $name) = @_; - die "invalid channel URL ‘$url’" unless $url =~ /^(file|http|https):\/\//; - die "invalid channel identifier ‘$name’" unless $name =~ /^[a-zA-Z0-9_][a-zA-Z0-9_\-\.]*$/; - readChannels; - $channels{$name} = $url; - writeChannels; -} - - -# Remove a channel. -sub removeChannel { - my ($name) = @_; - readChannels; - my $url = $channels{$name}; - delete $channels{$name}; - writeChannels; - - system("$Nix::Config::binDir/nix-env --profile '$profile' -e '$name'") == 0 - or die "cannot remove channel ‘$name’\n"; -} - - -# Fetch Nix expressions and binary cache URLs from the subscribed channels. -sub update { - my @channelNames = @_; - - readChannels; - - # Download each channel. - my $exprs = ""; - foreach my $name (keys %channels) { - next if scalar @channelNames > 0 && ! grep { $_ eq $name } @{channelNames}; - - my $url = $channels{$name}; - - # We want to download the url to a file to see if it's a tarball while also checking if we - # got redirected in the process, so that we can grab the various parts of a nix channel - # definition from a consistent location if the redirect changes mid-download. - my $tmpdir = tempdir( CLEANUP => 1 ); - my $filename; - ($url, $filename) = `cd $tmpdir && $Nix::Config::curl --silent --write-out '%{url_effective}\n%{filename_effective}' -L '$url' -O`; - chomp $url; - die "$0: unable to check ‘$url’\n" if $? != 0; - - # If the URL contains a version number, append it to the name - # attribute (so that "nix-env -q" on the channels profile - # shows something useful). - my $cname = $name; - $cname .= $1 if basename($url) =~ /(-\d.*)$/; - - my $path; - my $ret = -1; - if (-e "$tmpdir/$filename" && $filename =~ /\.tar\.(gz|bz2|xz)$/) { - # Get our temporary download into the store. - (my $hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url 'file://$tmpdir/$filename'`; - chomp $path; - - # Try unpacking the expressions to see if they'll be valid for us to process later. - # Like anything in nix, this will cache the result so we don't do it again outside of the loop below. - $ret = system("$Nix::Config::binDir/nix-build --no-out-link -E 'import <nix/unpack-channel.nix> " . - "{ name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; }'"); - } - - # The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it - my $extraAttrs = ""; - if ($ret != 0) { - # Check if the channel advertises a binary cache. - my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`; - $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; " - if $? == 0 && $binaryCacheURL ne ""; - - # Download the channel tarball. - my $fullURL = "$url/nixexprs.tar.xz"; - system("$Nix::Config::curl --fail --silent --head '$fullURL' > /dev/null") == 0 or - $fullURL = "$url/nixexprs.tar.bz2"; - print STDERR "downloading Nix expressions from ‘$fullURL’...\n"; - (my $hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url '$fullURL'`; - die "cannot fetch ‘$fullURL’\n" if $? != 0; - chomp $path; - } - - # Regardless of where it came from, add the expression representing this channel to accumulated expression - $exprs .= "'f: f { name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; $extraAttrs }' "; - } - - # Unpack the channel tarballs into the Nix store and install them - # into the channels profile. - print STDERR "unpacking channels...\n"; - system("$Nix::Config::binDir/nix-env --profile '$profile' " . - "-f '<nix/unpack-channel.nix>' -i -E $exprs --quiet") == 0 - or die "cannot unpack the channels"; - - # Make the channels appear in nix-env. - unlink $nixDefExpr if -l $nixDefExpr; # old-skool ~/.nix-defexpr - mkdir $nixDefExpr or die "cannot create directory ‘$nixDefExpr’" if !-e $nixDefExpr; - my $channelLink = "$nixDefExpr/channels"; - unlink $channelLink; # !!! not atomic - symlink($profile, $channelLink) or die "cannot symlink ‘$channelLink’ to ‘$profile’"; -} - - -die "$0: argument expected\n" if scalar @ARGV == 0; - - -while (scalar @ARGV) { - my $arg = shift @ARGV; - - if ($arg eq "--add") { - die "$0: ‘--add’ requires one or two arguments\n" if scalar @ARGV < 1 || scalar @ARGV > 2; - my $url = shift @ARGV; - my $name = shift @ARGV; - unless (defined $name) { - $name = basename $url; - $name =~ s/-unstable//; - $name =~ s/-stable//; - } - addChannel($url, $name); - last; - } - - if ($arg eq "--remove") { - die "$0: ‘--remove’ requires one argument\n" if scalar @ARGV != 1; - removeChannel(shift @ARGV); - last; - } - - if ($arg eq "--list") { - die "$0: ‘--list’ requires one argument\n" if scalar @ARGV != 0; - readChannels; - foreach my $name (keys %channels) { - print "$name $channels{$name}\n"; - } - last; - } - - elsif ($arg eq "--update") { - update(@ARGV); - last; - } - - elsif ($arg eq "--rollback") { - die "$0: ‘--rollback’ has at most one argument\n" if scalar @ARGV > 1; - my $generation = shift @ARGV; - my @args = ("$Nix::Config::binDir/nix-env", "--profile", $profile); - if (defined $generation) { - die "invalid channel generation number ‘$generation’" unless $generation =~ /^[0-9]+$/; - push @args, "--switch-generation", $generation; - } else { - push @args, "--rollback"; - } - system(@args) == 0 or exit 1; - last; - } - - elsif ($arg eq "--help") { - exec "man nix-channel" or die; - } - - elsif ($arg eq "--version") { - print "nix-channel (Nix) $Nix::Config::version\n"; - exit 0; - } - - else { - die "unknown argument ‘$arg’; try ‘--help’\n"; - } -} diff --git a/src/libstore/download.cc b/src/libstore/download.cc index cf3929cadd..9cc4332281 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -31,7 +31,7 @@ struct CurlDownloader : public Downloader { CURL * curl; ref<std::string> data; - string etag, status, expectedETag; + string etag, status, expectedETag, effectiveUrl; struct curl_slist * requestHeaders; @@ -199,6 +199,11 @@ struct CurlDownloader : public Downloader % url % curl_easy_strerror(res) % res); } + char *effectiveUrlCStr; + curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr); + if (effectiveUrlCStr) + effectiveUrl = effectiveUrlCStr; + if (httpStatus == 304) return false; return true; @@ -212,6 +217,7 @@ struct CurlDownloader : public Downloader res.data = data; } else res.cached = true; + res.effectiveUrl = effectiveUrl; res.etag = etag; return res; } @@ -224,6 +230,12 @@ ref<Downloader> makeDownloader() Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, const Hash & expectedHash) { + string ignored; + return downloadCached(store, url_, unpack, ignored, expectedHash); +} + +Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string & effectiveUrl, const Hash & expectedHash) +{ auto url = resolveUri(url_); string name; @@ -259,9 +271,10 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) + if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) { skip = true; - else if (!ss[1].empty()) { + effectiveUrl = url_; + } else if (!ss[1].empty()) { printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]); expectedETag = ss[1]; } @@ -276,6 +289,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa DownloadOptions options; options.expectedETag = expectedETag; auto res = download(url, options); + effectiveUrl = res.effectiveUrl; if (!res.cached) { ValidPathInfo info; diff --git a/src/libstore/download.hh b/src/libstore/download.hh index efddc55281..d17e14400f 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -19,6 +19,7 @@ struct DownloadResult { bool cached; string etag; + string effectiveUrl; std::shared_ptr<std::string> data; }; @@ -31,6 +32,11 @@ struct Downloader Path downloadCached(ref<Store> store, const string & url, bool unpack, const Hash & expectedHash = Hash()); + /* Need to overload because can't have an rvalue default value for non-const reference */ + + Path downloadCached(ref<Store> store, const string & url, bool unpack, + string & effectiveUrl, const Hash & expectedHash = Hash()); + enum Error { NotFound, Forbidden, Misc }; }; diff --git a/src/nix-channel/local.mk b/src/nix-channel/local.mk new file mode 100644 index 0000000000..49fc105c6f --- /dev/null +++ b/src/nix-channel/local.mk @@ -0,0 +1,7 @@ +programs += nix-channel + +nix-channel_DIR := $(d) + +nix-channel_LIBS = libmain libutil libformat libstore + +nix-channel_SOURCES := $(d)/nix-channel.cc diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc new file mode 100755 index 0000000000..79c9d0eceb --- /dev/null +++ b/src/nix-channel/nix-channel.cc @@ -0,0 +1,270 @@ +#include "shared.hh" +#include "globals.hh" +#include "download.hh" +#include <fcntl.h> +#include <regex> +#include "store-api.hh" +#include <pwd.h> + +using namespace nix; + +typedef std::map<string,string> Channels; + +static auto channels = Channels{}; +static auto channelsList = Path{}; + +// Reads the list of channels. +static void readChannels() +{ + if (!pathExists(channelsList)) return; + auto channelsFile = readFile(channelsList); + + for (const auto & line : tokenizeString<std::vector<string>>(channelsFile, "\n")) { + chomp(line); + if (std::regex_search(line, std::regex("^\\s*\\#"))) + continue; + auto split = tokenizeString<std::vector<string>>(line, " "); + auto url = std::regex_replace(split[0], std::regex("/*$"), ""); + auto name = split.size() > 1 ? split[1] : baseNameOf(url); + channels[name] = url; + } +} + +// Writes the list of channels. +static void writeChannels() +{ + auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; + if (!channelsFD) + throw SysError(format("opening ‘%1%’ for writing") % channelsList); + for (const auto & channel : channels) + writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); +} + +// Adds a channel. +static void addChannel(const string & url, const string & name) +{ + if (!regex_search(url, std::regex("^(file|http|https)://"))) + throw Error(format("invalid channel URL ‘%1%’") % url); + if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$"))) + throw Error(format("invalid channel identifier ‘%1%’") % name); + readChannels(); + channels[name] = url; + writeChannels(); +} + +static auto profile = Path{}; + +// Remove a channel. +static void removeChannel(const string & name) +{ + readChannels(); + channels.erase(name); + writeChannels(); + + runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name }); +} + +static auto nixDefExpr = Path{}; + +// Fetch Nix expressions and binary cache URLs from the subscribed channels. +static void update(const StringSet & channelNames) +{ + readChannels(); + + auto store = openStore(); + + // Download each channel. + auto exprs = Strings{}; + for (const auto & channel : channels) { + if (!channelNames.empty() && channelNames.find(channel.first) != channelNames.end()) + continue; + auto name = channel.first; + auto url = channel.second; + + // We want to download the url to a file to see if it's a tarball while also checking if we + // got redirected in the process, so that we can grab the various parts of a nix channel + // definition from a consistent location if the redirect changes mid-download. + auto effectiveUrl = string{}; + auto dl = makeDownloader(); + auto filename = dl->downloadCached(store, url, false, effectiveUrl); + url = chomp(std::move(effectiveUrl)); + + // If the URL contains a version number, append it to the name + // attribute (so that "nix-env -q" on the channels profile + // shows something useful). + auto cname = name; + std::smatch match; + auto urlBase = baseNameOf(url); + if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) { + cname = cname + (string) match[1]; + } + + auto extraAttrs = string{}; + + auto unpacked = false; + if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { + try { + runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> " + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); + unpacked = true; + } catch (ExecError & e) { + } + } + + if (!unpacked) { + // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it + // Check if the channel advertises a binary cache. + DownloadOptions opts; + opts.showProgress = DownloadOptions::no; + try { + auto dlRes = dl->download(url + "/binary-cache-url", opts); + extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";"; + } catch (DownloadError & e) { + } + + // Download the channel tarball. + auto fullURL = url + "/nixexprs.tar.xz"; + try { + filename = dl->downloadCached(store, fullURL, false); + } catch (DownloadError & e) { + fullURL = url + "/nixexprs.tar.bz2"; + filename = dl->downloadCached(store, fullURL, false); + } + chomp(filename); + } + + // Regardless of where it came from, add the expression representing this channel to accumulated expression + exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }"); + } + + // Unpack the channel tarballs into the Nix store and install them + // into the channels profile. + std::cerr << "unpacking channels...\n"; + auto envArgs = Strings{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" }; + for (auto & expr : exprs) + envArgs.push_back(std::move(expr)); + envArgs.push_back("--quiet"); + runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + + // Make the channels appear in nix-env. + struct stat st; + if (lstat(nixDefExpr.c_str(), &st) == 0) { + if (S_ISLNK(st.st_mode)) + // old-skool ~/.nix-defexpr + if (unlink(nixDefExpr.c_str()) == -1) + throw SysError(format("unlinking %1%") % nixDefExpr); + } else if (errno != ENOENT) { + throw SysError(format("getting status of %1%") % nixDefExpr); + } + createDirs(nixDefExpr); + auto channelLink = nixDefExpr + "/channels"; + replaceSymlink(profile, channelLink); +} + +int main(int argc, char ** argv) +{ + return handleExceptions(argv[0], [&]() { + initNix(); + + // Turn on caching in nix-prefetch-url. + auto channelCache = settings.nixStateDir + "/channel-cache"; + createDirs(channelCache); + setenv("NIX_DOWNLOAD_CACHE", channelCache.c_str(), 1); + + // Figure out the name of the `.nix-channels' file to use + auto home = getEnv("HOME"); + if (home.empty()) + throw Error("$HOME not set"); + channelsList = home + "/.nix-channels"; + nixDefExpr = home + "/.nix-defexpr"; + + // Figure out the name of the channels profile. + auto name = string{}; + auto pw = getpwuid(getuid()); + if (!pw) + name = getEnv("USER", ""); + else + name = pw->pw_name; + if (name.empty()) + throw Error("cannot figure out user name"); + profile = settings.nixStateDir + "/profiles/per-user/" + name + "/channels"; + createDirs(dirOf(profile)); + + enum { + cNone, + cAdd, + cRemove, + cList, + cUpdate, + cRollback + } cmd = cNone; + auto args = std::vector<string>{}; + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") { + showManPage("nix-channel"); + } else if (*arg == "--version") { + printVersion("nix-channel"); + } else if (*arg == "--add") { + cmd = cAdd; + } else if (*arg == "--remove") { + cmd = cRemove; + } else if (*arg == "--list") { + cmd = cList; + } else if (*arg == "--update") { + cmd = cUpdate; + } else if (*arg == "--rollback") { + cmd = cRollback; + } else { + args.push_back(std::move(*arg)); + } + return true; + }); + switch (cmd) { + case cNone: + throw UsageError("no command specified"); + case cAdd: + if (args.size() < 1 || args.size() > 2) + throw UsageError("‘--add’ requires one or two arguments"); + { + auto url = args[0]; + auto name = string{}; + if (args.size() == 2) { + name = args[1]; + } else { + name = baseNameOf(url); + name = std::regex_replace(name, std::regex("-unstable$"), ""); + name = std::regex_replace(name, std::regex("-stable$"), ""); + } + addChannel(url, name); + } + break; + case cRemove: + if (args.size() != 1) + throw UsageError("‘--remove’ requires one argument"); + removeChannel(args[0]); + break; + case cList: + if (!args.empty()) + throw UsageError("‘--list’ expects no arguments"); + readChannels(); + for (const auto & channel : channels) + std::cout << channel.first << ' ' << channel.second << '\n'; + break; + case cUpdate: + update(StringSet(args.begin(), args.end())); + break; + case cRollback: + if (args.size() > 1) + throw UsageError("‘--rollback’ has at most one argument"); + auto envArgs = Strings{"--profile", profile}; + if (args.size() == 1) { + envArgs.push_back("--switch-generation"); + envArgs.push_back(args[0]); + } else { + envArgs.push_back("--rollback"); + } + runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + break; + } + }); +} |