diff options
| -rw-r--r-- | configure.ac | 17 | ||||
| -rw-r--r-- | corepkgs/buildenv.nix | 5 | ||||
| -rw-r--r-- | scripts/local.mk | 2 | ||||
| -rwxr-xr-x | scripts/resolve-system-dependencies.pl.in | 122 | ||||
| -rw-r--r-- | src/libstore/build.cc | 76 | ||||
| -rw-r--r-- | src/libstore/builtins.cc | 12 | ||||
| -rw-r--r-- | src/libstore/download.cc | 30 | ||||
| -rw-r--r-- | src/libstore/download.hh | 9 | ||||
| -rw-r--r-- | src/libstore/globals.cc | 5 | ||||
| -rw-r--r-- | src/libstore/local.mk | 1 | ||||
| -rw-r--r-- | src/libstore/sandbox-defaults.sb.in | 55 | ||||
| -rw-r--r-- | src/nix-prefetch-url/nix-prefetch-url.cc | 2 |
12 files changed, 278 insertions, 58 deletions
diff --git a/configure.ac b/configure.ac index 1d4b7d3b8995..a30e0c3b2451 100644 --- a/configure.ac +++ b/configure.ac @@ -261,6 +261,23 @@ AC_MSG_RESULT(yes) AC_SUBST(perlFlags) +# Check for otool, an optional dependency on Darwin. +AC_PATH_PROG(otool, otool) +AC_MSG_CHECKING([that otool works]) +case $host_os in + darwin*) + if test -z "$otool" || ! $otool --version 2>/dev/null; then + AC_MSG_RESULT(no) + AC_MSG_ERROR([Can't get version from otool; do you need to install developer tools?]) + fi + AC_MSG_RESULT(yes) + ;; + *) + AC_MSG_RESULT(not needed) + ;; +esac + + # Whether to build the Perl bindings AC_MSG_CHECKING([whether to build the Perl bindings]) AC_ARG_ENABLE(perl-bindings, AC_HELP_STRING([--enable-perl-bindings], diff --git a/corepkgs/buildenv.nix b/corepkgs/buildenv.nix index 5bf7b4e56042..4a23d15a9ab8 100644 --- a/corepkgs/buildenv.nix +++ b/corepkgs/buildenv.nix @@ -23,5 +23,10 @@ derivation { # network traffic, so don't do that. preferLocalBuild = true; + __impureHostDeps = [ + "/usr/lib/libSystem.dylib" + "/usr/lib/system" + ]; + inherit chrootDeps; } diff --git a/scripts/local.mk b/scripts/local.mk index 3fb47676fa54..cdac56bf13cb 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -17,6 +17,7 @@ nix_substituters := \ nix_noinst_scripts := \ $(d)/build-remote.pl \ $(d)/find-runtime-roots.pl \ + $(d)/resolve-system-dependencies.pl \ $(d)/nix-http-export.cgi \ $(d)/nix-profile.sh \ $(d)/nix-reduce-build \ @@ -29,6 +30,7 @@ profiledir = $(sysconfdir)/profile.d $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) $(eval $(call install-program-in, $(d)/find-runtime-roots.pl, $(libexecdir)/nix)) $(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix)) +$(eval $(call install-program-in, $(d)/resolve-system-dependencies.pl, $(libexecdir)/nix)) $(foreach prog, $(nix_substituters), $(eval $(call install-program-in, $(prog), $(libexecdir)/nix/substituters))) $(eval $(call install-symlink, nix-build, $(bindir)/nix-shell)) diff --git a/scripts/resolve-system-dependencies.pl.in b/scripts/resolve-system-dependencies.pl.in new file mode 100755 index 000000000000..23416b75c023 --- /dev/null +++ b/scripts/resolve-system-dependencies.pl.in @@ -0,0 +1,122 @@ +#! @perl@ -w @perlFlags@ + +use utf8; +use strict; +use warnings; +use Cwd qw(realpath); +use Errno; +use File::Basename qw(dirname); +use File::Path qw(make_path); +use File::Spec::Functions qw(catfile); +use List::Util qw(reduce); +use IPC::Open3; +use Nix::Config; +use Nix::Store qw(derivationFromPath); +use POSIX qw(uname); +use Storable qw(lock_retrieve lock_store); + +my ($sysname, undef, $version, undef, $machine) = uname; +$sysname =~ /Darwin/ or die "This tool is only meant to be used on Darwin systems."; + +my $cache = "$Nix::Config::stateDir/dependency-maps/$machine-$sysname-$version.map"; + +make_path dirname($cache); + +our $DEPS; +eval { + $DEPS = lock_retrieve($cache); +}; + +if($!{ENOENT}) { + lock_store {}, $cache; + $DEPS = {}; +} elsif($@) { + die "Unable to obtain a lock on dependency-map file $cache: $@"; +} + +sub mkset(@) { + my %set; + @set{@_} = (); + \%set +} + +sub union($$) { + my ($set1, $set2) = @_; + my %new = (%$set1, %$set2); + \%new +} + +sub cache_filepath($) { + my $fp = shift; + $fp =~ s/-/--/g; + $fp =~ s/\//-/g; + $fp =~ s/^-//g; + catfile $cache, $fp +} + +sub resolve_tree { + sub resolve_tree_inner { + my ($lib, $TREE) = @_; + return if (defined $TREE->{$lib}); + $TREE->{$lib} = mkset(@{cache_get($lib)}); + foreach my $dep (keys %{$TREE->{$lib}}) { + resolve_tree_inner($dep, $TREE); + } + values %$TREE + } + + reduce { union($a, $b) } {}, resolve_tree_inner(@_) +} + +sub cache_get { + my $key = shift; + if (defined $DEPS->{$key}) { + $DEPS->{$key} + } else { + cache_insert($key); + cache_get($key) + } +} + +sub cache_insert($) { + my $key = shift; + print STDERR "Finding dependencies for $key...\n"; + my @deps = find_deps($key); + $DEPS->{$key} = \@deps; +} + +sub find_deps($) { + my $lib = shift; + my($chld_in, $chld_out, $chld_err); + my $pid = open3($chld_in, $chld_out, $chld_err, "@otool@", "-L", "-arch", "x86_64", $lib); + waitpid($pid, 0); + my $line = readline $chld_out; + if($? == 0 and $line !~ /not an object file/) { + my @libs; + while(<$chld_out>) { + my $dep = (split /\s+/)[1]; + push @libs, $dep unless $dep eq $lib or $dep =~ /\@rpath/; + } + @libs + } elsif (-l $lib) { + (realpath($lib)) + } else { + () + } +} + +if (defined $ARGV[0]) { + my $deps = derivationFromPath($ARGV[0])->{"env"}->{"__impureHostDeps"}; + if (defined $deps) { + my @files = split(/\s+/, $deps); + my $depcache = {}; + my $depset = reduce { union($a, $b) } (map { resolve_tree($_, $depcache) } @files); + print "extra-chroot-dirs\n"; + print join("\n", keys %$depset); + print "\n\n"; + } + lock_store($DEPS, $cache); +} else { + print STDERR "Usage: $0 path/to/derivation.drv\n"; + exit 1 +} diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5674e83c9e66..93bb2144425f 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -74,6 +74,7 @@ #if __linux__ #include <sys/personality.h> +#include <sys/mman.h> #endif #if HAVE_STATVFS @@ -1245,12 +1246,19 @@ void DerivationGoal::inputsRealised() } -static bool canBuildLocally(const string & platform) +static bool isBuiltin(const BasicDerivation & drv) +{ + return string(drv.builder, 0, 8) == "builtin:"; +} + + +static bool canBuildLocally(const BasicDerivation & drv) { - return platform == settings.thisSystem + return drv.platform == settings.thisSystem + || isBuiltin(drv) #if __linux__ - || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") - || (platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux") + || (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-linux") + || (drv.platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux") #endif ; } @@ -1265,7 +1273,7 @@ static string get(const StringPairs & map, const string & key, const string & de bool willBuildLocally(const BasicDerivation & drv) { - return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform); + return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv); } @@ -1275,12 +1283,6 @@ bool substitutesAllowed(const BasicDerivation & drv) } -static bool isBuiltin(const BasicDerivation & drv) -{ - return string(drv.builder, 0, 8) == "builtin:"; -} - - void DerivationGoal::tryToBuild() { trace("trying to build"); @@ -1682,7 +1684,7 @@ void DerivationGoal::startBuilder() "building path(s) %1%") % showPaths(missingPaths)); /* Right platform? */ - if (!canBuildLocally(drv->platform)) { + if (!canBuildLocally(*drv)) { if (settings.printBuildTrace) printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv->platform); throw Error( @@ -2057,7 +2059,7 @@ void DerivationGoal::startBuilder() auto lastPos = std::string::size_type{0}; for (auto nlPos = lines.find('\n'); nlPos != string::npos; nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos}; + auto line = std::string{lines, lastPos, nlPos - lastPos}; lastPos = nlPos + 1; if (state == stBegin) { if (line == "extra-chroot-dirs") { @@ -2128,14 +2130,17 @@ void DerivationGoal::startBuilder() ProcessOptions options; options.allowVfork = false; Pid helper = startProcess([&]() { - char stack[32 * 1024]; + size_t stackSize = 1 * 1024 * 1024; + char * stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (!stack) throw SysError("allocating stack"); int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (!fixedOutput) flags |= CLONE_NEWNET; - pid_t child = clone(childEntry, stack + sizeof(stack) - 8, flags, this); + pid_t child = clone(childEntry, stack + stackSize, flags, this); if (child == -1 && errno == EINVAL) /* Fallback for Linux < 2.13 where CLONE_NEWPID and CLONE_PARENT are not allowed together. */ - child = clone(childEntry, stack + sizeof(stack) - 8, flags & ~CLONE_NEWPID, this); + child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this); if (child == -1) throw SysError("cloning builder process"); writeFull(builderOut.writeSide, int2String(child) + "\n"); _exit(0); @@ -2434,9 +2439,11 @@ void DerivationGoal::runChild() for (auto & i : inputPaths) dirsInChroot[i] = i; - /* TODO: we should factor out the policy cleanly, so we don't have to repeat the constants every time... */ + /* This has to appear before import statements */ sandboxProfile += "(version 1)\n"; + sandboxProfile += (format("(import \"%1%/nix/sandbox-defaults.sb\")\n") % settings.nixDataDir).str(); + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ if (settings.get("darwin-log-sandbox-violations", false)) { sandboxProfile += "(deny default)\n"; @@ -2444,15 +2451,6 @@ void DerivationGoal::runChild() sandboxProfile += "(deny default (with no-log))\n"; } - sandboxProfile += "(allow file-read* file-write-data (literal \"/dev/null\"))\n"; - - sandboxProfile += "(allow file-read-metadata\n" - "\t(literal \"/var\")\n" - "\t(literal \"/tmp\")\n" - "\t(literal \"/etc\")\n" - "\t(literal \"/etc/nix\")\n" - "\t(literal \"/etc/nix/nix.conf\"))\n"; - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true); @@ -2460,20 +2458,6 @@ void DerivationGoal::runChild() /* They don't like trailing slashes on subpath directives */ if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - /* This is where our temp folders are and where most of the building will happen, so we want rwx on it. */ - sandboxProfile += (format("(allow file-read* file-write* process-exec (subpath \"%1%\") (subpath \"/private/tmp\"))\n") % globalTmpDir).str(); - - sandboxProfile += "(allow process-fork)\n"; - sandboxProfile += "(allow sysctl-read)\n"; - sandboxProfile += "(allow signal (target same-sandbox))\n"; - - /* Enables getpwuid (used by git and others) */ - sandboxProfile += "(allow mach-lookup (global-name \"com.apple.system.notification_center\") (global-name \"com.apple.system.opendirectoryd.libinfo\"))\n"; - - /* Allow local networking operations, mostly because lots of test suites use it and it seems mostly harmless */ - sandboxProfile += "(allow network* (local ip) (remote unix-socket))"; - - /* Our rwx outputs */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : missingPaths) { @@ -2487,7 +2471,7 @@ void DerivationGoal::runChild() denying it means the `access` syscall will return EPERM instead of EACCESS, which confuses a few programs that assume (understandably, since it appears to be a violation of the POSIX spec) that `access` won't do that, and don't deal with it nicely if it does. The most notable of these is the entire GHC Haskell ecosystem. */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; + sandboxProfile += "(allow file-read* file-write* process-exec mach-priv-task-port\n"; for (auto & i : dirsInChroot) { if (i.first != i.second) throw SysError(format("can't map '%1%' to '%2%': mismatched impure paths not supported on darwin")); @@ -2504,8 +2488,12 @@ void DerivationGoal::runChild() sandboxProfile += ")\n"; /* Our ancestry. N.B: this uses literal on folders, instead of subpath. Without that, - you open up the entire filesystem because you end up with (subpath "/") */ - sandboxProfile += "(allow file-read-metadata\n"; + you open up the entire filesystem because you end up with (subpath "/") + Note: file-read-metadata* is not sufficiently permissive for GHC. file-read* is but may + be a security hazard. + TODO: figure out a more appropriate directive. + */ + sandboxProfile += "(allow file-read*\n"; for (auto & i : ancestry) { sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); } @@ -2518,6 +2506,8 @@ void DerivationGoal::runChild() args.push_back("sandbox-exec"); args.push_back("-p"); args.push_back(sandboxProfile); + args.push_back("-D"); + args.push_back((format("_GLOBAL_TMP_DIR=%1%") % globalTmpDir).str()); args.push_back(drv->builder); } else { builder = drv->builder.c_str(); diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc index 25e2e7df30e7..2a4396308399 100644 --- a/src/libstore/builtins.cc +++ b/src/libstore/builtins.cc @@ -7,8 +7,16 @@ void builtinFetchurl(const BasicDerivation & drv) { auto url = drv.env.find("url"); if (url == drv.env.end()) throw Error("attribute ‘url’ missing"); - printMsg(lvlInfo, format("downloading ‘%1%’...") % url->second); - auto data = downloadFile(url->second); // FIXME: show progress + + /* No need to do TLS verification, because we check the hash of + the result anyway. */ + DownloadOptions options; + options.verifyTLS = false; + + /* Show a progress indicator, even though stderr is not a tty. */ + options.forceProgress = true; + + auto data = downloadFile(url->second, options); auto out = drv.env.find("out"); if (out == drv.env.end()) throw Error("attribute ‘url’ missing"); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index c6c2f611573f..94c13249d494 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -102,7 +102,6 @@ struct Curl if (!curl) throw Error("unable to initialize curl"); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str()); curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); @@ -115,8 +114,6 @@ struct Curl curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback_); curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, (void *) &curl); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); - - showProgress = isatty(STDERR_FILENO); } ~Curl() @@ -125,10 +122,19 @@ struct Curl if (requestHeaders) curl_slist_free_all(requestHeaders); } - bool fetch(const string & url, const string & expectedETag = "") + bool fetch(const string & url, const DownloadOptions & options) { + showProgress = options.forceProgress || isatty(STDERR_FILENO); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + if (options.verifyTLS) + curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str()); + else { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); + } + data.clear(); if (requestHeaders) { @@ -136,9 +142,9 @@ struct Curl requestHeaders = 0; } - if (!expectedETag.empty()) { - this->expectedETag = expectedETag; - requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + expectedETag).c_str()); + if (!options.expectedETag.empty()) { + this->expectedETag = options.expectedETag; + requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + options.expectedETag).c_str()); } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders); @@ -154,7 +160,7 @@ struct Curl //std::cerr << "\e[" << moveBack << "D\e[K\n"; std::cerr << "\n"; checkInterrupt(); - if (res == CURLE_WRITE_ERROR && etag == expectedETag) return false; + if (res == CURLE_WRITE_ERROR && etag == options.expectedETag) return false; if (res != CURLE_OK) throw DownloadError(format("unable to download ‘%1%’: %2% (%3%)") % url % curl_easy_strerror(res) % res); @@ -168,11 +174,11 @@ struct Curl }; -DownloadResult downloadFile(string url, string expectedETag) +DownloadResult downloadFile(string url, const DownloadOptions & options) { DownloadResult res; Curl curl; - if (curl.fetch(url, expectedETag)) { + if (curl.fetch(url, options)) { res.cached = false; res.data = curl.data; } else @@ -224,7 +230,9 @@ Path downloadFileCached(const string & url, bool unpack) if (!skip) { try { - auto res = downloadFile(url, expectedETag); + DownloadOptions options; + options.expectedETag = expectedETag; + auto res = downloadFile(url, options); if (!res.cached) storePath = store->addTextToStore(name, res.data, PathSet(), false); diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 28c9117e4227..c1cb25b90c32 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -5,13 +5,20 @@ namespace nix { +struct DownloadOptions +{ + string expectedETag; + bool verifyTLS{true}; + bool forceProgress{false}; +}; + struct DownloadResult { bool cached; string data, etag; }; -DownloadResult downloadFile(string url, string expectedETag = ""); +DownloadResult downloadFile(string url, const DownloadOptions & options); Path downloadFileCached(const string & url, bool unpack); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 73f8489438fc..e704837e8798 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -77,6 +77,11 @@ void Settings::processEnvironment() nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH); + + // should be set with the other config options, but depends on nixLibexecDir +#ifdef __APPLE__ + preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies.pl"; +#endif } diff --git a/src/libstore/local.mk b/src/libstore/local.mk index bf5c256c949e..f10981ad444c 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -33,3 +33,4 @@ $(d)/local-store.cc: $(d)/schema.sql.hh clean-files += $(d)/schema.sql.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) +$(eval $(call install-file-in, $(d)/sandbox-defaults.sb, $(datadir)/nix, 0644)) diff --git a/src/libstore/sandbox-defaults.sb.in b/src/libstore/sandbox-defaults.sb.in new file mode 100644 index 000000000000..12c39fa7f217 --- /dev/null +++ b/src/libstore/sandbox-defaults.sb.in @@ -0,0 +1,55 @@ +(allow file-read* file-write-data (literal "/dev/null")) +(allow ipc-posix*) +(allow mach-lookup (global-name "com.apple.SecurityServer")) + +(allow file-read* + (literal "/dev/dtracehelper") + (literal "/dev/tty") + (literal "/dev/autofs_nowait") + (literal "/System/Library/CoreServices/SystemVersion.plist") + (literal "/private/var/run/systemkeychaincheck.done") + (literal "/private/etc/protocols") + (literal "/private/var/tmp") + (literal "/private/var/db") + (subpath "/private/var/db/mds")) + +(allow file-write* + (literal "/dev/tty") + (literal "/dev/dtracehelper") + (literal "/mds")) + +(allow file-ioctl (literal "/dev/dtracehelper")) + +(allow file-read-metadata + (literal "/var") + (literal "/tmp") + ; symlinks + (literal "@sysconfdir@") + (literal "@sysconfdir@/nix") + (literal "@sysconfdir@/nix/nix.conf") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + +(allow file-read* + (literal "/private@sysconfdir@/nix/nix.conf") + (literal "/private/var/run/resolv.conf")) + +; some builders use filehandles other than stdin/stdout +(allow file* (subpath "/dev/fd")) + +; allow everything inside TMP +(allow file* process-exec + (subpath (param "_GLOBAL_TMP_DIR")) + (subpath "/private/tmp")) + +(allow process-fork) +(allow sysctl-read) +(allow signal (target same-sandbox)) + +; allow getpwuid (for git and other packages) +(allow mach-lookup + (global-name "com.apple.system.notification_center") + (global-name "com.apple.system.opendirectoryd.libinfo")) + +; allow local networking +(allow network* (local ip) (remote unix-socket)) diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 12aa7157204c..73a2845e07a5 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -158,7 +158,7 @@ int main(int argc, char * * argv) auto actualUri = resolveMirrorUri(state, uri); /* Download the file. */ - auto result = downloadFile(actualUri); + auto result = downloadFile(actualUri, DownloadOptions()); AutoDelete tmpDir(createTempDir(), true); Path tmpFile = (Path) tmpDir + "/tmp"; |