diff options
-rw-r--r-- | doc/manual/expressions/advanced-attributes.xml | 17 | ||||
-rw-r--r-- | doc/manual/expressions/builtins.xml | 35 | ||||
-rw-r--r-- | doc/manual/release-notes/rl-2.2.xml | 4 | ||||
-rw-r--r-- | release-common.nix | 2 | ||||
-rw-r--r-- | shell.nix | 2 | ||||
-rw-r--r-- | src/libexpr/get-drvs.hh | 2 | ||||
-rw-r--r-- | src/libstore/globals.cc | 6 | ||||
-rw-r--r-- | src/libstore/globals.hh | 8 | ||||
-rw-r--r-- | src/libstore/local-store.cc | 2 | ||||
-rw-r--r-- | src/libstore/s3-binary-cache-store.cc | 91 | ||||
-rw-r--r-- | src/libstore/ssh.cc | 3 | ||||
-rw-r--r-- | src/libstore/store-api.cc | 10 | ||||
-rw-r--r-- | src/libutil/util.cc | 32 | ||||
-rw-r--r-- | src/libutil/util.hh | 12 | ||||
-rwxr-xr-x | src/nix-build/nix-build.cc | 4 | ||||
-rw-r--r-- | src/nix-prefetch-url/nix-prefetch-url.cc | 9 | ||||
-rw-r--r-- | src/nix/edit.cc | 5 | ||||
-rw-r--r-- | src/nix/repl.cc | 2 | ||||
-rw-r--r-- | src/nix/run.cc | 4 | ||||
-rw-r--r-- | tests/common.sh.in | 15 | ||||
-rw-r--r-- | tests/init.sh | 1 |
21 files changed, 185 insertions, 81 deletions
diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml index 2af7a51acfbb..07b0d97d3f7d 100644 --- a/doc/manual/expressions/advanced-attributes.xml +++ b/doc/manual/expressions/advanced-attributes.xml @@ -312,9 +312,7 @@ big = "a very long string"; <varlistentry><term><varname>preferLocalBuild</varname></term> <listitem><para>If this attribute is set to - <literal>true</literal>, it has two effects. First, the - derivation will always be built, not substituted, even if a - substitute is available. Second, if <link + <literal>true</literal> and <link linkend="chap-distributed-builds">distributed building is enabled</link>, then, if possible, the derivaton will be built locally instead of forwarded to a remote machine. This is @@ -324,6 +322,19 @@ big = "a very long string"; </varlistentry> + + <varlistentry><term><varname>allowSubstitutes</varname></term> + + <listitem><para>If this attribute is set to + <literal>false</literal>, then Nix will always build this + derivation; it will not try to substitute its outputs. This is + useful for very trivial derivations (such as + <function>writeText</function> in Nixpkgs) that are cheaper to + build than to substitute from a binary cache.</para></listitem> + + </varlistentry> + + </variablelist> </section> diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 8d12da9b1356..412622714d40 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -436,7 +436,7 @@ stdenv.mkDerivation { … } <title>Fetching a repository's specific commit on an arbitrary branch</title> <para> If the revision you're looking for is in the default branch - of the gift repository you don't strictly need to specify + of the git repository you don't strictly need to specify the branch name in the <varname>ref</varname> attribute. </para> <para> @@ -465,7 +465,7 @@ stdenv.mkDerivation { … } <title>Fetching a repository's specific commit on the default branch</title> <para> If the revision you're looking for is in the default branch - of the gift repository you may omit the + of the git repository you may omit the <varname>ref</varname> attribute. </para> <programlisting>builtins.fetchGit { @@ -1077,22 +1077,8 @@ Evaluates to <literal>[ "foo" ]</literal>. <replaceable>path</replaceable></term> <listitem><para>Return <literal>true</literal> if the path - <replaceable>path</replaceable> exists, and - <literal>false</literal> otherwise. One application of this - function is to conditionally include a Nix expression containing - user configuration: - -<programlisting> -let - fileName = builtins.getEnv "CONFIG_FILE"; - config = - if fileName != "" && builtins.pathExists (builtins.toPath fileName) - then import (builtins.toPath fileName) - else { someSetting = false; }; <lineannotation># default configuration</lineannotation> -in config.someSetting</programlisting> - - (Note that <envar>CONFIG_FILE</envar> must be an absolute path for - this to work.)</para></listitem> + <replaceable>path</replaceable> exists at evaluation time, and + <literal>false</literal> otherwise.</para></listitem> </varlistentry> @@ -1409,13 +1395,10 @@ in foo</programlisting> <varlistentry xml:id='builtin-toPath'> <term><function>builtins.toPath</function> <replaceable>s</replaceable></term> - <listitem><para>Convert the string value - <replaceable>s</replaceable> into a path value. The string - <replaceable>s</replaceable> must represent an absolute path - (i.e., must start with <literal>/</literal>). The path need not - exist. The resulting path is canonicalised, e.g., - <literal>builtins.toPath "//foo/xyzzy/../bar/"</literal> returns - <literal>/foo/bar</literal>.</para></listitem> + <listitem><para> DEPRECATED. Use <literal>/. + "/path"</literal> + to convert a string into an absolute path. For relative paths, + use <literal>./. + "/path"</literal>. + </para></listitem> </varlistentry> @@ -1497,7 +1480,7 @@ stdenv.mkDerivation (rec { builder = builtins.toFile "builder.sh" " source $stdenv/setup mkdir $out - echo $servlets | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[ + echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[ "; stylesheet = builtins.toFile "stylesheet.xsl"]]> <co xml:id='ex-toxml-co-stylesheet' /> <![CDATA[ diff --git a/doc/manual/release-notes/rl-2.2.xml b/doc/manual/release-notes/rl-2.2.xml index bc28a56c9401..abe9b49adadd 100644 --- a/doc/manual/release-notes/rl-2.2.xml +++ b/doc/manual/release-notes/rl-2.2.xml @@ -19,6 +19,10 @@ </para> </listitem> + <listitem> + <para>Sandbox builds are now enabled by default on Linux.</para> + </listitem> + </itemizedlist> </section> diff --git a/release-common.nix b/release-common.nix index 690860abfdcf..f495df1440d6 100644 --- a/release-common.nix +++ b/release-common.nix @@ -66,10 +66,12 @@ rec { apis = ["s3" "transfer"]; customMemoryManagement = false; }).overrideDerivation (args: { + /* patches = args.patches or [] ++ [ (fetchpatch { url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch; sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2"; }) ]; + */ })); perlDeps = diff --git a/shell.nix b/shell.nix index c04bcd151309..817684b7646e 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ { useClang ? false }: -with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; }) {}; +with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }) {}; with import ./release-common.nix { inherit pkgs; }; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 4d9128e3f448..daaa635fe1b1 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -44,7 +44,7 @@ public: string queryDrvPath() const; string queryOutPath() const; string queryOutputName() const; - /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */ + /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ Outputs queryOutputs(bool onlyOutputsToInstall = false); StringSet queryMetaNames(); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index a9c07b23a6f3..1c2c08715a14 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -78,7 +78,11 @@ void loadConfFile() ~/.nix/nix.conf or the command line. */ globalConfig.resetOverriden(); - globalConfig.applyConfigFile(getConfigDir() + "/nix/nix.conf"); + auto dirs = getConfigDirs(); + // Iterate over them in reverse so that the ones appearing first in the path take priority + for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { + globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); + } } unsigned int Settings::getDefaultCores() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 6b3e204536f1..53efc6a90fb6 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -195,7 +195,13 @@ public: Setting<bool> showTrace{this, false, "show-trace", "Whether to show a stack trace on evaluation errors."}; - Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox", + Setting<SandboxMode> sandboxMode{this, + #if __linux__ + smEnabled + #else + smDisabled + #endif + , "sandbox", "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", {"build-use-chroot", "build-use-sandbox"}}; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 216f3417c4a8..e1cb423d151f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -366,6 +366,8 @@ void LocalStore::makeStoreWritable() throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { + saveMountNamespace(); + if (unshare(CLONE_NEWNS) == -1) throw SysError("setting up a private mount namespace"); diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ba11ce6bb6de..4f1e23198ffe 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -173,6 +173,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; + const Setting<bool> multipartUpload{ + this, false, "multipart-upload", "whether to use multi-part uploads"}; const Setting<uint64_t> bufferSize{ this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; @@ -261,48 +263,73 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads); - std::call_once(transferManagerCreated, [&]() { + std::call_once(transferManagerCreated, [&]() + { + if (multipartUpload) { + TransferManagerConfiguration transferConfig(executor.get()); + + transferConfig.s3Client = s3Helper.client; + transferConfig.bufferSize = bufferSize; + + transferConfig.uploadProgressCallback = + [](const TransferManager *transferManager, + const std::shared_ptr<const TransferHandle> + &transferHandle) + { + //FIXME: find a way to properly abort the multipart upload. + //checkInterrupt(); + debug("upload progress ('%s'): '%d' of '%d' bytes", + transferHandle->GetKey(), + transferHandle->GetBytesTransferred(), + transferHandle->GetBytesTotalSize()); + }; + + transferManager = TransferManager::Create(transferConfig); + } + }); - TransferManagerConfiguration transferConfig(executor.get()); + auto now1 = std::chrono::steady_clock::now(); - transferConfig.s3Client = s3Helper.client; - transferConfig.bufferSize = bufferSize; + if (transferManager) { - transferConfig.uploadProgressCallback = - [&](const TransferManager *transferManager, - const std::shared_ptr<const TransferHandle> - &transferHandle) - { - //FIXME: find a way to properly abort the multipart upload. - //checkInterrupt(); - debug("upload progress ('%s'): '%d' of '%d' bytes", - path, - transferHandle->GetBytesTransferred(), - transferHandle->GetBytesTotalSize()); - }; + if (contentEncoding != "") + throw Error("setting a content encoding is not supported with S3 multi-part uploads"); - transferManager = TransferManager::Create(transferConfig); - }); + std::shared_ptr<TransferHandle> transferHandle = + transferManager->UploadFile( + stream, bucketName, path, mimeType, + Aws::Map<Aws::String, Aws::String>(), + nullptr /*, contentEncoding */); - auto now1 = std::chrono::steady_clock::now(); + transferHandle->WaitUntilFinished(); - std::shared_ptr<TransferHandle> transferHandle = - transferManager->UploadFile( - stream, bucketName, path, mimeType, - Aws::Map<Aws::String, Aws::String>(), - nullptr, contentEncoding); + if (transferHandle->GetStatus() == TransferStatus::FAILED) + throw Error("AWS error: failed to upload 's3://%s/%s': %s", + bucketName, path, transferHandle->GetLastError().GetMessage()); - transferHandle->WaitUntilFinished(); + if (transferHandle->GetStatus() != TransferStatus::COMPLETED) + throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", + bucketName, path); - if (transferHandle->GetStatus() == TransferStatus::FAILED) - throw Error("AWS error: failed to upload 's3://%s/%s': %s", - bucketName, path, transferHandle->GetLastError().GetMessage()); + } else { - if (transferHandle->GetStatus() != TransferStatus::COMPLETED) - throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", - bucketName, path); + auto request = + Aws::S3::Model::PutObjectRequest() + .WithBucket(bucketName) + .WithKey(path); - printTalkative("upload of '%s' completed", path); + request.SetContentType(mimeType); + + if (contentEncoding != "") + request.SetContentEncoding(contentEncoding); + + auto stream = std::make_shared<istringstream_nocopy>(data); + + request.SetBody(stream); + + auto result = checkAws(fmt("AWS error uploading '%s'", path), + s3Helper.client->PutObject(request)); + } auto now2 = std::chrono::steady_clock::now(); diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 5e0e44935cca..cf133b57cb20 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,4 +1,5 @@ #include "ssh.hh" +#include "affinity.hh" namespace nix { @@ -34,7 +35,9 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string auto conn = std::make_unique<Connection>(); conn->sshPid = startProcess([&]() { + restoreAffinity(); restoreSignals(); + restoreMountNamespace(); close(in.writeSide.get()); close(out.readSide.get()); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 92e2685f7f66..dc54c735fdb1 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -588,15 +588,19 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, uint64_t total = 0; - // FIXME -#if 0 if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); auto info2 = make_ref<ValidPathInfo>(*info); info2->narHash = hashString(htSHA256, *sink.s); if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; info = info2; + + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; } -#endif if (info->ultimate) { auto info2 = make_ref<ValidPathInfo>(*info); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 03f0be705c1d..6e4536e6e4ea 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -496,6 +496,15 @@ Path getConfigDir() return configDir; } +std::vector<Path> getConfigDirs() +{ + Path configHome = getConfigDir(); + string configDirs = getEnv("XDG_CONFIG_DIRS"); + std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + Path getDataDir() { @@ -927,6 +936,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) throw SysError("setting death signal"); #endif restoreAffinity(); + restoreMountNamespace(); fun(); } catch (std::exception & e) { try { @@ -1495,4 +1505,26 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> return std::unique_ptr<InterruptCallback>(res.release()); } +static AutoCloseFD fdSavedMountNamespace; + +void saveMountNamespace() +{ +#if __linux__ + std::once_flag done; + std::call_once(done, []() { + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) + throw SysError("saving parent mount namespace"); + }); +#endif +} + +void restoreMountNamespace() +{ +#if __linux__ + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); +#endif +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fc25d27758c7..2689cbd8b412 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -131,6 +131,9 @@ Path getCacheDir(); /* Return $XDG_CONFIG_HOME or $HOME/.config. */ Path getConfigDir(); +/* Return the directories to search for user configuration files */ +std::vector<Path> getConfigDirs(); + /* Return $XDG_DATA_HOME or $HOME/.local/share. */ Path getDataDir(); @@ -511,4 +514,13 @@ typedef std::function<bool(const Path & path)> PathFilter; extern PathFilter defaultPathFilter; +/* Save the current mount namespace. Ignored if called more than + once. */ +void saveMountNamespace(); + +/* Restore the mount namespace saved by saveMountNamespace(). Ignored + if saveMountNamespace() was never called. */ +void restoreMountNamespace(); + + } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 618895d387d4..11ea3b1f7ae1 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -401,8 +401,6 @@ static void _main(int argc, char * * argv) } else env[var.first] = var.second; - restoreAffinity(); - /* Run a shell using the derivation's environment. For convenience, source $stdenv/setup to setup additional environment variables and shell functions. Also don't @@ -446,7 +444,9 @@ static void _main(int argc, char * * argv) auto argPtrs = stringsToCharPtrs(args); + restoreAffinity(); restoreSignals(); + restoreMountNamespace(); execvp(shell.c_str(), argPtrs.data()); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index ddb724913214..f54706a8a011 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -7,6 +7,8 @@ #include "common-eval-args.hh" #include "attr-path.hh" #include "legacy.hh" +#include "finally.hh" +#include "progress-bar.hh" #include <iostream> @@ -96,6 +98,11 @@ static int _main(int argc, char * * argv) if (args.size() > 2) throw UsageError("too many arguments"); + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + auto store = openStore(); auto state = std::make_unique<EvalState>(myArgs.searchPath, store); @@ -213,6 +220,8 @@ static int _main(int argc, char * * argv) assert(storePath == store->makeFixedOutputPath(unpack, hash, name)); } + stopProgressBar(); + if (!printPath) printInfo(format("path is '%1%'") % storePath); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index c9671f76d0fa..d8d5895bd867 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -3,6 +3,7 @@ #include "eval.hh" #include "attr-path.hh" #include "progress-bar.hh" +#include "affinity.hh" #include <unistd.h> @@ -72,6 +73,10 @@ struct CmdEdit : InstallableCommand stopProgressBar(); + restoreAffinity(); + restoreSignals(); + restoreMountNamespace(); + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); throw SysError("cannot run editor '%s'", editor); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 838aa64ae29a..ff8477865b5f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -330,6 +330,8 @@ static int runProgram(const string & program, const Strings & args) if (pid == -1) throw SysError("forking"); if (pid == 0) { restoreAffinity(); + restoreSignals(); + restoreMountNamespace(); execvp(program.c_str(), stringsToCharPtrs(args2).data()); _exit(1); } diff --git a/src/nix/run.cc b/src/nix/run.cc index 35b763345872..1297072989b9 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -153,9 +153,9 @@ struct CmdRun : InstallablesCommand stopProgressBar(); - restoreSignals(); - restoreAffinity(); + restoreSignals(); + restoreMountNamespace(); /* If this is a diverted store (i.e. its "logical" location (typically /nix/store) differs from its "physical" location diff --git a/tests/common.sh.in b/tests/common.sh.in index 2ee2f589dae4..6a523ca9d832 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -85,16 +85,13 @@ killDaemon() { trap "" EXIT } -canUseSandbox() { - if [[ $(uname) != Linux ]]; then return 1; fi - - if [ ! -L /proc/self/ns/user ]; then - echo "Kernel doesn't support user namespaces, skipping this test..." - return 1 - fi +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then + _canUseSandbox=1 +fi - if ! unshare --user true ; then - echo "Unprivileged user namespaces disabled by sysctl, skipping this test..." +canUseSandbox() { + if [[ ! $_canUseSandbox ]]; then + echo "Sandboxing not supported, skipping this test..." return 1 fi diff --git a/tests/init.sh b/tests/init.sh index e5353598bcc4..19a12c1e2d9e 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -16,6 +16,7 @@ mkdir "$NIX_CONF_DIR" cat > "$NIX_CONF_DIR"/nix.conf <<EOF build-users-group = keep-derivations = false +sandbox = false include nix.conf.extra EOF |