about summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/manual/command-ref/nix-shell.xml8
-rw-r--r--doc/manual/packages/s3-substituter.xml159
-rw-r--r--doc/manual/packages/sharing-packages.xml1
-rw-r--r--nix.spec.in20
-rw-r--r--release.nix4
-rw-r--r--src/libexpr/eval.cc20
-rw-r--r--src/libexpr/eval.hh1
-rw-r--r--src/libexpr/parser.y4
-rw-r--r--src/libexpr/primops.cc18
-rw-r--r--src/libexpr/primops/fromTOML.cc4
-rw-r--r--src/libstore/download.cc16
-rw-r--r--src/libstore/s3-binary-cache-store.cc14
-rw-r--r--src/libstore/s3.hh4
-rw-r--r--src/libstore/store-api.cc14
-rw-r--r--src/libutil/serialise.cc4
-rwxr-xr-xsrc/nix-build/nix-build.cc8
-rw-r--r--src/nix-prefetch-url/nix-prefetch-url.cc24
-rw-r--r--src/nix/copy.cc4
-rw-r--r--tests/nix-shell.sh7
19 files changed, 264 insertions, 70 deletions
diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
index 62d026ac238e..5c44c4a8f446 100644
--- a/doc/manual/command-ref/nix-shell.xml
+++ b/doc/manual/command-ref/nix-shell.xml
@@ -32,6 +32,7 @@
     <arg><option>--run</option> <replaceable>cmd</replaceable></arg>
     <arg><option>--exclude</option> <replaceable>regexp</replaceable></arg>
     <arg><option>--pure</option></arg>
+    <arg><option>--keep</option> <replaceable>name</replaceable></arg>
     <group choice='req'>
       <arg choice='plain'>
         <group choice='req'>
@@ -165,6 +166,13 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
     </listitem></varlistentry>
 
+  <varlistentry><term><option>--keep</option> <replaceable>name</replaceable></term>
+
+    <listitem><para>When a <option>--pure</option> shell is started,
+    keep the listed environment variables.</para></listitem>
+
+  </varlistentry>
+
 </variablelist>
 
 <para>The following common options are supported:</para>
diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml
new file mode 100644
index 000000000000..bcd91cfdbccd
--- /dev/null
+++ b/doc/manual/packages/s3-substituter.xml
@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="utf-8"?>
+<section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         version="5.0"
+         xml:id="ssec-s3-substituter">
+
+<title>Serving a Nix store via AWS S3 or S3-compatible Service</title>
+
+<para>Nix has built-in support for storing and fetching store paths
+from Amazon S3 and S3 compatible services. This uses the same
+<emphasis>binary</emphasis> cache mechanism that Nix usually uses to
+fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
+
+<para>In this example we will use the bucket named
+<literal>example-bucket</literal>.</para>
+
+<section xml:id="ssec-s3-substituter-anonymous-reads">
+  <title>Anonymous Reads to your S3-compatible binary cache</title>
+
+  <para>If your binary cache is publicly accessible and does not
+  require authentication, the simplest and easiest way to use Nix with
+  your S3 compatible binary cache is to use the HTTP URL for that
+  cache.</para>
+
+  <para>For AWS S3 the binary cache URL for example bucket will be
+  exactly <uri>https://example-bucket.s3.amazonaws.com</uri>. For S3
+  compatible binary caches ago have to consult your software's
+  documentation.</para>
+
+  <para>Your bucket will need the following bucket policy:</para>
+
+  <programlisting>
+<![CDATA[
+{
+  "Id": "DirectReads",
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "AlowDirectReads",
+      "Action": [
+        "s3:GetObject"
+      ],
+      "Effect": "Allow",
+      "Resource": "arn:aws:s3:::example-bucket/*",
+      "Principal": "*"
+    }
+  ]
+}
+]]>
+</programlisting>
+</section>
+
+<section xml:id="ssec-s3-substituter-authenticated-reads">
+  <title>Authenticated Reads to your S3 binary cache</title>
+
+  <para>For AWS S3 the binary cache URL for example bucket will be
+  exactly <uri>s3://example-bucket</uri>.</para>
+
+  <para>Nix will use the <link
+  xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
+  credential provider chain</link> for authenticating requests to
+  Amazon S3.</para>
+
+  <para>Nix supports authenticated writes to S3 compatible binary
+  caches but only supports Authenticated reads from Amazon S3.
+  Additionally, the following limitations are in place for
+  authenticated reads:</para>
+
+  <itemizedlist>
+    <listitem><para>The bucket must actually be hosted by Amazon S3 and
+    <emphasis>not</emphasis> an S3 compatible
+    service.</para></listitem>
+
+    <listitem><para>The bucket must be within the
+    <literal>us-east-1</literal> region.</para></listitem>
+
+    <listitem><para>The Amazon credentials, if stored in a credential
+    profile, must be stored in the <literal>default</literal>
+    profile.</para></listitem>
+  </itemizedlist>
+
+  <para>Your bucket will need a bucket policy allowing the desired
+  users to perform the <literal>s3:GetObject</literal> action on all
+  objects in the bucket.</para>
+</section>
+
+
+<section xml:id="ssec-s3-substituter-authenticated-writes">
+  <title>Authenticated Writes to your S3-compatible binary cache</title>
+
+  <para>Nix support fully supports writing to Amazon S3 and S3
+  compatible buckets. The binary cache URL for our example bucket will
+  be <uri>s3://example-bucket</uri>.</para>
+
+  <para>Nix will use the <link
+  xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
+  credential provider chain</link> for authenticating requests to
+  Amazon S3.</para>
+
+  <para>The following options can be specified as URL parameters to
+  the S3 URL:</para>
+  <variablelist>
+    <varlistentry><term><literal>profile</literal></term>
+    <listitem>
+      <para>
+        The name of the AWS configuration profile to use. By default
+        Nix will use the <literal>default</literal> profile.
+      </para>
+    </listitem>
+    </varlistentry>
+
+    <varlistentry><term><literal>region</literal></term>
+    <listitem>
+      <para>
+        The region of the S3 bucket. <literal>us–east-1</literal> by
+        default.
+      </para>
+    </listitem>
+    </varlistentry>
+
+    <varlistentry><term><literal>endpoint</literal></term>
+    <listitem>
+      <para>
+        The URL to your S3-compatible service, for when not using
+        Amazon S3. Do not specify this value if you're using Amazon
+        S3.
+      </para>
+      <note><para>This endpoint must support HTTPS and will use
+      path-based addressing instead of virtual host based
+      addressing.</para></note>
+    </listitem>
+    </varlistentry>
+  </variablelist>
+
+  <example><title>Uploading with non-default credential profile for Amazon S3</title>
+    <para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload</command></para>
+  </example>
+
+  <example><title>Uploading to an S3-Compatible Binary Cache</title>
+    <para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload&amp;endpoint=minio.example.com</command></para>
+  </example>
+
+  <para>The user writing to the bucket will need to perform the
+  following actions against the bucket:</para>
+
+  <itemizedlist>
+    <listitem><para><literal>s3:ListBucket</literal></para></listitem>
+    <listitem><para><literal>s3:GetBucketLocation</literal></para></listitem>
+    <listitem><para><literal>s3:ListObjects</literal></para></listitem>
+    <listitem><para><literal>s3:GetObject</literal></para></listitem>
+    <listitem><para><literal>s3:PutObject</literal></para></listitem>
+    <listitem><para><literal>s3:ListBucketMultipartUploads</literal></para></listitem>
+    <listitem><para><literal>s3:CreateMultipartUpload</literal></para></listitem>
+    <listitem><para><literal>s3:ListMultipartUploadParts</literal></para></listitem>
+    <listitem><para><literal>s3:AbortMultipartUpload</literal></para></listitem>
+  </itemizedlist>
+</section>
+</section>
diff --git a/doc/manual/packages/sharing-packages.xml b/doc/manual/packages/sharing-packages.xml
index 8465c182ee72..bb6c52b8f8c1 100644
--- a/doc/manual/packages/sharing-packages.xml
+++ b/doc/manual/packages/sharing-packages.xml
@@ -15,5 +15,6 @@ packages between machines.</para>
 <xi:include href="binary-cache-substituter.xml" />
 <xi:include href="copy-closure.xml" />
 <xi:include href="ssh-substituter.xml" />
+<xi:include href="s3-substituter.xml" />
 
 </chapter>
diff --git a/nix.spec.in b/nix.spec.in
index cd053dbfce5c..477768c6a68c 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -152,11 +152,11 @@ systemctl start  nix-daemon.socket
 %{_prefix}/lib/systemd/system/nix-daemon.service
 %endif
 %{_datadir}/nix
-%if ! %{without docgen}
-%{_mandir}/man1/*.1*
-%{_mandir}/man5/*.5*
-%{_mandir}/man8/*.8*
-%endif
+#%if ! %{without docgen}
+#%{_mandir}/man1/*.1*
+#%{_mandir}/man5/*.5*
+#%{_mandir}/man8/*.8*
+#%endif
 %config(noreplace) %{_sysconfdir}/profile.d/nix.sh
 %config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh
 /nix
@@ -166,8 +166,8 @@ systemctl start  nix-daemon.socket
 %{_prefix}/lib/pkgconfig/*.pc
 
 
-%if ! %{without docgen}
-%files doc
-%docdir %{_defaultdocdir}/%{name}-doc-%{version}
-%{_defaultdocdir}/%{name}-doc-%{version}
-%endif
+#%if ! %{without docgen}
+#%files doc
+#%docdir %{_defaultdocdir}/%{name}-doc-%{version}
+#%{_defaultdocdir}/%{name}-doc-%{version}
+#%endif
diff --git a/release.nix b/release.nix
index 321f1688367a..076f1de8dea3 100644
--- a/release.nix
+++ b/release.nix
@@ -206,7 +206,7 @@ let
       };
 
 
-    rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ];
+    #rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ];
 
 
     #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
@@ -332,7 +332,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" ]
+            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ]
             ++ extraPackages; };
       # At most 2047MB can be simulated in qemu-system-i386
       memSize = 2047;
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 3abde6c92961..f41905787f9e 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1082,6 +1082,8 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
 
 void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
+    forceValue(fun, pos);
+
     if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
         callPrimOp(fun, arg, v, pos);
         return;
@@ -1097,10 +1099,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         auto & fun2 = *allocValue();
         fun2 = fun;
         /* !!! Should we use the attr pos here? */
-        forceValue(*found->value, pos);
         Value v2;
         callFunction(*found->value, fun2, v2, pos);
-        forceValue(v2, pos);
         return callFunction(v2, arg, v, pos);
       }
     }
@@ -1187,7 +1187,6 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
     if (fun.type == tAttrs) {
         auto found = fun.attrs->find(sFunctor);
         if (found != fun.attrs->end()) {
-            forceValue(*found->value);
             Value * v = allocValue();
             callFunction(*found->value, fun, *v, noPos);
             forceValue(*v);
@@ -1571,7 +1570,6 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     if (v.type == tAttrs) {
         auto i = v.attrs->find(sToString);
         if (i != v.attrs->end()) {
-            forceValue(*i->value, pos);
             Value v1;
             callFunction(*i->value, v, v1, pos);
             return coerceToString(pos, v1, context, coerceMore, copyToStore);
@@ -1726,20 +1724,6 @@ bool EvalState::eqValues(Value & v1, Value & v2)
 }
 
 
-void EvalState::printStats2()
-{
-    struct rusage ru;
-    getrusage(RUSAGE_SELF, &ru);
-
-    GC_prof_stats_s gc;
-    GC_get_prof_stats(&gc, sizeof(gc));
-
-    printError("STATS %d %d %d %d %d %d",
-        nrValues, nrValuesFreed.load(), nrValues - nrValuesFreed,
-        ru.ru_maxrss,
-        gc.heapsize_full, gc.free_bytes_full);
-}
-
 void EvalState::printStats()
 {
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 46bda86d084e..d0f298e168e9 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -276,7 +276,6 @@ public:
 
     /* Print statistics. */
     void printStats();
-    void printStats2();
 
     void realiseContext(const PathSet & context);
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index eee48887dc22..cbd576d7d126 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -273,11 +273,11 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %token IND_STRING_OPEN IND_STRING_CLOSE
 %token ELLIPSIS
 
-%nonassoc IMPL
+%right IMPL
 %left OR
 %left AND
 %nonassoc EQ NEQ
-%left '<' '>' LEQ GEQ
+%nonassoc '<' '>' LEQ GEQ
 %right UPDATE
 %left NOT
 %left '+' '-'
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e71e3a6d46cb..8ace6db4d11d 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1359,7 +1359,6 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 /* Apply a function to every element of an attribute set. */
 static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     state.forceAttrs(*args[1], pos);
 
     state.mkAttrs(v, args[1]->attrs->size());
@@ -1368,7 +1367,7 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va
         Value * vName = state.allocValue();
         Value * vFun2 = state.allocValue();
         mkString(*vName, i.name);
-        state.callFunction(*args[0], *vName, *vFun2, pos);
+        mkApp(*vFun2, *args[0], *vName);
         mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value);
     }
 }
@@ -1429,7 +1428,6 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
 /* Apply a function to every element of a list. */
 static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     state.forceList(*args[1], pos);
 
     state.mkList(v, args[1]->listSize());
@@ -1508,19 +1506,20 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
     state.forceFunction(*args[0], pos);
     state.forceList(*args[2], pos);
 
-    Value * vCur = args[1];
+    if (args[2]->listSize()) {
+        Value * vCur = args[1];
 
-    if (args[2]->listSize())
         for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
             Value vTmp;
             state.callFunction(*args[0], *vCur, vTmp, pos);
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
             state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
         }
-    else
-        v = *vCur;
-
-    state.forceValue(v);
+        state.forceValue(v);
+    } else {
+        state.forceValue(*args[1]);
+        v = *args[1];
+    }
 }
 
 
@@ -1557,7 +1556,6 @@ static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     auto len = state.forceInt(*args[1], pos);
 
     if (len < 0)
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index ae8aba612bb9..4128de05d0cf 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -20,7 +20,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
         if (auto t2 = t->as_table()) {
 
             size_t size = 0;
-            for (auto & i : *t2) size++;
+            for (auto & i : *t2) { (void) i; size++; }
 
             state.mkAttrs(v, size);
 
@@ -50,7 +50,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
         }
 
         else if (t->is_value()) {
-            if (auto val = t->as<NixInt>())
+            if (auto val = t->as<int64_t>())
                 mkInt(v, val->get());
             else if (auto val = t->as<NixFloat>())
                 mkFloat(v, val->get());
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 07acd5d0e004..f80c9e45b82e 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -598,7 +598,7 @@ struct CurlDownloader : public Downloader
             // FIXME: do this on a worker thread
             try {
 #ifdef ENABLE_S3
-                S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable
+                S3Helper s3Helper("", Aws::Region::US_EAST_1, ""); // FIXME: make configurable
                 auto slash = request.uri.find('/', 5);
                 if (slash == std::string::npos)
                     throw nix::Error("bad S3 URI '%s'", request.uri);
@@ -718,15 +718,17 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
     while (true) {
         checkInterrupt();
 
-        if (state->quit) {
-            if (state->exc) std::rethrow_exception(state->exc);
-            break;
-        }
-
         /* If no data is available, then wait for the download thread
            to wake us up. */
-        if (state->data.empty())
+        if (state->data.empty()) {
+
+            if (state->quit) {
+                if (state->exc) std::rethrow_exception(state->exc);
+                break;
+            }
+
             state.wait(state->avail);
+        }
 
         /* If data is available, then flush it to the sink and wake up
            the download thread if it's blocked on a full buffer. */
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 26144ccb40cc..2f18e3f38c6e 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -84,8 +84,8 @@ static void initAWS()
     });
 }
 
-S3Helper::S3Helper(const std::string & profile, const std::string & region)
-    : config(makeConfig(region))
+S3Helper::S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint)
+    : config(makeConfig(region, endpoint))
     , client(make_ref<Aws::S3::S3Client>(
             profile == ""
             ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
@@ -99,7 +99,7 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region)
 #else
             Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
 #endif
-            false))
+            endpoint.empty()))
 {
 }
 
@@ -116,11 +116,14 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
     }
 };
 
-ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region)
+ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & endpoint)
 {
     initAWS();
     auto res = make_ref<Aws::Client::ClientConfiguration>();
     res->region = region;
+    if (!endpoint.empty()) {
+        res->endpointOverride = endpoint;
+    }
     res->requestTimeoutMs = 600 * 1000;
     res->retryStrategy = std::make_shared<RetryStrategy>();
     res->caFile = settings.caFile;
@@ -170,6 +173,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
     const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
     const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
+    const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
     const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
     const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
     const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
@@ -186,7 +190,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
-        , s3Helper(profile, region)
+        , s3Helper(profile, region, endpoint)
     {
         diskCache = getNarInfoDiskCache();
     }
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
index 4f996400343c..95d612b66335 100644
--- a/src/libstore/s3.hh
+++ b/src/libstore/s3.hh
@@ -14,9 +14,9 @@ struct S3Helper
     ref<Aws::Client::ClientConfiguration> config;
     ref<Aws::S3::S3Client> client;
 
-    S3Helper(const std::string & profile, const std::string & region);
+    S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint);
 
-    ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region);
+    ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & endpoint);
 
     struct DownloadResult
     {
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 9b0b7d6327e0..1854353828df 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -629,11 +629,12 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
     Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
 
     std::atomic<size_t> nrDone{0};
+    std::atomic<size_t> nrFailed{0};
     std::atomic<uint64_t> bytesExpected{0};
     std::atomic<uint64_t> nrRunning{0};
 
     auto showProgress = [&]() {
-        act.progress(nrDone, missing.size(), nrRunning);
+        act.progress(nrDone, missing.size(), nrRunning, nrFailed);
     };
 
     ThreadPool pool;
@@ -662,7 +663,16 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
             if (!dstStore->isValidPath(storePath)) {
                 MaintainCount<decltype(nrRunning)> mc(nrRunning);
                 showProgress();
-                copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
+                try {
+                    copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
+                } catch (Error &e) {
+                    nrFailed++;
+                    if (!settings.keepGoing)
+                        throw e;
+                    logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what());
+                    showProgress();
+                    return;
+                }
             }
 
             nrDone++;
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 21803edd056a..b2c49d911b34 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -157,6 +157,10 @@ size_t StringSource::read(unsigned char * data, size_t len)
 }
 
 
+#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600
+#error Coroutines are broken in this version of Boost!
+#endif
+
 std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun)
 {
     struct SinkToSource : Source
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index de0e9118fd21..34f1cba9ddac 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -85,7 +85,6 @@ void mainWrapped(int argc, char * * argv)
     BuildMode buildMode = bmNormal;
     bool readStdin = false;
 
-    auto shell = getEnv("SHELL", "/bin/sh");
     std::string envCommand; // interactive shell
     Strings envExclude;
 
@@ -99,6 +98,9 @@ void mainWrapped(int argc, char * * argv)
 
     std::string outLink = "./result";
 
+    // List of environment variables kept for --pure
+    std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"};
+
     Strings args;
     for (int i = 1; i < argc; ++i)
         args.push_back(argv[i]);
@@ -218,6 +220,9 @@ void mainWrapped(int argc, char * * argv)
             }
         }
 
+        else if (*arg == "--keep")
+            keepVars.insert(getArg(*arg, arg, end));
+
         else if (*arg == "-")
             readStdin = true;
 
@@ -368,7 +373,6 @@ void mainWrapped(int argc, char * * argv)
         auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp"));
 
         if (pure) {
-            std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"};
             decltype(env) newEnv;
             for (auto & i : env)
                 if (keepVars.count(i.first))
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 50b2c2803ec9..a3b025723cf1 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -9,6 +9,10 @@
 
 #include <iostream>
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
 using namespace nix;
 
 
@@ -160,14 +164,20 @@ int main(int argc, char * * argv)
 
             auto actualUri = resolveMirrorUri(*state, uri);
 
-            /* Download the file. */
-            DownloadRequest req(actualUri);
-            req.decompress = false;
-            auto result = getDownloader()->download(req);
-
             AutoDelete tmpDir(createTempDir(), true);
             Path tmpFile = (Path) tmpDir + "/tmp";
-            writeFile(tmpFile, *result.data);
+
+            /* Download the file. */
+            {
+                AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600);
+                if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
+
+                FdSink sink(fd.get());
+
+                DownloadRequest req(actualUri);
+                req.decompress = false;
+                getDownloader()->download(std::move(req), sink);
+            }
 
             /* Optionally unpack the file. */
             if (unpack) {
@@ -191,7 +201,7 @@ int main(int argc, char * * argv)
 
             /* FIXME: inefficient; addToStore() will also hash
                this. */
-            hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data);
+            hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
 
             if (expectedHash != Hash(ht) && expectedHash != hash)
                 throw Error(format("hash mismatch for '%1%'") % uri);
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index e4e6c3e303ed..91711c8b46da 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -72,6 +72,10 @@ struct CmdCopy : StorePathsCommand
                 "To populate the current folder build output to a S3 binary cache:",
                 "nix copy --to s3://my-bucket?region=eu-west-1"
             },
+            Example{
+                "To populate the current folder build output to an S3-compatible binary cache:",
+                "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com"
+            },
 #endif
         };
     }
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index d25c456cedfb..6024ea399750 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -4,12 +4,19 @@ clearStore
 
 # Test nix-shell -A
 export IMPURE_VAR=foo
+export SELECTED_IMPURE_VAR=baz
 export NIX_BUILD_SHELL=$SHELL
 output=$(nix-shell --pure shell.nix -A shellDrv --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"')
 
 [ "$output" = " - foo - bar" ]
 
+# Test --keep
+output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run \
+    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"')
+
+[ "$output" = " - foo - bar - baz" ]
+
 # Test nix-shell on a .drv
 [[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
generated by cgit-pink 1.4.1 (git 2.44.2) at 2024-12-25T11·11+0000