about summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/build-remote/build-remote.cc2
-rw-r--r--src/libexpr/eval.cc9
-rw-r--r--src/libmain/common-args.cc4
-rw-r--r--src/libstore/binary-cache-store.cc2
-rw-r--r--src/libstore/binary-cache-store.hh2
-rw-r--r--src/libstore/build.cc36
-rw-r--r--src/libstore/local-store.cc3
-rw-r--r--src/libstore/local-store.hh3
-rw-r--r--src/libstore/pathlocks.cc6
-rw-r--r--src/libstore/pathlocks.hh6
-rw-r--r--src/libstore/store-api.cc6
-rw-r--r--src/libutil/compression.cc32
-rw-r--r--src/libutil/compression.hh4
-rw-r--r--src/nix/build.cc2
-rw-r--r--src/nix/command.hh2
-rw-r--r--src/nix/installables.cc4
-rw-r--r--tests/nix-copy-closure.nix6
-rw-r--r--tests/remote-builds.nix10
-rw-r--r--tests/user-envs.sh3
19 files changed, 90 insertions, 52 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 50eb6b29e518..dbf8fe1b8f8a 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -243,7 +243,7 @@ connected:
 
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
-            setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */
+            store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
             copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute);
         }
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 0b0a0f7b1790..11195af77939 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1700,10 +1700,13 @@ void EvalState::printStats()
     printMsg(v, format("  time elapsed: %1%") % cpuTime);
     printMsg(v, format("  size of a value: %1%") % sizeof(Value));
     printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
-    printMsg(v, format("  environments allocated: %1% (%2% bytes)") % nrEnvs % bEnvs);
-    printMsg(v, format("  list elements: %1% (%2% bytes)") % nrListElems % bLists);
+    printMsg(v, format("  environments allocated count: %1%") % nrEnvs);
+    printMsg(v, format("  environments allocated bytes: %1%") % bEnvs);
+    printMsg(v, format("  list elements count: %1%") % nrListElems);
+    printMsg(v, format("  list elements bytes: %1%") % bLists);
     printMsg(v, format("  list concatenations: %1%") % nrListConcats);
-    printMsg(v, format("  values allocated: %1% (%2% bytes)") % nrValues % bValues);
+    printMsg(v, format("  values allocated count: %1%") % nrValues);
+    printMsg(v, format("  values allocated bytes: %1%") % bValues);
     printMsg(v, format("  sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets);
     printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
     printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index d3aac6aba1ff..bcc05c2cdad6 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -37,6 +37,10 @@ MixCommonArgs::MixCommonArgs(const string & programName)
 
     std::string cat = "config";
     settings.convertToArgs(*this, cat);
+
+    // Backward compatibility hack: nix-env already had a --system flag.
+    if (programName == "nix-env") longFlags.erase("system");
+
     hiddenCategories.insert(cat);
 }
 
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index ab971dd8b6d9..d1b278b8efbe 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -149,7 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     /* Compress the NAR. */
     narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, *nar);
+    auto narCompressed = compress(compression, *nar, parallelCompression);
     auto now2 = std::chrono::steady_clock::now();
     narInfo->fileHash = hashString(htSHA256, *narCompressed);
     narInfo->fileSize = narCompressed->size();
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 8492ff600eba..e20b968442b7 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -19,6 +19,8 @@ public:
     const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
     const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
     const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
+    const Setting<bool> parallelCompression{this, false, "parallel-compression",
+        "enable multi-threading compression, available for xz only currently"};
 
 private:
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9f669f7e4645..cc69ff1c74bf 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1335,19 +1335,6 @@ void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
 
-    /* Check for the possibility that some other goal in this process
-       has locked the output since we checked in haveDerivation().
-       (It can't happen between here and the lockPaths() call below
-       because we're not allowing multi-threading.)  If so, put this
-       goal to sleep until another goal finishes, then try again. */
-    for (auto & i : drv->outputs)
-        if (pathIsLockedByMe(worker.store.toRealPath(i.second.path))) {
-            debug(format("putting derivation '%1%' to sleep because '%2%' is locked by another goal")
-                % drvPath % i.second.path);
-            worker.waitForAnyGoal(shared_from_this());
-            return;
-        }
-
     /* Obtain locks on all output paths.  The locks are automatically
        released when we exit this function or Nix crashes.  If we
        can't acquire the lock, then continue; hopefully some other
@@ -3688,8 +3675,8 @@ void SubstitutionGoal::tryNext()
         && !sub->isTrusted
         && !info->checkSignatures(worker.store, worker.store.publicKeys))
     {
-        printInfo(format("warning: substituter '%s' does not have a valid signature for path '%s'")
-            % sub->getUri() % storePath);
+        printError("warning: substituter '%s' does not have a valid signature for path '%s'",
+            sub->getUri(), storePath);
         tryNext();
         return;
     }
@@ -3739,6 +3726,17 @@ void SubstitutionGoal::tryToRun()
         return;
     }
 
+    /* If the store path is already locked (probably by a
+       DerivationGoal), then put this goal to sleep. Note: we don't
+       acquire a lock here since that breaks addToStore(), so below we
+       handle an AlreadyLocked exception from addToStore(). The check
+       here is just an optimisation to prevent having to redo a
+       download due to a locked path. */
+    if (pathIsLockedByMe(worker.store.toRealPath(storePath))) {
+        worker.waitForAWhile(shared_from_this());
+        return;
+    }
+
     maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
     worker.updateProgress();
 
@@ -3778,8 +3776,14 @@ void SubstitutionGoal::finished()
 
     try {
         promise.get_future().get();
+    } catch (AlreadyLocked & e) {
+        /* Probably a DerivationGoal is already building this store
+           path. Sleep for a while and try again. */
+        state = &SubstitutionGoal::init;
+        worker.waitForAWhile(shared_from_this());
+        return;
     } catch (Error & e) {
-        printInfo(e.msg());
+        printError(e.msg());
 
         /* Try the next substitute. */
         state = &SubstitutionGoal::tryNext;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 7afecc1cfc62..4afe51ea91ec 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -992,8 +992,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
            lock on this path). */
-        Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
-        if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end())
+        if (!locksHeld.count(info.path))
             outputLock.lockPaths({realPath});
 
         if (repair || !isValidPath(info.path)) {
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 30bef3a799d4..bbd50e1c1451 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -104,6 +104,9 @@ private:
 
 public:
 
+    // Hack for build-remote.cc.
+    PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
+
     /* Initialise the local store, upgrading the schema if
        necessary. */
     LocalStore(const Params & params);
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index 587f29598851..08d1efdbeb01 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -113,8 +113,10 @@ bool PathLocks::lockPaths(const PathSet & _paths,
 
         {
             auto lockedPaths(lockedPaths_.lock());
-            if (lockedPaths->count(lockPath))
-                throw Error("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
+            if (lockedPaths->count(lockPath)) {
+                if (!wait) return false;
+                throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
+            }
             lockedPaths->insert(lockPath);
         }
 
diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh
index 2a7de611446e..db51f950a320 100644
--- a/src/libstore/pathlocks.hh
+++ b/src/libstore/pathlocks.hh
@@ -2,10 +2,8 @@
 
 #include "util.hh"
 
-
 namespace nix {
 
-
 /* Open (possibly create) a lock file and return the file descriptor.
    -1 is returned if create is false and the lock could not be opened
    because it doesn't exist.  Any other error throws an exception. */
@@ -18,6 +16,7 @@ enum LockType { ltRead, ltWrite, ltNone };
 
 bool lockFile(int fd, LockType lockType, bool wait);
 
+MakeError(AlreadyLocked, Error);
 
 class PathLocks
 {
@@ -38,9 +37,6 @@ public:
     void setDeletion(bool deletePaths);
 };
 
-
-// FIXME: not thread-safe!
 bool pathIsLockedByMe(const Path & path);
 
-
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7abb300a9bb8..4d43ef082d53 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -896,7 +896,11 @@ std::list<ref<Store>> getDefaultSubstituters()
         auto addStore = [&](const std::string & uri) {
             if (done.count(uri)) return;
             done.insert(uri);
-            stores.push_back(openStore(uri));
+            try {
+                stores.push_back(openStore(uri));
+            } catch (Error & e) {
+                printError("warning: %s", e.what());
+            }
         };
 
         for (auto uri : settings.substituters.get())
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 5e2631ba3408..ed15761b32a2 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -151,10 +151,10 @@ static ref<std::string> decompressBrotli(const std::string & in)
 #endif // HAVE_BROTLI
 }
 
-ref<std::string> compress(const std::string & method, const std::string & in)
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
 {
     StringSink ssink;
-    auto sink = makeCompressionSink(method, ssink);
+    auto sink = makeCompressionSink(method, ssink, parallel);
     (*sink)(in);
     sink->finish();
     return ssink.s;
@@ -189,10 +189,28 @@ struct XzSink : CompressionSink
     lzma_stream strm = LZMA_STREAM_INIT;
     bool finished = false;
 
-    XzSink(Sink & nextSink) : nextSink(nextSink)
+    XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink)
     {
-        lzma_ret ret = lzma_easy_encoder(
-            &strm, 6, LZMA_CHECK_CRC64);
+        lzma_ret ret;
+        if (parallel) {
+            lzma_mt mt_options = {};
+            mt_options.flags = 0;
+            mt_options.timeout = 300; // Using the same setting as the xz cmd line
+            mt_options.preset = LZMA_PRESET_DEFAULT;
+            mt_options.filters = NULL;
+            mt_options.check = LZMA_CHECK_CRC64;
+            mt_options.threads = lzma_cputhreads();
+            mt_options.block_size = 0;
+            if (mt_options.threads == 0)
+                mt_options.threads = 1;
+            // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
+            // number of threads.
+            ret = lzma_stream_encoder_mt(
+                &strm, &mt_options);
+        } else
+            ret = lzma_easy_encoder(
+                &strm, 6, LZMA_CHECK_CRC64);
+
         if (ret != LZMA_OK)
             throw CompressionError("unable to initialise lzma encoder");
         // FIXME: apply the x86 BCJ filter?
@@ -449,12 +467,12 @@ struct BrotliSink : CompressionSink
 };
 #endif // HAVE_BROTLI
 
-ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink)
+ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
 {
     if (method == "none")
         return make_ref<NoneSink>(nextSink);
     else if (method == "xz")
-        return make_ref<XzSink>(nextSink);
+        return make_ref<XzSink>(nextSink, parallel);
     else if (method == "bzip2")
         return make_ref<BzipSink>(nextSink);
     else if (method == "br")
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index e3e6f5a99303..a0d7530d74fc 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -8,7 +8,7 @@
 
 namespace nix {
 
-ref<std::string> compress(const std::string & method, const std::string & in);
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
 
 ref<std::string> decompress(const std::string & method, const std::string & in);
 
@@ -17,7 +17,7 @@ struct CompressionSink : BufferedSink
     virtual void finish() = 0;
 };
 
-ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink);
+ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false);
 
 MakeError(UnknownCompressionMethod, Error);
 
diff --git a/src/nix/build.cc b/src/nix/build.cc
index f7c99f12dbbf..b4f21b32d78f 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -50,7 +50,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand
 
     void run(ref<Store> store) override
     {
-        auto buildables = toBuildables(store, dryRun ? DryRun : Build, installables);
+        auto buildables = build(store, dryRun ? DryRun : Build, installables);
 
         for (size_t i = 0; i < buildables.size(); ++i) {
             auto & b(buildables[i]);
diff --git a/src/nix/command.hh b/src/nix/command.hh
index a7863c49f37a..97a6fee7fd27 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -198,7 +198,7 @@ std::shared_ptr<Installable> parseInstallable(
     SourceExprCommand & cmd, ref<Store> store, const std::string & installable,
     bool useDefaultInstallables);
 
-Buildables toBuildables(ref<Store> store, RealiseMode mode,
+Buildables build(ref<Store> store, RealiseMode mode,
     std::vector<std::shared_ptr<Installable>> installables);
 
 PathSet toStorePaths(ref<Store> store, RealiseMode mode,
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index c3b06c22eba8..a3fdd8a2808d 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -253,7 +253,7 @@ std::shared_ptr<Installable> parseInstallable(
     return installables.front();
 }
 
-Buildables toBuildables(ref<Store> store, RealiseMode mode,
+Buildables build(ref<Store> store, RealiseMode mode,
     std::vector<std::shared_ptr<Installable>> installables)
 {
     if (mode != Build)
@@ -291,7 +291,7 @@ PathSet toStorePaths(ref<Store> store, RealiseMode mode,
 {
     PathSet outPaths;
 
-    for (auto & b : toBuildables(store, mode, installables))
+    for (auto & b : build(store, mode, installables))
         for (auto & output : b.outputs)
             outPaths.insert(output.second);
 
diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix
index be0a4a683cda..0dc147fb34e9 100644
--- a/tests/nix-copy-closure.nix
+++ b/tests/nix-copy-closure.nix
@@ -29,10 +29,10 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
       startAll;
 
       # Create an SSH key on the client.
-      my $key = `${pkgs.openssh}/bin/ssh-keygen -t dsa -f key -N ""`;
+      my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`;
       $client->succeed("mkdir -m 700 /root/.ssh");
-      $client->copyFileFromHost("key", "/root/.ssh/id_dsa");
-      $client->succeed("chmod 600 /root/.ssh/id_dsa");
+      $client->copyFileFromHost("key", "/root/.ssh/id_ed25519");
+      $client->succeed("chmod 600 /root/.ssh/id_ed25519");
 
       # Install the SSH key on the server.
       $server->succeed("mkdir -m 700 /root/.ssh");
diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index 75704ace2dba..d7a4b21989e5 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -46,13 +46,13 @@ in
           nix.buildMachines =
             [ { hostName = "slave1";
                 sshUser = "root";
-                sshKey = "/root/.ssh/id_dsa";
+                sshKey = "/root/.ssh/id_ed25519";
                 system = "i686-linux";
                 maxJobs = 1;
               }
               { hostName = "slave2";
                 sshUser = "root";
-                sshKey = "/root/.ssh/id_dsa";
+                sshKey = "/root/.ssh/id_ed25519";
                 system = "i686-linux";
                 maxJobs = 1;
               }
@@ -70,10 +70,10 @@ in
       startAll;
 
       # Create an SSH key on the client.
-      my $key = `${pkgs.openssh}/bin/ssh-keygen -t dsa -f key -N ""`;
+      my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`;
       $client->succeed("mkdir -p -m 700 /root/.ssh");
-      $client->copyFileFromHost("key", "/root/.ssh/id_dsa");
-      $client->succeed("chmod 600 /root/.ssh/id_dsa");
+      $client->copyFileFromHost("key", "/root/.ssh/id_ed25519");
+      $client->succeed("chmod 600 /root/.ssh/id_ed25519");
 
       # Install the SSH key on the slaves.
       $client->waitForUnit("network.target");
diff --git a/tests/user-envs.sh b/tests/user-envs.sh
index c4192fdc59b2..ba63923113d8 100644
--- a/tests/user-envs.sh
+++ b/tests/user-envs.sh
@@ -24,6 +24,9 @@ rm -f $HOME/.nix-defexpr
 ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr
 nix-env -qa '*' --description | grep -q silly
 
+# Query the system.
+nix-env -qa '*' --system | grep -q $system
+
 # Install "foo-1.0".
 nix-env -i foo-1.0
generated by cgit-pink 1.4.1 (git 2.44.2) at 2024-09-21T10ยท38+0000