about summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--misc/docker/Dockerfile16
-rw-r--r--mk/tests.mk29
-rw-r--r--release-common.nix4
-rw-r--r--release.nix2
-rw-r--r--src/libutil/thread-pool.cc81
-rw-r--r--src/libutil/thread-pool.hh9
-rw-r--r--tests/binary-cache.sh4
-rw-r--r--tests/nix-channel.sh2
-rw-r--r--tests/repair.sh2
9 files changed, 95 insertions, 54 deletions
diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile
index fb6f73517bb6..d6b88c7e91a5 100644
--- a/misc/docker/Dockerfile
+++ b/misc/docker/Dockerfile
@@ -8,19 +8,19 @@ RUN wget -O- https://nixos.org/releases/nix/nix-1.11.14/nix-1.11.14-x86_64-linux
   && addgroup -g 30000 -S nixbld \
   && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \
   && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \
-  && ln -s /root/.nix-profile/etc/profile.d/nix.sh /etc/profile.d/ \
+  && ln -s /nix/var/nix/profiles/default/etc/profile.d/nix.sh /etc/profile.d/ \
   && rm -r /nix-*-x86_64-linux \
   && rm -r /var/cache/apk/*
 
 ONBUILD ENV \
     ENV=/etc/profile \
-    PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
-    GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \
-    NIX_SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt
+    PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
+    GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
+    NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
 
 ENV \
     ENV=/etc/profile \
-    PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
-    GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \
-    NIX_SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \
-    NIX_PATH=/nix/var/nix/profiles/per-user/root/channels/
+    PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
+    GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
+    NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
+    NIX_PATH=/nix/var/nix/profiles/per-user/root/channels
diff --git a/mk/tests.mk b/mk/tests.mk
index 004a48028616..1138857c3c16 100644
--- a/mk/tests.mk
+++ b/mk/tests.mk
@@ -7,20 +7,37 @@ define run-install-test
 
 endef
 
+# Color code from https://unix.stackexchange.com/a/10065
 installcheck:
-	@total=0; failed=0; for i in $(_installcheck-list); do \
+	@total=0; failed=0; \
+	red=""; \
+	green=""; \
+	normal=""; \
+	if [ -t 1 ]; then \
+		ncolors="$$(tput colors)"; \
+		if [ -n "$$ncolors" ] && [ "$$ncolors" -ge 8 ]; then \
+			red="$$(tput setaf 1)"; \
+			green="$$(tput setaf 2)"; \
+			normal="$$(tput sgr0)"; \
+		fi; \
+	fi; \
+	for i in $(_installcheck-list); do \
 	  total=$$((total + 1)); \
-	  echo "running test $$i"; \
-	  if (cd $$(dirname $$i) && $(tests-environment) $$(basename $$i)); then \
-	    echo "PASS: $$i"; \
+	  printf "running test $$i..."; \
+	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
+	  if [ $$? -eq 0 ]; then \
+	    echo " [$${green}PASS$$normal]"; \
 	  else \
-	    echo "FAIL: $$i"; \
+	    echo " [$${red}FAIL$$normal]"; \
+	    echo "$$log" | sed 's/^/    /'; \
 	    failed=$$((failed + 1)); \
 	  fi; \
 	done; \
 	if [ "$$failed" != 0 ]; then \
-	  echo "$$failed out of $$total tests failed "; \
+	  echo "$${red}$$failed out of $$total tests failed $$normal"; \
 	  exit 1; \
+	else \
+		echo "$${green}All tests succeeded"; \
 	fi
 
 .PHONY: check installcheck
diff --git a/release-common.nix b/release-common.nix
index c64fc619df6d..4553118e1f56 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -7,8 +7,8 @@ rec {
     enableMinimal = true;
     extraConfig = ''
       CONFIG_ASH y
-      CONFIG_ASH_BUILTIN_ECHO y
-      CONFIG_ASH_BUILTIN_TEST y
+      CONFIG_ASH_ECHO y
+      CONFIG_ASH_TEST y
       CONFIG_ASH_OPTIMIZE_FOR_SIZE y
     '';
   };
diff --git a/release.nix b/release.nix
index 29cd36860ad4..a98199258842 100644
--- a/release.nix
+++ b/release.nix
@@ -332,7 +332,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" ]
+            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "ncurses-bin" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;
diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc
index ce126d36db8e..857ee91f87d0 100644
--- a/src/libutil/thread-pool.cc
+++ b/src/libutil/thread-pool.cc
@@ -13,11 +13,16 @@ ThreadPool::ThreadPool(size_t _maxThreads)
         if (!maxThreads) maxThreads = 1;
     }
 
-    debug(format("starting pool of %d threads") % maxThreads);
+    debug("starting pool of %d threads", maxThreads - 1);
 }
 
 ThreadPool::~ThreadPool()
 {
+    shutdown();
+}
+
+void ThreadPool::shutdown()
+{
     std::vector<std::thread> workers;
     {
         auto state(state_.lock());
@@ -25,7 +30,9 @@ ThreadPool::~ThreadPool()
         std::swap(workers, state->workers);
     }
 
-    debug(format("reaping %d worker threads") % workers.size());
+    if (workers.empty()) return;
+
+    debug("reaping %d worker threads", workers.size());
 
     work.notify_all();
 
@@ -38,32 +45,43 @@ void ThreadPool::enqueue(const work_t & t)
     auto state(state_.lock());
     if (quit)
         throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down");
-    state->left.push(t);
-    if (state->left.size() > state->workers.size() && state->workers.size() < maxThreads)
-        state->workers.emplace_back(&ThreadPool::workerEntry, this);
+    state->pending.push(t);
+    /* Note: process() also executes items, so count it as a worker. */
+    if (state->pending.size() > state->workers.size() + 1 && state->workers.size() + 1 < maxThreads)
+        state->workers.emplace_back(&ThreadPool::doWork, this, false);
     work.notify_one();
 }
 
 void ThreadPool::process()
 {
-    /* Loop until there are no active work items *and* there either
-       are no queued items or there is an exception. The
-       post-condition is that no new items will become active. */
-    while (true) {
+    state_.lock()->draining = true;
+
+    /* Do work until no more work is pending or active. */
+    try {
+        doWork(true);
+
         auto state(state_.lock());
-        if (!state->active) {
-            if (state->exception)
-                std::rethrow_exception(state->exception);
-            if (state->left.empty())
-                break;
-        }
-        state.wait(done);
+
+        assert(quit);
+
+        if (state->exception)
+            std::rethrow_exception(state->exception);
+
+    } catch (...) {
+        /* In the exceptional case, some workers may still be
+           active. They may be referencing the stack frame of the
+           caller. So wait for them to finish. (~ThreadPool also does
+           this, but it might be destroyed after objects referenced by
+           the work item lambdas.) */
+        shutdown();
+        throw;
     }
 }
 
-void ThreadPool::workerEntry()
+void ThreadPool::doWork(bool mainThread)
 {
-    interruptCheck = [&]() { return (bool) quit; };
+    if (!mainThread)
+        interruptCheck = [&]() { return (bool) quit; };
 
     bool didWork = false;
     std::exception_ptr exc;
@@ -99,24 +117,27 @@ void ThreadPool::workerEntry()
                 }
             }
 
-            /* Wait until a work item is available or another thread
-               had an exception or we're asked to quit. */
+            /* Wait until a work item is available or we're asked to
+               quit. */
             while (true) {
-                if (quit) {
-                    if (!state->active)
-                        done.notify_one();
-                    return;
-                }
-                if (!state->left.empty()) break;
-                if (!state->active) {
-                    done.notify_one();
+                if (quit) return;
+
+                if (!state->pending.empty()) break;
+
+                /* If there are no active or pending items, and the
+                   main thread is running process(), then no new items
+                   can be added. So exit. */
+                if (!state->active && state->draining) {
+                    quit = true;
+                    work.notify_all();
                     return;
                 }
+
                 state.wait(work);
             }
 
-            w = std::move(state->left.front());
-            state->left.pop();
+            w = std::move(state->pending.front());
+            state->pending.pop();
             state->active++;
         }
 
diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh
index 06a097ab5ea7..bb16b639a591 100644
--- a/src/libutil/thread-pool.hh
+++ b/src/libutil/thread-pool.hh
@@ -44,19 +44,22 @@ private:
 
     struct State
     {
-        std::queue<work_t> left;
+        std::queue<work_t> pending;
         size_t active = 0;
         std::exception_ptr exception;
         std::vector<std::thread> workers;
+        bool draining = false;
     };
 
     std::atomic_bool quit{false};
 
     Sync<State> state_;
 
-    std::condition_variable work, done;
+    std::condition_variable work;
 
-    void workerEntry();
+    void doWork(bool mainThread);
+
+    void shutdown();
 };
 
 /* Process in parallel a set of items of type T that have a partial
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 532099d02142..30d7cc6ba88c 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -6,7 +6,7 @@ clearCache
 # Create the binary cache.
 outPath=$(nix-build dependencies.nix --no-out-link)
 
-nix copy --recursive --to file://$cacheDir $outPath
+nix copy --to file://$cacheDir $outPath
 
 
 basicTests() {
@@ -117,7 +117,7 @@ badKey="$(cat $TEST_ROOT/pk2)"
 res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3))
 otherKey="$(cat $TEST_ROOT/pk3)"
 
-nix copy --recursive --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
+nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
 
 
 # Downloading should fail if we don't provide a key.
diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh
index 553ada51d9f7..55f1695c4412 100644
--- a/tests/nix-channel.sh
+++ b/tests/nix-channel.sh
@@ -15,7 +15,7 @@ nix-channel --remove xyzzy
 # Create a channel.
 rm -rf $TEST_ROOT/foo
 mkdir -p $TEST_ROOT/foo
-nix copy --recursive --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r $(nix-instantiate dependencies.nix))
+nix copy --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r $(nix-instantiate dependencies.nix))
 rm -rf $TEST_ROOT/nixexprs
 mkdir -p $TEST_ROOT/nixexprs
 cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/
diff --git a/tests/repair.sh b/tests/repair.sh
index 57152d450a17..7c928e3be73c 100644
--- a/tests/repair.sh
+++ b/tests/repair.sh
@@ -46,7 +46,7 @@ fi
 # --verify can fix it.
 clearCache
 
-nix copy --recursive --to file://$cacheDir $path
+nix copy --to file://$cacheDir $path
 
 chmod u+w $path2
 rm -rf $path2
generated by cgit-pink 1.4.1 (git 2.44.2) at 2024-10-07T06ยท04+0000