diff options
| -rw-r--r-- | .gitignore | 4 | ||||
| -rw-r--r-- | misc/docker/Dockerfile | 16 | ||||
| -rw-r--r-- | mk/tests.mk | 11 | ||||
| -rw-r--r-- | release-common.nix | 4 | ||||
| -rw-r--r-- | release.nix | 2 | ||||
| -rw-r--r-- | scripts/install-darwin-multi-user.sh | 2 | ||||
| -rw-r--r-- | src/libstore/download.cc | 57 | ||||
| -rw-r--r-- | src/libutil/thread-pool.cc | 81 | ||||
| -rw-r--r-- | src/libutil/thread-pool.hh | 9 |
9 files changed, 115 insertions, 71 deletions
diff --git a/.gitignore b/.gitignore index 6163087384d8..ce22fa007dc7 100644 --- a/.gitignore +++ b/.gitignore @@ -99,16 +99,20 @@ perl/Makefile.config /misc/systemd/nix-daemon.socket /misc/upstart/nix-daemon.conf +/src/resolve-system-dependencies/resolve-system-dependencies + inst/ *.a *.o *.so +*.dylib *.dll *.exe *.dep *~ *.pc +*.plist # GNU Global GPATH diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile index fb6f73517bb6..d6b88c7e91a5 100644 --- a/misc/docker/Dockerfile +++ b/misc/docker/Dockerfile @@ -8,19 +8,19 @@ RUN wget -O- https://nixos.org/releases/nix/nix-1.11.14/nix-1.11.14-x86_64-linux && addgroup -g 30000 -S nixbld \ && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \ && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \ - && ln -s /root/.nix-profile/etc/profile.d/nix.sh /etc/profile.d/ \ + && ln -s /nix/var/nix/profiles/default/etc/profile.d/nix.sh /etc/profile.d/ \ && rm -r /nix-*-x86_64-linux \ && rm -r /var/cache/apk/* ONBUILD ENV \ ENV=/etc/profile \ - PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ - GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ - NIX_SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt + PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ + GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \ + NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt ENV \ ENV=/etc/profile \ - PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ - GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ - NIX_SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ - NIX_PATH=/nix/var/nix/profiles/per-user/root/channels/ + PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ + GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \ + NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \ + NIX_PATH=/nix/var/nix/profiles/per-user/root/channels diff --git a/mk/tests.mk b/mk/tests.mk index aa97d6427ccc..1138857c3c16 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -10,13 +10,12 @@ endef # Color code from https://unix.stackexchange.com/a/10065 installcheck: @total=0; failed=0; \ - pad=" "; \ red=""; \ green=""; \ normal=""; \ if [ -t 1 ]; then \ ncolors="$$(tput colors)"; \ - if [[ -n "$$ncolors" && $$ncolors -ge 8 ]]; then \ + if [ -n "$$ncolors" ] && [ "$$ncolors" -ge 8 ]; then \ red="$$(tput setaf 1)"; \ green="$$(tput setaf 2)"; \ normal="$$(tput sgr0)"; \ @@ -24,12 +23,12 @@ installcheck: fi; \ for i in $(_installcheck-list); do \ total=$$((total + 1)); \ - printf "running test $$i... $${pad:$${#i}}"; \ + printf "running test $$i..."; \ log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \ - if [ $$? == 0 ]; then \ - echo "[$${green}PASS$$normal]"; \ + if [ $$? -eq 0 ]; then \ + echo " [$${green}PASS$$normal]"; \ else \ - echo "[$${red}FAIL$$normal]"; \ + echo " [$${red}FAIL$$normal]"; \ echo "$$log" | sed 's/^/ /'; \ failed=$$((failed + 1)); \ fi; \ diff --git a/release-common.nix b/release-common.nix index c64fc619df6d..4553118e1f56 100644 --- a/release-common.nix +++ b/release-common.nix @@ -7,8 +7,8 @@ rec { enableMinimal = true; extraConfig = '' CONFIG_ASH y - CONFIG_ASH_BUILTIN_ECHO y - CONFIG_ASH_BUILTIN_TEST y + CONFIG_ASH_ECHO y + CONFIG_ASH_TEST y CONFIG_ASH_OPTIMIZE_FOR_SIZE y ''; }; diff --git a/release.nix b/release.nix index 29cd36860ad4..a98199258842 100644 --- a/release.nix +++ b/release.nix @@ -332,7 +332,7 @@ let src = jobs.tarball; diskImage = (diskImageFun vmTools.diskImageFuns) { extraPackages = - [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" ] + [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "ncurses-bin" ] ++ extraPackages; }; memSize = 1024; meta.schedulingPriority = 50; diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index cb255e83c63b..cea25eb8adf3 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -318,7 +318,7 @@ EOF for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do if [ -f "$file" ]; then - if grep -l ".nix-profile" "$file"; then + if grep -l "^[^#].*.nix-profile" "$file"; then failure <<EOF I found a reference to a ".nix-profile" in $file. This has a high chance of breaking a new nix installation. It was most diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 3f5e744dde19..608b8fd399b4 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -278,26 +278,43 @@ struct CurlDownloader : public Downloader callFailure(failure, std::current_exception()); } } else { - Error err = - (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) ? NotFound : - httpStatus == 403 ? Forbidden : - (httpStatus == 408 || httpStatus == 500 || httpStatus == 503 - || httpStatus == 504 || httpStatus == 522 || httpStatus == 524 - || code == CURLE_COULDNT_RESOLVE_HOST - || code == CURLE_RECV_ERROR - - // this seems to occur occasionally for retriable reasons, and shows up in an error like this: - // curl: (23) Failed writing body (315 != 16366) - || code == CURLE_WRITE_ERROR - - // this is a generic SSL failure that in some cases (e.g., certificate error) is permanent but also appears in transient cases, so we consider it retryable - || code == CURLE_SSL_CONNECT_ERROR -#if LIBCURL_VERSION_NUM >= 0x073200 - || code == CURLE_HTTP2 - || code == CURLE_HTTP2_STREAM -#endif - ) ? Transient : - Misc; + // We treat most errors as transient, but won't retry when hopeless + Error err = Transient; + + if (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) { + // The file is definitely not there + err = NotFound; + } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) { + // Don't retry on authentication/authorization failures + err = Forbidden; + } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408) { + // Most 4xx errors are client errors and are probably not worth retrying: + // * 408 means the server timed out waiting for us, so we try again + err = Misc; + } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) { + // Let's treat most 5xx (server) errors as transient, except for a handful: + // * 501 not implemented + // * 505 http version not supported + // * 511 we're behind a captive portal + err = Misc; + } else { + // Don't bother retrying on certain cURL errors either + switch (code) { + case CURLE_FAILED_INIT: + case CURLE_NOT_BUILT_IN: + case CURLE_REMOTE_ACCESS_DENIED: + case CURLE_FILE_COULDNT_READ_FILE: + case CURLE_FUNCTION_NOT_FOUND: + case CURLE_ABORTED_BY_CALLBACK: + case CURLE_BAD_FUNCTION_ARGUMENT: + case CURLE_INTERFACE_FAILED: + case CURLE_UNKNOWN_OPTION: + err = Misc; + break; + default: // Shut up warnings + break; + } + } attempt++; diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index ce126d36db8e..857ee91f87d0 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -13,11 +13,16 @@ ThreadPool::ThreadPool(size_t _maxThreads) if (!maxThreads) maxThreads = 1; } - debug(format("starting pool of %d threads") % maxThreads); + debug("starting pool of %d threads", maxThreads - 1); } ThreadPool::~ThreadPool() { + shutdown(); +} + +void ThreadPool::shutdown() +{ std::vector<std::thread> workers; { auto state(state_.lock()); @@ -25,7 +30,9 @@ ThreadPool::~ThreadPool() std::swap(workers, state->workers); } - debug(format("reaping %d worker threads") % workers.size()); + if (workers.empty()) return; + + debug("reaping %d worker threads", workers.size()); work.notify_all(); @@ -38,32 +45,43 @@ void ThreadPool::enqueue(const work_t & t) auto state(state_.lock()); if (quit) throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down"); - state->left.push(t); - if (state->left.size() > state->workers.size() && state->workers.size() < maxThreads) - state->workers.emplace_back(&ThreadPool::workerEntry, this); + state->pending.push(t); + /* Note: process() also executes items, so count it as a worker. */ + if (state->pending.size() > state->workers.size() + 1 && state->workers.size() + 1 < maxThreads) + state->workers.emplace_back(&ThreadPool::doWork, this, false); work.notify_one(); } void ThreadPool::process() { - /* Loop until there are no active work items *and* there either - are no queued items or there is an exception. The - post-condition is that no new items will become active. */ - while (true) { + state_.lock()->draining = true; + + /* Do work until no more work is pending or active. */ + try { + doWork(true); + auto state(state_.lock()); - if (!state->active) { - if (state->exception) - std::rethrow_exception(state->exception); - if (state->left.empty()) - break; - } - state.wait(done); + + assert(quit); + + if (state->exception) + std::rethrow_exception(state->exception); + + } catch (...) { + /* In the exceptional case, some workers may still be + active. They may be referencing the stack frame of the + caller. So wait for them to finish. (~ThreadPool also does + this, but it might be destroyed after objects referenced by + the work item lambdas.) */ + shutdown(); + throw; } } -void ThreadPool::workerEntry() +void ThreadPool::doWork(bool mainThread) { - interruptCheck = [&]() { return (bool) quit; }; + if (!mainThread) + interruptCheck = [&]() { return (bool) quit; }; bool didWork = false; std::exception_ptr exc; @@ -99,24 +117,27 @@ void ThreadPool::workerEntry() } } - /* Wait until a work item is available or another thread - had an exception or we're asked to quit. */ + /* Wait until a work item is available or we're asked to + quit. */ while (true) { - if (quit) { - if (!state->active) - done.notify_one(); - return; - } - if (!state->left.empty()) break; - if (!state->active) { - done.notify_one(); + if (quit) return; + + if (!state->pending.empty()) break; + + /* If there are no active or pending items, and the + main thread is running process(), then no new items + can be added. So exit. */ + if (!state->active && state->draining) { + quit = true; + work.notify_all(); return; } + state.wait(work); } - w = std::move(state->left.front()); - state->left.pop(); + w = std::move(state->pending.front()); + state->pending.pop(); state->active++; } diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 06a097ab5ea7..bb16b639a591 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -44,19 +44,22 @@ private: struct State { - std::queue<work_t> left; + std::queue<work_t> pending; size_t active = 0; std::exception_ptr exception; std::vector<std::thread> workers; + bool draining = false; }; std::atomic_bool quit{false}; Sync<State> state_; - std::condition_variable work, done; + std::condition_variable work; - void workerEntry(); + void doWork(bool mainThread); + + void shutdown(); }; /* Process in parallel a set of items of type T that have a partial |