about summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--misc/docker/Dockerfile4
-rw-r--r--misc/docker/README.md8
-rw-r--r--nix.spec.in61
-rw-r--r--release-common.nix2
-rw-r--r--src/libexpr/eval.cc12
-rw-r--r--src/libexpr/lexer.l8
-rw-r--r--src/libexpr/nixexpr.hh1
-rw-r--r--src/libexpr/parser.y9
-rw-r--r--src/libstore/local.mk2
-rw-r--r--src/libstore/s3-binary-cache-store.cc80
-rw-r--r--tests/lang/parse-fail-uft8.nix1
11 files changed, 134 insertions, 54 deletions
diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile
index 2f8e3dd7a679..0f69d02df25f 100644
--- a/misc/docker/Dockerfile
+++ b/misc/docker/Dockerfile
@@ -4,8 +4,8 @@ FROM alpine
 RUN apk add --update openssl
 
 # Download Nix and install it into the system.
-RUN wget https://nixos.org/releases/nix/nix-2.0/nix-2.0-x86_64-linux.tar.bz2 \
-  && echo "6312837aee33306cdbb351b75ba1638b89d21b30f0caf0346f9a742425f197ee  nix-2.0-x86_64-linux.tar.bz2" | sha256sum -c \
+RUN wget https://nixos.org/releases/nix/nix-2.0.2/nix-2.0.2-x86_64-linux.tar.bz2 \
+  && echo "d0c2492d7d8f824e3b1ace15a1a58f64a0a8faacc59936ebedfe18905d982d7c  nix-2.0.2-x86_64-linux.tar.bz2" | sha256sum -c \
   && tar xjf nix-*-x86_64-linux.tar.bz2 \
   && addgroup -g 30000 -S nixbld \
   && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \
diff --git a/misc/docker/README.md b/misc/docker/README.md
new file mode 100644
index 000000000000..491be7408964
--- /dev/null
+++ b/misc/docker/README.md
@@ -0,0 +1,8 @@
+To update https://hub.docker.com/r/nixos/nix/
+
+    $ docker build . -t nixos/nix:2.0
+    $ docker tag nixos/nix:2.0 nixos/nix:latest
+    $ docker push nixos/nix:latest
+    $ docker push nixos/nix:2.0
+
+Write access: @domenkozar
diff --git a/nix.spec.in b/nix.spec.in
index d962bcc857b9..cd053dbfce5c 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -3,33 +3,47 @@
 %global nixbld_user "nix-builder-"
 %global nixbld_group "nixbld"
 
+# NOTE: BUILD on EL7 requires
+# - Centos / RHEL7 software collection repository 
+#   yum install centos-release-scl
+#
+# - Recent boost backport
+#   curl https://copr.fedorainfracloud.org/coprs/whosthere/boost/repo/epel-7/whosthere-boost-epel-7.repo -o /etc/yum.repos.d/whosthere-boost-epel-7.repo
+#
+
+# Disable documentation generation
+# necessary on some platforms
+%bcond_without docgen
+
 Summary: The Nix software deployment system
 Name: nix
 Version: @PACKAGE_VERSION@
 Release: 2%{?dist}
 License: LGPLv2+
-%if 0%{?rhel} && 0%{?rhel} < 7
 Group: Applications/System
-%endif
 URL: http://nixos.org/
 Source0: %{name}-%{version}.tar.bz2
-%if 0%{?el5}
-BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
-%endif
+
 Requires: curl
 Requires: bzip2
 Requires: gzip
 Requires: xz
-Requires: libseccomp
-Requires: boost-context
+BuildRequires: bison
+BuildRequires: boost-devel >= 1.60
 BuildRequires: bzip2-devel
-BuildRequires: sqlite-devel
+
+# for RHEL <= 7, we need software collections for a C++14 compatible compatible compiler
+%if 0%{?rhel}
+BuildRequires: devtoolset-7-gcc
+BuildRequires: devtoolset-7-gcc-c++
+%endif
+
+BuildRequires: flex
 BuildRequires: libcurl-devel
 BuildRequires: libseccomp-devel
-BuildRequires: boost-devel
-
-# Hack to make that shitty RPM scanning hack shut up.
-Provides: perl(Nix::SSH)
+BuildRequires: openssl-devel
+BuildRequires: sqlite-devel
+BuildRequires: xz-devel
 
 %description
 Nix is a purely functional package manager. It allows multiple
@@ -41,9 +55,6 @@ it can be used equally well under other Unix systems.
 
 %package        devel
 Summary:        Development files for %{name}
-%if 0%{?rhel} && 0%{?rhel} < 7
-Group:          Development/Libraries
-%endif
 Requires:       %{name}%{?_isa} = %{version}-%{release}
 
 %description   devel
@@ -53,9 +64,6 @@ developing applications that use %{name}.
 
 %package doc
 Summary:        Documentation files for %{name}
-%if 0%{?rhel} && 0%{?rhel} < 7
-Group:          Documentation
-%endif
 BuildArch:      noarch
 Requires:       %{name} = %{version}-%{release}
 
@@ -67,20 +75,25 @@ The %{name}-doc package contains documentation files for %{name}.
 
 
 %build
+%if 0%{?rhel}
+source /opt/rh/devtoolset-7/enable 
+%endif
 extraFlags=
 # - override docdir so large documentation files are owned by the
 #   -doc subpackage
 # - set localstatedir by hand to the preferred nix value
 %configure --localstatedir=/nix/var \
+	   %{!?without_docgen:--disable-doc-gen} \
            --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \
            $extraFlags
-make -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
+make V=1 %{?_smp_mflags}
 
 
 %install
-%if 0%{?el5}
-rm -rf $RPM_BUILD_ROOT
+%if 0%{?rhel}
+source /opt/rh/devtoolset-7/enable 
 %endif
+
 make DESTDIR=$RPM_BUILD_ROOT install
 
 find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';'
@@ -130,6 +143,7 @@ systemctl start  nix-daemon.socket
 %endif
 
 %files
+%license COPYING
 %{_bindir}/nix*
 %{_libdir}/*.so
 %{_prefix}/libexec/*
@@ -138,9 +152,11 @@ systemctl start  nix-daemon.socket
 %{_prefix}/lib/systemd/system/nix-daemon.service
 %endif
 %{_datadir}/nix
+%if ! %{without docgen}
 %{_mandir}/man1/*.1*
 %{_mandir}/man5/*.5*
 %{_mandir}/man8/*.8*
+%endif
 %config(noreplace) %{_sysconfdir}/profile.d/nix.sh
 %config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh
 /nix
@@ -149,6 +165,9 @@ systemctl start  nix-daemon.socket
 %{_includedir}/nix
 %{_prefix}/lib/pkgconfig/*.pc
 
+
+%if ! %{without docgen}
 %files doc
 %docdir %{_defaultdocdir}/%{name}-doc-%{version}
 %{_defaultdocdir}/%{name}-doc-%{version}
+%endif
diff --git a/release-common.nix b/release-common.nix
index 0c12bc7ceb38..d7fb8125f25e 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -61,7 +61,7 @@ rec {
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
       (aws-sdk-cpp.override {
-        apis = ["s3"];
+        apis = ["s3" "transfer"];
         customMemoryManagement = false;
       });
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index a2cce162b90c..353097f89713 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -317,10 +317,20 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     if (settings.restrictEval || settings.pureEval) {
         allowedPaths = PathSet();
+
         for (auto & i : searchPath) {
             auto r = resolveSearchPathElem(i);
             if (!r.first) continue;
-            allowedPaths->insert(r.second);
+
+            auto path = r.second;
+
+            if (store->isInStore(r.second)) {
+                PathSet closure;
+                store->computeFSClosure(store->toStorePath(r.second), closure);
+                for (auto & path : closure)
+                    allowedPaths->insert(path);
+            } else
+                allowedPaths->insert(r.second);
         }
     }
 
diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 1e9c29afa133..29ca327c1e4e 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -209,11 +209,13 @@ or          { return OR_KW; }
 \#[^\r\n]*    /* single-line comments */
 \/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
 
-{ANY}           return yytext[0];
+{ANY}       {
+              /* Don't return a negative number, as this will cause
+                 Bison to stop parsing without an error. */
+              return (unsigned char) yytext[0];
+            }
 
 }
 
-<<EOF>> { data->atEnd = true; return 0; }
-
 %%
 
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index b486595f07ab..665a42987dc1 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -11,7 +11,6 @@ namespace nix {
 
 MakeError(EvalError, Error)
 MakeError(ParseError, Error)
-MakeError(IncompleteParseError, ParseError)
 MakeError(AssertionError, EvalError)
 MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index e3f4521844e8..eee48887dc22 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -31,12 +31,10 @@ namespace nix {
         Path basePath;
         Symbol path;
         string error;
-        bool atEnd;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
             , symbols(state.symbols)
-            , atEnd(false)
             , sLetBody(symbols.create("<let-body>"))
             { };
     };
@@ -541,12 +539,7 @@ Expr * EvalState::parse(const char * text,
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
 
-    if (res) {
-      if (data.atEnd)
-        throw IncompleteParseError(data.error);
-      else
-        throw ParseError(data.error);
-    }
+    if (res) throw ParseError(data.error);
 
     data.result->bindVars(staticEnv);
 
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index a7279aa3939f..3799257f83ff 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -18,7 +18,7 @@ libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
 $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
 
 ifeq ($(ENABLE_S3), 1)
-	libstore_LDFLAGS += -laws-cpp-sdk-s3 -laws-cpp-sdk-core
+	libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core
 endif
 
 ifeq ($(OS), SunOS)
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 23af452094cf..103f141a1a11 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -17,6 +17,7 @@
 #include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/core/utils/logging/FormattedLogSystem.h>
 #include <aws/core/utils/logging/LogMacros.h>
+#include <aws/core/utils/threading/Executor.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/CreateBucketRequest.h>
 #include <aws/s3/model/GetBucketLocationRequest.h>
@@ -24,6 +25,9 @@
 #include <aws/s3/model/HeadObjectRequest.h>
 #include <aws/s3/model/ListObjectsRequest.h>
 #include <aws/s3/model/PutObjectRequest.h>
+#include <aws/transfer/TransferManager.h>
+
+using namespace Aws::Transfer;
 
 namespace nix {
 
@@ -169,6 +173,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
     const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
     const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
+    const Setting<uint64_t> bufferSize{
+        this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
 
     std::string bucketName;
 
@@ -271,34 +277,76 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         const std::string & mimeType,
         const std::string & contentEncoding)
     {
-        auto request =
-            Aws::S3::Model::PutObjectRequest()
-            .WithBucket(bucketName)
-            .WithKey(path);
+        auto stream = std::make_shared<istringstream_nocopy>(data);
 
-        request.SetContentType(mimeType);
+        auto maxThreads = std::thread::hardware_concurrency();
 
-        if (contentEncoding != "")
-            request.SetContentEncoding(contentEncoding);
+        static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
+            executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
 
-        auto stream = std::make_shared<istringstream_nocopy>(data);
+        TransferManagerConfiguration transferConfig(executor.get());
 
-        request.SetBody(stream);
+        transferConfig.s3Client = s3Helper.client;
+        transferConfig.bufferSize = bufferSize;
 
-        stats.put++;
-        stats.putBytes += data.size();
+        if (contentEncoding != "")
+            transferConfig.createMultipartUploadTemplate.SetContentEncoding(
+                contentEncoding);
+
+        transferConfig.uploadProgressCallback =
+            [&](const TransferManager *transferManager,
+                const std::shared_ptr<const TransferHandle>
+                    &transferHandle) {
+              //FIXME: find a way to properly abort the multipart upload.
+              checkInterrupt();
+              printTalkative("upload progress ('%s'): '%d' of '%d' bytes",
+                             path,
+                             transferHandle->GetBytesTransferred(),
+                             transferHandle->GetBytesTotalSize());
+            };
+
+        transferConfig.transferStatusUpdatedCallback =
+            [&](const TransferManager *,
+                const std::shared_ptr<const TransferHandle>
+                    &transferHandle) {
+              switch (transferHandle->GetStatus()) {
+                  case TransferStatus::COMPLETED:
+                      printTalkative("upload of '%s' completed", path);
+                      stats.put++;
+                      stats.putBytes += data.size();
+                      break;
+                  case TransferStatus::IN_PROGRESS:
+                      break;
+                  case TransferStatus::FAILED:
+                      throw Error("AWS error: failed to upload 's3://%s/%s'",
+                                  bucketName, path);
+                      break;
+                  default:
+                      throw Error("AWS error: transfer status of 's3://%s/%s' "
+                                  "in unexpected state",
+                                  bucketName, path);
+              };
+            };
+
+        std::shared_ptr<TransferManager> transferManager =
+            TransferManager::Create(transferConfig);
 
         auto now1 = std::chrono::steady_clock::now();
 
-        auto result = checkAws(format("AWS error uploading '%s'") % path,
-            s3Helper.client->PutObject(request));
+        std::shared_ptr<TransferHandle> transferHandle =
+            transferManager->UploadFile(stream, bucketName, path, mimeType,
+                                        Aws::Map<Aws::String, Aws::String>());
+
+        transferHandle->WaitUntilFinished();
 
         auto now2 = std::chrono::steady_clock::now();
 
-        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+        auto duration =
+            std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
+                .count();
 
-        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms")
-            % bucketName % path % data.size() % duration);
+        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
+                  bucketName % path % data.size() % duration);
 
         stats.putTimeMs += duration;
     }
diff --git a/tests/lang/parse-fail-uft8.nix b/tests/lang/parse-fail-uft8.nix
new file mode 100644
index 000000000000..34948d48aed2
--- /dev/null
+++ b/tests/lang/parse-fail-uft8.nix
@@ -0,0 +1 @@
+123 é 4
generated by cgit-pink 1.4.1 (git 2.44.2) at 2025-01-24T09·47+0000