diff options
| -rw-r--r-- | doc/manual/packages/s3-substituter.xml | 14 | ||||
| -rw-r--r-- | src/libstore/build.cc | 4 | ||||
| -rw-r--r-- | src/libstore/download.cc | 6 | ||||
| -rw-r--r-- | src/libstore/local-store.cc | 4 | ||||
| -rw-r--r-- | src/libstore/nar-info-disk-cache.cc | 35 | ||||
| -rw-r--r-- | src/libstore/s3-binary-cache-store.cc | 12 | ||||
| -rw-r--r-- | src/libstore/s3.hh | 4 | ||||
| -rw-r--r-- | src/libutil/util.cc | 2 | ||||
| -rw-r--r-- | src/nix/repl.cc | 2 |
9 files changed, 51 insertions, 32 deletions
diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml index ea654392c6b1..2ec9687a0c60 100644 --- a/doc/manual/packages/s3-substituter.xml +++ b/doc/manual/packages/s3-substituter.xml @@ -51,6 +51,18 @@ the S3 URL:</para> addressing.</para></note> </listitem> </varlistentry> + + <varlistentry><term><literal>scheme</literal></term> + <listitem> + <para> + The scheme used for S3 requests, <literal>https</literal> + (default) or <literal>http</literal>. This option allows you to + disable HTTPS for binary caches which don't support it. + </para> + <note><para>HTTPS should be used if the cache might contain + sensitive information.</para></note> + </listitem> + </varlistentry> </variablelist> <para>In this example we will use the bucket named @@ -165,7 +177,7 @@ the S3 URL:</para> </example> <example><title>Uploading to an S3-Compatible Binary Cache</title> - <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&endpoint=minio.example.com' nixpkgs.hello</command></para> + <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' nixpkgs.hello</command></para> </example> </section> </section> diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9c408e29c06c..59abae9b90db 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3129,8 +3129,8 @@ void DerivationGoal::registerOutputs() /* Throw an error after registering the path as valid. */ delayedException = std::make_exception_ptr( - BuildError("fixed-output derivation produced path '%s' with %s hash '%s' instead of the expected hash '%s'", - dest, printHashType(h.type), printHash16or32(h2), printHash16or32(h))); + BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", + dest, h.to_string(), h2.to_string())); Path actualDest = worker.store.toRealPath(dest); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 7773d903265b..467f570bbf05 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -622,7 +622,7 @@ struct CurlDownloader : public Downloader // FIXME: do this on a worker thread try { #ifdef ENABLE_S3 - S3Helper s3Helper("", Aws::Region::US_EAST_1, ""); // FIXME: make configurable + S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable auto slash = request.uri.find('/', 5); if (slash == std::string::npos) throw nix::Error("bad S3 URI '%s'", request.uri); @@ -881,8 +881,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa Hash gotHash = unpack ? hashPath(expectedHash.type, store->toRealPath(storePath)).first : hashFile(expectedHash.type, store->toRealPath(storePath)); - throw nix::Error("hash mismatch in file downloaded from '%s': got hash '%s' instead of the expected hash '%s'", - url, gotHash.to_string(), expectedHash.to_string()); + throw nix::Error("hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + url, expectedHash.to_string(), gotHash.to_string()); } return store->toRealPath(storePath); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e1cb423d151f..5b4e7ca4ca99 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1022,11 +1022,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto hashResult = hashSink.finish(); if (hashResult.first != info.narHash) - throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'", + throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", info.path, info.narHash.to_string(), hashResult.first.to_string()); if (hashResult.second != info.narSize) - throw Error("size mismatch importing path '%s'; expected %s, got %s", + throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", info.path, info.narSize, hashResult.second); autoGC(); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 35403e5df56f..32ad7f2b27ff 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -31,6 +31,7 @@ create table if not exists NARs ( refs text, deriver text, sigs text, + ca text, timestamp integer not null, present integer not null, primary key (cache, hashPart), @@ -72,7 +73,7 @@ public: { auto state(_state.lock()); - Path dbPath = getCacheDir() + "/nix/binary-cache-v5.sqlite"; + Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); @@ -94,13 +95,13 @@ public: state->insertNAR.create(state->db, "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " - "narSize, refs, deriver, sigs, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); + "narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); state->insertMissingNAR.create(state->db, "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)"); state->queryNAR.create(state->db, - "select * from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); + "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); /* Periodically purge expired entries from the database. */ retrySQLite<void>([&]() { @@ -189,27 +190,28 @@ public: if (!queryNAR.next()) return {oUnknown, 0}; - if (!queryNAR.getInt(13)) + if (!queryNAR.getInt(0)) return {oInvalid, 0}; auto narInfo = make_ref<NarInfo>(); - auto namePart = queryNAR.getStr(2); + auto namePart = queryNAR.getStr(1); narInfo->path = cache.storeDir + "/" + hashPart + (namePart.empty() ? "" : "-" + namePart); - narInfo->url = queryNAR.getStr(3); - narInfo->compression = queryNAR.getStr(4); - if (!queryNAR.isNull(5)) - narInfo->fileHash = Hash(queryNAR.getStr(5)); - narInfo->fileSize = queryNAR.getInt(6); - narInfo->narHash = Hash(queryNAR.getStr(7)); - narInfo->narSize = queryNAR.getInt(8); - for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " ")) + narInfo->url = queryNAR.getStr(2); + narInfo->compression = queryNAR.getStr(3); + if (!queryNAR.isNull(4)) + narInfo->fileHash = Hash(queryNAR.getStr(4)); + narInfo->fileSize = queryNAR.getInt(5); + narInfo->narHash = Hash(queryNAR.getStr(6)); + narInfo->narSize = queryNAR.getInt(7); + for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) narInfo->references.insert(cache.storeDir + "/" + r); - if (!queryNAR.isNull(10)) - narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(10); - for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " ")) + if (!queryNAR.isNull(9)) + narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9); + for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); + narInfo->ca = queryNAR.getStr(11); return {oValid, narInfo}; }); @@ -243,6 +245,7 @@ public: (concatStringsSep(" ", info->shortRefs())) (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") (concatStringsSep(" ", info->sigs)) + (info->ca) (time(0)).exec(); } else { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 4f1e23198ffe..51de89e0d92f 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -82,8 +82,8 @@ static void initAWS() }); } -S3Helper::S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint) - : config(makeConfig(region, endpoint)) +S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint) + : config(makeConfig(region, scheme, endpoint)) , client(make_ref<Aws::S3::S3Client>( profile == "" ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( @@ -114,11 +114,14 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy } }; -ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & endpoint) +ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint) { initAWS(); auto res = make_ref<Aws::Client::ClientConfiguration>(); res->region = region; + if (!scheme.empty()) { + res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); + } if (!endpoint.empty()) { res->endpointOverride = endpoint; } @@ -169,6 +172,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."}; const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; @@ -188,7 +192,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore const Params & params, const std::string & bucketName) : S3BinaryCacheStore(params) , bucketName(bucketName) - , s3Helper(profile, region, endpoint) + , s3Helper(profile, region, scheme, endpoint) { diskCache = getNarInfoDiskCache(); } diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index 95d612b66335..ef5f23d0f253 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -14,9 +14,9 @@ struct S3Helper ref<Aws::Client::ClientConfiguration> config; ref<Aws::S3::S3Client> client; - S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint); + S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint); - ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & endpoint); + ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); struct DownloadResult { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e12c4b258c25..ce50334e1e62 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -202,7 +202,7 @@ bool isInDir(const Path & path, const Path & dir) bool isDirOrInDir(const Path & path, const Path & dir) { - return path == dir or isInDir(path, dir); + return path == dir || isInDir(path, dir); } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index d93fd770e807..d4806d74adb8 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -445,7 +445,7 @@ bool NixRepl::processLine(string line) /* We could do the build in this process using buildPaths(), but doing it in a child makes it easier to recover from problems / SIGINT. */ - if (runProgram(settings.nixBinDir + "/nix", Strings{"build", drvPath}) == 0) { + if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { Derivation drv = readDerivation(drvPath); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputs) |