diff options
Diffstat (limited to 'src/libstore/s3-binary-cache-store.cc')
| -rw-r--r-- | src/libstore/s3-binary-cache-store.cc | 44 |
1 files changed, 37 insertions, 7 deletions
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 800380c62d48..571bf7dfd801 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -5,6 +5,8 @@ #include "nar-info.hh" #include "nar-info-disk-cache.hh" #include "globals.hh" +#include "compression.hh" +#include "download.hh" #include <aws/core/Aws.h> #include <aws/core/client/ClientConfiguration.h> @@ -52,8 +54,8 @@ static void initAWS() }); } -S3Helper::S3Helper() - : config(makeConfig()) +S3Helper::S3Helper(const string & region) + : config(makeConfig(region)) , client(make_ref<Aws::S3::S3Client>(*config)) { } @@ -70,11 +72,11 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy } }; -ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig() +ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region) { initAWS(); auto res = make_ref<Aws::Client::ClientConfiguration>(); - res->region = Aws::Region::US_EAST_1; // FIXME: make configurable + res->region = region; res->requestTimeoutMs = 600 * 1000; res->retryStrategy = std::make_shared<RetryStrategy>(); res->caFile = settings.caFile; @@ -104,8 +106,10 @@ S3Helper::DownloadResult S3Helper::getObject( auto result = checkAws(fmt("AWS error fetching ‘%s’", key), client->GetObject(request)); - res.data = std::make_shared<std::string>( - dynamic_cast<std::stringstream &>(result.GetBody()).str()); + res.data = decodeContent( + result.GetContentEncoding(), + make_ref<std::string>( + dynamic_cast<std::stringstream &>(result.GetBody()).str())); } catch (S3Error & e) { if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; @@ -137,10 +141,16 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore S3Helper s3Helper; + std::string narinfoCompression, lsCompression, logCompression; + S3BinaryCacheStoreImpl( const Params & params, const std::string & bucketName) : S3BinaryCacheStore(params) , bucketName(bucketName) + , s3Helper(get(params, "aws-region", Aws::Region::US_EAST_1)) + , narinfoCompression(get(params, "narinfo-compression", "")) + , lsCompression(get(params, "ls-compression", "")) + , logCompression(get(params, "log-compression", "")) { diskCache = getNarInfoDiskCache(); } @@ -219,13 +229,20 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore return true; } - void upsertFile(const std::string & path, const std::string & data) override + void uploadFile(const std::string & path, const std::string & data, + const std::string & mimeType, + const std::string & contentEncoding) { auto request = Aws::S3::Model::PutObjectRequest() .WithBucket(bucketName) .WithKey(path); + request.SetContentType(mimeType); + + if (contentEncoding != "") + request.SetContentEncoding(contentEncoding); + auto stream = std::make_shared<istringstream_nocopy>(data); request.SetBody(stream); @@ -248,6 +265,19 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore stats.putTimeMs += duration; } + void upsertFile(const std::string & path, const std::string & data, + const std::string & mimeType) override + { + if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) + uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression); + else if (lsCompression != "" && hasSuffix(path, ".ls")) + uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); + else if (logCompression != "" && hasPrefix(path, "log/")) + uploadFile(path, *compress(logCompression, data), mimeType, logCompression); + else + uploadFile(path, data, mimeType, ""); + } + void getFile(const std::string & path, std::function<void(std::shared_ptr<std::string>)> success, std::function<void(std::exception_ptr exc)> failure) override |