From 28c7e926ea48e4f5e57b9b8d1a70ccc494b2c188 Mon Sep 17 00:00:00 2001 From: Griffin Smith Date: Mon, 3 Aug 2020 23:50:42 -0400 Subject: feat(3p/nix): Implement AddToStore rpc client Implement the AddToStore RPC client method, which uses an AddToStorePathWriterSink (the dual of the AddToStorePathReaderSource on the server side) to hook into the dumpPath machinery (which we should refactor not to use sinks or sources, but not yet) and write dumped paths as binary data over gRPC. With this commit and sandboxing disabled, the following derivation builds cleanly: derivation { name = "test"; builder = ./build.sh; system = "x86_64-linux"; } where build.sh has chmod +x and contains: #!/bin/sh echo 1 > $out Change-Id: I94cab86f0825a3a9993262a9807130645c13bf44 Reviewed-on: https://cl.tvl.fyi/c/depot/+/1638 Reviewed-by: kanepyork Tested-by: BuildkiteCI --- third_party/nix/src/libstore/rpc-store.cc | 52 +++++++++++++++++++++- third_party/nix/src/libutil/hash.cc | 15 +++++++ third_party/nix/src/libutil/hash.hh | 2 + third_party/nix/src/nix-daemon/nix-daemon-proto.cc | 2 +- 4 files changed, 69 insertions(+), 2 deletions(-) diff --git a/third_party/nix/src/libstore/rpc-store.cc b/third_party/nix/src/libstore/rpc-store.cc index f0abdd42efa9..0b57127610ad 100644 --- a/third_party/nix/src/libstore/rpc-store.cc +++ b/third_party/nix/src/libstore/rpc-store.cc @@ -1,6 +1,7 @@ #include "rpc-store.hh" #include +#include #include #include @@ -20,6 +21,7 @@ #include "libproto/worker.grpc.pb.h" #include "libproto/worker.pb.h" #include "libstore/store-api.hh" +#include "libutil/archive.hh" #include "libutil/hash.hh" #include "libutil/types.hh" @@ -54,6 +56,32 @@ T FillFrom(const U& src) { return result; } +class AddToStorePathWriterSink : public BufferedSink { + public: + explicit AddToStorePathWriterSink( + std::unique_ptr< + grpc_impl::ClientWriter>&& + writer) + : writer_(std::move(writer)), good_(true) {} + + bool good() override { return good_; } + + void write(const unsigned char* data, size_t len) override { + proto::AddToStoreRequest req; + req.set_data(data, len); + if (!writer_->Write(req)) { + good_ = false; + } + } + + grpc::Status Finish() { return writer_->Finish(); } + + private: + std::unique_ptr> + writer_; + bool good_; +}; + constexpr absl::StatusCode GRPCStatusCodeToAbsl(grpc::StatusCode code) { switch (code) { case grpc::StatusCode::OK: @@ -304,7 +332,29 @@ void RpcStore::addToStore(const ValidPathInfo& info, Path RpcStore::addToStore(const std::string& name, const Path& srcPath, bool recursive, HashType hashAlgo, PathFilter& filter, RepairFlag repair) { - throw Unsupported(absl::StrCat("Not implemented ", __func__)); + if (repair != 0u) { + throw Error( + "repairing is not supported when building through the Nix daemon"); + } + + ClientContext ctx; + proto::StorePath response; + auto writer = stub_->AddToStore(&ctx, &response); + + proto::AddToStoreRequest metadata_req; + metadata_req.mutable_meta()->set_base_name(name); + // TODO(grfn): what is fixed? + metadata_req.mutable_meta()->set_fixed(!(hashAlgo == htSHA256 && recursive)); + metadata_req.mutable_meta()->set_recursive(recursive); + metadata_req.mutable_meta()->set_hash_type(HashTypeToProto(hashAlgo)); + writer->Write(metadata_req); + + AddToStorePathWriterSink sink(std::move(writer)); + dumpPath(std::filesystem::absolute(srcPath), sink); + sink.flush(); + SuccessOrThrow(sink.Finish()); + + return response.path(); } Path RpcStore::addTextToStore(const std::string& name, diff --git a/third_party/nix/src/libutil/hash.cc b/third_party/nix/src/libutil/hash.cc index 97cc137b11a7..f7ee44728f60 100644 --- a/third_party/nix/src/libutil/hash.cc +++ b/third_party/nix/src/libutil/hash.cc @@ -35,6 +35,21 @@ std::optional hash_type_from(nix::proto::HashType hash_type) { } } +nix::proto::HashType HashTypeToProto(HashType hash_type) { + switch (hash_type) { + case HashType::htMD5: + return nix::proto::HashType::MD5; + case HashType::htSHA1: + return nix::proto::HashType::SHA1; + case HashType::htSHA256: + return nix::proto::HashType::SHA256; + case HashType::htSHA512: + return nix::proto::HashType::SHA512; + default: + return nix::proto::HashType::UNKNOWN; + } +} + void Hash::init() { if (type == htMD5) { hashSize = md5HashSize; diff --git a/third_party/nix/src/libutil/hash.hh b/third_party/nix/src/libutil/hash.hh index bc32f9146373..4d52702aee75 100644 --- a/third_party/nix/src/libutil/hash.hh +++ b/third_party/nix/src/libutil/hash.hh @@ -16,6 +16,8 @@ enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 }; std::optional hash_type_from(nix::proto::HashType hash_type); +nix::proto::HashType HashTypeToProto(HashType hash_type); + const int md5HashSize = 16; const int sha1HashSize = 20; const int sha256HashSize = 32; diff --git a/third_party/nix/src/nix-daemon/nix-daemon-proto.cc b/third_party/nix/src/nix-daemon/nix-daemon-proto.cc index b3ecc9d5b9a1..fe580b08a214 100644 --- a/third_party/nix/src/nix-daemon/nix-daemon-proto.cc +++ b/third_party/nix/src/nix-daemon/nix-daemon-proto.cc @@ -146,7 +146,7 @@ class WorkerServiceImpl final : public WorkerService::Service { proto::AddToStoreRequest metadata_request; auto has_metadata = reader->Read(&metadata_request); - if (!has_metadata || metadata_request.has_meta()) { + if (!has_metadata || !metadata_request.has_meta()) { return Status(grpc::StatusCode::INVALID_ARGUMENT, "Metadata must be set before sending file content"); } -- cgit 1.4.1