about summary refs log tree commit diff
path: root/third_party/nix/src/nix-daemon
diff options
context:
space:
mode:
authorGriffin Smith <grfn@gws.fyi>2020-08-21T22·58-0400
committerglittershark <grfn@gws.fyi>2020-08-29T14·29+0000
commit74a8c3d3591801eea4ad00c74b98f0043f20d4cc (patch)
treeccc3f974b8d79c3ae35e23d529cf77bb066f0cf6 /third_party/nix/src/nix-daemon
parent059d90dd6dd317b29bde5517901fa95695792d2c (diff)
fix(tvix): Chunk the AddTextToStore request r/1736
Rather than sending the entire AddTextToStore request along in a single
message, send it in a stream of chunks using the same metadata-first
approach we've been using for the other store gRPC requests. This fixes
a bug where certain builds could send more data than the maximum gRPC
request size (4194304 bytes, it would appear), resulting in a
RESOURCE_EXHAUSTED error.

The initial chunk size, which is currently constant but should be made
dynamic at some point in the future, has been chosen based on the IPC
bandwidth delay product for tazjin's desktop, rounded up.

Change-Id: I6f0232cdbc98653484816b39855126873fc59a03
Reviewed-on: https://cl.tvl.fyi/c/depot/+/1835
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>
Reviewed-by: kanepyork <rikingcoding@gmail.com>
Diffstat (limited to 'third_party/nix/src/nix-daemon')
-rw-r--r--third_party/nix/src/nix-daemon/nix-daemon-proto.cc33
1 files changed, 27 insertions, 6 deletions
diff --git a/third_party/nix/src/nix-daemon/nix-daemon-proto.cc b/third_party/nix/src/nix-daemon/nix-daemon-proto.cc
index 6d18fc40964f..ed859f584e39 100644
--- a/third_party/nix/src/nix-daemon/nix-daemon-proto.cc
+++ b/third_party/nix/src/nix-daemon/nix-daemon-proto.cc
@@ -271,17 +271,38 @@ class WorkerServiceImpl final : public WorkerService::Service {
         __FUNCTION__);
   }
 
-  Status AddTextToStore(grpc::ServerContext*,
-                        const nix::proto::AddTextToStoreRequest* request,
-                        nix::proto::StorePath* response) override {
+  Status AddTextToStore(
+      grpc::ServerContext*,
+      grpc::ServerReader<nix::proto::AddTextToStoreRequest>* reader,
+      nix::proto::StorePath* response) override {
     return HandleExceptions(
         [&]() -> Status {
+          proto::AddTextToStoreRequest request;
+          auto has_metadata = reader->Read(&request);
+          if (!has_metadata || !request.has_meta()) {
+            return Status(grpc::StatusCode::INVALID_ARGUMENT,
+                          "Metadata must be set before sending content");
+          }
+
+          proto::AddTextToStoreRequest_Metadata meta = request.meta();
+
           PathSet references;
-          for (const auto& ref : request->references()) {
+          for (const auto& ref : meta.references()) {
             references.insert(ref);
           }
-          auto path = store_->addTextToStore(request->name(),
-                                             request->content(), references);
+
+          std::string content;
+          content.reserve(meta.size());
+          while (reader->Read(&request)) {
+            if (request.add_oneof_case() != request.kData) {
+              return Status(grpc::StatusCode::INVALID_ARGUMENT,
+                            "All requests except the first must contain data");
+            }
+
+            content.append(request.data());
+          }
+
+          auto path = store_->addTextToStore(meta.name(), content, references);
           response->set_path(path);
           return Status::OK;
         },