diff options
author | Florian Klink <flokli@flokli.de> | 2022-11-19T20·34+0000 |
---|---|---|
committer | flokli <flokli@flokli.de> | 2023-09-17T13·24+0000 |
commit | 0ecd10bf307049b9833e69f331ec049ae8840d85 (patch) | |
tree | 1718b6e0cd7cb3177b951c88dff1dba11faecabf /tvix/nar-bridge/pkg/server/blob_upload.go | |
parent | 683d3e0d2d1de30eb7895861627203e62702a770 (diff) |
feat(tvix/nar-bridge): init r/6600
This provides a Nix HTTP Binary Cache interface in front of a tvix-store that's reachable via gRPC. TODOs: - remove import command, move serve up to toplevel. We have nix-copy- closure and tvix-store commands. - loop into CI. We should be able to fetch the protos as a third-party dependency. - Check if we can test nar-bridge slightly easier in an integration test. - Ensure we support connecting to unix sockets and grpc+http at least, using the same syntax as tvix-store. - Don't buffer the entire blob when rendering NAR Co-Authored-By: Connor Brewster <cbrewster@hey.com> Co-Authored-By: Márton Boros <martonboros@gmail.com> Co-Authored-By: Vo Minh Thu <noteed@gmail.com> Change-Id: I6064474e49dfe78cea67676957462d9f28658d4a Reviewed-on: https://cl.tvl.fyi/c/depot/+/9339 Tested-by: BuildkiteCI Reviewed-by: tazjin <tazjin@tvl.su>
Diffstat (limited to 'tvix/nar-bridge/pkg/server/blob_upload.go')
-rw-r--r-- | tvix/nar-bridge/pkg/server/blob_upload.go | 48 |
1 files changed, 48 insertions, 0 deletions
diff --git a/tvix/nar-bridge/pkg/server/blob_upload.go b/tvix/nar-bridge/pkg/server/blob_upload.go new file mode 100644 index 000000000000..fe554f5a5a2b --- /dev/null +++ b/tvix/nar-bridge/pkg/server/blob_upload.go @@ -0,0 +1,48 @@ +package server + +import ( + storev1pb "code.tvl.fyi/tvix/store/protos" + "context" + "encoding/base64" + "fmt" + log "github.com/sirupsen/logrus" + "io" +) + +// this returns a callback function that can be used as fileCb +// for the reader.Import function call +func genBlobServiceWriteCb(ctx context.Context, blobServiceClient storev1pb.BlobServiceClient) func(io.Reader) error { + return func(fileReader io.Reader) error { + // Read from fileReader into a buffer. + // We currently buffer all contents and send them to blobServiceClient at once, + // but that's about to change. + contents, err := io.ReadAll(fileReader) + if err != nil { + return fmt.Errorf("unable to read all contents from file reader: %w", err) + } + + log := log.WithField("blob_size", len(contents)) + + log.Infof("about to upload blob") + + putter, err := blobServiceClient.Put(ctx) + if err != nil { + // return error to the importer + return fmt.Errorf("error from blob service: %w", err) + } + err = putter.Send(&storev1pb.BlobChunk{ + Data: contents, + }) + if err != nil { + return fmt.Errorf("putting blob chunk: %w", err) + } + resp, err := putter.CloseAndRecv() + if err != nil { + return fmt.Errorf("close blob putter: %w", err) + } + + log.WithField("digest", base64.StdEncoding.EncodeToString(resp.GetDigest())).Info("uploaded blob") + + return nil + } +} |