From 0c031461c3732fd925668a22a042aa5f1b2764f3 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Sun, 17 Sep 2023 21:47:11 +0300 Subject: fix(tvix/nar-bridge): chunk blobs Instead of creating one big BlobChunk containing all data, and creating way too large proto messages, chunk blobs up to a reasonable (1MiB) chunk size, and send them to the server like that. Change-Id: Ia45a53956a6d7c0599cc59ac516ba37e9fb1b30e Reviewed-on: https://cl.tvl.fyi/c/depot/+/9357 Reviewed-by: Connor Brewster Tested-by: BuildkiteCI --- tvix/nar-bridge/pkg/reader/reader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'tvix/nar-bridge/pkg/reader/reader.go') diff --git a/tvix/nar-bridge/pkg/reader/reader.go b/tvix/nar-bridge/pkg/reader/reader.go index 04e1e6f2ae..76e0b8b63e 100644 --- a/tvix/nar-bridge/pkg/reader/reader.go +++ b/tvix/nar-bridge/pkg/reader/reader.go @@ -41,7 +41,7 @@ func New(r io.Reader) *Reader { func (r *Reader) Import( ctx context.Context, // callback function called with each regular file content - fileCb func(fileReader io.Reader) error, + blobCb func(fileReader io.Reader) error, // callback function called with each finalized directory node directoryCb func(directory *storev1pb.Directory) error, ) (*storev1pb.PathInfo, error) { @@ -219,9 +219,9 @@ func (r *Reader) Import( // wrap reader with a reader calculating the blake3 hash fileReader := NewHasher(narReader, blake3.New(32, nil)) - err := fileCb(fileReader) + err := blobCb(fileReader) if err != nil { - return nil, fmt.Errorf("failure from fileCb: %w", err) + return nil, fmt.Errorf("failure from blobCb: %w", err) } // drive the file reader to the end, in case the CB function doesn't read -- cgit 1.4.1