about summary refs log tree commit diff
path: root/tvix/nar-bridge/pkg/importer/roundtrip_test.go
diff options
context:
space:
mode:
authorConnor Brewster <cbrewster@hey.com>2024-01-23T02·37-0600
committerConnor Brewster <cbrewster@hey.com>2024-01-23T17·01+0000
commit85421b7f973e64ce9ec6f8d675854b0970967794 (patch)
tree08804ea51b9a3033dbc8b1dc4888dd76a30eb019 /tvix/nar-bridge/pkg/importer/roundtrip_test.go
parentd0563294121a85ecddbcc44474373b9293c31e7f (diff)
feat(tvix/store/nar-info): Support async blob upload r/7443
Right now all blob uploads are performed synchronously, this means if a
NAR contains many small files, the import time is dominated by round
trip time to the blob service. For small files, we can buffer them in
memory and upload them asynchronously to the blob service. Before
returning we make sure to join all the uploads to make sure they
complete successfully before responding OK.

This reduces time to import a bash-interactive closure on my machine
from 1m19s to 7s.

Change-Id: Ica3695c159e6c8ad8769281ac20d037e3143e856
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10679
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
Autosubmit: Connor Brewster <cbrewster@hey.com>
Diffstat (limited to 'tvix/nar-bridge/pkg/importer/roundtrip_test.go')
-rw-r--r--tvix/nar-bridge/pkg/importer/roundtrip_test.go4
1 files changed, 4 insertions, 0 deletions
diff --git a/tvix/nar-bridge/pkg/importer/roundtrip_test.go b/tvix/nar-bridge/pkg/importer/roundtrip_test.go
index b16c310522f5..6d6fcb9ee220 100644
--- a/tvix/nar-bridge/pkg/importer/roundtrip_test.go
+++ b/tvix/nar-bridge/pkg/importer/roundtrip_test.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"sync"
 	"testing"
 
 	castorev1pb "code.tvl.fyi/tvix/castore-go"
@@ -27,6 +28,7 @@ func TestRoundtrip(t *testing.T) {
 	narContents, err := io.ReadAll(f)
 	require.NoError(t, err)
 
+	var mu sync.Mutex
 	blobsMap := make(map[string][]byte, 0)
 	directoriesMap := make(map[string]*castorev1pb.Directory)
 
@@ -41,7 +43,9 @@ func TestRoundtrip(t *testing.T) {
 			dgst := mustBlobDigest(bytes.NewReader(contents))
 
 			// put it in filesMap
+			mu.Lock()
 			blobsMap[base64.StdEncoding.EncodeToString(dgst)] = contents
+			mu.Unlock()
 
 			return dgst, nil
 		},