chore(tvix/nar-bridge): move to Export from storev1pb r/6773

This removes the Export method in nar-bridge, and updates all users to
the version now in storev1pb.

It moves the roundtrip test to the importer crate, and some of the
utility functions into a separate util_test.go file.

Change-Id: I81d9e0b35dfd78ef1042bed307281eecd2aaa2a8
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9603
Reviewed-by: Brian McGee <brian@bmcgee.ie>
Tested-by: BuildkiteCI

3 files changed, 115 insertions, 26 deletions

diff --git a/tvix/nar-bridge/pkg/importer/importer_test.go b/tvix/nar-bridge/pkg/importer/importer_test.go
index de0548da9398..6754fd005551 100644
--- a/tvix/nar-bridge/pkg/importer/importer_test.go
+++ b/tvix/nar-bridge/pkg/importer/importer_test.go
@@ -11,35 +11,9 @@ import (
 	castorev1pb "code.tvl.fyi/tvix/castore/protos"
 	"code.tvl.fyi/tvix/nar-bridge/pkg/importer"
 	storev1pb "code.tvl.fyi/tvix/store/protos"
-	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/require"
-	"google.golang.org/protobuf/testing/protocmp"
-	"lukechampine.com/blake3"
 )
 
-func requireProtoEq(t *testing.T, expected interface{}, actual interface{}) {
-	if diff := cmp.Diff(expected, actual, protocmp.Transform()); diff != "" {
-		t.Errorf("unexpected difference:\n%v", diff)
-	}
-}
-
-func mustDirectoryDigest(d *castorev1pb.Directory) []byte {
-	dgst, err := d.Digest()
-	if err != nil {
-		panic(err)
-	}
-	return dgst
-}
-
-func mustBlobDigest(r io.Reader) []byte {
-	hasher := blake3.New(32, nil)
-	_, err := io.Copy(hasher, r)
-	if err != nil {
-		panic(err)
-	}
-	return hasher.Sum([]byte{})
-}
-
 func TestSymlink(t *testing.T) {
 	f, err := os.Open("../../testdata/symlink.nar")
 	require.NoError(t, err)
diff --git a/tvix/nar-bridge/pkg/importer/roundtrip_test.go b/tvix/nar-bridge/pkg/importer/roundtrip_test.go
new file mode 100644
index 000000000000..89603cfcfdbe
--- /dev/null
+++ b/tvix/nar-bridge/pkg/importer/roundtrip_test.go
@@ -0,0 +1,81 @@
+package importer_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	castorev1pb "code.tvl.fyi/tvix/castore/protos"
+	"code.tvl.fyi/tvix/nar-bridge/pkg/importer"
+	storev1pb "code.tvl.fyi/tvix/store/protos"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRoundtrip(t *testing.T) {
+	// We pipe nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar to
+	// storev1pb.Export, and store all the file contents and directory objects
+	// received in two hashmaps.
+	// We then feed it to the writer, and test we come up with the same NAR file.
+
+	f, err := os.Open("../../testdata/nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar")
+	require.NoError(t, err)
+
+	narContents, err := io.ReadAll(f)
+	require.NoError(t, err)
+
+	blobsMap := make(map[string][]byte, 0)
+	directoriesMap := make(map[string]*castorev1pb.Directory)
+
+	pathInfo, err := importer.Import(
+		context.Background(),
+		bytes.NewBuffer(narContents),
+		func(blobReader io.Reader) ([]byte, error) {
+			// read in contents, we need to put it into filesMap later.
+			contents, err := io.ReadAll(blobReader)
+			require.NoError(t, err)
+
+			dgst := mustBlobDigest(bytes.NewReader(contents))
+
+			// put it in filesMap
+			blobsMap[base64.StdEncoding.EncodeToString(dgst)] = contents
+
+			return dgst, nil
+		},
+		func(directory *castorev1pb.Directory) ([]byte, error) {
+			dgst := mustDirectoryDigest(directory)
+
+			directoriesMap[base64.StdEncoding.EncodeToString(dgst)] = directory
+			return dgst, nil
+		},
+	)
+
+	require.NoError(t, err)
+
+	// done populating everything, now actually test the export :-)
+	var buf bytes.Buffer
+	err = storev1pb.Export(
+		&buf,
+		pathInfo,
+		func(directoryDgst []byte) (*castorev1pb.Directory, error) {
+			d, found := directoriesMap[base64.StdEncoding.EncodeToString(directoryDgst)]
+			if !found {
+				panic(fmt.Sprintf("directory %v not found", base64.StdEncoding.EncodeToString(directoryDgst)))
+			}
+			return d, nil
+		},
+		func(blobDgst []byte) (io.ReadCloser, error) {
+			blobContents, found := blobsMap[base64.StdEncoding.EncodeToString(blobDgst)]
+			if !found {
+				panic(fmt.Sprintf("blob      %v not found", base64.StdEncoding.EncodeToString(blobDgst)))
+			}
+			return io.NopCloser(bytes.NewReader(blobContents)), nil
+		},
+	)
+
+	require.NoError(t, err, "exporter shouldn't fail")
+	require.Equal(t, narContents, buf.Bytes())
+}
diff --git a/tvix/nar-bridge/pkg/importer/util_test.go b/tvix/nar-bridge/pkg/importer/util_test.go
new file mode 100644
index 000000000000..623253ed1a35
--- /dev/null
+++ b/tvix/nar-bridge/pkg/importer/util_test.go
@@ -0,0 +1,34 @@
+package importer_test
+
+import (
+	"io"
+	"testing"
+
+	castorev1pb "code.tvl.fyi/tvix/castore/protos"
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/protobuf/testing/protocmp"
+	"lukechampine.com/blake3"
+)
+
+func requireProtoEq(t *testing.T, expected interface{}, actual interface{}) {
+	if diff := cmp.Diff(expected, actual, protocmp.Transform()); diff != "" {
+		t.Errorf("unexpected difference:\n%v", diff)
+	}
+}
+
+func mustDirectoryDigest(d *castorev1pb.Directory) []byte {
+	dgst, err := d.Digest()
+	if err != nil {
+		panic(err)
+	}
+	return dgst
+}
+
+func mustBlobDigest(r io.Reader) []byte {
+	hasher := blake3.New(32, nil)
+	_, err := io.Copy(hasher, r)
+	if err != nil {
+		panic(err)
+	}
+	return hasher.Sum([]byte{})
+}