feat(tvix/nar-bridge): init r/6600

This provides a Nix HTTP Binary Cache interface in front of a tvix-store
that's reachable via gRPC.

TODOs:

 - remove import command, move serve up to toplevel. We have nix-copy-
   closure and tvix-store commands.
 - loop into CI. We should be able to fetch the protos as a third-party
   dependency.
 - Check if we can test nar-bridge slightly easier in an integration
   test.
 - Ensure we support connecting to unix sockets and grpc+http at least,
   using the same syntax as tvix-store.
 - Don't buffer the entire blob when rendering NAR

Co-Authored-By: Connor Brewster <cbrewster@hey.com>
Co-Authored-By: Márton Boros <martonboros@gmail.com>
Co-Authored-By: Vo Minh Thu <noteed@gmail.com>

Change-Id: I6064474e49dfe78cea67676957462d9f28658d4a
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9339
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>

1 files changed, 211 insertions, 0 deletions

diff --git a/tvix/nar-bridge/pkg/writer/writer_test.go b/tvix/nar-bridge/pkg/writer/writer_test.go
new file mode 100644
index 000000000000..5914e1792bcb
--- /dev/null
+++ b/tvix/nar-bridge/pkg/writer/writer_test.go
@@ -0,0 +1,211 @@
+package writer_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/hex"
+	"io"
+	"os"
+	"testing"
+
+	"code.tvl.fyi/tvix/nar-bridge/pkg/reader"
+	"code.tvl.fyi/tvix/nar-bridge/pkg/writer"
+	storev1pb "code.tvl.fyi/tvix/store/protos"
+	"github.com/stretchr/testify/require"
+	"lukechampine.com/blake3"
+)
+
+func mustDigest(d *storev1pb.Directory) []byte {
+	dgst, err := d.Digest()
+	if err != nil {
+		panic(err)
+	}
+	return dgst
+}
+
+func TestSymlink(t *testing.T) {
+	pathInfo := &storev1pb.PathInfo{
+
+		Node: &storev1pb.Node{
+			Node: &storev1pb.Node_Symlink{
+				Symlink: &storev1pb.SymlinkNode{
+					Name:   []byte("doesntmatter"),
+					Target: []byte("/nix/store/somewhereelse"),
+				},
+			},
+		},
+	}
+
+	var buf bytes.Buffer
+
+	err := writer.Export(&buf, pathInfo, func([]byte) (*storev1pb.Directory, error) {
+		panic("no directories expected")
+	}, func([]byte) (io.ReadCloser, error) {
+		panic("no files expected")
+	})
+	require.NoError(t, err, "exporter shouldn't fail")
+
+	f, err := os.Open("../../testdata/symlink.nar")
+	require.NoError(t, err)
+
+	bytesExpected, err := io.ReadAll(f)
+	if err != nil {
+		panic(err)
+	}
+
+	require.Equal(t, bytesExpected, buf.Bytes(), "expected nar contents to match")
+}
+
+func TestRegular(t *testing.T) {
+	// The blake3 digest of the 0x01 byte.
+	BLAKE3_DIGEST_0X01 := []byte{
+		0x48, 0xfc, 0x72, 0x1f, 0xbb, 0xc1, 0x72, 0xe0, 0x92, 0x5f, 0xa2, 0x7a, 0xf1, 0x67, 0x1d,
+		0xe2, 0x25, 0xba, 0x92, 0x71, 0x34, 0x80, 0x29, 0x98, 0xb1, 0x0a, 0x15, 0x68, 0xa1, 0x88,
+		0x65, 0x2b,
+	}
+
+	pathInfo := &storev1pb.PathInfo{
+		Node: &storev1pb.Node{
+			Node: &storev1pb.Node_File{
+				File: &storev1pb.FileNode{
+					Name:       []byte("doesntmatter"),
+					Digest:     BLAKE3_DIGEST_0X01,
+					Size:       1,
+					Executable: false,
+				},
+			},
+		},
+	}
+
+	var buf bytes.Buffer
+
+	err := writer.Export(&buf, pathInfo, func([]byte) (*storev1pb.Directory, error) {
+		panic("no directories expected")
+	}, func(blobRef []byte) (io.ReadCloser, error) {
+		if !bytes.Equal(blobRef, BLAKE3_DIGEST_0X01) {
+			panic("unexpected blobref")
+		}
+		return io.NopCloser(bytes.NewBuffer([]byte{0x01})), nil
+	})
+	require.NoError(t, err, "exporter shouldn't fail")
+
+	f, err := os.Open("../../testdata/onebyteregular.nar")
+	require.NoError(t, err)
+
+	bytesExpected, err := io.ReadAll(f)
+	if err != nil {
+		panic(err)
+	}
+
+	require.Equal(t, bytesExpected, buf.Bytes(), "expected nar contents to match")
+}
+
+func TestEmptyDirectory(t *testing.T) {
+	// construct empty directory node this refers to
+	emptyDirectory := &storev1pb.Directory{
+		Directories: []*storev1pb.DirectoryNode{},
+		Files:       []*storev1pb.FileNode{},
+		Symlinks:    []*storev1pb.SymlinkNode{},
+	}
+	emptyDirectoryDigest := mustDigest(emptyDirectory)
+
+	pathInfo := &storev1pb.PathInfo{
+		Node: &storev1pb.Node{
+			Node: &storev1pb.Node_Directory{
+				Directory: &storev1pb.DirectoryNode{
+					Name:   []byte("doesntmatter"),
+					Digest: emptyDirectoryDigest,
+					Size:   0,
+				},
+			},
+		},
+	}
+
+	var buf bytes.Buffer
+
+	err := writer.Export(&buf, pathInfo, func(directoryRef []byte) (*storev1pb.Directory, error) {
+		if !bytes.Equal(directoryRef, emptyDirectoryDigest) {
+			panic("unexpected directoryRef")
+		}
+		return emptyDirectory, nil
+	}, func([]byte) (io.ReadCloser, error) {
+		panic("no files expected")
+	})
+	require.NoError(t, err, "exporter shouldn't fail")
+
+	f, err := os.Open("../../testdata/emptydirectory.nar")
+	require.NoError(t, err)
+
+	bytesExpected, err := io.ReadAll(f)
+	if err != nil {
+		panic(err)
+	}
+
+	require.Equal(t, bytesExpected, buf.Bytes(), "expected nar contents to match")
+}
+
+func TestFull(t *testing.T) {
+	// We pipe nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar to the exporter,
+	// and store all the file contents and directory objects received in two hashmaps.
+	// We then feed it to the writer, and test we come up with the same NAR file.
+
+	f, err := os.Open("../../testdata/nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar")
+	require.NoError(t, err)
+
+	narContents, err := io.ReadAll(f)
+	require.NoError(t, err)
+
+	filesMap := make(map[string][]byte, 0)
+	directoriesMap := make(map[string]*storev1pb.Directory)
+
+	r := reader.New(bytes.NewBuffer(narContents))
+	pathInfo, err := r.Import(
+		context.Background(),
+		func(fileReader io.Reader) error {
+			fileContents, err := io.ReadAll(fileReader)
+			require.NoError(t, err)
+
+			b3Writer := blake3.New(32, nil)
+			_, err = io.Copy(b3Writer, bytes.NewReader(fileContents))
+			require.NoError(t, err)
+
+			// put it in filesMap
+			filesMap[hex.EncodeToString(b3Writer.Sum(nil))] = fileContents
+
+			return nil
+		},
+		func(directory *storev1pb.Directory) error {
+			dgst := mustDigest(directory)
+
+			directoriesMap[hex.EncodeToString(dgst)] = directory
+			return nil
+		},
+	)
+
+	require.NoError(t, err)
+
+	// done populating everything, now actually test the export :-)
+
+	var buf bytes.Buffer
+	err = writer.Export(
+		&buf,
+		pathInfo,
+		func(directoryRef []byte) (*storev1pb.Directory, error) {
+			d, found := directoriesMap[hex.EncodeToString(directoryRef)]
+			if !found {
+				panic("directories not found")
+			}
+			return d, nil
+		},
+		func(fileRef []byte) (io.ReadCloser, error) {
+			fileContents, found := filesMap[hex.EncodeToString(fileRef)]
+			if !found {
+				panic("file not found")
+			}
+			return io.NopCloser(bytes.NewReader(fileContents)), nil
+		},
+	)
+
+	require.NoError(t, err, "exporter shouldn't fail")
+	require.Equal(t, narContents, buf.Bytes())
+}