From d3c44ab03d14bc143e790426bede465feb871cd5 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Tue, 3 Oct 2023 14:29:24 +0300 Subject: refactor(tvix/nar-bridge): rename writer package to exporter Change-Id: I7cf6a3ed0dca4c450fadd9228dc0154ad6254f51 Reviewed-on: https://cl.tvl.fyi/c/depot/+/9530 Tested-by: BuildkiteCI Reviewed-by: Connor Brewster Autosubmit: flokli --- tvix/nar-bridge/pkg/exporter/export_test.go | 220 ++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 tvix/nar-bridge/pkg/exporter/export_test.go (limited to 'tvix/nar-bridge/pkg/exporter/export_test.go') diff --git a/tvix/nar-bridge/pkg/exporter/export_test.go b/tvix/nar-bridge/pkg/exporter/export_test.go new file mode 100644 index 000000000000..ffee62f29183 --- /dev/null +++ b/tvix/nar-bridge/pkg/exporter/export_test.go @@ -0,0 +1,220 @@ +package exporter_test + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "os" + "testing" + + castorev1pb "code.tvl.fyi/tvix/castore/protos" + "code.tvl.fyi/tvix/nar-bridge/pkg/exporter" + "code.tvl.fyi/tvix/nar-bridge/pkg/importer" + storev1pb "code.tvl.fyi/tvix/store/protos" + "github.com/stretchr/testify/require" + "lukechampine.com/blake3" +) + +func mustDirectoryDigest(d *castorev1pb.Directory) []byte { + dgst, err := d.Digest() + if err != nil { + panic(err) + } + return dgst +} + +func mustBlobDigest(r io.Reader) []byte { + hasher := blake3.New(32, nil) + _, err := io.Copy(hasher, r) + if err != nil { + panic(err) + } + return hasher.Sum([]byte{}) +} + +func TestSymlink(t *testing.T) { + pathInfo := &storev1pb.PathInfo{ + + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_Symlink{ + Symlink: &castorev1pb.SymlinkNode{ + Name: []byte("doesntmatter"), + Target: []byte("/nix/store/somewhereelse"), + }, + }, + }, + } + + var buf bytes.Buffer + + err := exporter.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) { + panic("no directories expected") + }, func([]byte) (io.ReadCloser, error) { + panic("no files expected") + }) + require.NoError(t, err, "exporter shouldn't fail") + + f, err := os.Open("../../testdata/symlink.nar") + require.NoError(t, err) + + bytesExpected, err := io.ReadAll(f) + if err != nil { + panic(err) + } + + require.Equal(t, bytesExpected, buf.Bytes(), "expected nar contents to match") +} + +func TestRegular(t *testing.T) { + // The blake3 digest of the 0x01 byte. + BLAKE3_DIGEST_0X01 := []byte{ + 0x48, 0xfc, 0x72, 0x1f, 0xbb, 0xc1, 0x72, 0xe0, 0x92, 0x5f, 0xa2, 0x7a, 0xf1, 0x67, 0x1d, + 0xe2, 0x25, 0xba, 0x92, 0x71, 0x34, 0x80, 0x29, 0x98, 0xb1, 0x0a, 0x15, 0x68, 0xa1, 0x88, + 0x65, 0x2b, + } + + pathInfo := &storev1pb.PathInfo{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_File{ + File: &castorev1pb.FileNode{ + Name: []byte("doesntmatter"), + Digest: BLAKE3_DIGEST_0X01, + Size: 1, + Executable: false, + }, + }, + }, + } + + var buf bytes.Buffer + + err := exporter.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) { + panic("no directories expected") + }, func(blobRef []byte) (io.ReadCloser, error) { + if !bytes.Equal(blobRef, BLAKE3_DIGEST_0X01) { + panic("unexpected blobref") + } + return io.NopCloser(bytes.NewBuffer([]byte{0x01})), nil + }) + require.NoError(t, err, "exporter shouldn't fail") + + f, err := os.Open("../../testdata/onebyteregular.nar") + require.NoError(t, err) + + bytesExpected, err := io.ReadAll(f) + if err != nil { + panic(err) + } + + require.Equal(t, bytesExpected, buf.Bytes(), "expected nar contents to match") +} + +func TestEmptyDirectory(t *testing.T) { + // construct empty directory node this refers to + emptyDirectory := &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + emptyDirectoryDigest := mustDirectoryDigest(emptyDirectory) + + pathInfo := &storev1pb.PathInfo{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_Directory{ + Directory: &castorev1pb.DirectoryNode{ + Name: []byte("doesntmatter"), + Digest: emptyDirectoryDigest, + Size: 0, + }, + }, + }, + } + + var buf bytes.Buffer + + err := exporter.Export(&buf, pathInfo, func(directoryRef []byte) (*castorev1pb.Directory, error) { + if !bytes.Equal(directoryRef, emptyDirectoryDigest) { + panic("unexpected directoryRef") + } + return emptyDirectory, nil + }, func([]byte) (io.ReadCloser, error) { + panic("no files expected") + }) + require.NoError(t, err, "exporter shouldn't fail") + + f, err := os.Open("../../testdata/emptydirectory.nar") + require.NoError(t, err) + + bytesExpected, err := io.ReadAll(f) + if err != nil { + panic(err) + } + + require.Equal(t, bytesExpected, buf.Bytes(), "expected nar contents to match") +} + +func TestFull(t *testing.T) { + // We pipe nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar to the exporter, + // and store all the file contents and directory objects received in two hashmaps. + // We then feed it to the writer, and test we come up with the same NAR file. + + f, err := os.Open("../../testdata/nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar") + require.NoError(t, err) + + narContents, err := io.ReadAll(f) + require.NoError(t, err) + + blobsMap := make(map[string][]byte, 0) + directoriesMap := make(map[string]*castorev1pb.Directory) + + pathInfo, err := importer.Import( + context.Background(), + bytes.NewBuffer(narContents), + func(blobReader io.Reader) ([]byte, error) { + // read in contents, we need to put it into filesMap later. + contents, err := io.ReadAll(blobReader) + require.NoError(t, err) + + dgst := mustBlobDigest(bytes.NewReader(contents)) + + // put it in filesMap + blobsMap[base64.StdEncoding.EncodeToString(dgst)] = contents + + return dgst, nil + }, + func(directory *castorev1pb.Directory) ([]byte, error) { + dgst := mustDirectoryDigest(directory) + + directoriesMap[base64.StdEncoding.EncodeToString(dgst)] = directory + return dgst, nil + }, + ) + + require.NoError(t, err) + + // done populating everything, now actually test the export :-) + var buf bytes.Buffer + err = exporter.Export( + &buf, + pathInfo, + func(directoryDgst []byte) (*castorev1pb.Directory, error) { + d, found := directoriesMap[base64.StdEncoding.EncodeToString(directoryDgst)] + if !found { + panic(fmt.Sprintf("directory %v not found", base64.StdEncoding.EncodeToString(directoryDgst))) + } + return d, nil + }, + func(blobDgst []byte) (io.ReadCloser, error) { + blobContents, found := blobsMap[base64.StdEncoding.EncodeToString(blobDgst)] + if !found { + panic(fmt.Sprintf("blob %v not found", base64.StdEncoding.EncodeToString(blobDgst))) + } + return io.NopCloser(bytes.NewReader(blobContents)), nil + }, + ) + + require.NoError(t, err, "exporter shouldn't fail") + require.Equal(t, narContents, buf.Bytes()) +} -- cgit 1.4.1