diff options
author | Florian Klink <flokli@flokli.de> | 2023-10-10T18·04+0200 |
---|---|---|
committer | clbot <clbot@tvl.fyi> | 2023-10-10T18·55+0000 |
commit | e6ba84ea50217c83d0aafdbbe7c5a659ba3e6586 (patch) | |
tree | fa84598af1ad6fe2e45cf72cc99209a46191b1a2 /tvix/store/protos/export_test.go | |
parent | 6fe34b7ba03c5e2406a0659da651134e675a55d6 (diff) |
feat(tvix/store/protos): add Export r/6772
Export will traverse a given PathInfo structure, and write the contents in NAR format to the passed Writer. It uses directoryLookupFn and blobLookupFn to resolve references. This is being moved over from nar-bridge. We need to keep the code there around until we can bump go.mod to storev1 with this merged, but the tests can already be moved entirely. Change-Id: Ie0de3077b09344cafa00ff1e2ddb8b52e9e631bc Reviewed-on: https://cl.tvl.fyi/c/depot/+/9602 Tested-by: BuildkiteCI Reviewed-by: Brian McGee <brian@bmcgee.ie> Autosubmit: flokli <flokli@flokli.de>
Diffstat (limited to '')
-rw-r--r-- | tvix/store/protos/export_test.go (renamed from tvix/nar-bridge/pkg/exporter/export_test.go) | 83 |
1 files changed, 7 insertions, 76 deletions
diff --git a/tvix/nar-bridge/pkg/exporter/export_test.go b/tvix/store/protos/export_test.go index ffee62f291..d45dd1ee7c 100644 --- a/tvix/nar-bridge/pkg/exporter/export_test.go +++ b/tvix/store/protos/export_test.go @@ -1,17 +1,12 @@ -package exporter_test +package storev1_test import ( "bytes" - "context" - "encoding/base64" - "fmt" "io" "os" "testing" castorev1pb "code.tvl.fyi/tvix/castore/protos" - "code.tvl.fyi/tvix/nar-bridge/pkg/exporter" - "code.tvl.fyi/tvix/nar-bridge/pkg/importer" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/stretchr/testify/require" "lukechampine.com/blake3" @@ -49,14 +44,14 @@ func TestSymlink(t *testing.T) { var buf bytes.Buffer - err := exporter.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) { + err := storev1pb.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) { panic("no directories expected") }, func([]byte) (io.ReadCloser, error) { panic("no files expected") }) require.NoError(t, err, "exporter shouldn't fail") - f, err := os.Open("../../testdata/symlink.nar") + f, err := os.Open("testdata/symlink.nar") require.NoError(t, err) bytesExpected, err := io.ReadAll(f) @@ -90,7 +85,7 @@ func TestRegular(t *testing.T) { var buf bytes.Buffer - err := exporter.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) { + err := storev1pb.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) { panic("no directories expected") }, func(blobRef []byte) (io.ReadCloser, error) { if !bytes.Equal(blobRef, BLAKE3_DIGEST_0X01) { @@ -100,7 +95,7 @@ func TestRegular(t *testing.T) { }) require.NoError(t, err, "exporter shouldn't fail") - f, err := os.Open("../../testdata/onebyteregular.nar") + f, err := os.Open("testdata/onebyteregular.nar") require.NoError(t, err) bytesExpected, err := io.ReadAll(f) @@ -134,7 +129,7 @@ func TestEmptyDirectory(t *testing.T) { var buf bytes.Buffer - err := exporter.Export(&buf, pathInfo, func(directoryRef []byte) (*castorev1pb.Directory, error) { + err := storev1pb.Export(&buf, pathInfo, func(directoryRef []byte) (*castorev1pb.Directory, error) { if !bytes.Equal(directoryRef, emptyDirectoryDigest) { panic("unexpected directoryRef") } @@ -144,7 +139,7 @@ func TestEmptyDirectory(t *testing.T) { }) require.NoError(t, err, "exporter shouldn't fail") - f, err := os.Open("../../testdata/emptydirectory.nar") + f, err := os.Open("testdata/emptydirectory.nar") require.NoError(t, err) bytesExpected, err := io.ReadAll(f) @@ -154,67 +149,3 @@ func TestEmptyDirectory(t *testing.T) { require.Equal(t, bytesExpected, buf.Bytes(), "expected nar contents to match") } - -func TestFull(t *testing.T) { - // We pipe nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar to the exporter, - // and store all the file contents and directory objects received in two hashmaps. - // We then feed it to the writer, and test we come up with the same NAR file. - - f, err := os.Open("../../testdata/nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar") - require.NoError(t, err) - - narContents, err := io.ReadAll(f) - require.NoError(t, err) - - blobsMap := make(map[string][]byte, 0) - directoriesMap := make(map[string]*castorev1pb.Directory) - - pathInfo, err := importer.Import( - context.Background(), - bytes.NewBuffer(narContents), - func(blobReader io.Reader) ([]byte, error) { - // read in contents, we need to put it into filesMap later. - contents, err := io.ReadAll(blobReader) - require.NoError(t, err) - - dgst := mustBlobDigest(bytes.NewReader(contents)) - - // put it in filesMap - blobsMap[base64.StdEncoding.EncodeToString(dgst)] = contents - - return dgst, nil - }, - func(directory *castorev1pb.Directory) ([]byte, error) { - dgst := mustDirectoryDigest(directory) - - directoriesMap[base64.StdEncoding.EncodeToString(dgst)] = directory - return dgst, nil - }, - ) - - require.NoError(t, err) - - // done populating everything, now actually test the export :-) - var buf bytes.Buffer - err = exporter.Export( - &buf, - pathInfo, - func(directoryDgst []byte) (*castorev1pb.Directory, error) { - d, found := directoriesMap[base64.StdEncoding.EncodeToString(directoryDgst)] - if !found { - panic(fmt.Sprintf("directory %v not found", base64.StdEncoding.EncodeToString(directoryDgst))) - } - return d, nil - }, - func(blobDgst []byte) (io.ReadCloser, error) { - blobContents, found := blobsMap[base64.StdEncoding.EncodeToString(blobDgst)] - if !found { - panic(fmt.Sprintf("blob %v not found", base64.StdEncoding.EncodeToString(blobDgst))) - } - return io.NopCloser(bytes.NewReader(blobContents)), nil - }, - ) - - require.NoError(t, err, "exporter shouldn't fail") - require.Equal(t, narContents, buf.Bytes()) -} |