From 732dc68727caa09c2e430fa6822b40ed9b6153f7 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Fri, 22 Sep 2023 16:38:10 +0300 Subject: refactor(tvix/nar-bridge): update to new separated protos Change-Id: I5eaadc837a4d3a7f635574437127a22de88f556b Reviewed-on: https://cl.tvl.fyi/c/depot/+/9407 Tested-by: BuildkiteCI Reviewed-by: Connor Brewster Autosubmit: flokli --- tvix/nar-bridge/pkg/reader/reader.go | 39 +++++------ tvix/nar-bridge/pkg/reader/reader_test.go | 107 +++++++++++++++--------------- 2 files changed, 74 insertions(+), 72 deletions(-) (limited to 'tvix/nar-bridge/pkg/reader') diff --git a/tvix/nar-bridge/pkg/reader/reader.go b/tvix/nar-bridge/pkg/reader/reader.go index 76e0b8b63e86..ff9804934ff4 100644 --- a/tvix/nar-bridge/pkg/reader/reader.go +++ b/tvix/nar-bridge/pkg/reader/reader.go @@ -9,6 +9,7 @@ import ( "path" "strings" + castorev1pb "code.tvl.fyi/tvix/castore/protos" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/nix-community/go-nix/pkg/nar" "lukechampine.com/blake3" @@ -21,7 +22,7 @@ type Reader struct { // An item on the directories stack type item struct { path string - directory *storev1pb.Directory + directory *castorev1pb.Directory } func New(r io.Reader) *Reader { @@ -43,7 +44,7 @@ func (r *Reader) Import( // callback function called with each regular file content blobCb func(fileReader io.Reader) error, // callback function called with each finalized directory node - directoryCb func(directory *storev1pb.Directory) error, + directoryCb func(directory *castorev1pb.Directory) error, ) (*storev1pb.PathInfo, error) { // construct a NAR reader, by reading through hrSha256 @@ -55,9 +56,9 @@ func (r *Reader) Import( // If we store a symlink or regular file at the root, these are not nil. // If they are nil, we instead have a stackDirectory. - var rootSymlink *storev1pb.SymlinkNode - var rootFile *storev1pb.FileNode - var stackDirectory *storev1pb.Directory + var rootSymlink *castorev1pb.SymlinkNode + var rootFile *castorev1pb.FileNode + var stackDirectory *castorev1pb.Directory var stack = []item{} @@ -81,7 +82,7 @@ func (r *Reader) Import( } topOfStack := stack[len(stack)-1].directory - topOfStack.Directories = append(topOfStack.Directories, &storev1pb.DirectoryNode{ + topOfStack.Directories = append(topOfStack.Directories, &castorev1pb.DirectoryNode{ Name: []byte(path.Base(toPop.path)), Digest: dgst, Size: toPop.directory.Size(), @@ -154,15 +155,15 @@ func (r *Reader) Import( // assemble pathInfo with these and return. pi := assemblePathInfo() if rootFile != nil { - pi.Node = &storev1pb.Node{ - Node: &storev1pb.Node_File{ + pi.Node = &castorev1pb.Node{ + Node: &castorev1pb.Node_File{ File: rootFile, }, } } if rootSymlink != nil { - pi.Node = &storev1pb.Node{ - Node: &storev1pb.Node_Symlink{ + pi.Node = &castorev1pb.Node{ + Node: &castorev1pb.Node_Symlink{ Symlink: rootSymlink, }, } @@ -174,9 +175,9 @@ func (r *Reader) Import( return nil, fmt.Errorf("unable to calculate root directory digest: %w", err) } - pi.Node = &storev1pb.Node{ - Node: &storev1pb.Node_Directory{ - Directory: &storev1pb.DirectoryNode{ + pi.Node = &castorev1pb.Node{ + Node: &castorev1pb.Node_Directory{ + Directory: &castorev1pb.DirectoryNode{ Name: []byte{}, Digest: dgst, Size: stackDirectory.Size(), @@ -203,7 +204,7 @@ func (r *Reader) Import( } if hdr.Type == nar.TypeSymlink { - symlinkNode := &storev1pb.SymlinkNode{ + symlinkNode := &castorev1pb.SymlinkNode{ Name: []byte(getBasename(hdr.Path)), Target: []byte(hdr.LinkTarget), } @@ -236,7 +237,7 @@ func (r *Reader) Import( // read the blake3 hash dgst := fileReader.Sum(nil) - fileNode := &storev1pb.FileNode{ + fileNode := &castorev1pb.FileNode{ Name: []byte(getBasename(hdr.Path)), Digest: dgst, Size: uint32(hdr.Size), @@ -250,10 +251,10 @@ func (r *Reader) Import( } } if hdr.Type == nar.TypeDirectory { - directory := &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + directory := &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } stack = append(stack, item{ directory: directory, diff --git a/tvix/nar-bridge/pkg/reader/reader_test.go b/tvix/nar-bridge/pkg/reader/reader_test.go index 1ba9c9651cdc..560b06ebdd4e 100644 --- a/tvix/nar-bridge/pkg/reader/reader_test.go +++ b/tvix/nar-bridge/pkg/reader/reader_test.go @@ -7,6 +7,7 @@ import ( "os" "testing" + castorev1pb "code.tvl.fyi/tvix/castore/protos" "code.tvl.fyi/tvix/nar-bridge/pkg/reader" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/google/go-cmp/cmp" @@ -20,7 +21,7 @@ func requireProtoEq(t *testing.T, expected interface{}, actual interface{}) { } } -func mustDigest(d *storev1pb.Directory) []byte { +func mustDigest(d *castorev1pb.Directory) []byte { dgst, err := d.Digest() if err != nil { panic(err) @@ -38,16 +39,16 @@ func TestSymlink(t *testing.T) { context.Background(), func(fileReader io.Reader) error { panic("no file contents expected!") - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { panic("no directories expected!") }, ) require.NoError(t, err) expectedPathInfo := &storev1pb.PathInfo{ - Node: &storev1pb.Node{ - Node: &storev1pb.Node_Symlink{ - Symlink: &storev1pb.SymlinkNode{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_Symlink{ + Symlink: &castorev1pb.SymlinkNode{ Name: []byte(""), Target: []byte("/nix/store/somewhereelse"), }, @@ -80,7 +81,7 @@ func TestRegular(t *testing.T) { require.NoError(t, err, "reading fileReader should not error") require.Equal(t, []byte{0x01}, contents, "contents read from fileReader should match expectations") return nil - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { panic("no directories expected!") }, ) @@ -94,9 +95,9 @@ func TestRegular(t *testing.T) { } expectedPathInfo := &storev1pb.PathInfo{ - Node: &storev1pb.Node{ - Node: &storev1pb.Node_File{ - File: &storev1pb.FileNode{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_File{ + File: &castorev1pb.FileNode{ Name: []byte(""), Digest: BLAKE3_DIGEST_0X01, Size: 1, @@ -124,16 +125,16 @@ func TestEmptyDirectory(t *testing.T) { r := reader.New(f) - expectedDirectory := &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + expectedDirectory := &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } actualPathInfo, err := r.Import( context.Background(), func(fileReader io.Reader) error { panic("no file contents expected!") - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { requireProtoEq(t, expectedDirectory, directory) return nil }, @@ -141,9 +142,9 @@ func TestEmptyDirectory(t *testing.T) { require.NoError(t, err) expectedPathInfo := &storev1pb.PathInfo{ - Node: &storev1pb.Node{ - Node: &storev1pb.Node_Directory{ - Directory: &storev1pb.DirectoryNode{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_Directory{ + Directory: &castorev1pb.DirectoryNode{ Name: []byte(""), Digest: mustDigest(expectedDirectory), Size: expectedDirectory.Size(), @@ -178,12 +179,12 @@ func TestFull(t *testing.T) { "/share", "/", } - expectedDirectories := make(map[string]*storev1pb.Directory, len(expectedDirectoryPaths)) + expectedDirectories := make(map[string]*castorev1pb.Directory, len(expectedDirectoryPaths)) // /bin is a leaf directory - expectedDirectories["/bin"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{ + expectedDirectories["/bin"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{ { Name: []byte("arp"), Digest: []byte{ @@ -257,7 +258,7 @@ func TestFull(t *testing.T) { Executable: true, }, }, - Symlinks: []*storev1pb.SymlinkNode{ + Symlinks: []*castorev1pb.SymlinkNode{ { Name: []byte("dnsdomainname"), Target: []byte("hostname"), @@ -279,9 +280,9 @@ func TestFull(t *testing.T) { // /share/man/man1 is a leaf directory. // The parser traversed over /sbin, but only added it to / which is still on the stack. - expectedDirectories["/share/man/man1"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{ + expectedDirectories["/share/man/man1"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{ { Name: []byte("dnsdomainname.1.gz"), Digest: []byte{ @@ -323,13 +324,13 @@ func TestFull(t *testing.T) { Executable: false, }, }, - Symlinks: []*storev1pb.SymlinkNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } // /share/man/man5 is a leaf directory - expectedDirectories["/share/man/man5"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{ + expectedDirectories["/share/man/man5"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{ { Name: []byte("ethers.5.gz"), Digest: []byte{ @@ -339,13 +340,13 @@ func TestFull(t *testing.T) { Executable: false, }, }, - Symlinks: []*storev1pb.SymlinkNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } // /share/man/man8 is a leaf directory - expectedDirectories["/share/man/man8"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{ + expectedDirectories["/share/man/man8"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{ { Name: []byte("arp.8.gz"), Digest: []byte{ @@ -411,12 +412,12 @@ func TestFull(t *testing.T) { Executable: false, }, }, - Symlinks: []*storev1pb.SymlinkNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } // /share/man holds /share/man/man{1,5,8}. - expectedDirectories["/share/man"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{ + expectedDirectories["/share/man"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{ { Name: []byte("man1"), Digest: mustDigest(expectedDirectories["/share/man/man1"]), @@ -433,26 +434,26 @@ func TestFull(t *testing.T) { Size: expectedDirectories["/share/man/man8"].Size(), }, }, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } // /share holds /share/man. - expectedDirectories["/share"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{ + expectedDirectories["/share"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{ { Name: []byte("man"), Digest: mustDigest(expectedDirectories["/share/man"]), Size: expectedDirectories["/share/man"].Size(), }, }, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } // / holds /bin, /share, and a /sbin symlink. - expectedDirectories["/"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{ + expectedDirectories["/"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{ { Name: []byte("bin"), Digest: mustDigest(expectedDirectories["/bin"]), @@ -464,8 +465,8 @@ func TestFull(t *testing.T) { Size: expectedDirectories["/share"].Size(), }, }, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{ + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{ { Name: []byte("sbin"), Target: []byte("bin"), @@ -486,7 +487,7 @@ func TestFull(t *testing.T) { // This also covers the case when the client doesn't read from the reader, and that the // importer will take care of reading all the way to the end no matter what. return nil - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { // use actualDirectoryOrder to look up the Directory object we expect at this specific invocation. currentDirectoryPath := expectedDirectoryPaths[numDirectoriesReceived] @@ -502,9 +503,9 @@ func TestFull(t *testing.T) { require.NoError(t, err) expectedPathInfo := &storev1pb.PathInfo{ - Node: &storev1pb.Node{ - Node: &storev1pb.Node_Directory{ - Directory: &storev1pb.DirectoryNode{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_Directory{ + Directory: &castorev1pb.DirectoryNode{ Name: []byte(""), Digest: mustDigest(expectedDirectories["/"]), Size: expectedDirectories["/"].Size(), @@ -540,7 +541,7 @@ func TestCallbackErrors(t *testing.T) { context.Background(), func(fileReader io.Reader) error { return targetErr - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { panic("no directories expected!") }, ) @@ -559,7 +560,7 @@ func TestCallbackErrors(t *testing.T) { context.Background(), func(fileReader io.Reader) error { panic("no file contents expected!") - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { return targetErr }, ) @@ -588,7 +589,7 @@ func TestPopDirectories(t *testing.T) { _, err = r.Import( context.Background(), func(fileReader io.Reader) error { return nil }, - func(directory *storev1pb.Directory) error { + func(directory *castorev1pb.Directory) error { return directory.Validate() }, ) -- cgit 1.4.1