From 732dc68727caa09c2e430fa6822b40ed9b6153f7 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Fri, 22 Sep 2023 16:38:10 +0300 Subject: refactor(tvix/nar-bridge): update to new separated protos Change-Id: I5eaadc837a4d3a7f635574437127a22de88f556b Reviewed-on: https://cl.tvl.fyi/c/depot/+/9407 Tested-by: BuildkiteCI Reviewed-by: Connor Brewster Autosubmit: flokli --- tvix/nar-bridge/cmd/nar_bridge/main.go | 5 +- tvix/nar-bridge/default.nix | 2 +- tvix/nar-bridge/go.mod | 3 +- tvix/nar-bridge/go.sum | 6 +- tvix/nar-bridge/pkg/reader/reader.go | 39 ++++---- tvix/nar-bridge/pkg/reader/reader_test.go | 107 +++++++++++---------- tvix/nar-bridge/pkg/server/blob_upload.go | 6 +- tvix/nar-bridge/pkg/server/directory_upload.go | 12 +-- tvix/nar-bridge/pkg/server/nar_get.go | 15 +-- tvix/nar-bridge/pkg/server/nar_put.go | 4 +- tvix/nar-bridge/pkg/server/narinfo_get.go | 7 +- tvix/nar-bridge/pkg/server/narinfo_put.go | 25 ++--- tvix/nar-bridge/pkg/server/server.go | 9 +- tvix/nar-bridge/pkg/writer/writer.go | 27 +++--- .../pkg/writer/writer_pick_next_node_test.go | 20 ++-- tvix/nar-bridge/pkg/writer/writer_test.go | 41 ++++---- 16 files changed, 170 insertions(+), 158 deletions(-) diff --git a/tvix/nar-bridge/cmd/nar_bridge/main.go b/tvix/nar-bridge/cmd/nar_bridge/main.go index a1732433c050..a1986740c16a 100644 --- a/tvix/nar-bridge/cmd/nar_bridge/main.go +++ b/tvix/nar-bridge/cmd/nar_bridge/main.go @@ -11,6 +11,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + castorev1pb "code.tvl.fyi/tvix/castore/protos" "code.tvl.fyi/tvix/nar-bridge/pkg/server" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/sirupsen/logrus" @@ -47,8 +48,8 @@ func main() { defer conn.Close() s := server.New( - storev1pb.NewDirectoryServiceClient(conn), - storev1pb.NewBlobServiceClient(conn), + castorev1pb.NewDirectoryServiceClient(conn), + castorev1pb.NewBlobServiceClient(conn), storev1pb.NewPathInfoServiceClient(conn), cli.EnableAccessLog, 30, diff --git a/tvix/nar-bridge/default.nix b/tvix/nar-bridge/default.nix index f8e762a0a1f2..06c43bb961c1 100644 --- a/tvix/nar-bridge/default.nix +++ b/tvix/nar-bridge/default.nix @@ -6,5 +6,5 @@ pkgs.buildGoModule { name = "nar-bridge"; src = depot.third_party.gitignoreSource ./.; - vendorHash = "sha256-xaNf/bnSuQpt1vadFmYt4NcpJQD3KmiYQ4SrdtiK33U="; + vendorHash = "sha256-sxyoVyHq/FJbNzVa6Xlnv0+/vxLxf0SAY4ZbvRySoFQ="; } diff --git a/tvix/nar-bridge/go.mod b/tvix/nar-bridge/go.mod index c6ee208d21f9..6a987adaccf5 100644 --- a/tvix/nar-bridge/go.mod +++ b/tvix/nar-bridge/go.mod @@ -1,7 +1,8 @@ module code.tvl.fyi/tvix/nar-bridge require ( - code.tvl.fyi/tvix/store/protos v0.0.0-20230918103330-20a5285e3e82 + code.tvl.fyi/tvix/castore/protos v0.0.0-20230922125121-72355662d742 + code.tvl.fyi/tvix/store/protos v0.0.0-20230922125121-4251671e09c1 github.com/alecthomas/kong v0.7.1 github.com/go-chi/chi v1.5.4 github.com/go-chi/chi/v5 v5.0.7 diff --git a/tvix/nar-bridge/go.sum b/tvix/nar-bridge/go.sum index 3db4f9581bba..f8b9a9559c53 100644 --- a/tvix/nar-bridge/go.sum +++ b/tvix/nar-bridge/go.sum @@ -1,6 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -code.tvl.fyi/tvix/store/protos v0.0.0-20230918103330-20a5285e3e82 h1:DffXv1J4znl2BT6bATBaR3lnb8P4cPr02fNOn9RuGYY= -code.tvl.fyi/tvix/store/protos v0.0.0-20230918103330-20a5285e3e82/go.mod h1:uOGlmwyEraDd9kzAWHdDeZdNgKrpOAmtWCdMeY+DVKs= +code.tvl.fyi/tvix/castore/protos v0.0.0-20230922125121-72355662d742 h1:x7LsxggggaN3acnCMNDO5LZLAV+A+rZ+R8TXzr+Lgsk= +code.tvl.fyi/tvix/castore/protos v0.0.0-20230922125121-72355662d742/go.mod h1:Ejhyvc0dJUWQMxtJxddfFuAF5N8IKIO94q5CP4czY8Y= +code.tvl.fyi/tvix/store/protos v0.0.0-20230922125121-4251671e09c1 h1:ttMO8cE4rVbqJlqLsoirePWEtXhT4MxrGvLMFyJ+UhA= +code.tvl.fyi/tvix/store/protos v0.0.0-20230922125121-4251671e09c1/go.mod h1:uOGlmwyEraDd9kzAWHdDeZdNgKrpOAmtWCdMeY+DVKs= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0= github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4= diff --git a/tvix/nar-bridge/pkg/reader/reader.go b/tvix/nar-bridge/pkg/reader/reader.go index 76e0b8b63e86..ff9804934ff4 100644 --- a/tvix/nar-bridge/pkg/reader/reader.go +++ b/tvix/nar-bridge/pkg/reader/reader.go @@ -9,6 +9,7 @@ import ( "path" "strings" + castorev1pb "code.tvl.fyi/tvix/castore/protos" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/nix-community/go-nix/pkg/nar" "lukechampine.com/blake3" @@ -21,7 +22,7 @@ type Reader struct { // An item on the directories stack type item struct { path string - directory *storev1pb.Directory + directory *castorev1pb.Directory } func New(r io.Reader) *Reader { @@ -43,7 +44,7 @@ func (r *Reader) Import( // callback function called with each regular file content blobCb func(fileReader io.Reader) error, // callback function called with each finalized directory node - directoryCb func(directory *storev1pb.Directory) error, + directoryCb func(directory *castorev1pb.Directory) error, ) (*storev1pb.PathInfo, error) { // construct a NAR reader, by reading through hrSha256 @@ -55,9 +56,9 @@ func (r *Reader) Import( // If we store a symlink or regular file at the root, these are not nil. // If they are nil, we instead have a stackDirectory. - var rootSymlink *storev1pb.SymlinkNode - var rootFile *storev1pb.FileNode - var stackDirectory *storev1pb.Directory + var rootSymlink *castorev1pb.SymlinkNode + var rootFile *castorev1pb.FileNode + var stackDirectory *castorev1pb.Directory var stack = []item{} @@ -81,7 +82,7 @@ func (r *Reader) Import( } topOfStack := stack[len(stack)-1].directory - topOfStack.Directories = append(topOfStack.Directories, &storev1pb.DirectoryNode{ + topOfStack.Directories = append(topOfStack.Directories, &castorev1pb.DirectoryNode{ Name: []byte(path.Base(toPop.path)), Digest: dgst, Size: toPop.directory.Size(), @@ -154,15 +155,15 @@ func (r *Reader) Import( // assemble pathInfo with these and return. pi := assemblePathInfo() if rootFile != nil { - pi.Node = &storev1pb.Node{ - Node: &storev1pb.Node_File{ + pi.Node = &castorev1pb.Node{ + Node: &castorev1pb.Node_File{ File: rootFile, }, } } if rootSymlink != nil { - pi.Node = &storev1pb.Node{ - Node: &storev1pb.Node_Symlink{ + pi.Node = &castorev1pb.Node{ + Node: &castorev1pb.Node_Symlink{ Symlink: rootSymlink, }, } @@ -174,9 +175,9 @@ func (r *Reader) Import( return nil, fmt.Errorf("unable to calculate root directory digest: %w", err) } - pi.Node = &storev1pb.Node{ - Node: &storev1pb.Node_Directory{ - Directory: &storev1pb.DirectoryNode{ + pi.Node = &castorev1pb.Node{ + Node: &castorev1pb.Node_Directory{ + Directory: &castorev1pb.DirectoryNode{ Name: []byte{}, Digest: dgst, Size: stackDirectory.Size(), @@ -203,7 +204,7 @@ func (r *Reader) Import( } if hdr.Type == nar.TypeSymlink { - symlinkNode := &storev1pb.SymlinkNode{ + symlinkNode := &castorev1pb.SymlinkNode{ Name: []byte(getBasename(hdr.Path)), Target: []byte(hdr.LinkTarget), } @@ -236,7 +237,7 @@ func (r *Reader) Import( // read the blake3 hash dgst := fileReader.Sum(nil) - fileNode := &storev1pb.FileNode{ + fileNode := &castorev1pb.FileNode{ Name: []byte(getBasename(hdr.Path)), Digest: dgst, Size: uint32(hdr.Size), @@ -250,10 +251,10 @@ func (r *Reader) Import( } } if hdr.Type == nar.TypeDirectory { - directory := &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + directory := &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } stack = append(stack, item{ directory: directory, diff --git a/tvix/nar-bridge/pkg/reader/reader_test.go b/tvix/nar-bridge/pkg/reader/reader_test.go index 1ba9c9651cdc..560b06ebdd4e 100644 --- a/tvix/nar-bridge/pkg/reader/reader_test.go +++ b/tvix/nar-bridge/pkg/reader/reader_test.go @@ -7,6 +7,7 @@ import ( "os" "testing" + castorev1pb "code.tvl.fyi/tvix/castore/protos" "code.tvl.fyi/tvix/nar-bridge/pkg/reader" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/google/go-cmp/cmp" @@ -20,7 +21,7 @@ func requireProtoEq(t *testing.T, expected interface{}, actual interface{}) { } } -func mustDigest(d *storev1pb.Directory) []byte { +func mustDigest(d *castorev1pb.Directory) []byte { dgst, err := d.Digest() if err != nil { panic(err) @@ -38,16 +39,16 @@ func TestSymlink(t *testing.T) { context.Background(), func(fileReader io.Reader) error { panic("no file contents expected!") - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { panic("no directories expected!") }, ) require.NoError(t, err) expectedPathInfo := &storev1pb.PathInfo{ - Node: &storev1pb.Node{ - Node: &storev1pb.Node_Symlink{ - Symlink: &storev1pb.SymlinkNode{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_Symlink{ + Symlink: &castorev1pb.SymlinkNode{ Name: []byte(""), Target: []byte("/nix/store/somewhereelse"), }, @@ -80,7 +81,7 @@ func TestRegular(t *testing.T) { require.NoError(t, err, "reading fileReader should not error") require.Equal(t, []byte{0x01}, contents, "contents read from fileReader should match expectations") return nil - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { panic("no directories expected!") }, ) @@ -94,9 +95,9 @@ func TestRegular(t *testing.T) { } expectedPathInfo := &storev1pb.PathInfo{ - Node: &storev1pb.Node{ - Node: &storev1pb.Node_File{ - File: &storev1pb.FileNode{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_File{ + File: &castorev1pb.FileNode{ Name: []byte(""), Digest: BLAKE3_DIGEST_0X01, Size: 1, @@ -124,16 +125,16 @@ func TestEmptyDirectory(t *testing.T) { r := reader.New(f) - expectedDirectory := &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + expectedDirectory := &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } actualPathInfo, err := r.Import( context.Background(), func(fileReader io.Reader) error { panic("no file contents expected!") - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { requireProtoEq(t, expectedDirectory, directory) return nil }, @@ -141,9 +142,9 @@ func TestEmptyDirectory(t *testing.T) { require.NoError(t, err) expectedPathInfo := &storev1pb.PathInfo{ - Node: &storev1pb.Node{ - Node: &storev1pb.Node_Directory{ - Directory: &storev1pb.DirectoryNode{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_Directory{ + Directory: &castorev1pb.DirectoryNode{ Name: []byte(""), Digest: mustDigest(expectedDirectory), Size: expectedDirectory.Size(), @@ -178,12 +179,12 @@ func TestFull(t *testing.T) { "/share", "/", } - expectedDirectories := make(map[string]*storev1pb.Directory, len(expectedDirectoryPaths)) + expectedDirectories := make(map[string]*castorev1pb.Directory, len(expectedDirectoryPaths)) // /bin is a leaf directory - expectedDirectories["/bin"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{ + expectedDirectories["/bin"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{ { Name: []byte("arp"), Digest: []byte{ @@ -257,7 +258,7 @@ func TestFull(t *testing.T) { Executable: true, }, }, - Symlinks: []*storev1pb.SymlinkNode{ + Symlinks: []*castorev1pb.SymlinkNode{ { Name: []byte("dnsdomainname"), Target: []byte("hostname"), @@ -279,9 +280,9 @@ func TestFull(t *testing.T) { // /share/man/man1 is a leaf directory. // The parser traversed over /sbin, but only added it to / which is still on the stack. - expectedDirectories["/share/man/man1"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{ + expectedDirectories["/share/man/man1"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{ { Name: []byte("dnsdomainname.1.gz"), Digest: []byte{ @@ -323,13 +324,13 @@ func TestFull(t *testing.T) { Executable: false, }, }, - Symlinks: []*storev1pb.SymlinkNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } // /share/man/man5 is a leaf directory - expectedDirectories["/share/man/man5"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{ + expectedDirectories["/share/man/man5"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{ { Name: []byte("ethers.5.gz"), Digest: []byte{ @@ -339,13 +340,13 @@ func TestFull(t *testing.T) { Executable: false, }, }, - Symlinks: []*storev1pb.SymlinkNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } // /share/man/man8 is a leaf directory - expectedDirectories["/share/man/man8"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{ + expectedDirectories["/share/man/man8"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{ { Name: []byte("arp.8.gz"), Digest: []byte{ @@ -411,12 +412,12 @@ func TestFull(t *testing.T) { Executable: false, }, }, - Symlinks: []*storev1pb.SymlinkNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } // /share/man holds /share/man/man{1,5,8}. - expectedDirectories["/share/man"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{ + expectedDirectories["/share/man"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{ { Name: []byte("man1"), Digest: mustDigest(expectedDirectories["/share/man/man1"]), @@ -433,26 +434,26 @@ func TestFull(t *testing.T) { Size: expectedDirectories["/share/man/man8"].Size(), }, }, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } // /share holds /share/man. - expectedDirectories["/share"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{ + expectedDirectories["/share"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{ { Name: []byte("man"), Digest: mustDigest(expectedDirectories["/share/man"]), Size: expectedDirectories["/share/man"].Size(), }, }, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } // / holds /bin, /share, and a /sbin symlink. - expectedDirectories["/"] = &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{ + expectedDirectories["/"] = &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{ { Name: []byte("bin"), Digest: mustDigest(expectedDirectories["/bin"]), @@ -464,8 +465,8 @@ func TestFull(t *testing.T) { Size: expectedDirectories["/share"].Size(), }, }, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{ + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{ { Name: []byte("sbin"), Target: []byte("bin"), @@ -486,7 +487,7 @@ func TestFull(t *testing.T) { // This also covers the case when the client doesn't read from the reader, and that the // importer will take care of reading all the way to the end no matter what. return nil - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { // use actualDirectoryOrder to look up the Directory object we expect at this specific invocation. currentDirectoryPath := expectedDirectoryPaths[numDirectoriesReceived] @@ -502,9 +503,9 @@ func TestFull(t *testing.T) { require.NoError(t, err) expectedPathInfo := &storev1pb.PathInfo{ - Node: &storev1pb.Node{ - Node: &storev1pb.Node_Directory{ - Directory: &storev1pb.DirectoryNode{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_Directory{ + Directory: &castorev1pb.DirectoryNode{ Name: []byte(""), Digest: mustDigest(expectedDirectories["/"]), Size: expectedDirectories["/"].Size(), @@ -540,7 +541,7 @@ func TestCallbackErrors(t *testing.T) { context.Background(), func(fileReader io.Reader) error { return targetErr - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { panic("no directories expected!") }, ) @@ -559,7 +560,7 @@ func TestCallbackErrors(t *testing.T) { context.Background(), func(fileReader io.Reader) error { panic("no file contents expected!") - }, func(directory *storev1pb.Directory) error { + }, func(directory *castorev1pb.Directory) error { return targetErr }, ) @@ -588,7 +589,7 @@ func TestPopDirectories(t *testing.T) { _, err = r.Import( context.Background(), func(fileReader io.Reader) error { return nil }, - func(directory *storev1pb.Directory) error { + func(directory *castorev1pb.Directory) error { return directory.Validate() }, ) diff --git a/tvix/nar-bridge/pkg/server/blob_upload.go b/tvix/nar-bridge/pkg/server/blob_upload.go index cfb65eddb14a..87d3918efa87 100644 --- a/tvix/nar-bridge/pkg/server/blob_upload.go +++ b/tvix/nar-bridge/pkg/server/blob_upload.go @@ -8,7 +8,7 @@ import ( "fmt" "io" - storev1pb "code.tvl.fyi/tvix/store/protos" + castorev1pb "code.tvl.fyi/tvix/castore/protos" log "github.com/sirupsen/logrus" ) @@ -17,7 +17,7 @@ const chunkSize = 1024 * 1024 // this produces a callback function that can be used as blobCb for the // reader.Import function call -func genBlobServiceWriteCb(ctx context.Context, blobServiceClient storev1pb.BlobServiceClient) func(io.Reader) error { +func genBlobServiceWriteCb(ctx context.Context, blobServiceClient castorev1pb.BlobServiceClient) func(io.Reader) error { return func(blobReader io.Reader) error { // Ensure the blobReader is buffered to at least the chunk size. blobReader = bufio.NewReaderSize(blobReader, chunkSize) @@ -42,7 +42,7 @@ func genBlobServiceWriteCb(ctx context.Context, blobServiceClient storev1pb.Blob blobSize += n // send the blob chunk to the server. The err is only valid in the inner scope - if err := putter.Send(&storev1pb.BlobChunk{ + if err := putter.Send(&castorev1pb.BlobChunk{ Data: chunk[:n], }); err != nil { return fmt.Errorf("sending blob chunk: %w", err) diff --git a/tvix/nar-bridge/pkg/server/directory_upload.go b/tvix/nar-bridge/pkg/server/directory_upload.go index d679d3c0b31d..27dedc888bc2 100644 --- a/tvix/nar-bridge/pkg/server/directory_upload.go +++ b/tvix/nar-bridge/pkg/server/directory_upload.go @@ -5,17 +5,17 @@ import ( "encoding/base64" "fmt" - storev1pb "code.tvl.fyi/tvix/store/protos" + castorev1pb "code.tvl.fyi/tvix/castore/protos" log "github.com/sirupsen/logrus" ) type DirectoriesUploader struct { ctx context.Context - directoryServiceClient storev1pb.DirectoryServiceClient - directoryServicePutStream storev1pb.DirectoryService_PutClient + directoryServiceClient castorev1pb.DirectoryServiceClient + directoryServicePutStream castorev1pb.DirectoryService_PutClient } -func NewDirectoriesUploader(ctx context.Context, directoryServiceClient storev1pb.DirectoryServiceClient) *DirectoriesUploader { +func NewDirectoriesUploader(ctx context.Context, directoryServiceClient castorev1pb.DirectoryServiceClient) *DirectoriesUploader { return &DirectoriesUploader{ ctx: ctx, directoryServiceClient: directoryServiceClient, @@ -23,7 +23,7 @@ func NewDirectoriesUploader(ctx context.Context, directoryServiceClient storev1p } } -func (du *DirectoriesUploader) Put(directory *storev1pb.Directory) error { +func (du *DirectoriesUploader) Put(directory *castorev1pb.Directory) error { directoryDgst, err := directory.Digest() if err != nil { return fmt.Errorf("failed calculating directory digest: %w", err) @@ -50,7 +50,7 @@ func (du *DirectoriesUploader) Put(directory *storev1pb.Directory) error { } // Done is called whenever we're -func (du *DirectoriesUploader) Done() (*storev1pb.PutDirectoryResponse, error) { +func (du *DirectoriesUploader) Done() (*castorev1pb.PutDirectoryResponse, error) { // only close once, and only if we opened. if du.directoryServicePutStream == nil { return nil, nil diff --git a/tvix/nar-bridge/pkg/server/nar_get.go b/tvix/nar-bridge/pkg/server/nar_get.go index d31fa1fbeeab..7397992611fc 100644 --- a/tvix/nar-bridge/pkg/server/nar_get.go +++ b/tvix/nar-bridge/pkg/server/nar_get.go @@ -12,6 +12,7 @@ import ( "net/http" "sync" + castorev1pb "code.tvl.fyi/tvix/castore/protos" "code.tvl.fyi/tvix/nar-bridge/pkg/writer" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/go-chi/chi/v5" @@ -27,8 +28,8 @@ const ( func renderNar( ctx context.Context, log *log.Entry, - directoryServiceClient storev1pb.DirectoryServiceClient, - blobServiceClient storev1pb.BlobServiceClient, + directoryServiceClient castorev1pb.DirectoryServiceClient, + blobServiceClient castorev1pb.BlobServiceClient, narHashToPathInfoMu *sync.Mutex, narHashToPathInfo map[string]*storev1pb.PathInfo, w io.Writer, @@ -50,15 +51,15 @@ func renderNar( return nil } - directories := make(map[string]*storev1pb.Directory) + directories := make(map[string]*castorev1pb.Directory) // If the root node is a directory, ask the directory service for all directories if pathInfoDirectory := pathInfo.GetNode().GetDirectory(); pathInfoDirectory != nil { rootDirectoryDigest := pathInfoDirectory.GetDigest() log = log.WithField("root_directory", base64.StdEncoding.EncodeToString(rootDirectoryDigest)) - directoryStream, err := directoryServiceClient.Get(ctx, &storev1pb.GetDirectoryRequest{ - ByWhat: &storev1pb.GetDirectoryRequest_Digest{ + directoryStream, err := directoryServiceClient.Get(ctx, &castorev1pb.GetDirectoryRequest{ + ByWhat: &castorev1pb.GetDirectoryRequest_Digest{ Digest: rootDirectoryDigest, }, Recursive: true, @@ -96,7 +97,7 @@ func renderNar( err := writer.Export( w, pathInfo, - func(directoryDigest []byte) (*storev1pb.Directory, error) { + func(directoryDigest []byte) (*castorev1pb.Directory, error) { log.WithField("directory", base64.StdEncoding.EncodeToString(directoryDigest)).Debug("Get directory") directoryRefStr := hex.EncodeToString(directoryDigest) directory, found := directories[directoryRefStr] @@ -112,7 +113,7 @@ func renderNar( }, func(blobDigest []byte) (io.ReadCloser, error) { log.WithField("blob", base64.StdEncoding.EncodeToString(blobDigest)).Debug("Get blob") - resp, err := blobServiceClient.Read(ctx, &storev1pb.ReadBlobRequest{ + resp, err := blobServiceClient.Read(ctx, &castorev1pb.ReadBlobRequest{ Digest: blobDigest, }) if err != nil { diff --git a/tvix/nar-bridge/pkg/server/nar_put.go b/tvix/nar-bridge/pkg/server/nar_put.go index aecad571cdaa..68529d6c3a17 100644 --- a/tvix/nar-bridge/pkg/server/nar_put.go +++ b/tvix/nar-bridge/pkg/server/nar_put.go @@ -6,8 +6,8 @@ import ( "fmt" "net/http" + castorev1pb "code.tvl.fyi/tvix/castore/protos" "code.tvl.fyi/tvix/nar-bridge/pkg/reader" - storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/go-chi/chi/v5" nixhash "github.com/nix-community/go-nix/pkg/hash" "github.com/nix-community/go-nix/pkg/nixbase32" @@ -44,7 +44,7 @@ func registerNarPut(s *Server) { pathInfo, err := rd.Import( ctx, genBlobServiceWriteCb(ctx, s.blobServiceClient), - func(directory *storev1pb.Directory) error { + func(directory *castorev1pb.Directory) error { return directoriesUploader.Put(directory) }, ) diff --git a/tvix/nar-bridge/pkg/server/narinfo_get.go b/tvix/nar-bridge/pkg/server/narinfo_get.go index ec850670b40d..f2276400c61a 100644 --- a/tvix/nar-bridge/pkg/server/narinfo_get.go +++ b/tvix/nar-bridge/pkg/server/narinfo_get.go @@ -12,6 +12,7 @@ import ( "strings" "sync" + castorev1pb "code.tvl.fyi/tvix/castore/protos" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/go-chi/chi/v5" nixhash "github.com/nix-community/go-nix/pkg/hash" @@ -82,11 +83,11 @@ func renderNarinfo( // extract the name of the node in the pathInfo structure, which will become the output path var nodeName []byte switch v := (pathInfo.GetNode().GetNode()).(type) { - case *storev1pb.Node_File: + case *castorev1pb.Node_File: nodeName = v.File.GetName() - case *storev1pb.Node_Symlink: + case *castorev1pb.Node_Symlink: nodeName = v.Symlink.GetName() - case *storev1pb.Node_Directory: + case *castorev1pb.Node_Directory: nodeName = v.Directory.GetName() } diff --git a/tvix/nar-bridge/pkg/server/narinfo_put.go b/tvix/nar-bridge/pkg/server/narinfo_put.go index 1ad85d8a8bcb..55543c9bc297 100644 --- a/tvix/nar-bridge/pkg/server/narinfo_put.go +++ b/tvix/nar-bridge/pkg/server/narinfo_put.go @@ -4,6 +4,7 @@ import ( "net/http" "path" + castorev1pb "code.tvl.fyi/tvix/castore/protos" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/go-chi/chi/v5" "github.com/nix-community/go-nix/pkg/narinfo" @@ -123,10 +124,10 @@ func registerNarinfoPut(s *Server) { // We need to add the basename of the storepath from the .narinfo // to the pathInfo to be sent. switch v := (pathInfo.GetNode().GetNode()).(type) { - case *storev1pb.Node_File: - pathInfoToUpload.Node = &storev1pb.Node{ - Node: &storev1pb.Node_File{ - File: &storev1pb.FileNode{ + case *castorev1pb.Node_File: + pathInfoToUpload.Node = &castorev1pb.Node{ + Node: &castorev1pb.Node_File{ + File: &castorev1pb.FileNode{ Name: []byte(path.Base(narInfo.StorePath)), Digest: v.File.Digest, Size: v.File.Size, @@ -134,19 +135,19 @@ func registerNarinfoPut(s *Server) { }, }, } - case *storev1pb.Node_Symlink: - pathInfoToUpload.Node = &storev1pb.Node{ - Node: &storev1pb.Node_Symlink{ - Symlink: &storev1pb.SymlinkNode{ + case *castorev1pb.Node_Symlink: + pathInfoToUpload.Node = &castorev1pb.Node{ + Node: &castorev1pb.Node_Symlink{ + Symlink: &castorev1pb.SymlinkNode{ Name: []byte(path.Base(narInfo.StorePath)), Target: v.Symlink.Target, }, }, } - case *storev1pb.Node_Directory: - pathInfoToUpload.Node = &storev1pb.Node{ - Node: &storev1pb.Node_Directory{ - Directory: &storev1pb.DirectoryNode{ + case *castorev1pb.Node_Directory: + pathInfoToUpload.Node = &castorev1pb.Node{ + Node: &castorev1pb.Node_Directory{ + Directory: &castorev1pb.DirectoryNode{ Name: []byte(path.Base(narInfo.StorePath)), Digest: v.Directory.Digest, Size: v.Directory.Size, diff --git a/tvix/nar-bridge/pkg/server/server.go b/tvix/nar-bridge/pkg/server/server.go index f58842bfa792..e50c63754130 100644 --- a/tvix/nar-bridge/pkg/server/server.go +++ b/tvix/nar-bridge/pkg/server/server.go @@ -7,6 +7,7 @@ import ( "sync" "time" + castorev1pb "code.tvl.fyi/tvix/castore/protos" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/go-chi/chi/middleware" "github.com/go-chi/chi/v5" @@ -17,8 +18,8 @@ type Server struct { srv *http.Server handler chi.Router - directoryServiceClient storev1pb.DirectoryServiceClient - blobServiceClient storev1pb.BlobServiceClient + directoryServiceClient castorev1pb.DirectoryServiceClient + blobServiceClient castorev1pb.BlobServiceClient pathInfoServiceClient storev1pb.PathInfoServiceClient // When uploading NAR files to a HTTP binary cache, the .nar @@ -32,8 +33,8 @@ type Server struct { } func New( - directoryServiceClient storev1pb.DirectoryServiceClient, - blobServiceClient storev1pb.BlobServiceClient, + directoryServiceClient castorev1pb.DirectoryServiceClient, + blobServiceClient castorev1pb.BlobServiceClient, pathInfoServiceClient storev1pb.PathInfoServiceClient, enableAccessLog bool, priority int, diff --git a/tvix/nar-bridge/pkg/writer/writer.go b/tvix/nar-bridge/pkg/writer/writer.go index e69cf27f2a50..fd154e061bf0 100644 --- a/tvix/nar-bridge/pkg/writer/writer.go +++ b/tvix/nar-bridge/pkg/writer/writer.go @@ -5,11 +5,12 @@ import ( "io" "path" + castorev1pb "code.tvl.fyi/tvix/castore/protos" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/nix-community/go-nix/pkg/nar" ) -type DirectoryLookupFn func([]byte) (*storev1pb.Directory, error) +type DirectoryLookupFn func([]byte) (*castorev1pb.Directory, error) type BlobLookupFn func([]byte) (io.ReadCloser, error) // Export will traverse a given pathInfo structure, and write the contents @@ -38,13 +39,13 @@ func Export( // and emit individual elements to the NAR writer, draining the Directory object. // once it's empty, we can pop it off the stack. var stackPaths = []string{} - var stackDirectories = []*storev1pb.Directory{} + var stackDirectories = []*castorev1pb.Directory{} // peek at the pathInfo root and assemble the root node and write to writer // in the case of a regular file, we retrieve and write the contents, close and exit // in the case of a symlink, we write the symlink, close and exit switch v := (pathInfo.GetNode().GetNode()).(type) { - case *storev1pb.Node_File: + case *castorev1pb.Node_File: rootHeader.Type = nar.TypeRegular rootHeader.Size = int64(v.File.GetSize()) rootHeader.Executable = v.File.GetExecutable() @@ -77,7 +78,7 @@ func Export( return nil - case *storev1pb.Node_Symlink: + case *castorev1pb.Node_Symlink: rootHeader.Type = nar.TypeSymlink rootHeader.LinkTarget = string(v.Symlink.GetTarget()) err := narWriter.WriteHeader(rootHeader) @@ -91,7 +92,7 @@ func Export( } return nil - case *storev1pb.Node_Directory: + case *castorev1pb.Node_Directory: // We have a directory at the root, look it up and put in on the stack. directory, err := directoryLookupFn(v.Directory.Digest) if err != nil { @@ -136,7 +137,7 @@ func Export( } switch n := (nextNode).(type) { - case *storev1pb.DirectoryNode: + case *castorev1pb.DirectoryNode: err := narWriter.WriteHeader(&nar.Header{ Path: path.Join(topOfStackPath, string(n.GetName())), Type: nar.TypeDirectory, @@ -153,7 +154,7 @@ func Export( // add to stack stackDirectories = append(stackDirectories, d) stackPaths = append(stackPaths, path.Join(topOfStackPath, string(n.GetName()))) - case *storev1pb.FileNode: + case *castorev1pb.FileNode: err := narWriter.WriteHeader(&nar.Header{ Path: path.Join(topOfStackPath, string(n.GetName())), Type: nar.TypeRegular, @@ -180,7 +181,7 @@ func Export( if err != nil { return fmt.Errorf("unable to close content reader: %w", err) } - case *storev1pb.SymlinkNode: + case *castorev1pb.SymlinkNode: err := narWriter.WriteHeader(&nar.Header{ Path: path.Join(topOfStackPath, string(n.GetName())), Type: nar.TypeSymlink, @@ -198,15 +199,15 @@ func Export( // drainNextNode will drain a directory message with one of its child nodes, // whichever comes first alphabetically. -func drainNextNode(d *storev1pb.Directory) interface{} { +func drainNextNode(d *castorev1pb.Directory) interface{} { switch v := (smallestNode(d)).(type) { - case *storev1pb.DirectoryNode: + case *castorev1pb.DirectoryNode: d.Directories = d.Directories[1:] return v - case *storev1pb.FileNode: + case *castorev1pb.FileNode: d.Files = d.Files[1:] return v - case *storev1pb.SymlinkNode: + case *castorev1pb.SymlinkNode: d.Symlinks = d.Symlinks[1:] return v case nil: @@ -218,7 +219,7 @@ func drainNextNode(d *storev1pb.Directory) interface{} { // smallestNode will return the node from a directory message, // whichever comes first alphabetically. -func smallestNode(d *storev1pb.Directory) interface{} { +func smallestNode(d *castorev1pb.Directory) interface{} { childDirectories := d.GetDirectories() childFiles := d.GetFiles() childSymlinks := d.GetSymlinks() diff --git a/tvix/nar-bridge/pkg/writer/writer_pick_next_node_test.go b/tvix/nar-bridge/pkg/writer/writer_pick_next_node_test.go index 3601e9fb5296..d69adca89cb7 100644 --- a/tvix/nar-bridge/pkg/writer/writer_pick_next_node_test.go +++ b/tvix/nar-bridge/pkg/writer/writer_pick_next_node_test.go @@ -3,7 +3,7 @@ package writer import ( "testing" - storev1pb "code.tvl.fyi/tvix/store/protos" + castorev1pb "code.tvl.fyi/tvix/castore/protos" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" "google.golang.org/protobuf/testing/protocmp" @@ -17,18 +17,18 @@ func requireProtoEq(t *testing.T, expected interface{}, actual interface{}) { func TestPopNextNode(t *testing.T) { t.Run("empty directory", func(t *testing.T) { - d := &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + d := &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } n := drainNextNode(d) require.Equal(t, nil, n) }) t.Run("only directories", func(t *testing.T) { - ds := &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{{ + ds := &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ Name: []byte("a"), Digest: []byte{}, Size: 0, @@ -37,12 +37,12 @@ func TestPopNextNode(t *testing.T) { Digest: []byte{}, Size: 0, }}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } n := drainNextNode(ds) - requireProtoEq(t, &storev1pb.DirectoryNode{ + requireProtoEq(t, &castorev1pb.DirectoryNode{ Name: []byte("a"), Digest: []byte{}, Size: 0, diff --git a/tvix/nar-bridge/pkg/writer/writer_test.go b/tvix/nar-bridge/pkg/writer/writer_test.go index 5914e1792bcb..83ac717aa3d6 100644 --- a/tvix/nar-bridge/pkg/writer/writer_test.go +++ b/tvix/nar-bridge/pkg/writer/writer_test.go @@ -8,6 +8,7 @@ import ( "os" "testing" + castorev1pb "code.tvl.fyi/tvix/castore/protos" "code.tvl.fyi/tvix/nar-bridge/pkg/reader" "code.tvl.fyi/tvix/nar-bridge/pkg/writer" storev1pb "code.tvl.fyi/tvix/store/protos" @@ -15,7 +16,7 @@ import ( "lukechampine.com/blake3" ) -func mustDigest(d *storev1pb.Directory) []byte { +func mustDigest(d *castorev1pb.Directory) []byte { dgst, err := d.Digest() if err != nil { panic(err) @@ -26,9 +27,9 @@ func mustDigest(d *storev1pb.Directory) []byte { func TestSymlink(t *testing.T) { pathInfo := &storev1pb.PathInfo{ - Node: &storev1pb.Node{ - Node: &storev1pb.Node_Symlink{ - Symlink: &storev1pb.SymlinkNode{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_Symlink{ + Symlink: &castorev1pb.SymlinkNode{ Name: []byte("doesntmatter"), Target: []byte("/nix/store/somewhereelse"), }, @@ -38,7 +39,7 @@ func TestSymlink(t *testing.T) { var buf bytes.Buffer - err := writer.Export(&buf, pathInfo, func([]byte) (*storev1pb.Directory, error) { + err := writer.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) { panic("no directories expected") }, func([]byte) (io.ReadCloser, error) { panic("no files expected") @@ -65,9 +66,9 @@ func TestRegular(t *testing.T) { } pathInfo := &storev1pb.PathInfo{ - Node: &storev1pb.Node{ - Node: &storev1pb.Node_File{ - File: &storev1pb.FileNode{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_File{ + File: &castorev1pb.FileNode{ Name: []byte("doesntmatter"), Digest: BLAKE3_DIGEST_0X01, Size: 1, @@ -79,7 +80,7 @@ func TestRegular(t *testing.T) { var buf bytes.Buffer - err := writer.Export(&buf, pathInfo, func([]byte) (*storev1pb.Directory, error) { + err := writer.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) { panic("no directories expected") }, func(blobRef []byte) (io.ReadCloser, error) { if !bytes.Equal(blobRef, BLAKE3_DIGEST_0X01) { @@ -102,17 +103,17 @@ func TestRegular(t *testing.T) { func TestEmptyDirectory(t *testing.T) { // construct empty directory node this refers to - emptyDirectory := &storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + emptyDirectory := &castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } emptyDirectoryDigest := mustDigest(emptyDirectory) pathInfo := &storev1pb.PathInfo{ - Node: &storev1pb.Node{ - Node: &storev1pb.Node_Directory{ - Directory: &storev1pb.DirectoryNode{ + Node: &castorev1pb.Node{ + Node: &castorev1pb.Node_Directory{ + Directory: &castorev1pb.DirectoryNode{ Name: []byte("doesntmatter"), Digest: emptyDirectoryDigest, Size: 0, @@ -123,7 +124,7 @@ func TestEmptyDirectory(t *testing.T) { var buf bytes.Buffer - err := writer.Export(&buf, pathInfo, func(directoryRef []byte) (*storev1pb.Directory, error) { + err := writer.Export(&buf, pathInfo, func(directoryRef []byte) (*castorev1pb.Directory, error) { if !bytes.Equal(directoryRef, emptyDirectoryDigest) { panic("unexpected directoryRef") } @@ -156,7 +157,7 @@ func TestFull(t *testing.T) { require.NoError(t, err) filesMap := make(map[string][]byte, 0) - directoriesMap := make(map[string]*storev1pb.Directory) + directoriesMap := make(map[string]*castorev1pb.Directory) r := reader.New(bytes.NewBuffer(narContents)) pathInfo, err := r.Import( @@ -174,7 +175,7 @@ func TestFull(t *testing.T) { return nil }, - func(directory *storev1pb.Directory) error { + func(directory *castorev1pb.Directory) error { dgst := mustDigest(directory) directoriesMap[hex.EncodeToString(dgst)] = directory @@ -190,7 +191,7 @@ func TestFull(t *testing.T) { err = writer.Export( &buf, pathInfo, - func(directoryRef []byte) (*storev1pb.Directory, error) { + func(directoryRef []byte) (*castorev1pb.Directory, error) { d, found := directoriesMap[hex.EncodeToString(directoryRef)] if !found { panic("directories not found") -- cgit 1.4.1