From 32f41458c0a0f62bf906021ef096c465ccc45581 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Thu, 21 Sep 2023 22:32:44 +0300 Subject: refactor(tvix): move castore into tvix-castore crate This splits the pure content-addressed layers from tvix-store into a `castore` crate, and only leaves PathInfo related things, as well as the CLI entrypoint in the tvix-store crate. Notable changes: - `fixtures` and `utils` had to be moved out of the `test` cfg, so they can be imported from tvix-store. - Some ad-hoc fixtures in the test were moved to proper fixtures in the same step. - The protos are now created by a (more static) recipe in the protos/ directory. The (now two) golang targets are commented out, as it's not possible to update them properly in the same CL. This will be done by a followup CL once this is merged (and whitby deployed) Bug: https://b.tvl.fyi/issues/301 Change-Id: I8d675d4bf1fb697eb7d479747c1b1e3635718107 Reviewed-on: https://cl.tvl.fyi/c/depot/+/9370 Reviewed-by: tazjin Reviewed-by: flokli Autosubmit: flokli Tested-by: BuildkiteCI Reviewed-by: Connor Brewster --- tvix/castore/protos/castore_test.go | 271 ++++++++++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 tvix/castore/protos/castore_test.go (limited to 'tvix/castore/protos/castore_test.go') diff --git a/tvix/castore/protos/castore_test.go b/tvix/castore/protos/castore_test.go new file mode 100644 index 000000000000..958d399d76cc --- /dev/null +++ b/tvix/castore/protos/castore_test.go @@ -0,0 +1,271 @@ +package castorev1_test + +import ( + "testing" + + castorev1pb "code.tvl.fyi/tvix/castore/protos" + "github.com/stretchr/testify/assert" +) + +var ( + dummyDigest = []byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } +) + +func TestDirectorySize(t *testing.T) { + t.Run("empty", func(t *testing.T) { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + + assert.Equal(t, uint32(0), d.Size()) + }) + + t.Run("containing single empty directory", func(t *testing.T) { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ + Name: []byte([]byte("foo")), + Digest: dummyDigest, + Size: 0, + }}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + + assert.Equal(t, uint32(1), d.Size()) + }) + + t.Run("containing single non-empty directory", func(t *testing.T) { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ + Name: []byte("foo"), + Digest: dummyDigest, + Size: 4, + }}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + + assert.Equal(t, uint32(5), d.Size()) + }) + + t.Run("containing single file", func(t *testing.T) { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{{ + Name: []byte("foo"), + Digest: dummyDigest, + Size: 42, + Executable: false, + }}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + + assert.Equal(t, uint32(1), d.Size()) + }) + + t.Run("containing single symlink", func(t *testing.T) { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{{ + Name: []byte("foo"), + Target: []byte("bar"), + }}, + } + + assert.Equal(t, uint32(1), d.Size()) + }) + +} +func TestDirectoryDigest(t *testing.T) { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + + dgst, err := d.Digest() + assert.NoError(t, err, "calling Digest() on a directory shouldn't error") + assert.Equal(t, []byte{ + 0xaf, 0x13, 0x49, 0xb9, 0xf5, 0xf9, 0xa1, 0xa6, 0xa0, 0x40, 0x4d, 0xea, 0x36, 0xdc, + 0xc9, 0x49, 0x9b, 0xcb, 0x25, 0xc9, 0xad, 0xc1, 0x12, 0xb7, 0xcc, 0x9a, 0x93, 0xca, + 0xe4, 0x1f, 0x32, 0x62, + }, dgst) +} + +func TestDirectoryValidate(t *testing.T) { + t.Run("empty", func(t *testing.T) { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + + assert.NoError(t, d.Validate()) + }) + + t.Run("invalid names", func(t *testing.T) { + { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ + Name: []byte{}, + Digest: dummyDigest, + Size: 42, + }}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + + assert.ErrorContains(t, d.Validate(), "invalid name") + } + { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ + Name: []byte("."), + Digest: dummyDigest, + Size: 42, + }}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + + assert.ErrorContains(t, d.Validate(), "invalid name") + } + { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{{ + Name: []byte(".."), + Digest: dummyDigest, + Size: 42, + Executable: false, + }}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + + assert.ErrorContains(t, d.Validate(), "invalid name") + } + { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{{ + Name: []byte("\x00"), + Target: []byte("foo"), + }}, + } + + assert.ErrorContains(t, d.Validate(), "invalid name") + } + { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{{ + Name: []byte("foo/bar"), + Target: []byte("foo"), + }}, + } + + assert.ErrorContains(t, d.Validate(), "invalid name") + } + }) + + t.Run("invalid digest", func(t *testing.T) { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ + Name: []byte("foo"), + Digest: nil, + Size: 42, + }}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + + assert.ErrorContains(t, d.Validate(), "invalid digest length") + }) + + t.Run("sorting", func(t *testing.T) { + // "b" comes before "a", bad. + { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ + Name: []byte("b"), + Digest: dummyDigest, + Size: 42, + }, { + Name: []byte("a"), + Digest: dummyDigest, + Size: 42, + }}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + assert.ErrorContains(t, d.Validate(), "is not in sorted order") + } + + // "a" exists twice, bad. + { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ + Name: []byte("a"), + Digest: dummyDigest, + Size: 42, + }}, + Files: []*castorev1pb.FileNode{{ + Name: []byte("a"), + Digest: dummyDigest, + Size: 42, + Executable: false, + }}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + assert.ErrorContains(t, d.Validate(), "duplicate name") + } + + // "a" comes before "b", all good. + { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ + Name: []byte("a"), + Digest: dummyDigest, + Size: 42, + }, { + Name: []byte("b"), + Digest: dummyDigest, + Size: 42, + }}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, + } + assert.NoError(t, d.Validate(), "shouldn't error") + } + + // [b, c] and [a] are both properly sorted. + { + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ + Name: []byte("b"), + Digest: dummyDigest, + Size: 42, + }, { + Name: []byte("c"), + Digest: dummyDigest, + Size: 42, + }}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{{ + Name: []byte("a"), + Target: []byte("foo"), + }}, + } + assert.NoError(t, d.Validate(), "shouldn't error") + } + }) +} -- cgit 1.4.1