From 98c17147c64f9898b656f56ab139d4d52248743e Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Wed, 11 Oct 2023 12:28:10 +0200 Subject: refactor(tvix/nar-bridge): have Export return root node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … and nar size / sha256 digest. Instead of producing sparse PathInfo messages when NARs are sent to nar-bridge, the nar-bridge http server now keeps a lookup table (narsha256) -> (rootNode, narSize) This removes a whole bunch of noise, because we don't need to keep sparse fields around. A convenience function `GenPathInfo(rootNode *castorev1pb.Node, narInfo *narinfo.NarInfo)` is added, which is used to produce PathInfo messages, either when receiving a NAR file over http and uploading it to a remote PathInfoService, or to synthesize the PathInfoMessage to return to the client, if nar-bridge is acting as a PathInfoService for a remove Nix HTTP Binary cache. Change-Id: Ibba1ab6238a050816c4fab29cb21ae88877d8613 Reviewed-on: https://cl.tvl.fyi/c/depot/+/9651 Tested-by: BuildkiteCI Reviewed-by: Brian McGee --- tvix/nar-bridge/pkg/http/nar_put.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'tvix/nar-bridge/pkg/http/nar_put.go') diff --git a/tvix/nar-bridge/pkg/http/nar_put.go b/tvix/nar-bridge/pkg/http/nar_put.go index 52ab425dfcf9..16e257537898 100644 --- a/tvix/nar-bridge/pkg/http/nar_put.go +++ b/tvix/nar-bridge/pkg/http/nar_put.go @@ -39,7 +39,7 @@ func registerNarPut(s *Server) { directoriesUploader := importer.NewDirectoriesUploader(ctx, s.directoryServiceClient) defer directoriesUploader.Done() //nolint:errcheck - pathInfo, err := importer.Import( + rootNode, narSize, narSha256, err := importer.Import( ctx, // buffer the body by 10MiB bufio.NewReaderSize(r.Body, 10*1024*1024), @@ -80,7 +80,7 @@ func registerNarPut(s *Server) { // This check ensures the server-side came up with the same root hash. if directoriesPutResponse != nil { - rootDigestPathInfo := pathInfo.GetNode().GetDirectory().GetDigest() + rootDigestPathInfo := rootNode.GetDirectory().GetDigest() rootDigestDirectoriesPutResponse := directoriesPutResponse.GetRootDigest() log := log.WithFields(logrus.Fields{ @@ -102,17 +102,18 @@ func registerNarPut(s *Server) { // Compare the nar hash specified in the URL with the one that has been // calculated while processing the NAR file - piNarHash, err := nixhash.ParseNixBase32( - "sha256:" + nixbase32.EncodeToString(pathInfo.GetNarinfo().NarSha256), + // TODO: bump go-nix and remove the parsing + narHash, err := nixhash.ParseNixBase32( + "sha256:" + nixbase32.EncodeToString(narSha256), ) if err != nil { panic("must parse nixbase32") } - if !bytes.Equal(narHashFromUrl.Digest(), piNarHash.Digest()) { + if !bytes.Equal(narHashFromUrl.Digest(), narHash.Digest()) { log := log.WithFields(logrus.Fields{ - "narhash_received_sha256": piNarHash.SRIString(), - "narsize": pathInfo.GetNarinfo().GetNarSize(), + "narhash_received_sha256": narHash.SRIString(), + "narsize": narSize, }) log.Error("received bytes don't match narhash from URL") @@ -123,7 +124,6 @@ func registerNarPut(s *Server) { } return - } // Insert the partial pathinfo structs into our lookup map, @@ -131,9 +131,12 @@ func registerNarPut(s *Server) { // The same might exist already, but it'll have the same contents (so // replacing will be a no-op), except maybe the root node Name field value, which // is safe to ignore (as not part of the NAR). - s.narHashToPathInfoMu.Lock() - s.narHashToPathInfo[piNarHash.SRIString()] = pathInfo - s.narHashToPathInfoMu.Unlock() + s.narDbMu.Lock() + s.narDb[narHash.SRIString()] = &narData{ + rootNode: rootNode, + narSize: narSize, + } + s.narDbMu.Unlock() // Done! }) -- cgit 1.4.1