From c04041a0017359811d91406e9484ed65b76ba33b Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Tue, 3 Oct 2023 19:08:34 +0300 Subject: feat(tvix/nar-bridge): add nar-bridge-pathinfo command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds an additional nar-bridge-pathinfo command. It exposes a PathInfoService for a HTTP Binary Cache, ingesting data into a BlobService/DirectoryService as it goes through the NAR file. It does this whenever it receives a Get request for a specific output path, and waits returning with the PathInfo response until it ingested the data. It does not do any sort of caching - this means it re-downloads NAR files again whenever the PathInfo is requested again, so you most likely do not want to use this currently. It's one building component as soon as we have store composition (which we currently don't, so don't use this). It can be used as an alternative mechanism to ingest data (Blobs and Directories) of a given store path from a binary cache into tvix-store. ``` ❯ nix-build -A third_party.nixpkgs.hello /nix/store/mdi7lvrn2mx7rfzv3fdq3v5yw8swiks6-hello-2.12.1 ❯ nix hash to-sri --type sha1 mdi7lvrn2mx7rfzv3fdq3v5yw8swiks6 sha1-Rs/INeK+7IGbG/u7fHoVNm96Yqs= ❯ out=$(mg build //tvix/nar-bridge) $out/bin/nar-bridge-pathinfo --log-level debug & INFO[0000] Starting nar-bridge-pathinfosvc at [::]:8001 ❯ mg run //tvix:store -- daemon & [mg] building target //tvix:store [mg] running target //tvix:store 2023-10-03T16:21:57.433739Z INFO tvix_store: tvix-store listening on [::]:8000 at src/bin/tvix-store.rs:229 ❯ evans --host localhost --port 8001 -r repl […] tvix.store.v1.PathInfoService@localhost:8001> call Get ✔ by_output_hash by_output_hash (TYPE_BYTES) => Rs/INeK+7IGbG/u7fHoVNm96Yqs= { "narinfo": { "narSha256": "sXrPtjqhSoc2u0YfM1HVZThknkSYuRuHdtKCB6wkDFo=", "narSize": "226552", "referenceNames": [ "aw2fw9ag10wr9pf0qk4nk5sxi0q0bn56-glibc-2.37-8", "mdi7lvrn2mx7rfzv3fdq3v5yw8swiks6-hello-2.12.1" ], "signatures": [ { "data": "7guDbfaF2Q29HY0c5axhtuacfxN6uxuEqeUfncDiSvMSAWvfHVMppB89ILqV8FE58pEQ04tSbMnRhR3FGPV0AA==", "name": "cache.nixos.org-1" } ] }, "node": { "directory": { "digest": "xvo6BYbYaDw76IibLu5sr+VZoj9iM0ET2RUuYSYLwKE=", "name": "bWRpN2x2cm4ybXg3cmZ6djNmZHEzdjV5dzhzd2lrczYtaGVsbG8tMi4xMi4x", "size": 141 } }, "references": [ "ptgFMIhdl2nJxMDdlDkITyXuBFc=", "Rs/INeK+7IGbG/u7fHoVNm96Yqs=" ] } ``` Change-Id: I50167d0ac081c91adf5cf2733bbc4dc0993bd46e Reviewed-on: https://cl.tvl.fyi/c/depot/+/9539 Autosubmit: flokli Tested-by: BuildkiteCI Reviewed-by: Connor Brewster Reviewed-by: Brian Olsen --- tvix/nar-bridge/go.sum | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tvix/nar-bridge/go.sum') diff --git a/tvix/nar-bridge/go.sum b/tvix/nar-bridge/go.sum index f8b9a9559c..f148e8318b 100644 --- a/tvix/nar-bridge/go.sum +++ b/tvix/nar-bridge/go.sum @@ -53,6 +53,7 @@ github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2 github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/nix-community/go-nix v0.0.0-20230825195510-c72199eca18e h1:15CPg2PQMyBl+TTEKuonrQqS9uOJyi7JcuU0FpvV088= github.com/nix-community/go-nix v0.0.0-20230825195510-c72199eca18e/go.mod h1:y3eASc0gMh26jjoP9Xz+qqMKjTnqJgG1RG8xvKvFR8s= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -68,6 +69,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= +github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -- cgit 1.4.1