From 28417afbb4d8776501f9ae3ecead5859707488b9 Mon Sep 17 00:00:00 2001 From: talyz Date: Mon, 19 Sep 2022 09:30:28 +0200 Subject: fix(nixery): Avoid race when the same image is fetched in parallel Remove a race condition which appears when uploadHashLayer is called with the same key from multiple threads simultaneously. This can easily happen when the same image path is requested by multiple clients at the same time. When it does, a 500 status is returned and the following error message is logged: { "context": { "filePath": "github.com/google/nixery/builder/builder.go", "lineNumber": 440, "functionName": "github.com/google/nixery/builder.uploadHashLayer" }, "error": "rename /var/lib/nixery/staging/ /var/lib/nixery/layers/: no such file or directory", "eventTime": "...", "layer": "", "message": "failed to move layer from staging", ... } To solve this issue, introduce a mutex keyed on the uploaded hash and move all layer caching into uploadHashLayer. This could additionally provide a small performance benefit when an already built image is requested and NIXERY_PKGS_PATH is set, since symlink layers and config layers are now also cached. Change-Id: I50788a7ec7940cb5e5760f244692e361019a9bb7 Reviewed-on: https://cl.tvl.fyi/c/depot/+/6695 Reviewed-by: tazjin Tested-by: BuildkiteCI --- tools/nixery/cmd/server/main.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'tools/nixery/cmd/server') diff --git a/tools/nixery/cmd/server/main.go b/tools/nixery/cmd/server/main.go index 8fe1679cfad8..24aec6391c45 100644 --- a/tools/nixery/cmd/server/main.go +++ b/tools/nixery/cmd/server/main.go @@ -30,6 +30,7 @@ import ( "github.com/google/nixery/logs" mf "github.com/google/nixery/manifest" "github.com/google/nixery/storage" + "github.com/im7mortal/kmutex" log "github.com/sirupsen/logrus" ) @@ -257,10 +258,11 @@ func main() { } state := builder.State{ - Cache: &cache, - Cfg: cfg, - Pop: pop, - Storage: s, + Cache: &cache, + Cfg: cfg, + Pop: pop, + Storage: s, + UploadMutex: kmutex.New(), } log.WithFields(log.Fields{ -- cgit 1.4.1