From a9c47ffc506a0f3bc3fd54120f9a58e77176e94c Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Mon, 18 Sep 2023 12:26:03 +0300 Subject: refactor(tvix/nar-bridge): don't buffer blob in memory Create a pipe, pass the read end, and have a goroutine write to the write end. Change-Id: I301c273355705e60113b018e7e84b76972200e8c Reviewed-on: https://cl.tvl.fyi/c/depot/+/9361 Autosubmit: flokli Tested-by: BuildkiteCI Reviewed-by: Connor Brewster --- tvix/nar-bridge/pkg/server/nar_get.go | 41 +++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 16 deletions(-) (limited to 'tvix/nar-bridge') diff --git a/tvix/nar-bridge/pkg/server/nar_get.go b/tvix/nar-bridge/pkg/server/nar_get.go index b1eb70b624..d31fa1fbee 100644 --- a/tvix/nar-bridge/pkg/server/nar_get.go +++ b/tvix/nar-bridge/pkg/server/nar_get.go @@ -117,25 +117,34 @@ func renderNar( }) if err != nil { return nil, fmt.Errorf("unable to get blob: %w", err) - } - // TODO: spin up a goroutine producing this. - data := &bytes.Buffer{} - for { - chunk, err := resp.Recv() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return nil, fmt.Errorf("read chunk: %w", err) - } - _, err = data.Write(chunk.GetData()) - if err != nil { - return nil, fmt.Errorf("buffer chunk: %w", err) + // set up a pipe, let a goroutine write, return the reader. + pR, pW := io.Pipe() + + go func() { + for { + chunk, err := resp.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + pW.CloseWithError(fmt.Errorf("receiving chunk: %w", err)) + return + } + + // write the received chunk to the writer part of the pipe + if _, err := io.Copy(pW, bytes.NewReader(chunk.GetData())); err != nil { + log.WithError(err).Error("writing chunk to pipe") + pW.CloseWithError(fmt.Errorf("writing chunk to pipe: %w", err)) + return + } } - } - return io.NopCloser(data), nil + pW.Close() + + }() + + return io.NopCloser(pR), nil }, ) if err != nil { -- cgit 1.4.1