about summary refs log tree commit diff
path: root/tvix
diff options
context:
space:
mode:
authorFlorian Klink <flokli@flokli.de>2023-09-18T09·26+0300
committerflokli <flokli@flokli.de>2023-09-18T14·02+0000
commita9c47ffc506a0f3bc3fd54120f9a58e77176e94c (patch)
treed99bfd718d0aa542aadf7fe4e403ef75d99f9347 /tvix
parent07af692ecb1a9cabf03af1575dff7a82cf18a7ac (diff)
refactor(tvix/nar-bridge): don't buffer blob in memory r/6614
Create a pipe, pass the read end, and have a goroutine write to the
write end.

Change-Id: I301c273355705e60113b018e7e84b76972200e8c
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9361
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Diffstat (limited to 'tvix')
-rw-r--r--tvix/nar-bridge/pkg/server/nar_get.go41
1 files changed, 25 insertions, 16 deletions
diff --git a/tvix/nar-bridge/pkg/server/nar_get.go b/tvix/nar-bridge/pkg/server/nar_get.go
index b1eb70b624f0..d31fa1fbeeab 100644
--- a/tvix/nar-bridge/pkg/server/nar_get.go
+++ b/tvix/nar-bridge/pkg/server/nar_get.go
@@ -117,25 +117,34 @@ func renderNar(
 			})
 			if err != nil {
 				return nil, fmt.Errorf("unable to get blob: %w", err)
-
 			}
 
-			// TODO: spin up a goroutine producing this.
-			data := &bytes.Buffer{}
-			for {
-				chunk, err := resp.Recv()
-				if errors.Is(err, io.EOF) {
-					break
-				}
-				if err != nil {
-					return nil, fmt.Errorf("read chunk: %w", err)
-				}
-				_, err = data.Write(chunk.GetData())
-				if err != nil {
-					return nil, fmt.Errorf("buffer chunk: %w", err)
+			// set up a pipe, let a goroutine write, return the reader.
+			pR, pW := io.Pipe()
+
+			go func() {
+				for {
+					chunk, err := resp.Recv()
+					if errors.Is(err, io.EOF) {
+						break
+					}
+					if err != nil {
+						pW.CloseWithError(fmt.Errorf("receiving chunk: %w", err))
+						return
+					}
+
+					// write the received chunk to the writer part of the pipe
+					if _, err := io.Copy(pW, bytes.NewReader(chunk.GetData())); err != nil {
+						log.WithError(err).Error("writing chunk to pipe")
+						pW.CloseWithError(fmt.Errorf("writing chunk to pipe: %w", err))
+						return
+					}
 				}
-			}
-			return io.NopCloser(data), nil
+				pW.Close()
+
+			}()
+
+			return io.NopCloser(pR), nil
 		},
 	)
 	if err != nil {