From 5fc403587ffb38635e767697f1c18e7e3559aea6 Mon Sep 17 00:00:00 2001
From: Florian Klink <flokli@flokli.de>
Date: Sat, 20 Apr 2024 19:32:30 +0300
Subject: refactor(tvix/castore): ingest filesystem entries in parallel

Rather than carrying around an Future in the IngestionEntry::Regular,
simply carry the plain B3Digest.

Code reading through a non-seekable data stream has no choice but to
read and upload blobs immediately, and code seeking through something
seekable (like a filesystem) probably knows better what concurrency to
pick when ingesting, rather than the consuming side.

(Our only) one of these seekable source implementations is now doing
exactly that. We produce a stream of futures, and then use
[StreamExt::buffered] to process more than one, concurrently.

We still keep the same order, to avoid shuffling things and violating
the stream order.

This also cleans up walk_path_for_ingestion in castore/import, as well
as ingest_dir_entries in glue/tvix_store_io.

Change-Id: I5eb70f3e1e372c74bcbfcf6b6e2653eba36e151d
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11491
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Tested-by: BuildkiteCI
---
 tvix/glue/src/tvix_store_io.rs | 26 +++-----------------------
 1 file changed, 3 insertions(+), 23 deletions(-)

(limited to 'tvix/glue/src/tvix_store_io.rs')

diff --git a/tvix/glue/src/tvix_store_io.rs b/tvix/glue/src/tvix_store_io.rs
index 87e9c85a53..3c86f42630 100644
--- a/tvix/glue/src/tvix_store_io.rs
+++ b/tvix/glue/src/tvix_store_io.rs
@@ -20,10 +20,8 @@ use tokio_util::io::{InspectReader, SyncIoBridge};
 use tracing::{error, instrument, warn, Level};
 use tvix_build::buildservice::BuildService;
 use tvix_castore::import::archive::ingest_archive;
-use tvix_castore::import::fs::dir_entry_iter_to_ingestion_stream;
 use tvix_eval::{ErrorKind, EvalIO, FileType, StdIO};
 use tvix_store::utils::AsyncIoBridge;
-use walkdir::DirEntry;
 
 use tvix_castore::{
     blobservice::BlobService,
@@ -54,9 +52,9 @@ use crate::tvix_build::derivation_to_build_request;
 /// implementation of "Tvix Store IO" which does not necessarily bring the concept of blob service,
 /// directory service or path info service.
 pub struct TvixStoreIO {
-    blob_service: Arc<dyn BlobService>,
-    directory_service: Arc<dyn DirectoryService>,
-    // This is public so builtins can put PathInfos directly.
+    // This is public so helper functions can interact with the stores directly.
+    pub(crate) blob_service: Arc<dyn BlobService>,
+    pub(crate) directory_service: Arc<dyn DirectoryService>,
     pub(crate) path_info_service: Arc<dyn PathInfoService>,
     std_io: StdIO,
     #[allow(dead_code)]
@@ -277,24 +275,6 @@ impl TvixStoreIO {
             .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e))
     }
 
-    /// This forwards the ingestion to the [`tvix_castore::import::ingest_entries`],
-    /// passing the blob_service and directory_service that's used.
-    /// The error is mapped to std::io::Error for simplicity.
-    pub(crate) async fn ingest_dir_entries<'a, I>(
-        &'a self,
-        iter: I,
-        root: &Path,
-    ) -> io::Result<Node>
-    where
-        I: Iterator<Item = Result<DirEntry, walkdir::Error>> + Send + Sync + 'a,
-    {
-        let entries_stream = dir_entry_iter_to_ingestion_stream(&self.blob_service, iter, root);
-
-        tvix_castore::import::ingest_entries(&self.directory_service, entries_stream)
-            .await
-            .map_err(|err| std::io::Error::new(io::ErrorKind::Other, err))
-    }
-
     pub(crate) async fn node_to_path_info(
         &self,
         name: &str,
-- 
cgit 1.4.1