From f22d5b3d114cb3c3c73b41c738342fd9b6b2d04a Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Sat, 2 Mar 2024 18:12:40 +0200 Subject: feat(tvix/castore/blobsvc/from_addr): support object_store The object_store crate supports a ton of different stores, with different schemes. For now, use a objectstore+ scheme prefix to enable these. Change-Id: I946f76e32a0fb0867ef59060217894cda5b959b9 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11080 Tested-by: BuildkiteCI Reviewed-by: Connor Brewster Autosubmit: flokli --- tvix/castore/src/blobservice/from_addr.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'tvix') diff --git a/tvix/castore/src/blobservice/from_addr.rs b/tvix/castore/src/blobservice/from_addr.rs index 1f73c802cc14..6803b415ec82 100644 --- a/tvix/castore/src/blobservice/from_addr.rs +++ b/tvix/castore/src/blobservice/from_addr.rs @@ -3,7 +3,8 @@ use url::Url; use crate::{proto::blob_service_client::BlobServiceClient, Error}; use super::{ - BlobService, GRPCBlobService, MemoryBlobService, SimpleFilesystemBlobService, SledBlobService, + BlobService, GRPCBlobService, MemoryBlobService, ObjectStoreBlobService, + SimpleFilesystemBlobService, SledBlobService, }; /// Constructs a new instance of a [BlobService] from an URI. @@ -62,6 +63,17 @@ pub async fn from_addr(uri: &str) -> Result, crate::Error> } Box::new(SimpleFilesystemBlobService::new(url.path().into()).await?) + } else if let Some(_trimmed_scheme) = url.scheme().strip_prefix("objectstore+") { + // We need to convert the URL to string, strip the prefix there, and then + // parse it back as url, as Url::set_scheme() rejects some of the transitions we want to do. + let trimmed_url = { + let s = url.to_string(); + Url::parse(s.strip_prefix("objectstore+").unwrap()).unwrap() + }; + return Ok(Box::new( + ObjectStoreBlobService::parse_url(&trimmed_url) + .map_err(|e| Error::StorageError(e.to_string()))?, + )); } else { Err(crate::Error::StorageError(format!( "unknown scheme: {}", @@ -114,6 +126,14 @@ mod tests { #[test_case("grpc+https://localhost", true; "grpc valid https host without port")] /// Correct scheme to connect to localhost over http, but with additional path, which is invalid. #[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")] + /// An example for object store (Memory) + #[test_case("objectstore+memory:///", true; "objectstore valid memory url")] + /// An example for object store (File) + #[test_case("objectstore+file:///foo/bar", true; "objectstore valid file url")] // ?? + /// An example for object store (S3) + #[test_case("objectstore+s3://bucket/path", true; "objectstore valid s3 url")] + /// An example for object store (GCS) + #[test_case("objectstore+gs://bucket/path", true; "objectstore valid gcs url")] #[tokio::test] async fn test_from_addr_tokio(uri_str: &str, exp_succeed: bool) { if exp_succeed { -- cgit 1.4.1