add join_paths method to storage backend

rust/src/storage/file/mod.rs

@@ -1,4 +1,4 @@
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::pin::Pin;
 
 use chrono::DateTime;
@@ -36,6 +36,13 @@ impl StorageBackend for FileStorageBackend {
             .unwrap()
     }
 
+    fn join_paths(&self, paths: &[&str]) -> String {
+        let mut iter = paths.iter();
+        let mut path = PathBuf::from(iter.next().unwrap_or(&""));
+        iter.for_each(|s| path.push(s));
+        path.into_os_string().into_string().unwrap()
+    }
+
     async fn head_obj(&self, path: &str) -> Result<ObjectMeta, StorageError> {
         let attr = fs::metadata(path).await?;
 
@@ -107,3 +114,23 @@ async fn create_tmp_file_with_retry(
 
     Err(StorageError::AlreadyExists(String::from(path)))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn join_multiple_paths() {
+        let backend = FileStorageBackend::new("./");
+        assert_eq!(
+            Path::new(&backend.join_paths(&["abc", "efg/", "123"])),
+            Path::new("abc").join("efg").join("123"),
+        );
+        assert_eq!(
+            &backend.join_paths(&["abc", "efg"]),
+            &backend.join_path("abc", "efg"),
+        );
+        assert_eq!(&backend.join_paths(&["foo"]), "foo",);
+        assert_eq!(&backend.join_paths(&[]), "",);
+    }
+}

rust/src/storage/mod.rs

@@ -235,16 +235,37 @@ pub struct ObjectMeta {
 
 #[async_trait::async_trait]
 pub trait StorageBackend: Send + Sync + Debug {
+    /// Create a new path by appending `path_to_join` as a new component to `path`.
     fn join_path(&self, path: &str, path_to_join: &str) -> String {
         let normalized_path = path.trim_end_matches('/');
         format!("{}/{}", normalized_path, path_to_join)
     }
+
+    /// More efficient path join for multiple path components. Use this method if you need to
+    /// combine more than two path components.
+    fn join_paths(&self, paths: &[&str]) -> String {
+        paths
+            .iter()
+            .map(|s| s.trim_end_matches('/'))
+            .collect::<Vec<_>>()
+            .join("/")
+    }
+
+    /// Fetch object metadata without reading the actual content
     async fn head_obj(&self, path: &str) -> Result<ObjectMeta, StorageError>;
+
+    /// Fetch object content
     async fn get_obj(&self, path: &str) -> Result<Vec<u8>, StorageError>;
+
+    /// Return a list of objects by `path` prefix in an async stream.
     async fn list_objs<'a>(
         &'a self,
         path: &'a str,
     ) -> Result<Pin<Box<dyn Stream<Item = Result<ObjectMeta, StorageError>> + 'a>>, StorageError>;
+
+    /// Create new object with `obj_bytes` as content.
+    ///
+    /// For a multi-writer safe backend, put_obj needs to implement create if not exists semantic.
     async fn put_obj(&self, path: &str, obj_bytes: &[u8]) -> Result<(), StorageError>;
 }
 

rust/src/storage/s3.rs

@@ -300,3 +300,17 @@ impl StorageBackend for S3StorageBackend {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn join_multiple_paths() {
+        let backend = S3StorageBackend::new();
+        assert_eq!(&backend.join_paths(&["abc", "efg/", "123"]), "abc/efg/123",);
+        assert_eq!(&backend.join_paths(&["abc", "efg/"]), "abc/efg",);
+        assert_eq!(&backend.join_paths(&["foo"]), "foo",);
+        assert_eq!(&backend.join_paths(&[]), "",);
+    }
+}