Export object_store integration tests (#5709)

* Export object_store integration tests

* Clippy

* Clippy

* Even more clippy

* Format

Author: tustvold

object_store/Cargo.toml

@@ -66,6 +66,7 @@ gcp = ["cloud", "rustls-pemfile"]
 aws = ["cloud", "md-5"]
 http = ["cloud"]
 tls-webpki-roots = ["reqwest?/rustls-tls-webpki-roots"]
+integration = []
 
 [dev-dependencies] # In alphabetical order
 futures-test = "0.3"

object_store/src/aws/builder.rs

@@ -70,21 +70,9 @@ enum Error {
     #[snafu(display("Configuration key: '{}' is not known.", key))]
     UnknownConfigurationKey { key: String },
 
-    #[snafu(display("Bucket '{}' not found", bucket))]
-    BucketNotFound { bucket: String },
-
-    #[snafu(display("Failed to resolve region for bucket '{}'", bucket))]
-    ResolveRegion {
-        bucket: String,
-        source: reqwest::Error,
-    },
-
     #[snafu(display("Invalid Zone suffix for bucket '{bucket}'"))]
     ZoneSuffix { bucket: String },
 
-    #[snafu(display("Failed to parse the region for bucket '{}'", bucket))]
-    RegionParse { bucket: String },
-
     #[snafu(display("Invalid encryption type: {}. Valid values are \"AES256\", \"sse:kms\", and \"sse:kms:dsse\".", passed))]
     InvalidEncryptionType { passed: String },
 

object_store/src/aws/client.rs

@@ -66,12 +66,6 @@ const SHA256_CHECKSUM: &str = "x-amz-checksum-sha256";
 #[derive(Debug, Snafu)]
 #[allow(missing_docs)]
 pub(crate) enum Error {
-    #[snafu(display("Error fetching get response body {}: {}", path, source))]
-    GetResponseBody {
-        source: reqwest::Error,
-        path: String,
-    },
-
     #[snafu(display("Error performing DeleteObjects request: {}", source))]
     DeleteObjectsRequest { source: crate::client::retry::Error },
 

object_store/src/aws/dynamo.rs

@@ -451,18 +451,6 @@ struct PutItem<'a> {
     return_values_on_condition_check_failure: Option<ReturnValues>,
 }
 
-/// A DynamoDB [GetItem] payload
-///
-/// [GetItem]: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_GetItem.html
-#[derive(Serialize)]
-#[serde(rename_all = "PascalCase")]
-struct GetItem<'a> {
-    /// The table name
-    table_name: &'a str,
-    /// The primary key
-    key: Map<'a, &'a str, AttributeValue<'a>>,
-}
-
 #[derive(Deserialize)]
 struct ErrorResponse<'a> {
     #[serde(rename = "__type")]

object_store/src/aws/mod.rs

@@ -409,14 +409,16 @@ impl MultipartStore for AmazonS3 {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{client::get::GetClient, tests::*};
+    use crate::client::get::GetClient;
+    use crate::integration::*;
+    use crate::tests::*;
     use hyper::HeaderMap;
 
     const NON_EXISTENT_NAME: &str = "nonexistentname";
 
     #[tokio::test]
     async fn s3_test() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let config = AmazonS3Builder::from_env();
 
         let integration = config.build().unwrap();
@@ -475,7 +477,7 @@ mod tests {
 
     #[tokio::test]
     async fn s3_test_get_nonexistent_location() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let integration = AmazonS3Builder::from_env().build().unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
@@ -488,7 +490,7 @@ mod tests {
 
     #[tokio::test]
     async fn s3_test_get_nonexistent_bucket() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let config = AmazonS3Builder::from_env().with_bucket_name(NON_EXISTENT_NAME);
         let integration = config.build().unwrap();
 
@@ -500,7 +502,7 @@ mod tests {
 
     #[tokio::test]
     async fn s3_test_put_nonexistent_bucket() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let config = AmazonS3Builder::from_env().with_bucket_name(NON_EXISTENT_NAME);
         let integration = config.build().unwrap();
 
@@ -513,7 +515,7 @@ mod tests {
 
     #[tokio::test]
     async fn s3_test_delete_nonexistent_location() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let integration = AmazonS3Builder::from_env().build().unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
@@ -523,7 +525,7 @@ mod tests {
 
     #[tokio::test]
     async fn s3_test_delete_nonexistent_bucket() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let config = AmazonS3Builder::from_env().with_bucket_name(NON_EXISTENT_NAME);
         let integration = config.build().unwrap();
 
@@ -560,7 +562,7 @@ mod tests {
     }
 
     async fn s3_encryption(store: &AmazonS3) {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
 
         let data = PutPayload::from(vec![3u8; 1024]);
 

object_store/src/azure/builder.rs

@@ -89,11 +89,6 @@ enum Error {
 
     #[snafu(display("Configuration key: '{}' is not known.", key))]
     UnknownConfigurationKey { key: String },
-
-    #[snafu(display("Unable to extract metadata from headers: {}", source))]
-    Metadata {
-        source: crate::client::header::Error,
-    },
 }
 
 impl From<Error> for crate::Error {

object_store/src/azure/client.rs

@@ -67,12 +67,6 @@ pub(crate) enum Error {
         path: String,
     },
 
-    #[snafu(display("Error getting get response body {}: {}", path, source))]
-    GetResponseBody {
-        source: reqwest::Error,
-        path: String,
-    },
-
     #[snafu(display("Error performing put request {}: {}", path, source))]
     PutRequest {
         source: crate::client::retry::Error,
@@ -94,11 +88,6 @@ pub(crate) enum Error {
     #[snafu(display("Got invalid list response: {}", source))]
     InvalidListResponse { source: quick_xml::de::DeError },
 
-    #[snafu(display("Error authorizing request: {}", source))]
-    Authorization {
-        source: crate::azure::credential::Error,
-    },
-
     #[snafu(display("Unable to extract metadata from headers: {}", source))]
     Metadata {
         source: crate::client::header::Error,

object_store/src/azure/mod.rs

@@ -276,12 +276,13 @@ impl MultipartStore for MicrosoftAzure {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::integration::*;
     use crate::tests::*;
     use bytes::Bytes;
 
     #[tokio::test]
     async fn azure_blob_test() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let integration = MicrosoftAzureBuilder::from_env().build().unwrap();
 
         put_get_delete_list(&integration).await;

object_store/src/chunked.rs

@@ -178,10 +178,10 @@ impl ObjectStore for ChunkedStore {
 mod tests {
     use futures::StreamExt;
 
+    use crate::integration::*;
     use crate::local::LocalFileSystem;
     use crate::memory::InMemory;
     use crate::path::Path;
-    use crate::tests::*;
 
     use super::*;
 

object_store/src/client/list.rs

@@ -48,6 +48,7 @@ pub trait ListClientExt {
 
     fn list(&self, prefix: Option<&Path>) -> BoxStream<'_, Result<ObjectMeta>>;
 
+    #[allow(unused)]
     fn list_with_offset(
         &self,
         prefix: Option<&Path>,

object_store/src/gcp/builder.rs

@@ -60,11 +60,6 @@ enum Error {
     #[snafu(display("Configuration key: '{}' is not known.", key))]
     UnknownConfigurationKey { key: String },
 
-    #[snafu(display("Unable to extract metadata from headers: {}", source))]
-    Metadata {
-        source: crate::client::header::Error,
-    },
-
     #[snafu(display("GCP credential error: {}", source))]
     Credential { source: credential::Error },
 }

object_store/src/gcp/client.rs

@@ -81,12 +81,6 @@ enum Error {
     #[snafu(display("Got invalid put response: {}", source))]
     InvalidPutResponse { source: quick_xml::de::DeError },
 
-    #[snafu(display("Error performing post request {}: {}", path, source))]
-    PostRequest {
-        source: crate::client::retry::Error,
-        path: String,
-    },
-
     #[snafu(display("Unable to extract metadata from headers: {}", source))]
     Metadata {
         source: crate::client::header::Error,

object_store/src/gcp/credential.rs

@@ -536,7 +536,7 @@ impl ApplicationDefaultCredentials {
             let path = Path::new(&home).join(Self::CREDENTIALS_PATH);
 
             // It's expected for this file to not exist unless it has been explicitly configured by the user.
-            if path.try_exists().unwrap_or(false) {
+            if path.exists() {
                 return read_credentials_file::<Self>(path).map(Some);
             }
         }

object_store/src/gcp/mod.rs

@@ -273,6 +273,7 @@ mod test {
 
     use credential::DEFAULT_GCS_BASE_URL;
 
+    use crate::integration::*;
     use crate::tests::*;
 
     use super::*;
@@ -281,7 +282,7 @@ mod test {
 
     #[tokio::test]
     async fn gcs_test() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let integration = GoogleCloudStorageBuilder::from_env().build().unwrap();
 
         put_get_delete_list(&integration).await;
@@ -307,7 +308,7 @@ mod test {
     #[tokio::test]
     #[ignore]
     async fn gcs_test_sign() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let integration = GoogleCloudStorageBuilder::from_env().build().unwrap();
 
         let client = reqwest::Client::new();
@@ -336,7 +337,7 @@ mod test {
 
     #[tokio::test]
     async fn gcs_test_get_nonexistent_location() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let integration = GoogleCloudStorageBuilder::from_env().build().unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
@@ -351,7 +352,7 @@ mod test {
 
     #[tokio::test]
     async fn gcs_test_get_nonexistent_bucket() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let config = GoogleCloudStorageBuilder::from_env();
         let integration = config.with_bucket_name(NON_EXISTENT_NAME).build().unwrap();
 
@@ -369,7 +370,7 @@ mod test {
 
     #[tokio::test]
     async fn gcs_test_delete_nonexistent_location() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let integration = GoogleCloudStorageBuilder::from_env().build().unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
@@ -383,7 +384,7 @@ mod test {
 
     #[tokio::test]
     async fn gcs_test_delete_nonexistent_bucket() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let config = GoogleCloudStorageBuilder::from_env();
         let integration = config.with_bucket_name(NON_EXISTENT_NAME).build().unwrap();
 
@@ -398,7 +399,7 @@ mod test {
 
     #[tokio::test]
     async fn gcs_test_put_nonexistent_bucket() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let config = GoogleCloudStorageBuilder::from_env();
         let integration = config.with_bucket_name(NON_EXISTENT_NAME).build().unwrap();
 

object_store/src/http/mod.rs

@@ -64,9 +64,6 @@ enum Error {
     Metadata {
         source: crate::client::header::Error,
     },
-
-    #[snafu(display("Request error: {}", source))]
-    Reqwest { source: reqwest::Error },
 }
 
 impl From<Error> for crate::Error {
@@ -249,13 +246,14 @@ impl HttpBuilder {
 
 #[cfg(test)]
 mod tests {
+    use crate::integration::*;
     use crate::tests::*;
 
     use super::*;
 
     #[tokio::test]
     async fn http_test() {
-        crate::test_util::maybe_skip_integration!();
+        maybe_skip_integration!();
         let url = std::env::var("HTTP_URL").expect("HTTP_URL must be set");
         let options = ClientOptions::new().with_allow_http(true);
         let integration = HttpBuilder::new()