From e2e987d6020a0e0c8b04cd7a9e04abb825fbafbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tara=C5=9B?= Date: Sun, 11 May 2025 13:49:47 +0200 Subject: [PATCH 01/10] added information about setting up ide --- CONTRIBUTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0cc703db..8da1ab1a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,6 +19,8 @@ # Development instructions +Many dependencies are behind [cargo feature flags](https://doc.rust-lang.org/cargo/reference/features.html). When developing locally you may want to enable some or all of them. You can use `--all-features` option with cargo. If you're using VSCode you may want to add: `"rust-analyzer.cargo.features": "all"` to your `.vscode/settings.json` file to have full IDE support. + ## Running Tests Tests can be run using `cargo` From f04e0df80ea49efc474a704ad24b8ec618110c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tara=C5=9B?= Date: Sun, 11 May 2025 13:52:52 +0200 Subject: [PATCH 02/10] add flag to ignore paths that throw error on Path::parse --- src/azure/client.rs | 36 ++++++++++++++++++++++++++++-------- src/azure/mod.rs | 8 ++++---- src/client/list.rs | 20 +++++++++++++------- 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/src/azure/client.rs b/src/azure/client.rs index 2d5db131..71d17053 100644 --- a/src/azure/client.rs +++ b/src/azure/client.rs @@ -964,6 +964,7 @@ impl ListClient for Arc { delimiter: bool, token: Option<&str>, offset: Option<&str>, + ignore_unparsable_paths: bool ) -> Result<(ListResult, Option)> { assert!(offset.is_none()); // Not yet supported @@ -1010,7 +1011,7 @@ impl ListClient for Arc { let token = response.next_marker.take(); - Ok((to_list_result(response, prefix)?, token)) + Ok((to_list_result(response, prefix, ignore_unparsable_paths)?, token)) } } @@ -1025,7 +1026,7 @@ struct ListResultInternal { pub blobs: Blobs, } -fn to_list_result(value: ListResultInternal, prefix: Option<&str>) -> Result { +fn to_list_result(value: ListResultInternal, prefix: Option<&str>, ignore_unparsable_paths: bool) -> Result { let prefix = prefix.unwrap_or_default(); let common_prefixes = value .blobs @@ -1045,6 +1046,9 @@ fn to_list_result(value: ListResultInternal, prefix: Option<&str>) -> Result prefix.len() }) + .map(BlobInternal::try_from) + .filter(|parsed| !ignore_unparsable_paths || parsed.is_ok()) + .map(|parsed| parsed.unwrap()) .map(ObjectMeta::try_from) .collect::>()?; @@ -1083,15 +1087,31 @@ struct Blob { pub metadata: Option>, } -impl TryFrom for ObjectMeta { +struct BlobInternal { + pub blob: Blob, + pub path: Path, +} + +impl TryFrom for BlobInternal { + type Error = crate::path::Error; + + fn try_from(value: Blob) -> Result { + Ok(BlobInternal { + path: Path::parse(&value.name)?, + blob: value, + }) + } +} + +impl TryFrom for ObjectMeta { type Error = crate::Error; - fn try_from(value: Blob) -> Result { + fn try_from(value: BlobInternal) -> Result { Ok(Self { - location: Path::parse(value.name)?, - last_modified: value.properties.last_modified, - size: value.properties.content_length, - e_tag: value.properties.e_tag, + location: value.path, + last_modified: value.blob.properties.last_modified, + size: value.blob.properties.content_length, + e_tag: value.blob.properties.e_tag, version: None, // For consistency with S3 and GCP which don't include this }) } diff --git a/src/azure/mod.rs b/src/azure/mod.rs index b4243dda..65254f42 100644 --- a/src/azure/mod.rs +++ b/src/azure/mod.rs @@ -119,8 +119,8 @@ impl ObjectStore for MicrosoftAzure { self.client.delete_request(location, &()).await } - fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { - self.client.list(prefix) + fn list(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> BoxStream<'static, Result> { + self.client.list(prefix, ignore_unparsable_paths) } fn delete_stream<'a>( &'a self, @@ -142,8 +142,8 @@ impl ObjectStore for MicrosoftAzure { .boxed() } - async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { - self.client.list_with_delimiter(prefix).await + async fn list_with_delimiter(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> Result { + self.client.list_with_delimiter(prefix, ignore_unparsable_paths).await } async fn copy(&self, from: &Path, to: &Path) -> Result<()> { diff --git a/src/client/list.rs b/src/client/list.rs index fe9bfebf..f1f652f2 100644 --- a/src/client/list.rs +++ b/src/client/list.rs @@ -33,6 +33,7 @@ pub(crate) trait ListClient: Send + Sync + 'static { delimiter: bool, token: Option<&str>, offset: Option<&str>, + ignore_unparsable_paths: bool, ) -> Result<(ListResult, Option)>; } @@ -44,18 +45,20 @@ pub(crate) trait ListClientExt { prefix: Option<&Path>, delimiter: bool, offset: Option<&Path>, + ignore_unparsable_paths: bool, ) -> BoxStream<'static, Result>; - fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result>; + fn list(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> BoxStream<'static, Result>; #[allow(unused)] fn list_with_offset( &self, prefix: Option<&Path>, offset: &Path, + ignore_unparsable_paths: bool, ) -> BoxStream<'static, Result>; - async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result; + async fn list_with_delimiter(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> Result; } #[async_trait] @@ -65,6 +68,7 @@ impl ListClientExt for T { prefix: Option<&Path>, delimiter: bool, offset: Option<&Path>, + ignore_unparsable_paths: bool, ) -> BoxStream<'static, Result> { let offset = offset.map(|x| x.to_string()); let prefix = prefix @@ -81,6 +85,7 @@ impl ListClientExt for T { delimiter, token.as_deref(), offset.as_deref(), + ignore_unparsable_paths, ) .await?; Ok((r, (prefix, offset), next_token)) @@ -89,8 +94,8 @@ impl ListClientExt for T { .boxed() } - fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { - self.list_paginated(prefix, false, None) + fn list(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> BoxStream<'static, Result> { + self.list_paginated(prefix, false, None, ignore_unparsable_paths) .map_ok(|r| futures::stream::iter(r.objects.into_iter().map(Ok))) .try_flatten() .boxed() @@ -100,15 +105,16 @@ impl ListClientExt for T { &self, prefix: Option<&Path>, offset: &Path, + ignore_unparsable_paths: bool, ) -> BoxStream<'static, Result> { - self.list_paginated(prefix, false, Some(offset)) + self.list_paginated(prefix, false, Some(offset), ignore_unparsable_paths) .map_ok(|r| futures::stream::iter(r.objects.into_iter().map(Ok))) .try_flatten() .boxed() } - async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { - let mut stream = self.list_paginated(prefix, true, None); + async fn list_with_delimiter(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> Result { + let mut stream = self.list_paginated(prefix, true, None, ignore_unparsable_paths); let mut common_prefixes = BTreeSet::new(); let mut objects = Vec::new(); From acc82040f5ecd3d48e2534a2d87ff2ef1dcf0ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tara=C5=9B?= Date: Sat, 17 May 2025 11:59:59 +0200 Subject: [PATCH 03/10] changed location of flag for ignoring paths to be part of AzureConfig rather than parameter on public list trait --- src/azure/builder.rs | 28 +++++++++ src/azure/client.rs | 136 ++++++++++++++++++++++++++++++++++++++++++- src/azure/mod.rs | 8 +-- src/client/list.rs | 20 +++---- 4 files changed, 173 insertions(+), 19 deletions(-) diff --git a/src/azure/builder.rs b/src/azure/builder.rs index f176fc68..68bc6f17 100644 --- a/src/azure/builder.rs +++ b/src/azure/builder.rs @@ -180,6 +180,8 @@ pub struct MicrosoftAzureBuilder { fabric_cluster_identifier: Option, /// The [`HttpConnector`] to use http_connector: Option>, + /// When listing objects, ignore paths that throw and error when parsing + ignore_unparsable_paths: ConfigValue, } /// Configuration keys for [`MicrosoftAzureBuilder`] @@ -380,6 +382,18 @@ pub enum AzureConfigKey { /// - `fabric_cluster_identifier` FabricClusterIdentifier, + /// When listing objects, ignore paths that do not conform to the Path class assumptions + /// + /// e.g. Azure allows to have paths like `foo/bar//baz` which would throw an error + /// when trying to parse them into a Path. + /// + /// When set to true, these paths will be ignored. + /// + /// Supported keys: + /// - `azure_ignore_unparsable_paths` + /// - `ignore_unparsable_paths` + IgnoreUnparsablePaths, + /// Client options Client(ClientConfigKey), } @@ -410,6 +424,7 @@ impl AsRef for AzureConfigKey { Self::FabricWorkloadHost => "azure_fabric_workload_host", Self::FabricSessionToken => "azure_fabric_session_token", Self::FabricClusterIdentifier => "azure_fabric_cluster_identifier", + Self::IgnoreUnparsablePaths => "azure_ignore_unparsable_paths", Self::Client(key) => key.as_ref(), } } @@ -466,6 +481,9 @@ impl FromStr for AzureConfigKey { "azure_fabric_cluster_identifier" | "fabric_cluster_identifier" => { Ok(Self::FabricClusterIdentifier) } + "azure_ignore_unparsable_paths" | "ignore_unparsable_paths" => { + Ok(Self::IgnoreUnparsablePaths) + } // Backwards compatibility "azure_allow_http" => Ok(Self::Client(ClientConfigKey::AllowHttp)), _ => match s.strip_prefix("azure_").unwrap_or(s).parse() { @@ -594,6 +612,7 @@ impl MicrosoftAzureBuilder { AzureConfigKey::FabricClusterIdentifier => { self.fabric_cluster_identifier = Some(value.into()) } + AzureConfigKey::IgnoreUnparsablePaths => self.ignore_unparsable_paths.parse(value), }; self } @@ -635,6 +654,7 @@ impl MicrosoftAzureBuilder { AzureConfigKey::FabricWorkloadHost => self.fabric_workload_host.clone(), AzureConfigKey::FabricSessionToken => self.fabric_session_token.clone(), AzureConfigKey::FabricClusterIdentifier => self.fabric_cluster_identifier.clone(), + AzureConfigKey::IgnoreUnparsablePaths => Some(self.ignore_unparsable_paths.to_string()), } } @@ -897,6 +917,13 @@ impl MicrosoftAzureBuilder { self } + /// If set to `true` list operations will ignore paths that do not conform to the Path class assumptions + /// e.g. foo/bar//baz.txt + pub fn with_ignore_unparsable_paths(mut self, ignore_unparsable_paths: bool) -> Self { + self.ignore_unparsable_paths = ignore_unparsable_paths.into(); + self + } + /// Configure a connection to container with given name on Microsoft Azure Blob store. pub fn build(mut self) -> Result { if let Some(url) = self.url.take() { @@ -1040,6 +1067,7 @@ impl MicrosoftAzureBuilder { client_options: self.client_options, service: storage_url, credentials: auth, + ignore_unparsable_paths: self.ignore_unparsable_paths.get()?, }; let http_client = http.connect(&config.client_options)?; diff --git a/src/azure/client.rs b/src/azure/client.rs index 71d17053..ac387563 100644 --- a/src/azure/client.rs +++ b/src/azure/client.rs @@ -172,6 +172,7 @@ pub(crate) struct AzureConfig { pub skip_signature: bool, pub disable_tagging: bool, pub client_options: ClientOptions, + pub ignore_unparsable_paths: bool, } impl AzureConfig { @@ -964,7 +965,6 @@ impl ListClient for Arc { delimiter: bool, token: Option<&str>, offset: Option<&str>, - ignore_unparsable_paths: bool ) -> Result<(ListResult, Option)> { assert!(offset.is_none()); // Not yet supported @@ -1011,7 +1011,7 @@ impl ListClient for Arc { let token = response.next_marker.take(); - Ok((to_list_result(response, prefix, ignore_unparsable_paths)?, token)) + Ok((to_list_result(response, prefix, self.config.ignore_unparsable_paths)?, token)) } } @@ -1410,6 +1410,7 @@ mod tests { skip_signature: false, disable_tagging: false, client_options: Default::default(), + ignore_unparsable_paths: Default::default(), }; let client = AzureClient::new(config, HttpClient::new(Client::new())); @@ -1547,4 +1548,135 @@ Time:2018-06-14T16:46:54.6040685Z\r assert_eq!("404", code); assert_eq!("The specified blob does not exist.", reason); } + + #[tokio::test] + async fn test_list_blobs() { + let fake_properties = BlobProperties { + last_modified: Utc::now(), + content_length: 8, + content_type: "text/plain".to_string(), + content_encoding: None, + content_language: None, + e_tag: Some("etag".to_string()), + resource_type: Some("resource".to_string()), + }; + let fake_result = ListResultInternal { + prefix: None, + max_results: None, + delimiter: None, + next_marker: None, + blobs: Blobs { + blob_prefix: vec![], + blobs: vec![ + Blob { + name: "blob0.txt".to_string(), + version_id: None, + is_current_version: None, + deleted: None, + properties: fake_properties.clone(), + metadata: None, + }, + Blob { + name: "blob1.txt".to_string(), + version_id: None, + is_current_version: None, + deleted: None, + properties: fake_properties.clone(), + metadata: None, + }, + ], + }, + }; + let result = to_list_result(fake_result, None, false).unwrap(); + assert_eq!(result.common_prefixes.len(), 0); + assert_eq!(result.objects.len(), 2); + assert_eq!(result.objects[0].location, Path::from("blob0.txt")); + assert_eq!(result.objects[1].location, Path::from("blob1.txt")); + } + + #[tokio::test] + #[should_panic] + async fn test_list_blobs_invalid_paths() { + let fake_properties = BlobProperties { + last_modified: Utc::now(), + content_length: 8, + content_type: "text/plain".to_string(), + content_encoding: None, + content_language: None, + e_tag: Some("etag".to_string()), + resource_type: Some("resource".to_string()), + }; + let fake_result = ListResultInternal { + prefix: None, + max_results: None, + delimiter: None, + next_marker: None, + blobs: Blobs { + blob_prefix: vec![], + blobs: vec![ + Blob { + name: "foo/blob0.txt".to_string(), + version_id: None, + is_current_version: None, + deleted: None, + properties: fake_properties.clone(), + metadata: None, + }, + Blob { + name: "foo//blob1.txt".to_string(), + version_id: None, + is_current_version: None, + deleted: None, + properties: fake_properties.clone(), + metadata: None, + }, + ], + }, + }; + to_list_result(fake_result, None, false).unwrap(); + } + + #[tokio::test] + async fn test_list_blobs_ignore_invalid_paths() { + let fake_properties = BlobProperties { + last_modified: Utc::now(), + content_length: 8, + content_type: "text/plain".to_string(), + content_encoding: None, + content_language: None, + e_tag: Some("etag".to_string()), + resource_type: Some("resource".to_string()), + }; + let fake_result = ListResultInternal { + prefix: None, + max_results: None, + delimiter: None, + next_marker: None, + blobs: Blobs { + blob_prefix: vec![], + blobs: vec![ + Blob { + name: "foo/blob0.txt".to_string(), + version_id: None, + is_current_version: None, + deleted: None, + properties: fake_properties.clone(), + metadata: None, + }, + Blob { + name: "foo//blob1.txt".to_string(), + version_id: None, + is_current_version: None, + deleted: None, + properties: fake_properties.clone(), + metadata: None, + }, + ], + }, + }; + let result = to_list_result(fake_result, None, true).unwrap(); + assert_eq!(result.common_prefixes.len(), 0); + assert_eq!(result.objects.len(), 1); + assert_eq!(result.objects[0].location, Path::from("foo/blob0.txt")); + } } diff --git a/src/azure/mod.rs b/src/azure/mod.rs index 65254f42..b4243dda 100644 --- a/src/azure/mod.rs +++ b/src/azure/mod.rs @@ -119,8 +119,8 @@ impl ObjectStore for MicrosoftAzure { self.client.delete_request(location, &()).await } - fn list(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> BoxStream<'static, Result> { - self.client.list(prefix, ignore_unparsable_paths) + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { + self.client.list(prefix) } fn delete_stream<'a>( &'a self, @@ -142,8 +142,8 @@ impl ObjectStore for MicrosoftAzure { .boxed() } - async fn list_with_delimiter(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> Result { - self.client.list_with_delimiter(prefix, ignore_unparsable_paths).await + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { + self.client.list_with_delimiter(prefix).await } async fn copy(&self, from: &Path, to: &Path) -> Result<()> { diff --git a/src/client/list.rs b/src/client/list.rs index f1f652f2..fe9bfebf 100644 --- a/src/client/list.rs +++ b/src/client/list.rs @@ -33,7 +33,6 @@ pub(crate) trait ListClient: Send + Sync + 'static { delimiter: bool, token: Option<&str>, offset: Option<&str>, - ignore_unparsable_paths: bool, ) -> Result<(ListResult, Option)>; } @@ -45,20 +44,18 @@ pub(crate) trait ListClientExt { prefix: Option<&Path>, delimiter: bool, offset: Option<&Path>, - ignore_unparsable_paths: bool, ) -> BoxStream<'static, Result>; - fn list(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> BoxStream<'static, Result>; + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result>; #[allow(unused)] fn list_with_offset( &self, prefix: Option<&Path>, offset: &Path, - ignore_unparsable_paths: bool, ) -> BoxStream<'static, Result>; - async fn list_with_delimiter(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> Result; + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result; } #[async_trait] @@ -68,7 +65,6 @@ impl ListClientExt for T { prefix: Option<&Path>, delimiter: bool, offset: Option<&Path>, - ignore_unparsable_paths: bool, ) -> BoxStream<'static, Result> { let offset = offset.map(|x| x.to_string()); let prefix = prefix @@ -85,7 +81,6 @@ impl ListClientExt for T { delimiter, token.as_deref(), offset.as_deref(), - ignore_unparsable_paths, ) .await?; Ok((r, (prefix, offset), next_token)) @@ -94,8 +89,8 @@ impl ListClientExt for T { .boxed() } - fn list(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> BoxStream<'static, Result> { - self.list_paginated(prefix, false, None, ignore_unparsable_paths) + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { + self.list_paginated(prefix, false, None) .map_ok(|r| futures::stream::iter(r.objects.into_iter().map(Ok))) .try_flatten() .boxed() @@ -105,16 +100,15 @@ impl ListClientExt for T { &self, prefix: Option<&Path>, offset: &Path, - ignore_unparsable_paths: bool, ) -> BoxStream<'static, Result> { - self.list_paginated(prefix, false, Some(offset), ignore_unparsable_paths) + self.list_paginated(prefix, false, Some(offset)) .map_ok(|r| futures::stream::iter(r.objects.into_iter().map(Ok))) .try_flatten() .boxed() } - async fn list_with_delimiter(&self, prefix: Option<&Path>, ignore_unparsable_paths: bool) -> Result { - let mut stream = self.list_paginated(prefix, true, None, ignore_unparsable_paths); + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { + let mut stream = self.list_paginated(prefix, true, None); let mut common_prefixes = BTreeSet::new(); let mut objects = Vec::new(); From c703ef18b878efe0d283e9ba1a9047544842beae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tara=C5=9B?= Date: Sun, 25 May 2025 14:33:30 +0200 Subject: [PATCH 04/10] changed type to Self to appease cargo clippy --- src/azure/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure/client.rs b/src/azure/client.rs index ac387563..96b9b885 100644 --- a/src/azure/client.rs +++ b/src/azure/client.rs @@ -1096,7 +1096,7 @@ impl TryFrom for BlobInternal { type Error = crate::path::Error; fn try_from(value: Blob) -> Result { - Ok(BlobInternal { + Ok(Self { path: Path::parse(&value.name)?, blob: value, }) From 1b5b5cfdd32d65690407ee78f314a47784a52be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tara=C5=9B?= Date: Sun, 25 May 2025 14:35:52 +0200 Subject: [PATCH 05/10] add note about clippy --- CONTRIBUTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8da1ab1a..f714c1be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,8 @@ Many dependencies are behind [cargo feature flags](https://doc.rust-lang.org/cargo/reference/features.html). When developing locally you may want to enable some or all of them. You can use `--all-features` option with cargo. If you're using VSCode you may want to add: `"rust-analyzer.cargo.features": "all"` to your `.vscode/settings.json` file to have full IDE support. +Before submitting PR run linter clippy with command like: `cargo clippy --all-features -- -D warnings` or `cargo clippy --features azure -- -D warnings` for a specific feature. It's part of CI suite so any warnings here would be flagged in a PR. + ## Running Tests Tests can be run using `cargo` From 146af11ae069a493cc3dc6f05459185f2a390012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tara=C5=9B?= Date: Sun, 25 May 2025 17:29:30 +0200 Subject: [PATCH 06/10] format code using cargo fmt --- src/azure/builder.rs | 4 ++-- src/azure/client.rs | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/azure/builder.rs b/src/azure/builder.rs index 0874e4ab..14447f85 100644 --- a/src/azure/builder.rs +++ b/src/azure/builder.rs @@ -383,10 +383,10 @@ pub enum AzureConfigKey { FabricClusterIdentifier, /// When listing objects, ignore paths that do not conform to the Path class assumptions - /// + /// /// e.g. Azure allows to have paths like `foo/bar//baz` which would throw an error /// when trying to parse them into a Path. - /// + /// /// When set to true, these paths will be ignored. /// /// Supported keys: diff --git a/src/azure/client.rs b/src/azure/client.rs index b26a2682..c675bd91 100644 --- a/src/azure/client.rs +++ b/src/azure/client.rs @@ -1039,7 +1039,11 @@ struct ListResultInternal { pub blobs: Blobs, } -fn to_list_result(value: ListResultInternal, prefix: Option<&str>, ignore_unparsable_paths: bool) -> Result { +fn to_list_result( + value: ListResultInternal, + prefix: Option<&str>, + ignore_unparsable_paths: bool, +) -> Result { let prefix = prefix.unwrap_or_default(); let common_prefixes = value .blobs From 8fdeda91ecadf131cfab93a7a1658809aff622de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tara=C5=9B?= Date: Sun, 25 May 2025 17:30:50 +0200 Subject: [PATCH 07/10] add info about cargo fmt --- CONTRIBUTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f714c1be..b3af4939 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,6 +23,8 @@ Many dependencies are behind [cargo feature flags](https://doc.rust-lang.org/car Before submitting PR run linter clippy with command like: `cargo clippy --all-features -- -D warnings` or `cargo clippy --features azure -- -D warnings` for a specific feature. It's part of CI suite so any warnings here would be flagged in a PR. +Additionally remember to run code formatter with command: `cargo fmt --all` as part of CI suite is running: `cargo fmt --all -- --check` which will throw error if it detects that some code is not formatted properly. + ## Running Tests Tests can be run using `cargo` From 5bb90973806a8af225056b37964256faa37f10e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tara=C5=9B?= Date: Sun, 1 Jun 2025 17:31:17 +0200 Subject: [PATCH 08/10] specify expected error message --- src/azure/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure/client.rs b/src/azure/client.rs index c675bd91..f65acf2f 100644 --- a/src/azure/client.rs +++ b/src/azure/client.rs @@ -1612,7 +1612,7 @@ Time:2018-06-14T16:46:54.6040685Z\r } #[tokio::test] - #[should_panic] + #[should_panic(expected = "EmptySegment { path: \"foo//blob1.txt\" }")] async fn test_list_blobs_invalid_paths() { let fake_properties = BlobProperties { last_modified: Utc::now(), From 429ae6bdd9ece2ecd98e191278b07626b3f3feda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tara=C5=9B?= Date: Sun, 1 Jun 2025 17:41:49 +0200 Subject: [PATCH 09/10] changed separate filter and map into single step --- src/azure/client.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/azure/client.rs b/src/azure/client.rs index f65acf2f..b2299125 100644 --- a/src/azure/client.rs +++ b/src/azure/client.rs @@ -1064,8 +1064,11 @@ fn to_list_result( && blob.name.len() > prefix.len() }) .map(BlobInternal::try_from) - .filter(|parsed| !ignore_unparsable_paths || parsed.is_ok()) - .map(|parsed| parsed.unwrap()) + .filter_map(|parsed| match parsed { + Ok(parsed) => Some(parsed), + Err(_) if ignore_unparsable_paths => None, + Err(e) => panic!("cannot parse path: {e}"), + }) .map(ObjectMeta::try_from) .collect::>()?; @@ -1612,7 +1615,7 @@ Time:2018-06-14T16:46:54.6040685Z\r } #[tokio::test] - #[should_panic(expected = "EmptySegment { path: \"foo//blob1.txt\" }")] + #[should_panic(expected = "cannot parse path: Path \"foo//blob1.txt\" contained empty path segment")] async fn test_list_blobs_invalid_paths() { let fake_properties = BlobProperties { last_modified: Utc::now(), From fa05a87bc606a29001879cd37dcb9cce51d1a79d Mon Sep 17 00:00:00 2001 From: ttomasz <31594210+ttomasz@users.noreply.github.com> Date: Sat, 14 Jun 2025 21:04:55 +0200 Subject: [PATCH 10/10] fix typo Co-authored-by: Kyle Barron --- src/azure/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure/builder.rs b/src/azure/builder.rs index 14447f85..1eff810e 100644 --- a/src/azure/builder.rs +++ b/src/azure/builder.rs @@ -180,7 +180,7 @@ pub struct MicrosoftAzureBuilder { fabric_cluster_identifier: Option, /// The [`HttpConnector`] to use http_connector: Option>, - /// When listing objects, ignore paths that throw and error when parsing + /// When listing objects, ignore paths that throw an error when parsing ignore_unparsable_paths: ConfigValue, }