[ENH]: limit concurrency on operators spawned by GC

Author: codetheweb

rust/garbage_collector/src/bin/garbage_collector_tool.rs

@@ -245,6 +245,7 @@ async fn main() {
                 config.min_versions_to_keep,
                 enable_log_gc,
                 enable_dangerous_option_to_ignore_min_versions_for_wal3,
+                10,
             );
 
             let result = orchestrator.run(system.clone()).await;

rust/garbage_collector/src/config.rs

@@ -34,6 +34,10 @@ pub struct GarbageCollectorConfig {
     )]
     pub(super) version_cutoff_time: Duration,
     pub(super) max_collections_to_gc: u32,
+    #[serde(
+        default = "GarbageCollectorConfig::default_max_concurrent_list_files_operations_per_collection"
+    )]
+    pub(super) max_concurrent_list_files_operations_per_collection: usize,
     pub(super) max_collections_to_fetch: Option<u32>,
     pub(super) gc_interval_mins: u32,
     #[serde(default = "GarbageCollectorConfig::default_min_versions_to_keep")]
@@ -70,6 +74,10 @@ impl GarbageCollectorConfig {
         2
     }
 
+    fn default_max_concurrent_list_files_operations_per_collection() -> usize {
+        10
+    }
+
     fn default_filter_min_versions_if_alive() -> Option<u64> {
         None
     }

rust/garbage_collector/src/garbage_collector_component.rs

@@ -187,6 +187,8 @@ impl GarbageCollector {
                 self.config.min_versions_to_keep,
                 enable_log_gc,
                 enable_dangerous_option_to_ignore_min_versions_for_wal3,
+                self.config
+                    .max_concurrent_list_files_operations_per_collection,
             );
 
         let started_at = SystemTime::now();
@@ -865,6 +867,7 @@ mod tests {
             enable_log_gc_for_tenant_threshold: "tenant-threshold".to_string(),
             log: LogConfig::Grpc(GrpcLogConfig::default()),
             enable_dangerous_option_to_ignore_min_versions_for_wal3: false,
+            max_concurrent_list_files_operations_per_collection: 10,
         };
         let registry = Registry::new();