refactor: improve config loading

.dockerignore

@@ -39,3 +39,4 @@ test-results/
 # Keep these for build
 !.cargo/config.toml
 !migrations/
+!configuration/

Cargo.toml

@@ -10,6 +10,7 @@ test-utils = ["dep:testcontainers", "dep:testcontainers-modules"]
 [dependencies]
 anyhow = { version = "1.0.98", default-features = false, features = ["std"] }
 chrono = { version = "0.4.41", default-features = false }
+config = { version = "0.14", default-features = false, features = ["yaml", "json"] }
 const-oid = { version = "0.9.6", default-features = false }
 futures = { version = "0.3.31", default-features = false }
 metrics = { version = "0.24.2", default-features = false }
@@ -28,6 +29,7 @@ sqlx = { version = "0.8.6", default-features = false, features = [
   "chrono",
   "uuid",
 ] }
+thiserror = { version = "2.0", default-features = false }
 tokio = { version = "1.47.0", default-features = false, features = ["rt-multi-thread", "macros", "signal"] }
 tokio-postgres = { git = "https://github.com/MaterializeInc/rust-postgres", default-features = false, rev = "c4b473b478b3adfbf8667d2fbe895d8423f1290b", features = [
   "runtime",
@@ -41,7 +43,6 @@ tracing-subscriber = { version = "0.3", default-features = false, features = ["f
 x509-cert = { version = "0.2.2", default-features = false }
 
 etl          = { git = "https://github.com/supabase/etl", rev = "483c7cfaf2b3730413acce8e0b4ad79a8cf28bbd" }
-etl-config   = { git = "https://github.com/supabase/etl", rev = "483c7cfaf2b3730413acce8e0b4ad79a8cf28bbd" }
 etl-postgres = { git = "https://github.com/supabase/etl", rev = "483c7cfaf2b3730413acce8e0b4ad79a8cf28bbd" }
 uuid         = { version = "1.19.0", default-features = false, features = ["v4"] }
 
@@ -57,6 +58,8 @@ testcontainers         = { version = "0.23", optional = true }
 testcontainers-modules = { version = "0.11", optional = true, features = ["postgres"] }
 
 [dev-dependencies]
+temp-env = "0.3"
+tempfile = "3.13"
 
 [lints.clippy]
 fallible_impl_from        = "deny"

Dockerfile

@@ -47,8 +47,9 @@ WORKDIR /app
 # Create non-root user (distroless already has nonroot user)
 USER nonroot:nonroot
 
-# Copy binary
+# Copy binary and configuration files
 COPY --from=builder /app/target/release/postgres-stream ./postgres-stream
+COPY configuration/ ./configuration/
 
 # Use exec form for proper signal handling
 ENTRYPOINT ["./postgres-stream"]

configuration/base.yml

@@ -1,6 +1,3 @@
-# Base configuration for daemon
-# This file contains default settings that apply to all environments
-
 stream:
   id: 1
   pg_connection:
@@ -12,8 +9,8 @@ stream:
       enabled: false
       trusted_root_certs: ""
   batch:
-    max_size: 100
-    max_fill_ms: 50
+    max_size: 1000
+    max_fill_ms: 1000
 
 sink:
   type: memory

src/concurrency/stream.rs

@@ -1,6 +1,6 @@
 use core::pin::Pin;
 use core::task::{Context, Poll};
-use etl_config::shared::BatchConfig;
+use etl::config::BatchConfig;
 use futures::{Future, Stream, ready};
 use pin_project_lite::pin_project;
 use std::time::Duration;