Merge branch 'main' into feat/support-register-deregister-listing-schema-provider

Author: rtyler

.cargo/config.toml

@@ -1,4 +1,10 @@
 [target.x86_64-apple-darwin]
+rustflags = [
+  "-C", "link-arg=-undefined",
+  "-C", "link-arg=dynamic_lookup",
+]
+
+[target.aarch64-apple-darwin]
 rustflags = [
   "-C", "link-arg=-undefined",
   "-C", "link-arg=dynamic_lookup",

.github/workflows/build.yml

@@ -20,7 +20,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.81'
+          toolchain: '1.82'
           override: true
 
       - name: Format
@@ -42,7 +42,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.81'
+          toolchain: '1.82'
           override: true
 
       - name: build and lint with clippy
@@ -79,7 +79,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.81'
+          toolchain: '1.82'
           override: true
 
       - name: Run tests
@@ -114,7 +114,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.81'
+          toolchain: '1.82'
           override: true
 
       # Install Java and Hadoop for HDFS integration tests
@@ -160,7 +160,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.81'
+          toolchain: '1.82'
           override: true
 
       - name: Download Lakectl

.github/workflows/codecov.yml

@@ -20,7 +20,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.81'
+          toolchain: '1.82'
           override: true
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov

.github/workflows/python_build.yml

@@ -22,7 +22,7 @@ jobs:
 
       - name: Check Python
         run: |
-          uv pip install ruff==0.5.2 mypy==1.10.1 types-dataclasses typing-extensions --system
+          uv sync --only-group dev --no-install-project
           make check-python
 
       - name: Check Rust
@@ -46,7 +46,7 @@ jobs:
       - name: Build and install deltalake
         run: |
           # Install minimum PyArrow version
-          uv sync --extra devel --extra pandas --extra polars
+          make develop
           uv pip install pyarrow==16.0.0
         env:
           RUSTFLAGS: "-C debuginfo=line-tables-only"
@@ -213,7 +213,7 @@ jobs:
 
       - name: Build and install deltalake
         run: |
-          uv sync
+          make develop
 
       - name: Run deltalake
         run: |

Cargo.toml

@@ -5,7 +5,7 @@ resolver = "2"
 
 [workspace.package]
 authors = ["Qingping Hou <dave2008713@gmail.com>"]
-rust-version = "1.81"
+rust-version = "1.82"
 keywords = ["deltalake", "delta", "datalake"]
 readme = "README.md"
 edition = "2021"
@@ -26,7 +26,7 @@ debug = true
 debug = "line-tables-only"
 
 [workspace.dependencies]
-delta_kernel = { version = "=0.6.1", features = ["default-engine-rustls"] }
+delta_kernel = { version = "0.8.0", features = ["arrow_54", "default-engine-rustls"] }
 #delta_kernel = { path = "../delta-kernel-rs/kernel", features = ["sync-engine"]  }
 
 # arrow
@@ -41,20 +41,20 @@ arrow-ord = { version = "54" }
 arrow-row = { version = "54" }
 arrow-schema = { version = "54" }
 arrow-select = { version = "54" }
-object_store = { version = "0.11.2" , features = ["cloud"]}
+object_store = { version = "0.11.2" }
 parquet = { version = "54" }
 
 # datafusion
-datafusion = "45"
-datafusion-expr = "45"
-datafusion-common = "45"
-datafusion-ffi = "45"
-datafusion-functions = "45"
-datafusion-functions-aggregate = "45"
-datafusion-physical-expr = "45"
-datafusion-physical-plan = "45"
-datafusion-proto = "45"
-datafusion-sql = "45"
+datafusion = "46"
+datafusion-expr = "46"
+datafusion-common = "46"
+datafusion-ffi = "46"
+datafusion-functions = "46"
+datafusion-functions-aggregate = "46"
+datafusion-physical-expr = "46"
+datafusion-physical-plan = "46"
+datafusion-proto = "46"
+datafusion-sql = "46"
 
 # serde
 serde = { version = "1.0.194", features = ["derive"] }
@@ -77,4 +77,3 @@ async-trait = { version = "0.1" }
 futures = { version = "0.3" }
 tokio = { version = "1" }
 num_cpus = { version = "1" }
-

crates/aws/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltalake-aws"
-version = "0.7.0"
+version = "0.8.0"
 authors.workspace = true
 keywords.workspace = true
 readme.workspace = true
@@ -12,7 +12,7 @@ repository.workspace = true
 rust-version.workspace = true
 
 [dependencies]
-deltalake-core = { version = "0.24.0", path = "../core" }
+deltalake-core = { version = "0.25.0", path = "../core", features = ["cloud"] }
 aws-smithy-runtime-api = { version="1.7" }
 aws-smithy-runtime = { version="1.7", optional = true}
 aws-credential-types = { version="1.2", features = ["hardcoded-credentials"]}

crates/azure/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltalake-azure"
-version = "0.7.0"
+version = "0.8.0"
 authors.workspace = true
 keywords.workspace = true
 readme.workspace = true
@@ -12,8 +12,8 @@ repository.workspace = true
 rust-version.workspace = true
 
 [dependencies]
-deltalake-core = { version = "0.24.0", path = "../core", features = [
-    "datafusion",
+deltalake-core = { version = "0.25.0", path = "../core", features = [
+    "datafusion", "cloud",
 ]}
 
 # workspace depenndecies

crates/catalog-glue/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltalake-catalog-glue"
-version = "0.8.0"
+version = "0.9.0"
 authors.workspace = true
 keywords.workspace = true
 readme.workspace = true
@@ -15,7 +15,7 @@ rust-version.workspace = true
 async-trait = { workspace = true }
 aws-config = "1"
 aws-sdk-glue = "1"
-deltalake-core = { version = "0.24.0", path = "../core" }
+deltalake-core = { version = "0.25.0", path = "../core" }
 thiserror = { workspace = true }
 
 [dev-dependencies]

crates/catalog-unity/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltalake-catalog-unity"
-version = "0.8.0"
+version = "0.9.0"
 authors.workspace = true
 keywords.workspace = true
 readme.workspace = true
@@ -20,12 +20,12 @@ thiserror.workspace = true
 futures.workspace = true
 chrono.workspace = true
 tracing.workspace = true
-deltalake-core = { version = "0.24.0", path = "../core", features = [
+deltalake-core = { version = "0.25.0", path = "../core", features = [
     "datafusion",
 ] }
-deltalake-aws = { version = "0.7.0", path = "../aws", optional = true }
-deltalake-azure = { version = "0.7.0", path = "../azure", optional = true }
-deltalake-gcp = { version = "0.8.0", path = "../gcp", optional = true }
+deltalake-aws = { version = "0.8.0", path = "../aws", optional = true }
+deltalake-azure = { version = "0.8.0", path = "../azure", optional = true }
+deltalake-gcp = { version = "0.9.0", path = "../gcp", optional = true }
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "http2"] }
 reqwest-retry = "0.7"
 reqwest-middleware = { version = "0.4.0", features = ["json"] }

crates/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltalake-core"
-version = "0.24.0"
+version = "0.25.0"
 authors.workspace = true
 keywords.workspace = true
 readme.workspace = true
@@ -128,3 +128,4 @@ datafusion = [
 datafusion-ext = ["datafusion"]
 json = ["parquet/json"]
 python = ["arrow/pyarrow"]
+cloud = ["object_store/cloud"]