Support icechunk (#101)

Author: jbusecke

.github/workflows/ci.yml

@@ -49,10 +49,10 @@ jobs:
       - name: run pre-commit
         if: ${{ matrix.python-version == env.LATEST_PY_VERSION }}
         run: |
-          uv run pre-commit run --all-files
+          uv run pre-commit run --all-files --verbose --show-diff-on-failure
 
       - name: Run tests
-        run: uv run pytest
+        run: uv run pytest -n auto
 
   cdk-checks:
     needs: [tests]

.github/workflows/deploy-dev.yml

@@ -31,6 +31,8 @@ jobs:
       TITILER_MULTIDIM_PYTHONWARNINGS: ignore
       TITILER_MULTIDIM_DEBUG: true
       TITILER_MULTIDIM_READER_ROLE_ARN: ${{ vars.TITILER_MULTIDIM_READER_ROLE_ARN }}
+      TITILER_MULTIDIM_AUTHORIZED_CHUNK_ACCESS: '{"s3://nasa-waterinsight/NLDAS3/forcing/daily/": {"anonymous": true}, "s3://podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/": {"from_env": true}, "s3://nasa-waterinsight/RASI/": {"anonymous": true}}'
+
 
     steps:
       - name: Determine branch to deploy

.gitignore

@@ -105,5 +105,7 @@ cdk.out/
 node_modules
 cdk.context.json
 *.nc
+.DS_Store
 
 .test-deploy-env
+.vscode/settings.json

.pre-commit-config.yaml

@@ -9,13 +9,19 @@ repos:
     hooks:
       - id: ruff
         args: ["--fix"]
+        exclude: ^notebooks/
       - id: ruff-format
+        exclude: ^notebooks/
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.991
+    rev: v1.13.0
     hooks:
       - id: mypy
-        language_version: python
+        language_version: python3.12
         exclude: tests/.*
         additional_dependencies:
         - types-attrs
+        - types-redis
+        - types-requests
+        - boto3-stubs
+        - types-aiobotocore

README.md

@@ -14,6 +14,8 @@ Example of application built with `titiler.xarray` [package](https://development
 # It's recommended to install dependencies in a virtual environment
 uv sync --dev
 export TEST_ENVIRONMENT=true  # set this when running locally to mock redis
+#optional: Disable caching
+#export TITILER_MULTIDIM_ENABLE_CACHE=false
 uv run uvicorn titiler.multidim.main:app --reload
 ```
 
@@ -94,3 +96,4 @@ The following steps detail how to to setup and deploy the CDK stack from your lo
 
 In AWS Lambda environment we need to have specific version of botocore, S3FS, FSPEC and other libraries.
 To make sure the application will both work locally and in AWS Lambda environment you can install the dependencies using `python -m pip install -r infrastructure/aws/requirement-lambda.txt`
+

infrastructure/aws/cdk/config.py

@@ -96,6 +96,20 @@ class AppSettings(BaseSettings):
         description="Optional root path for all api endpoints",
     )
 
+    # Authorization config for icechunk virtual chunks
+    # This will be passed through to the Lambda as an environment variable
+    authorized_chunk_access: Optional[str] = Field(
+        None,
+        description="JSON string for authorizing virtual chunk access in icechunk datasets",
+    )
+
+    def model_post_init(self, __context):
+        """Add authorized_chunk_access to additional_env if set."""
+        if self.authorized_chunk_access:
+            self.additional_env["TITILER_MULTIDIM_AUTHORIZED_CHUNK_ACCESS"] = (
+                self.authorized_chunk_access
+            )
+
     class Config:
         """model config"""
 

infrastructure/aws/lambda/Dockerfile

@@ -15,7 +15,7 @@ EOF
 FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} AS builder
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
-RUN dnf install -y gcc-c++ && dnf clean all
+RUN dnf install -y gcc-c++ git && dnf clean all
 
 WORKDIR /build
 

pyproject.toml

@@ -25,8 +25,8 @@ classifiers = [
 ]
 dynamic = ["version"]
 dependencies = [
-    "titiler.core>=0.23.0,<0.24",
-    "titiler.xarray>=0.23.0,<0.24",
+    "titiler-core>=0.25.0,<0.26",
+    "titiler-xarray>=0.25.0,<0.26",
     "aiohttp",
     "aiobotocore>=2.24.0",
     "boto3>=1.39.0",
@@ -41,8 +41,9 @@ dependencies = [
     "requests",
     "rioxarray",
     "s3fs",
-    "xarray",
-    "zarr>=2,<3",
+    "xarray>=2025.10.1",
+    "zarr>3.1.0",
+    "icechunk>=1.1.9",
 ]
 
 [project.optional-dependencies]
@@ -56,6 +57,7 @@ lambda = [
 
 [dependency-groups]
 dev = [
+    "dask>=2025.9.1",
     "fakeredis>=2.23.5",
     "httpx",
     "ipykernel>=6.30.1",
@@ -65,15 +67,23 @@ dev = [
     "pytest-asyncio>=0.24.0",
     "pytest-cov>=5.0.0",
     "pytest>=8.3.2",
+    "pytest-xdist",
     "uvicorn>=0.34.0",
     "yappi>=1.6.0",
+    "virtualizarr",
+    "obstore",
 ]
 deployment = [
     "aws-cdk-lib~=2.177.0",
     "constructs>=10.4.2",
     "pydantic-settings~=2.0",
     "python-dotenv>=1.0.1",
 ]
+notebooks = [
+    "folium",
+    "httpx",
+    "matplotlib",
+]
 
 [project.urls]
 Homepage = "https://github.com/developmentseed/titiler-multidim"
@@ -97,6 +107,8 @@ known_first_party = ["titiler"]
 default_section = "THIRDPARTY"
 
 [tool.ruff]
+
+[tool.ruff.lint]
 select = [
     "D1",  # pydocstyle errors
     "E",  # pycodestyle errors
@@ -111,7 +123,7 @@ ignore = [
     "B905",  # ignore zip() without an explicit strict= parameter, only support with python >3.10
 ]
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "tests/test_app.py" = ["D1"]
 "tests/fixtures/generate_test_zarr.py" = ["D1"]
 "tests/fixtures/generate_test_reference.py" = ["D1"]
@@ -126,8 +138,6 @@ explicit_package_bases = true
 requires = ["pdm-backend"]
 build-backend = "pdm.backend"
 
-
-
 [tool.pdm.version]
 source = "file"
 path = "src/titiler/multidim/__init__.py"
@@ -142,3 +152,4 @@ excludes = ["tests/", "**/.mypy_cache", "**/.DS_Store"]
 
 [tool.pytest.ini_options]
 addopts = "--cov=titiler.multidim --cov-report term-missing -s -vv"
+pythonpath = ["src", "tests"]

src/titiler/multidim/main.py

@@ -90,7 +90,7 @@
     app.add_middleware(
         ServerTimingMiddleware,
         calls_to_track={
-            "1-xarray-open_dataset": (reader.xarray_open_dataset,),
+            "1-xarray-open_dataset": (reader.guess_opener,),
             "2-rioxarray-reproject": (rioxarray.raster_array.RasterArray.reproject,),
         },
     )