WIP: Support icechunk stores

.gitignore

@@ -105,3 +105,4 @@ cdk.out/
 node_modules
 cdk.context.json
 *.nc
+.DS_Store

README.md

@@ -14,6 +14,8 @@ Example of application built with `titiler.xarray` [package](https://development
 # It's recommended to install dependencies in a virtual environment
 uv sync --dev
 export TEST_ENVIRONMENT=true  # set this when running locally to mock redis
+#optional: Disable caching
+#export TITILER_MULTIDIM_ENABLE_CACHE=false
 uv run uvicorn titiler.multidim.main:app --reload
 ```
 

pyproject.toml

@@ -25,8 +25,8 @@ classifiers = [
 ]
 dynamic = ["version"]
 dependencies = [
-    "titiler.core>=0.23.0,<0.24",
-    "titiler.xarray>=0.23.0,<0.24",
+    "titiler-core>=0.23.0,<0.25",
+    "titiler-xarray>=0.23.0,<0.25",
     "aiohttp",
     "aiobotocore>=2.24.0",
     "boto3>=1.39.0",
@@ -39,10 +39,12 @@ dependencies = [
     "pydantic>=2.4,<3.0",
     "redis",
     "requests",
+    "rio-tiler==7.8.1", #see https://github.com/developmentseed/titiler-multidim/pull/96#issuecomment-3383102442
     "rioxarray",
     "s3fs",
-    "xarray",
-    "zarr>=2,<3",
+    "xarray>=2025.10.1",
+    "zarr>3.1.0",
+    "icechunk>=1.1.9",
 ]
 
 [project.optional-dependencies]
@@ -56,6 +58,7 @@ lambda = [
 
 [dependency-groups]
 dev = [
+    "dask>=2025.9.1",
     "fakeredis>=2.23.5",
     "httpx",
     "ipykernel>=6.30.1",
@@ -80,6 +83,10 @@ Homepage = "https://github.com/developmentseed/titiler-xarray"
 Issues = "https://github.com/developmentseed/titiler-xarray/issues"
 Source = "https://github.com/developmentseed/titiler-xarray"
 
+[tool.uv.sources]
+titiler-xarray = { git = "https://github.com/jbusecke/titiler.git", branch = "jbusecke-icechunk-reader", subdirectory = "src/titiler/xarray" }
+titiler-core = { git = "https://github.com/jbusecke/titiler.git", branch = "jbusecke-icechunk-reader", subdirectory = "src/titiler/core" }
+
 [tool.coverage.run]
 branch = true
 parallel = true
@@ -126,8 +133,6 @@ explicit_package_bases = true
 requires = ["pdm-backend"]
 build-backend = "pdm.backend"
 
-
-
 [tool.pdm.version]
 source = "file"
 path = "src/titiler/multidim/__init__.py"

tests/conftest.py

@@ -1,15 +1,37 @@
-"""titiler.multidim tests configuration."""
+"""Auto-parametrized fixture that runs both cache configurations."""
 
+import sys
 import pytest
 from fastapi.testclient import TestClient
 
 
-@pytest.fixture
-def app(monkeypatch):
-    """App fixture."""
+# This fixture will automatically parametrize ALL tests that use it
+@pytest.fixture(
+    params=[
+        pytest.param({"cache": True}, id="with_cache"),
+        pytest.param({"cache": False}, id="without_cache"),
+    ]
+)
+def app(request, monkeypatch):
+    """Auto-parametrized app fixture that runs tests with both cache configurations."""
+    config = request.param
+    enable_cache = config.get("cache", False)
+
+    # Set environment variables using monkeypatch (auto-cleanup)
     monkeypatch.setenv("TITILER_MULTIDIM_DEBUG", "TRUE")
     monkeypatch.setenv("TEST_ENVIRONMENT", "1")
+    monkeypatch.setenv(
+        "TITILER_MULTIDIM_ENABLE_CACHE", "TRUE" if enable_cache else "FALSE"
+    )
+
+    # Clear module cache to ensure fresh import
+    modules_to_clear = [
+        key for key in sys.modules.keys() if key.startswith("titiler.multidim")
+    ]
+    for module in modules_to_clear:
+        del sys.modules[module]
 
+    # Import and return the app
     from titiler.multidim.main import app
 
     with TestClient(app) as client:

tests/fixtures/generate_test_icechunk.py

@@ -0,0 +1,60 @@
+"""Create icechunk fixtures (native and later virtual)."""
+# TODO: these files could also be generated together with the zarr files using the same data
+
+import numpy as np
+import xarray as xr
+import icechunk as ic
+
+# Define dimensions and chunk sizes
+res = 5
+time_dim = 10
+lat_dim = 36
+lon_dim = 72
+chunk_size = {"time": 10, "lat": 10, "lon": 10}
+
+# Create coordinates
+time = np.arange(time_dim)
+lat = np.linspace(-90.0 + res / 2, 90.0 - res / 2, lat_dim)
+lon = np.linspace(-180.0 + res / 2, 180.0 - res / 2, lon_dim)
+
+dtype = np.float64
+# Initialize variables with random data
+CDD0 = xr.DataArray(
+    np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype),
+    dims=("time", "lat", "lon"),
+    name="CDD0",
+)
+DISPH = xr.DataArray(
+    np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype),
+    dims=("time", "lat", "lon"),
+    name="DISPH",
+)
+FROST_DAYS = xr.DataArray(
+    np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype),
+    dims=("time", "lat", "lon"),
+    name="FROST_DAYS",
+)
+GWETPROF = xr.DataArray(
+    np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype),
+    dims=("time", "lat", "lon"),
+    name="GWETPROF",
+)
+
+# Create dataset
+ds = xr.Dataset(
+    {
+        "CDD0": CDD0.chunk(chunk_size),
+        "DISPH": DISPH.chunk(chunk_size),
+        "FROST_DAYS": FROST_DAYS.chunk(chunk_size),
+        "GWETPROF": GWETPROF.chunk(chunk_size),
+    },
+    coords={"time": time, "lat": lat, "lon": lon},
+)
+storage = ic.local_filesystem_storage("tests/fixtures/icechunk_native")
+config = ic.RepositoryConfig.default()
+repo = ic.Repository.create(storage=storage, config=config)
+session = repo.writable_session("main")
+store = session.store
+
+ds.to_zarr(store, consolidated=False)
+session.commit("Add initial data")

tests/fixtures/generate_test_zarr.py

@@ -1,4 +1,4 @@
-"""Create zarr fixture."""
+"""Create zarr fixtures for v2 and v3."""
 
 import numpy as np
 import xarray as xr
@@ -8,31 +8,32 @@
 time_dim = 10
 lat_dim = 36
 lon_dim = 72
-chunk_size = (10, 10, 10)
+chunk_size = {"time": 10, "lat": 10, "lon": 10}
 
 # Create coordinates
 time = np.arange(time_dim)
-lat = np.linspace(-90 + res / 2, 90 - res / 2, lat_dim)
-lon = np.linspace(-180 + res / 2, 180 - res / 2, lon_dim)
+lat = np.linspace(-90.0 + res / 2, 90.0 - res / 2, lat_dim)
+lon = np.linspace(-180.0 + res / 2, 180.0 - res / 2, lon_dim)
 
+dtype = np.float64
 # Initialize variables with random data
 CDD0 = xr.DataArray(
-    np.random.rand(time_dim, lat_dim, lon_dim).astype(np.uint8),
+    np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype),
     dims=("time", "lat", "lon"),
     name="CDD0",
 )
 DISPH = xr.DataArray(
-    np.random.rand(time_dim, lat_dim, lon_dim).astype(np.uint8),
+    np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype),
     dims=("time", "lat", "lon"),
     name="DISPH",
 )
 FROST_DAYS = xr.DataArray(
-    np.random.rand(time_dim, lat_dim, lon_dim).astype(np.uint8),
+    np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype),
     dims=("time", "lat", "lon"),
     name="FROST_DAYS",
 )
 GWETPROF = xr.DataArray(
-    np.random.rand(time_dim, lat_dim, lon_dim).astype(np.uint8),
+    np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype),
     dims=("time", "lat", "lon"),
     name="GWETPROF",
 )
@@ -49,4 +50,17 @@
 )
 
 # Save dataset to a local Zarr store
-ds.to_zarr("tests/fixtures/test_zarr_store.zarr", mode="w")
+ds.to_zarr(
+    "tests/fixtures/zarr_store_v3.zarr",
+    mode="w",
+    zarr_format=3,
+    consolidated=False,
+)
+
+# Save dataset to a local Zarr store
+ds.to_zarr(
+    "tests/fixtures/zarr_store_v2.zarr",
+    mode="w",
+    zarr_format=2,
+    consolidated=True,
+)

tests/fixtures/icechunk_native/refs/branch.main/ref.json

@@ -0,0 +1 @@
+{"snapshot":"X7NF54E8W362EQT4PJDG"}

tests/fixtures/responses/icechunk_native_histogram.json

@@ -0,0 +1,42 @@
+[
+{
+  "bucket": [0.0007906359070823932, 0.10060333282002616],
+  "value": 278
+},
+{
+  "bucket": [0.10060333282002616, 0.20041602973296993],
+  "value": 229
+},
+{
+  "bucket": [0.20041602973296993, 0.3002287266459137],
+  "value": 265
+},
+{
+  "bucket": [0.3002287266459137, 0.40004142355885747],
+  "value": 298
+},
+{
+  "bucket": [0.40004142355885747, 0.4998541204718012],
+  "value": 257
+},
+{
+  "bucket": [0.4998541204718012, 0.599666817384745],
+  "value": 245
+},
+{
+  "bucket": [0.599666817384745, 0.6994795142976887],
+  "value": 252
+},
+{
+  "bucket": [0.6994795142976887, 0.7992922112106325],
+  "value": 244
+},
+{
+  "bucket": [0.7992922112106325, 0.8991049081235764],
+  "value": 242
+},
+{
+  "bucket": [0.8991049081235764, 0.99891760503652],
+  "value": 282
+}
+]

tests/fixtures/responses/icechunk_native_info.json

@@ -0,0 +1,14 @@
+{
+    "bounds": [-180.0, -90.0, 180.0, 90.0],
+    "band_metadata": [["b1", {}]],
+    "band_descriptions": [["b1", "0"]],
+    "dtype": "float64",
+    "nodata_type": "Nodata",
+    "height": 36,
+    "count": 1,
+    "width": 72,
+    "attrs": {},
+    "crs": "http://www.opengis.net/def/crs/EPSG/0/4326",
+    "dimensions": ["y", "x"],
+    "name": "CDD0"
+}

tests/fixtures/responses/test_zarr_store_zarr_tilejson.json → tests/fixtures/responses/icechunk_native_tilejson.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "scheme": "xyz",
   "tiles": [
-    "http://testserver/tiles/WebMercatorQuad/{z}/{x}/{y}@1x?url=tests%2Ffixtures%2Ftest_zarr_store.zarr&variable=CDD0&decode_times=false&sel=time%3D0"
+    "http://testserver/tiles/WebMercatorQuad/{z}/{x}/{y}@1x?url=tests%2Ffixtures%2Ficechunk_native&variable=CDD0&decode_times=false&sel=time%3D0"
   ],
   "minzoom": 0,
   "maxzoom": 0,