Merge branch 'main' of github.com:stac-utils/stac-fastapi-elasticsearch into patch_endpoints

Author: rhysrevans3

.github/workflows/cicd.yml

@@ -12,10 +12,9 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 20
 
     services:
-
       elasticsearch_8_svc:
         image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
         env:
@@ -63,21 +62,25 @@ jobs:
           OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
         ports:
           - 9202:9202
+
     strategy:
       matrix:
         python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12"]
+        backend: [ "elasticsearch7", "elasticsearch8", "opensearch"]
 
-    name: Python ${{ matrix.python-version }} testing
+    name: Python ${{ matrix.python-version }} testing with ${{ matrix.backend }}
 
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
 
-      # Setup Python (faster than using Python container)
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: |
+            **/setup.py
 
       - name: Lint code
         if: ${{ matrix.python-version == 3.11 }}
@@ -101,35 +104,17 @@ jobs:
         run: |
           pip install ./stac_fastapi/opensearch[dev,server]
 
-      - name: Run test suite against Elasticsearch 7.x
-        run: |
-          pipenv run pytest -svvv
-        env:
-          ENVIRONMENT: testing
-          ES_PORT: 9400
-          ES_HOST: 172.17.0.1
-          ES_USE_SSL: false
-          ES_VERIFY_CERTS: false
-          BACKEND: elasticsearch
-
-      - name: Run test suite against Elasticsearch 8.x
+      - name: Install pytest-timeout
         run: |
-          pipenv run pytest -svvv
-        env:
-          ENVIRONMENT: testing
-          ES_PORT: 9200
-          ES_HOST: 172.17.0.1
-          ES_USE_SSL: false
-          ES_VERIFY_CERTS: false
-          BACKEND: elasticsearch
+          pip install pytest-timeout
 
-      - name: Run test suite against OpenSearch 2.11.1
+      - name: Run test suite
         run: |
-          pipenv run pytest -svvv
+          pipenv run pytest -svvv --timeout=300
         env:
           ENVIRONMENT: testing
-          ES_PORT: 9202
+          ES_PORT: ${{ matrix.backend == 'elasticsearch7' && '9400' || matrix.backend == 'elasticsearch8' && '9200' || '9202' }}
           ES_HOST: 172.17.0.1
           ES_USE_SSL: false
           ES_VERIFY_CERTS: false
-          BACKEND: opensearch
+          BACKEND: ${{ matrix.backend == 'elasticsearch7' && 'elasticsearch' || matrix.backend == 'elasticsearch8' && 'elasticsearch' || 'opensearch' }}

CHANGELOG.md

@@ -7,7 +7,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Changed
+- Support escaped backslashes in CQL2 `LIKE` queries, and reject invalid (or incomplete) escape sequences. [#286](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/286)
+
+## [v3.0.0] - 2024-08-14
+
+### Changed
+- Aggregation bug fixes [#281](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/281)
+- Updated stac-fastapi libraries to v3.0.0 [#282](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/282)
+
+## [v3.0.0a3] - 2024-07-17
+
 ### Added
+ - Added an implementation of the Aggregation Extension. Enables spatial, frequency distribution, and datetime distribution aggregations. [#276](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/276)
  - Added support for route depndencies configuration through the STAC_FASTAPI_ROUTE_DEPENDENCIES environment variable, directly or via json file. Allows for fastapi's inbuilt OAuth2 flows to be used as dependencies. Custom dependencies can also be written, see Basic Auth for an example. [#251](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/251)
  - Added docker-compose.route_dependencies_file.yml that gives an example of OAuth2 workflow using keycloak as the identity provider. [#251](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/251)
  - Added docker-compose.route_dependencies_env.yml that gives an example using the STAC_FASTAPI_ROUTE_DEPENDENCIES environment variable. [#251](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/251)
@@ -232,10 +244,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Added db_to_stac serializer to item_collection method in core.py.
 
 
-[Unreleased]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.0.0a2...main>
-[v3.0.0a2]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.0.0a1...v3.0.0a2>
-[v3.0.0a1]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.0.0a0...v3.0.0a1>
-[v3.0.0a0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.1...v3.0.0a0>
+[Unreleased]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.0.0...main>
+[v3.0.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.1...v3.0.0>
 [v2.4.1]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.0...v2.4.1>
 [v2.4.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.3.0...v2.4.0>
 [v2.3.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.2.0...v2.3.0>

Makefile

@@ -1,6 +1,6 @@
 #!make
 APP_HOST ?= 0.0.0.0
-EXTERNAL_APP_PORT ?= ${APP_PORT}
+EXTERNAL_APP_PORT ?= 8080
 
 ES_APP_PORT ?= 8080
 ES_HOST ?= docker.for.mac.localhost

README.md

@@ -278,3 +278,109 @@ The modified Items with lowercase identifiers will now be visible to users acces
 ## Auth
 
 Authentication is an optional feature that can be enabled through `Route Dependencies` examples can be found and a more detailed explanation in [examples/auth](examples/auth).
+
+
+## Aggregation
+
+Sfeos supports the STAC API [Aggregation Extension](https://github.com/stac-api-extensions/aggregation). This enables geospatial aggregation of points and geometries, as well as frequency distribution aggregation of any other property including dates. Aggregations can be defined at the root Catalog level (`/aggregations`) and at the Collection level (`/<collection_id>/aggregations`). The `/aggregate` route also fully supports base search and the STAC API [Filter Extension](https://github.com/stac-api-extensions/filter). Any query made with `/search` may also be executed with `/aggregate`, provided that the relevant aggregation fields are available,
+
+
+A field named `aggregations` should be added to the Collection object for the collection for which the aggregations are available, for example:
+
+```json
+"aggregations": [
+    {
+      "name": "total_count",
+      "data_type": "integer"
+    },
+    {
+      "name": "datetime_max",
+      "data_type": "datetime"
+    },
+    {
+      "name": "datetime_min",
+      "data_type": "datetime"
+    },
+    {
+      "name": "datetime_frequency",
+      "data_type": "frequency_distribution",
+      "frequency_distribution_data_type": "datetime"
+    },
+    {
+      "name": "sun_elevation_frequency",
+      "data_type": "frequency_distribution",
+      "frequency_distribution_data_type": "numeric"
+    },
+    {
+      "name": "platform_frequency", 
+      "data_type": "frequency_distribution",
+      "frequency_distribution_data_type": "string"
+    },
+    {
+      "name": "sun_azimuth_frequency",
+      "data_type": "frequency_distribution",
+      "frequency_distribution_data_type": "numeric"
+    },
+    {
+      "name": "off_nadir_frequency",
+      "data_type": "frequency_distribution",
+      "frequency_distribution_data_type": "numeric"
+    },
+    {
+      "name": "cloud_cover_frequency",
+      "data_type": "frequency_distribution",
+      "frequency_distribution_data_type": "numeric"
+    },
+    {
+      "name": "grid_code_frequency",
+      "data_type": "frequency_distribution",
+      "frequency_distribution_data_type": "string"
+    },
+    {
+      "name": "centroid_geohash_grid_frequency",
+      "data_type": "frequency_distribution",
+      "frequency_distribution_data_type": "string"
+    },
+    {
+        "name": "centroid_geohex_grid_frequency",
+        "data_type": "frequency_distribution",
+        "frequency_distribution_data_type": "string"
+    },
+    {
+        "name": "centroid_geotile_grid_frequency",
+        "data_type": "frequency_distribution",
+        "frequency_distribution_data_type": "string"
+    },
+    {
+      "name": "geometry_geohash_grid_frequency",
+      "data_type": "frequency_distribution",
+      "frequency_distribution_data_type": "numeric"
+    },
+    {
+      "name": "geometry_geotile_grid_frequency",
+      "data_type": "frequency_distribution",
+      "frequency_distribution_data_type": "string"
+    }
+]
+  ```
+
+Available aggregations are:
+
+- total_count (count of total items)
+- collection_frequency (Item `collection` field)
+- platform_frequency (Item.Properties.platform)
+- cloud_cover_frequency (Item.Properties.eo:cloud_cover)
+- datetime_frequency (Item.Properties.datetime, monthly interval)
+- datetime_min (earliest Item.Properties.datetime)
+- datetime_max (latest Item.Properties.datetime)
+- sun_elevation_frequency (Item.Properties.view:sun_elevation)
+- sun_azimuth_frequency (Item.Properties.view:sun_azimuth)
+- off_nadir_frequency (Item.Properties.view:off_nadir)
+- grid_code_frequency (Item.Properties.grid:code)
+- centroid_geohash_grid_frequency ([geohash grid](https://opensearch.org/docs/latest/aggregations/bucket/geohash-grid/)  on Item.Properties.proj:centroid)
+- centroid_geohex_grid_frequency ([geohex grid](https://opensearch.org/docs/latest/aggregations/bucket/geohex-grid/) on Item.Properties.proj:centroid)
+- centroid_geotile_grid_frequency (geotile on Item.Properties.proj:centroid)
+- geometry_geohash_grid_frequency ([geohash grid](https://opensearch.org/docs/latest/aggregations/bucket/geohash-grid/) on Item.geometry)
+- geometry_geotile_grid_frequency ([geotile grid](https://opensearch.org/docs/latest/aggregations/bucket/geotile-grid/) on Item.geometry)
+
+Support for additional fields and new aggregations can be added in the associated `database_logic.py` file.