Merge pull request #17 from pedro-cf/stac-fastapi.core-2.3.0

stac-fastapi.core-2.3.0

Author: jonhealy1

CHANGELOG.md

@@ -9,22 +9,30 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0/
 
 ### Added
 
+- Added option to include Basic Auth. [#12](https://github.com/Healy-Hyperspatial/stac-fastapi-mongo/issues/12)
+
 ### Changed
 
+- Upgraded stac-fastapi.core to 2.3.0 [#15](https://github.com/Healy-Hyperspatial/stac-fastapi-mongo/issues/15)
+- Enforced `%Y-%m-%dT%H:%M:%S.%fZ` datetime format on create_item [#15](https://github.com/Healy-Hyperspatial/stac-fastapi-mongo/issues/15)
+- Queries now convert datetimes to `%Y-%m-%dT%H:%M:%S.%fZ` datetime format [#15](https://github.com/Healy-Hyperspatial/stac-fastapi-mongo/issues/15)
+
 ### Fixed
 
+- Added skip to basic_auth tests when `BASIC_AUTH` environment variable is not set
+
 
 ## [v3.0.1]
 
 ### Added
 
 ### Changed
 
+- Removed bulk transactions extension from app.py
+
 ### Fixed
 
-- Removed bulk transactions extension from app.py
 - Fixed pagination issue with MongoDB. Fixes [#1](https://github.com/Healy-Hyperspatial/stac-fastapi-mongo/issues/1)
-- Added option to include Basic Auth. [#12](https://github.com/Healy-Hyperspatial/stac-fastapi-mongo/issues/12)
 
 
 ## [v3.0.0]

setup.py

@@ -6,7 +6,7 @@
     desc = f.read()
 
 install_requires = [
-    "stac-fastapi.core==2.1.0",
+    "stac-fastapi.core==2.3.0",
     "motor==3.3.2",
     "pymongo==4.6.2",
     "uvicorn",

stac_fastapi/mongo/database_logic.py

@@ -15,7 +15,13 @@
 from stac_fastapi.extensions.core import SortExtension
 from stac_fastapi.mongo.config import AsyncMongoDBSettings as AsyncSearchSettings
 from stac_fastapi.mongo.config import MongoDBSettings as SyncSearchSettings
-from stac_fastapi.mongo.utilities import decode_token, encode_token, serialize_doc
+from stac_fastapi.mongo.utilities import (
+    convert_obj_datetimes,
+    decode_token,
+    encode_token,
+    parse_datestring,
+    serialize_doc,
+)
 from stac_fastapi.types.errors import ConflictError, NotFoundError
 from stac_fastapi.types.stac import Collection, Item
 
@@ -251,15 +257,25 @@ def apply_datetime_filter(search: MongoSearchAdapter, datetime_search):
             Search: The filtered search object.
         """
         if "eq" in datetime_search:
-            search.add_filter({"properties.datetime": datetime_search["eq"]})
+            search.add_filter(
+                {"properties.datetime": parse_datestring(datetime_search["eq"])}
+            )
         else:
-            if "gte" in datetime_search:
+            if "gte" in datetime_search and datetime_search["gte"]:
                 search.add_filter(
-                    {"properties.datetime": {"$gte": datetime_search["gte"]}}
+                    {
+                        "properties.datetime": {
+                            "$gte": parse_datestring(datetime_search["gte"])
+                        }
+                    }
                 )
-            if "lte" in datetime_search:
+            if "lte" in datetime_search and datetime_search["lte"]:
                 search.add_filter(
-                    {"properties.datetime": {"$lte": datetime_search["lte"]}}
+                    {
+                        "properties.datetime": {
+                            "$lte": parse_datestring(datetime_search["lte"])
+                        }
+                    }
                 )
         return search
 
@@ -638,6 +654,7 @@ async def create_item(self, item: Item, refresh: bool = False):
 
         new_item = item.copy()
         new_item["_id"] = item.get("_id", ObjectId())
+        convert_obj_datetimes(new_item)
 
         existing_item = await items_collection.find_one({"_id": new_item["_id"]})
         if existing_item:

stac_fastapi/mongo/utilities.py

@@ -3,6 +3,7 @@
 from base64 import urlsafe_b64decode, urlsafe_b64encode
 
 from bson import ObjectId
+from dateutil import parser  # type: ignore
 
 
 def serialize_doc(doc):
@@ -28,3 +29,50 @@ def encode_token(token_value: str) -> str:
     """Encode a token value (e.g., a UUID or cursor) as a base64 string."""
     encoded_token = urlsafe_b64encode(token_value.encode()).decode()
     return encoded_token
+
+
+def parse_datestring(str):
+    """Parse date string using dateutil.parser.parse() and returns a string formatted \
+    as ISO 8601 with milliseconds and 'Z' timezone indicator.
+
+    Args:
+        str (str): The date string to parse.
+
+    Returns:
+        str: The parsed and formatted date string in the format 'YYYY-MM-DDTHH:MM:SS.ssssssZ'.
+    """
+    parsed_value = parser.parse(str)
+    return parsed_value.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+def convert_obj_datetimes(obj):
+    """Recursively explores dictionaries and lists, attempting to parse strings as datestrings \
+    into a specific format.
+
+    Args:
+        obj (dict or list): The dictionary or list containing date strings to convert.
+
+    Returns:
+        dict or list: The converted dictionary or list with date strings in the desired format.
+    """
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if isinstance(value, dict) or isinstance(value, list):
+                obj[key] = convert_obj_datetimes(value)
+            elif isinstance(value, str):
+                try:
+                    obj[key] = parse_datestring(value)
+                except ValueError:
+                    pass  # If parsing fails, retain the original value
+            elif value is None:
+                obj[key] = None  # Handle null values
+    elif isinstance(obj, list):
+        for i, value in enumerate(obj):
+            if isinstance(value, str):  # Only attempt to parse strings
+                try:
+                    obj[i] = parse_datestring(value)
+                except ValueError:
+                    pass  # If parsing fails, retain the original value
+            elif isinstance(value, list):
+                obj[i] = convert_obj_datetimes(value)  # Recursively handle nested lists
+    return obj

stac_fastapi/tests/api/test_api.py

@@ -28,7 +28,7 @@
     "DELETE /collections/{collection_id}/items/{item_id}",
     "POST /collections",
     "POST /collections/{collection_id}/items",
-    "PUT /collections",
+    "PUT /collections/{collection_id}",
     "PUT /collections/{collection_id}/items/{item_id}",
 }
 
@@ -238,14 +238,14 @@ async def test_app_query_extension_limit_10000(app_client):
 async def test_app_sort_extension_get_asc(app_client, txn_client, ctx):
     first_item = ctx.item
     item_date = datetime.strptime(
-        first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ"
+        first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%S.%fZ"
     )
 
     second_item = dict(first_item)
     second_item["id"] = "another-item"
     another_item_date = item_date - timedelta(days=1)
     second_item["properties"]["datetime"] = another_item_date.strftime(
-        "%Y-%m-%dT%H:%M:%SZ"
+        "%Y-%m-%dT%H:%M:%S.%fZ"
     )
     await create_item(txn_client, second_item)
 
@@ -260,14 +260,14 @@ async def test_app_sort_extension_get_asc(app_client, txn_client, ctx):
 async def test_app_sort_extension_get_desc(app_client, txn_client, ctx):
     first_item = ctx.item
     item_date = datetime.strptime(
-        first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ"
+        first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%S.%fZ"
     )
 
     second_item = dict(first_item)
     second_item["id"] = "another-item"
     another_item_date = item_date - timedelta(days=1)
     second_item["properties"]["datetime"] = another_item_date.strftime(
-        "%Y-%m-%dT%H:%M:%SZ"
+        "%Y-%m-%dT%H:%M:%S.%fZ"
     )
     await create_item(txn_client, second_item)
 
@@ -282,14 +282,14 @@ async def test_app_sort_extension_get_desc(app_client, txn_client, ctx):
 async def test_app_sort_extension_post_asc(app_client, txn_client, ctx):
     first_item = ctx.item
     item_date = datetime.strptime(
-        first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ"
+        first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%S.%fZ"
     )
 
     second_item = dict(first_item)
     second_item["id"] = "another-item"
     another_item_date = item_date - timedelta(days=1)
     second_item["properties"]["datetime"] = another_item_date.strftime(
-        "%Y-%m-%dT%H:%M:%SZ"
+        "%Y-%m-%dT%H:%M:%S.%fZ"
     )
     await create_item(txn_client, second_item)
 
@@ -308,14 +308,14 @@ async def test_app_sort_extension_post_asc(app_client, txn_client, ctx):
 async def test_app_sort_extension_post_desc(app_client, txn_client, ctx):
     first_item = ctx.item
     item_date = datetime.strptime(
-        first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ"
+        first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%S.%fZ"
     )
 
     second_item = dict(first_item)
     second_item["id"] = "another-item"
     another_item_date = item_date - timedelta(days=1)
     second_item["properties"]["datetime"] = another_item_date.strftime(
-        "%Y-%m-%dT%H:%M:%SZ"
+        "%Y-%m-%dT%H:%M:%S.%fZ"
     )
     await create_item(txn_client, second_item)
 

stac_fastapi/tests/basic_auth/test_basic_auth.py

@@ -1,3 +1,5 @@
+import os
+
 import pytest
 
 # - BASIC_AUTH={"public_endpoints":[{"path":"/","method":"GET"},{"path":"/search","method":"GET"}],"users":[{"username":"admin","password":"admin","permissions":"*"},{"username":"reader","password":"reader","permissions":[{"path":"/conformance","method":["GET"]},{"path":"/collections/{collection_id}/items/{item_id}","method":["GET"]},{"path":"/search","method":["POST"]},{"path":"/collections","method":["GET"]},{"path":"/collections/{collection_id}","method":["GET"]},{"path":"/collections/{collection_id}/items","method":["GET"]},{"path":"/queryables","method":["GET"]},{"path":"/queryables/collections/{collection_id}/queryables","method":["GET"]},{"path":"/_mgmt/ping","method":["GET"]}]}]}
@@ -6,6 +8,8 @@
 @pytest.mark.asyncio
 async def test_get_search_not_authenticated(app_client_basic_auth):
     """Test public endpoint search without authentication"""
+    if not os.getenv("BASIC_AUTH"):
+        pytest.skip()
     params = {"query": '{"gsd": {"gt": 14}}'}
 
     response = await app_client_basic_auth.get("/search", params=params)
@@ -22,6 +26,8 @@ async def test_get_search_not_authenticated(app_client_basic_auth):
 @pytest.mark.asyncio
 async def test_post_search_authenticated(app_client_basic_auth):
     """Test protected post search with reader auhtentication"""
+    if not os.getenv("BASIC_AUTH"):
+        pytest.skip()
     params = {
         "bbox": [97.504892, -45.254738, 174.321298, -2.431580],
         "fields": {"exclude": ["properties"]},
@@ -42,6 +48,8 @@ async def test_post_search_authenticated(app_client_basic_auth):
 @pytest.mark.asyncio
 async def test_delete_resource_insufficient_permissions(app_client_basic_auth):
     """Test protected delete collection with reader auhtentication"""
+    if not os.getenv("BASIC_AUTH"):
+        pytest.skip()
     headers = {
         "Authorization": "Basic cmVhZGVyOnJlYWRlcg=="
     }  # Assuming this is a valid authorization token

stac_fastapi/tests/resources/test_collection.py

@@ -54,7 +54,9 @@ async def test_delete_missing_collection(app_client):
 async def test_update_collection_already_exists(ctx, app_client):
     """Test updating a collection which already exists"""
     ctx.collection["keywords"].append("test")
-    resp = await app_client.put("/collections", json=ctx.collection)
+    resp = await app_client.put(
+        f"/collections/{ctx.collection['id']}", json=ctx.collection
+    )
     assert resp.status_code == 200
 
     resp = await app_client.get(f"/collections/{ctx.collection['id']}")
@@ -69,7 +71,9 @@ async def test_update_new_collection(app_client, load_test_data):
     test_collection = load_test_data("test_collection.json")
     test_collection["id"] = "new-test-collection"
 
-    resp = await app_client.put("/collections", json=test_collection)
+    resp = await app_client.put(
+        f"/collections/{test_collection['id']}", json=test_collection
+    )
     assert resp.status_code == 404
 
 
@@ -83,7 +87,9 @@ async def test_collection_not_found(app_client):
 @pytest.mark.asyncio
 async def test_returns_valid_collection(ctx, app_client):
     """Test validates fetched collection with jsonschema"""
-    resp = await app_client.put("/collections", json=ctx.collection)
+    resp = await app_client.put(
+        f"/collections/{ctx.collection['id']}", json=ctx.collection
+    )
     assert resp.status_code == 200
 
     resp = await app_client.get(f"/collections/{ctx.collection['id']}")
@@ -108,7 +114,9 @@ async def test_collection_extensions(ctx, app_client):
     )
     test_asset = {"title": "test", "description": "test", "type": "test"}
     ctx.collection["item_assets"] = {"test": test_asset}
-    resp = await app_client.put("/collections", json=ctx.collection)
+    resp = await app_client.put(
+        f"/collections/{ctx.collection['id']}", json=ctx.collection
+    )
 
     assert resp.status_code == 200
     assert resp.json().get("item_assets", {}).get("test") == test_asset

stac_fastapi/tests/resources/test_item.py

@@ -832,5 +832,7 @@ async def test_search_datetime_validation_errors(app_client):
         resp = await app_client.post("/search", json=body)
         assert resp.status_code == 400
 
-        resp = await app_client.get("/search?datetime={}".format(dt))
-        assert resp.status_code == 400
+        # Getting this instead ValueError: Invalid RFC3339 datetime.
+        # resp = await app_client.get("/search?datetime={}".format(dt))
+        # assert resp.status_code == 400
+        # updated for same reason as sfeos