From 237bd05272229dd5e3fe92785f0e55bd483b2c44 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 21 Oct 2025 20:52:01 -0400 Subject: [PATCH 1/4] add support for custom formatters --- docs/source/configuration.rst | 4 ++ docs/source/plugins.rst | 1 + pygeoapi/api/itemtypes.py | 45 +++++++++++++------ pygeoapi/formatter/base.py | 8 ++-- pygeoapi/formatter/csv_.py | 5 +-- .../schemas/config/pygeoapi-config-0.x.yml | 17 +++++++ tests/api/test_itemtypes.py | 36 ++++++++------- tests/load_tinydb_records.py | 4 +- 8 files changed, 82 insertions(+), 38 deletions(-) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 6f8e3f3ac..9f9df454c 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -257,6 +257,10 @@ default. storage_crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 # optional CRS in which data is stored, default: as 'crs' field storage_crs_coordinate_epoch: 2017.23 # optional, if storage_crs is a dynamic coordinate reference system always_xy: false # optional should CRS respect axis ordering + formatters: # list of 1..n formatter definitions + - name: path.to.formatter + attachment: true # whether or not to provide as an attachment or normal response + geom: False # whether or not to include geometry hello-world: # name of process type: process # REQUIRED (collection, process, or stac-collection) diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index dcaa48435..403230ad1 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -436,6 +436,7 @@ The below template provides a minimal example (let's call the file ``mycooljsonf super().__init__({'name': 'cooljson', 'geom': None}) self.mimetype = 'application/json; subtype:mycooljson' + self.attachment = False def write(self, options={}, data=None): """custom writer""" diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index a1e1bda16..193d0506e 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -37,7 +37,7 @@ from collections import ChainMap from copy import deepcopy -from datetime import datetime +import datetime from http import HTTPStatus import logging from typing import Any, Tuple, Union @@ -241,9 +241,6 @@ def get_collection_items( :returns: tuple of headers, status code, content """ - if not request.is_valid(PLUGINS['formatter'].keys()): - return api.get_format_exception(request) - # Set Content-Language to system locale until provider locale # has been determined headers = request.get_response_headers(SYSTEM_LOCALE, @@ -352,6 +349,18 @@ def get_collection_items( err.http_status_code, headers, request.format, err.ogc_exception_code, err.message) + LOGGER.debug('Validating requested format') + dataset_formatters = {} + for key, value in PLUGINS['formatter'].items(): + df2 = load_plugin('formatter', {'name': key}) + dataset_formatters[df2.name] = df2 + for df in collections[dataset].get('formatters', []): + df2 = load_plugin('formatter', df) + dataset_formatters[df2.name] = df2 + + if not request.is_valid(dataset_formatters.keys()): + return api.get_format_exception(request) + crs_transform_spec = None if provider_type == 'feature': # crs query parameter is only available for OGC API - Features @@ -581,6 +590,14 @@ def get_collection_items( 'href': f'{uri}?f={F_HTML}{serialized_query_params}' }]) + for key, value in dataset_formatters.items(): + content['links'].append({ + 'type': value.mimetype, + 'rel': 'alternate', + 'title': f'This document as {key}', + 'href': f'{uri}?f={value.name}{serialized_query_params}' + }) + next_link = False prev_link = False @@ -625,7 +642,7 @@ def get_collection_items( 'href': '/'.join(uri.split('/')[:-1]) }) - content['timeStamp'] = datetime.utcnow().strftime( + content['timeStamp'] = datetime.datetime.now(datetime.UTC).strftime( '%Y-%m-%dT%H:%M:%S.%fZ') # Set response language to requested provider locale @@ -656,9 +673,8 @@ def get_collection_items( 'collections/items/index.html', content, request.locale) return headers, HTTPStatus.OK, content - elif request.format == 'csv': # render - formatter = load_plugin('formatter', - {'name': 'CSV', 'geom': True}) + elif request.format in dataset_formatters: # render + formatter = dataset_formatters[request.format] try: content = formatter.write( @@ -677,13 +693,14 @@ def get_collection_items( headers['Content-Type'] = formatter.mimetype - if p.filename is None: - filename = f'{dataset}.csv' - else: - filename = f'{p.filename}' + if formatter.attachment: + if p.filename is None: + filename = f'{dataset}.{formatter.extension}' + else: + filename = f'{p.filename}' - cd = f'attachment; filename="{filename}"' - headers['Content-Disposition'] = cd + cd = f'attachment; filename="{filename}"' + headers['Content-Disposition'] = cd return headers, HTTPStatus.OK, content diff --git a/pygeoapi/formatter/base.py b/pygeoapi/formatter/base.py index 95f20491e..d7084ce12 100644 --- a/pygeoapi/formatter/base.py +++ b/pygeoapi/formatter/base.py @@ -44,18 +44,18 @@ def __init__(self, formatter_def: dict): :returns: pygeoapi.formatter.base.BaseFormatter """ + self.extension = None self.mimetype = None - self.geom = False self.name = formatter_def['name'] - if 'geom' in formatter_def: - self.geom = formatter_def['geom'] + self.geom = formatter_def.get('geom', False) + self.attachment = formatter_def.get('attachment', False) def write(self, options: dict = {}, data: dict | None = None) -> str: """ Generate data in specified format - :param options: CSV formatting options + :param options: formatting options :param data: dict representation of GeoJSON object :returns: string representation of format diff --git a/pygeoapi/formatter/csv_.py b/pygeoapi/formatter/csv_.py index d5e856665..f9f238204 100644 --- a/pygeoapi/formatter/csv_.py +++ b/pygeoapi/formatter/csv_.py @@ -48,12 +48,11 @@ def __init__(self, formatter_def: dict): :returns: `pygeoapi.formatter.csv_.CSVFormatter` """ - geom = False - if 'geom' in formatter_def: - geom = formatter_def['geom'] + geom = formatter_def.get('geom', False) super().__init__({'name': 'csv', 'geom': geom}) self.mimetype = 'text/csv; charset=utf-8' + self.extension = 'csv' def write(self, options: dict = {}, data: dict = None) -> str: """ diff --git a/pygeoapi/resources/schemas/config/pygeoapi-config-0.x.yml b/pygeoapi/resources/schemas/config/pygeoapi-config-0.x.yml index c1e925c39..814cad12c 100644 --- a/pygeoapi/resources/schemas/config/pygeoapi-config-0.x.yml +++ b/pygeoapi/resources/schemas/config/pygeoapi-config-0.x.yml @@ -581,6 +581,23 @@ properties: - type - name - data + formatters: + type: object + description: custom formatters to apply to output + properties: + name: + type: string + description: name of formatter + geom: + type: boolean + default: true + description: whether to include geometry + attachment: + type: boolean + default: false + description: whether to provide as an attachment + required: + - name required: - type - title diff --git a/tests/api/test_itemtypes.py b/tests/api/test_itemtypes.py index a601a5259..3f70b9d8f 100644 --- a/tests/api/test_itemtypes.py +++ b/tests/api/test_itemtypes.py @@ -218,7 +218,7 @@ def test_get_collection_items(config, api_): assert features['features'][1]['properties']['stn_id'] == 35 links = features['links'] - assert len(links) == 5 + assert len(links) == 6 assert '/collections/obs/items?f=json' in links[0]['href'] assert links[0]['rel'] == 'self' assert '/collections/obs/items?f=jsonld' in links[1]['href'] @@ -226,8 +226,9 @@ def test_get_collection_items(config, api_): assert '/collections/obs/items?f=html' in links[2]['href'] assert links[2]['rel'] == 'alternate' assert '/collections/obs' in links[3]['href'] - assert links[3]['rel'] == 'next' - assert links[4]['rel'] == 'collection' + assert links[3]['rel'] == 'alternate' + assert links[4]['rel'] == 'next' + assert links[5]['rel'] == 'collection' # Invalid offset req = mock_api_request({'offset': -1}) @@ -244,17 +245,19 @@ def test_get_collection_items(config, api_): assert features['features'][1]['properties']['stn_id'] == 2147 links = features['links'] - assert len(links) == 5 + assert len(links) == 6 assert '/collections/obs/items?f=json' in links[0]['href'] assert links[0]['rel'] == 'self' assert '/collections/obs/items?f=jsonld' in links[1]['href'] assert links[1]['rel'] == 'alternate' assert '/collections/obs/items?f=html' in links[2]['href'] assert links[2]['rel'] == 'alternate' - assert '/collections/obs/items?offset=0' in links[3]['href'] - assert links[3]['rel'] == 'prev' - assert '/collections/obs' in links[4]['href'] - assert links[4]['rel'] == 'collection' + assert '/collections/obs/items?f=csv' in links[3]['href'] + assert links[3]['rel'] == 'alternate' + assert '/collections/obs/items?offset=0' in links[4]['href'] + assert links[4]['rel'] == 'prev' + assert '/collections/obs' in links[5]['href'] + assert links[5]['rel'] == 'collection' req = mock_api_request({ 'offset': '1', @@ -267,7 +270,7 @@ def test_get_collection_items(config, api_): assert len(features['features']) == 1 links = features['links'] - assert len(links) == 6 + assert len(links) == 7 assert '/collections/obs/items?f=json&limit=1&bbox=-180,-90,180,90' in \ links[0]['href'] assert links[0]['rel'] == 'self' @@ -277,13 +280,16 @@ def test_get_collection_items(config, api_): assert '/collections/obs/items?f=html&limit=1&bbox=-180,-90,180,90' in \ links[2]['href'] assert links[2]['rel'] == 'alternate' - assert '/collections/obs/items?offset=0&limit=1&bbox=-180,-90,180,90' \ + assert '/collections/obs/items?f=csv&limit=1&bbox=-180,-90,180,90' \ in links[3]['href'] - assert links[3]['rel'] == 'prev' - assert '/collections/obs' in links[4]['href'] - assert links[3]['rel'] == 'prev' - assert links[4]['rel'] == 'next' - assert links[5]['rel'] == 'collection' + assert links[3]['rel'] == 'alternate' + assert '/collections/obs/items?offset=0&limit=1&bbox=-180,-90,180,90' \ + in links[4]['href'] + assert links[4]['rel'] == 'prev' + assert '/collections/obs' in links[5]['href'] + assert links[4]['rel'] == 'prev' + assert links[5]['rel'] == 'next' + assert links[6]['rel'] == 'collection' req = mock_api_request({ 'sortby': 'bad-property', diff --git a/tests/load_tinydb_records.py b/tests/load_tinydb_records.py index 4c01fedf1..5a4ceb01c 100644 --- a/tests/load_tinydb_records.py +++ b/tests/load_tinydb_records.py @@ -27,7 +27,7 @@ # # ================================================================= -from datetime import datetime +import datetime from pathlib import Path import sys from typing import Union @@ -226,7 +226,7 @@ def get_anytext(bag: Union[list, str]) -> str: 'geometry': geometry, 'properties': { 'created': issued, - 'updated': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'), + 'updated': datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%dT%H:%M:%SZ'), # noqa 'type': type_, 'title': title, 'description': description, From 6e281462b5e3681befeef00000cbe0610f1c2cdf Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 26 Oct 2025 07:16:32 -0400 Subject: [PATCH 2/4] update --- pygeoapi/api/__init__.py | 11 ++++++++++- pygeoapi/api/itemtypes.py | 19 ++++++++++--------- pygeoapi/formatter/base.py | 2 +- pygeoapi/util.py | 25 +++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 34ddcd3ec..00891b476 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -68,7 +68,8 @@ TEMPLATESDIR, UrlPrefetcher, dategetter, filter_dict_by_key_value, filter_providers_by_type, get_api_rules, get_base_url, get_provider_by_type, get_provider_default, get_typed_value, - render_j2_template, to_json, get_choice_from_headers, get_from_headers + render_j2_template, to_json, get_choice_from_headers, get_from_headers, + get_dataset_formatters ) LOGGER = logging.getLogger(__name__) @@ -1042,6 +1043,14 @@ def describe_collections(api: API, request: APIRequest, 'href': f'{api.get_collections_url()}/{k}/items?f={F_HTML}' # noqa }) + for key, value in get_dataset_formatters(v).items(): + collection['links'].append({ + 'type': value.mimetype, + 'rel': 'items', + 'title': l10n.translate(f'Items as {key}', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}/items?f={value.f}' # noqa + }) + # OAPIF Part 2 - list supported CRSs and StorageCRS if collection_data_type in ['edr', 'feature']: collection['crs'] = get_supported_crs_list(collection_data) diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 193d0506e..284f2b8fc 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -55,13 +55,15 @@ set_content_crs_header) from pygeoapi.formatter.base import FormatterSerializationError from pygeoapi.linked_data import geojson2jsonld +from pygeoapi.openapi import get_oas_30_parameters from pygeoapi.plugin import load_plugin, PLUGINS from pygeoapi.provider.base import ( ProviderGenericError, ProviderTypeError, SchemaType) from pygeoapi.util import (filter_providers_by_type, to_json, filter_dict_by_key_value, str2bool, - get_provider_by_type, render_j2_template) + get_provider_by_type, render_j2_template, + get_dataset_formatters) from . import ( APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, @@ -350,13 +352,7 @@ def get_collection_items( err.ogc_exception_code, err.message) LOGGER.debug('Validating requested format') - dataset_formatters = {} - for key, value in PLUGINS['formatter'].items(): - df2 = load_plugin('formatter', {'name': key}) - dataset_formatters[df2.name] = df2 - for df in collections[dataset].get('formatters', []): - df2 = load_plugin('formatter', df) - dataset_formatters[df2.name] = df2 + dataset_formatters = get_dataset_formatters(collections[dataset]) if not request.is_valid(dataset_formatters.keys()): return api.get_format_exception(request) @@ -1090,6 +1086,11 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, v.get('limits', {}) ) + dataset_formatters = get_dataset_formatters(v) + coll_f_parameter = deepcopy(get_oas_30_parameters(cfg, locale))['f'] # noqa + for key, value in dataset_formatters.items(): + coll_f_parameter['schema']['enum'].append(key) + paths[items_path] = { 'get': { 'summary': f'Get {title} items', @@ -1097,7 +1098,7 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, 'tags': [k], 'operationId': f'get{k.capitalize()}Features', 'parameters': [ - {'$ref': '#/components/parameters/f'}, + coll_f_parameter, {'$ref': '#/components/parameters/lang'}, {'$ref': '#/components/parameters/bbox'}, coll_limit, diff --git a/pygeoapi/formatter/base.py b/pygeoapi/formatter/base.py index d7084ce12..a186cdebc 100644 --- a/pygeoapi/formatter/base.py +++ b/pygeoapi/formatter/base.py @@ -39,7 +39,7 @@ def __init__(self, formatter_def: dict): """ Initialize object - :param formatter_def: formatter definition + param formatter_def: formatter definition :returns: pygeoapi.formatter.base.BaseFormatter """ diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 30a18de57..29d1eda49 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -64,6 +64,7 @@ from pygeoapi import __version__ from pygeoapi import l10n from pygeoapi.models import config as config_models +from pygeoapi.plugin import load_plugin, PLUGINS from pygeoapi.provider.base import ProviderTypeError @@ -751,3 +752,27 @@ def get_choice_from_headers(headers: dict, # Return one or all choices return sorted_choices if all else sorted_choices[0] + + +def get_dataset_formatters(dataset: dict) -> dict: + """ + Helper function to derive all formatters for an itemtype + + :param dataset: `dict` of dataset resource definition + + :returns: `dict` of formatters + """ + + dataset_formatters = {} + + print(dataset) + for key, value in PLUGINS['formatter'].items(): + print("PLUGINS") + df2 = load_plugin('formatter', {'name': key}) + dataset_formatters[df2.name] = df2 + for df in dataset.get('formatters', []): + print("CUSTOM") + df2 = load_plugin('formatter', df) + dataset_formatters[df2.name] = df2 + + return dataset_formatters From 97723671b411403845bba90da09aabd5b51463e4 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 26 Oct 2025 08:33:22 -0400 Subject: [PATCH 3/4] update --- pygeoapi/__init__.py | 2 +- pygeoapi/util.py | 3 --- tests/api/test_api.py | 4 ++-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pygeoapi/__init__.py b/pygeoapi/__init__.py index cd97b1bb3..ffb62a90d 100644 --- a/pygeoapi/__init__.py +++ b/pygeoapi/__init__.py @@ -68,7 +68,7 @@ def decorator(click_group): try: click_group.add_command(entry_point.load()) except Exception as err: - print(err) + click.echo(err) return click_group return decorator diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 29d1eda49..452466c33 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -765,13 +765,10 @@ def get_dataset_formatters(dataset: dict) -> dict: dataset_formatters = {} - print(dataset) for key, value in PLUGINS['formatter'].items(): - print("PLUGINS") df2 = load_plugin('formatter', {'name': key}) dataset_formatters[df2.name] = df2 for df in dataset.get('formatters', []): - print("CUSTOM") df2 = load_plugin('formatter', df) dataset_formatters[df2.name] = df2 diff --git a/tests/api/test_api.py b/tests/api/test_api.py index fc1f03d00..3625eac94 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -591,7 +591,7 @@ def test_describe_collections(config, api_): assert collection['id'] == 'obs' assert collection['title'] == 'Observations' assert collection['description'] == 'My cool observations' - assert len(collection['links']) == 14 + assert len(collection['links']) == 15 assert collection['extent'] == { 'spatial': { 'bbox': [[-180, -90, 180, 90]], @@ -682,7 +682,7 @@ def test_describe_collections_json_ld(config, api_): assert len(expanded['http://schema.org/dataset']) == 1 dataset = expanded['http://schema.org/dataset'][0] assert dataset['@type'][0] == 'http://schema.org/Dataset' - assert len(dataset['http://schema.org/distribution']) == 14 + assert len(dataset['http://schema.org/distribution']) == 15 assert all(dist['@type'][0] == 'http://schema.org/DataDownload' for dist in dataset['http://schema.org/distribution']) From b364419469deab650e7f8b0ede76a3eedca56c26 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 26 Oct 2025 19:58:29 -0400 Subject: [PATCH 4/4] update --- docs/source/plugins.rst | 6 ++++-- pygeoapi/api/itemtypes.py | 7 ++++--- pygeoapi/formatter/csv_.py | 1 + pygeoapi/util.py | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 403230ad1..4d02699ca 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -435,8 +435,10 @@ The below template provides a minimal example (let's call the file ``mycooljsonf """Inherit from parent class""" super().__init__({'name': 'cooljson', 'geom': None}) - self.mimetype = 'application/json; subtype:mycooljson' - self.attachment = False + self.f = 'cooljson' # f= value + self.mimetype = 'application/json; subtype:mycooljson' # response media type + self.attachment = False # whether to provide as an attachment (default False) + self.extension = 'cooljson' # filename extension if providing as an attachment def write(self, options={}, data=None): """custom writer""" diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 284f2b8fc..5e9accf05 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -669,8 +669,9 @@ def get_collection_items( 'collections/items/index.html', content, request.locale) return headers, HTTPStatus.OK, content - elif request.format in dataset_formatters: # render - formatter = dataset_formatters[request.format] + elif request.format in [df.f for df in dataset_formatters.values()]: + formatter = [v for k, v in dataset_formatters.items() if + v.f == request.format][0] try: content = formatter.write( @@ -1089,7 +1090,7 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dataset_formatters = get_dataset_formatters(v) coll_f_parameter = deepcopy(get_oas_30_parameters(cfg, locale))['f'] # noqa for key, value in dataset_formatters.items(): - coll_f_parameter['schema']['enum'].append(key) + coll_f_parameter['schema']['enum'].append(value.f) paths[items_path] = { 'get': { diff --git a/pygeoapi/formatter/csv_.py b/pygeoapi/formatter/csv_.py index f9f238204..2dd8c9dfb 100644 --- a/pygeoapi/formatter/csv_.py +++ b/pygeoapi/formatter/csv_.py @@ -52,6 +52,7 @@ def __init__(self, formatter_def: dict): super().__init__({'name': 'csv', 'geom': geom}) self.mimetype = 'text/csv; charset=utf-8' + self.f = 'csv' self.extension = 'csv' def write(self, options: dict = {}, data: dict = None) -> str: diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 452466c33..f0fc16993 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -767,7 +767,7 @@ def get_dataset_formatters(dataset: dict) -> dict: for key, value in PLUGINS['formatter'].items(): df2 = load_plugin('formatter', {'name': key}) - dataset_formatters[df2.name] = df2 + dataset_formatters[key] = df2 for df in dataset.get('formatters', []): df2 = load_plugin('formatter', df) dataset_formatters[df2.name] = df2