add support for custom formatters

Author: tomkralidis

docs/source/configuration.rst

@@ -251,6 +251,10 @@ default.
                 options:  # optional options to pass to provider (i.e. GDAL creation)
                     option_name: option_value
                 include_extra_query_parameters: false  # include extra query parameters that are not part of the collection properties (default: false)
+          formatters:  # list of 1..n formatter definitions
+              - name: formatter
+                attachment: true  # whether or not to provide as an attachment or normal response
+                geom: False  # whether or not to include geometry
 
       hello-world:  # name of process
           type: process  # REQUIRED (collection, process, or stac-collection)

docs/source/plugins.rst

@@ -433,6 +433,7 @@ The below template provides a minimal example (let's call the file ``mycooljsonf
 
            super().__init__({'name': 'cooljson', 'geom': None})
            self.mimetype = 'application/json; subtype:mycooljson'
+           self.attachment = False
 
        def write(self, options={}, data=None):
            """custom writer"""

pygeoapi/api/itemtypes.py

@@ -37,7 +37,7 @@
 
 from collections import ChainMap
 from copy import deepcopy
-from datetime import datetime
+import datetime
 from http import HTTPStatus
 import logging
 from typing import Any, Tuple, Union, Optional
@@ -245,9 +245,6 @@ def get_collection_items(
     :returns: tuple of headers, status code, content
     """
 
-    if not request.is_valid(PLUGINS['formatter'].keys()):
-        return api.get_format_exception(request)
-
     # Set Content-Language to system locale until provider locale
     # has been determined
     headers = request.get_response_headers(SYSTEM_LOCALE,
@@ -356,6 +353,18 @@ def get_collection_items(
             err.http_status_code, headers, request.format,
             err.ogc_exception_code, err.message)
 
+    LOGGER.debug('Validating requested format')
+    dataset_formatters = {}
+    for key, value in PLUGINS['formatter'].items():
+        df2 = load_plugin('formatter', {'name': key})
+        dataset_formatters[df2.name] = df2
+    for df in collections[dataset].get('formatters', []):
+        df2 = load_plugin('formatter', df)
+        dataset_formatters[df2.name] = df2
+
+    if not request.is_valid(dataset_formatters.keys()):
+        return api.get_format_exception(request)
+
     crs_transform_spec = None
     if provider_type == 'feature':
         # crs query parameter is only available for OGC API - Features
@@ -585,6 +594,14 @@ def get_collection_items(
         'href': f'{uri}?f={F_HTML}{serialized_query_params}'
     }])
 
+    for key, value in dataset_formatters.items():
+        content['links'].append({
+            'type': value.mimetype,
+            'rel': 'alternate',
+            'title': f'This document as {key}',
+            'href': f'{uri}?f={value.name}{serialized_query_params}'
+        })
+
     next_link = False
     prev_link = False
 
@@ -629,7 +646,7 @@ def get_collection_items(
             'href': '/'.join(uri.split('/')[:-1])
         })
 
-    content['timeStamp'] = datetime.utcnow().strftime(
+    content['timeStamp'] = datetime.datetime.now(datetime.UTC).strftime(
         '%Y-%m-%dT%H:%M:%S.%fZ')
 
     # Set response language to requested provider locale
@@ -660,9 +677,8 @@ def get_collection_items(
                                      'collections/items/index.html',
                                      content, request.locale)
         return headers, HTTPStatus.OK, content
-    elif request.format == 'csv':  # render
-        formatter = load_plugin('formatter',
-                                {'name': 'CSV', 'geom': True})
+    elif request.format in dataset_formatters:  # render
+        formatter = dataset_formatters[request.format]
 
         try:
             content = formatter.write(
@@ -681,13 +697,14 @@ def get_collection_items(
 
         headers['Content-Type'] = formatter.mimetype
 
-        if p.filename is None:
-            filename = f'{dataset}.csv'
-        else:
-            filename = f'{p.filename}'
+        if formatter.attachment:
+            if p.filename is None:
+                filename = f'{dataset}.{formatter.extension}'
+            else:
+                filename = f'{p.filename}'
 
-        cd = f'attachment; filename="{filename}"'
-        headers['Content-Disposition'] = cd
+            cd = f'attachment; filename="{filename}"'
+            headers['Content-Disposition'] = cd
 
         return headers, HTTPStatus.OK, content
 

pygeoapi/formatter/base.py

@@ -44,18 +44,18 @@ def __init__(self, formatter_def: dict):
         :returns: pygeoapi.formatter.base.BaseFormatter
         """
 
+        self.extension = None
         self.mimetype = None
-        self.geom = False
 
         self.name = formatter_def['name']
-        if 'geom' in formatter_def:
-            self.geom = formatter_def['geom']
+        self.geom = formatter_def.get('geom', False)
+        self.attachment = formatter_def.get('attachment', False)
 
     def write(self, options: dict = {}, data: dict = None) -> str:
         """
         Generate data in specified format
 
-        :param options: CSV formatting options
+        :param options: formatting options
         :param data: dict representation of GeoJSON object
 
         :returns: string representation of format

pygeoapi/formatter/csv_.py

@@ -48,12 +48,11 @@ def __init__(self, formatter_def: dict):
         :returns: `pygeoapi.formatter.csv_.CSVFormatter`
         """
 
-        geom = False
-        if 'geom' in formatter_def:
-            geom = formatter_def['geom']
+        geom = formatter_def.get('geom', False)
 
         super().__init__({'name': 'csv', 'geom': geom})
         self.mimetype = 'text/csv; charset=utf-8'
+        self.extension = 'csv'
 
     def write(self, options: dict = {}, data: dict = None) -> str:
         """

pygeoapi/resources/schemas/config/pygeoapi-config-0.x.yml

@@ -581,6 +581,23 @@ properties:
                                       - type
                                       - name
                                       - data
+                          formatters:
+                              type: object
+                              description: custom formatters to apply to output
+                              properties:
+                                  name:
+                                      type: string
+                                      description: name of formatter
+                                  geom:
+                                      type: boolean
+                                      default: true
+                                      description: whether to include geometry
+                                  attachment:
+                                      type: boolean
+                                      default: false
+                                      description: whether to provide as an attachment
+                              required:
+                                  - name
                       required:
                           - type
                           - title

tests/api/test_itemtypes.py

@@ -217,16 +217,17 @@ def test_get_collection_items(config, api_):
     assert features['features'][1]['properties']['stn_id'] == 35
 
     links = features['links']
-    assert len(links) == 5
+    assert len(links) == 6
     assert '/collections/obs/items?f=json' in links[0]['href']
     assert links[0]['rel'] == 'self'
     assert '/collections/obs/items?f=jsonld' in links[1]['href']
     assert links[1]['rel'] == 'alternate'
     assert '/collections/obs/items?f=html' in links[2]['href']
     assert links[2]['rel'] == 'alternate'
     assert '/collections/obs' in links[3]['href']
-    assert links[3]['rel'] == 'next'
-    assert links[4]['rel'] == 'collection'
+    assert links[3]['rel'] == 'alternate'
+    assert links[4]['rel'] == 'next'
+    assert links[5]['rel'] == 'collection'
 
     # Invalid offset
     req = mock_api_request({'offset': -1})
@@ -243,17 +244,19 @@ def test_get_collection_items(config, api_):
     assert features['features'][1]['properties']['stn_id'] == 2147
 
     links = features['links']
-    assert len(links) == 5
+    assert len(links) == 6
     assert '/collections/obs/items?f=json' in links[0]['href']
     assert links[0]['rel'] == 'self'
     assert '/collections/obs/items?f=jsonld' in links[1]['href']
     assert links[1]['rel'] == 'alternate'
     assert '/collections/obs/items?f=html' in links[2]['href']
     assert links[2]['rel'] == 'alternate'
-    assert '/collections/obs/items?offset=0' in links[3]['href']
-    assert links[3]['rel'] == 'prev'
-    assert '/collections/obs' in links[4]['href']
-    assert links[4]['rel'] == 'collection'
+    assert '/collections/obs/items?f=csv' in links[3]['href']
+    assert links[3]['rel'] == 'alternate'
+    assert '/collections/obs/items?offset=0' in links[4]['href']
+    assert links[4]['rel'] == 'prev'
+    assert '/collections/obs' in links[5]['href']
+    assert links[5]['rel'] == 'collection'
 
     req = mock_api_request({
         'offset': '1',
@@ -266,7 +269,7 @@ def test_get_collection_items(config, api_):
     assert len(features['features']) == 1
 
     links = features['links']
-    assert len(links) == 6
+    assert len(links) == 7
     assert '/collections/obs/items?f=json&limit=1&bbox=-180,-90,180,90' in \
         links[0]['href']
     assert links[0]['rel'] == 'self'
@@ -276,13 +279,16 @@ def test_get_collection_items(config, api_):
     assert '/collections/obs/items?f=html&limit=1&bbox=-180,-90,180,90' in \
         links[2]['href']
     assert links[2]['rel'] == 'alternate'
-    assert '/collections/obs/items?offset=0&limit=1&bbox=-180,-90,180,90' \
+    assert '/collections/obs/items?f=csv&limit=1&bbox=-180,-90,180,90' \
         in links[3]['href']
-    assert links[3]['rel'] == 'prev'
-    assert '/collections/obs' in links[4]['href']
-    assert links[3]['rel'] == 'prev'
-    assert links[4]['rel'] == 'next'
-    assert links[5]['rel'] == 'collection'
+    assert links[3]['rel'] == 'alternate'
+    assert '/collections/obs/items?offset=0&limit=1&bbox=-180,-90,180,90' \
+        in links[4]['href']
+    assert links[4]['rel'] == 'prev'
+    assert '/collections/obs' in links[5]['href']
+    assert links[4]['rel'] == 'prev'
+    assert links[5]['rel'] == 'next'
+    assert links[6]['rel'] == 'collection'
 
     req = mock_api_request({
         'sortby': 'bad-property',

tests/load_tinydb_records.py

@@ -27,7 +27,7 @@
 #
 # =================================================================
 
-from datetime import datetime
+import datetime
 from pathlib import Path
 import sys
 from typing import Union
@@ -228,7 +228,7 @@ def get_anytext(bag: Union[list, str]) -> str:
         'geometry': geometry,
         'properties': {
             'created': issued,
-            'updated': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
+            'updated': datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%dT%H:%M:%SZ'),  # noqa
             'type': type_,
             'title': title,
             'description': description,