Merge pull request #73 from Cumulocity-IoT/feature/tuple-parsing-improvements

Feature/tuple parsing improvements

Author: chsou

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+* Added code coverage reporting to `test` target for _invoke_.
+* Updated `as_tuple` for  complex objects as well as the `as_tuples` parameter for `select`
+  and `get_all` functions to work with strings or 2-tuples. The use of a dictionary
+  was removed as dictionaries don't define an order.
+* Added `as_tuples` parameter to the Measurements API `select` and `get_all` functions.
 * Adding `c8y_tk.analytics` package with `to_numpy`, `to_series` and `to_data_frame` functions to
   ease incorporating Cumulocity data into standard analytics pipelines.
 

c8y_api/model/_base.py

@@ -525,11 +525,16 @@ def get(self, path: str, default=None) -> Any:
         segments = path.split('.')
         value = self
         for segment in segments:
-            if segment in value:
+            # try to drill down (assuming dict-like)
+            try:
                 value = value[segment]
                 continue
+            except (KeyError, TypeError):
+                pass
+            # if the segment is an actual attribute it should be the target
             if hasattr(value, segment):
                 return value.__getattribute__(segment)
+            # otherwise use the default
             return default
         return value
 

c8y_api/model/_parser.py

@@ -5,8 +5,42 @@
 from typing import Set
 
 from c8y_api.model._base import ComplexObject
+from c8y_api.model._util import _StringUtil
 
 
+def as_tuples(json_data, *path: str | tuple):
+    """Parse a JSON structure as tuples from paths.
+
+    Args:
+        json_data (dict):  A JSON structure as Python dict
+        path (*str|tuple):  Path(s) to extract from
+            the structure; Use dots to separate JSON levels; Arrays are not
+            supported.
+
+    Mote: This function automatically converts path segments from
+    Python snake_case to pascalCase, e.g. `creation_time` will match
+    both `creation_time` and `creationTime` fields.
+
+    Returns:
+        A tuple with `len(path)` elements containing the values as-is defined
+        in the JSON structure; an invalid path will result in None.
+    """
+    def resolve(segments, default=None):
+        json_level = json_data
+        for segment in segments[:-1]:
+            if segment in json_level:
+                json_level = json_level[segment]
+                continue
+            pascal_segment = _StringUtil.to_pascal_case(segment)
+            if pascal_segment in json_level:
+                json_level = json_level[pascal_segment]
+                continue
+            return default
+        return json_level.get(segments[-1], json_level.get(_StringUtil.to_pascal_case(segments[-1]), default))
+
+    # each p in path(s) can be a string or a tuple
+    return tuple(resolve(p[0].split('.'), p[1]) if isinstance(p, tuple) else resolve(p.split('.')) for p in path )
+
 class SimpleObjectParser(object):
     """A parser for simple (without fragments) Cumulocity database objects.
 

c8y_api/model/alarms.py

@@ -3,11 +3,11 @@
 from __future__ import annotations
 
 from datetime import datetime, timedelta
-from typing import List, Generator, Any
+from typing import List, Generator
 
 from c8y_api._base_api import CumulocityRestApi
-from c8y_api.model._base import CumulocityResource, SimpleObject, ComplexObject, get_all_by_path
-from c8y_api.model._parser import ComplexObjectParser
+from c8y_api.model._base import CumulocityResource, SimpleObject, ComplexObject
+from c8y_api.model._parser import as_tuples as parse_as_tuples, ComplexObjectParser
 from c8y_api.model._util import _DateUtil
 
 
@@ -255,7 +255,7 @@ def select(self,
                reverse: bool = False, limit: int = None,
                with_source_assets: bool = None, with_source_devices: bool = None,
                page_size: int = 1000, page_number: int = None,
-               as_tuples: list[str] | dict[str, Any] = None,
+               as_tuples: str | tuple | list[str | tuple] = None,
                **kwargs) -> Generator[Alarm]:
         """Query the database for alarms and iterate over the results.
 
@@ -307,10 +307,10 @@ def select(self,
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
                 automatic follow-up page retrieval.
-            as_tuples: (list[str] or dict[str, Any]):  Don't parse Alarms, but
-                extract the values at certain JSON paths as tuples; If the
-                path is not defined in a result, None is used; Specify a
-                dictionary to define proper default values for each path.
+            as_tuples: (*str|tuple):  Don't parse objects, but directly extract
+                the values at certain JSON paths as tuples; If the path is not
+                defined in a result, None is used; Specify a tuple to define
+                a proper default value for each path.
 
         Returns:
             Generator of Alarm objects
@@ -333,7 +333,8 @@ def select(self,
             base_query,
             page_number,
             limit,
-            Alarm.from_json if not as_tuples else (lambda x: get_all_by_path(x, as_tuples)))
+            Alarm.from_json if not as_tuples else
+            lambda x: parse_as_tuples(x, *([as_tuples] if isinstance(as_tuples, str) else as_tuples)))
 
     def get_all(
             self,
@@ -350,7 +351,7 @@ def get_all(
             with_source_assets: bool = None, with_source_devices: bool = None,
             reverse: bool = False, limit: int = None,
             page_size: int = 1000, page_number: int = None,
-            as_tuples: list[str] | dict[str, Any] = None,
+            as_tuples: str | tuple | list[str|tuple] = None,
             **kwargs) -> List[Alarm]:
         """Query the database for alarms and return the results as list.
 

c8y_api/model/events.py

@@ -6,11 +6,11 @@
 from __future__ import annotations
 
 from datetime import datetime, timedelta
-from typing import Generator, List, BinaryIO, Any
+from typing import Generator, List, BinaryIO
 
 from c8y_api._base_api import CumulocityRestApi
-from c8y_api.model._base import CumulocityResource, SimpleObject, ComplexObject, get_all_by_path
-from c8y_api.model._parser import ComplexObjectParser
+from c8y_api.model._base import CumulocityResource, SimpleObject, ComplexObject
+from c8y_api.model._parser import as_tuples as parse_as_tuples, ComplexObjectParser
 from c8y_api.model._util import _DateUtil
 
 
@@ -264,7 +264,7 @@ def select(self,
                reverse: bool = False, limit: int = None,
                with_source_assets: bool = None, with_source_devices: bool = None,
                page_size: int = 1000, page_number: int = None,
-               as_tuples: list[str] | dict[str, Any] = None,
+               as_tuples: str | tuple | list[str|tuple] = None,
                **kwargs) -> Generator[Event]:
         """Query the database for events and iterate over the results.
 
@@ -315,10 +315,10 @@ def select(self,
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
                 automatic follow-up page retrieval.
-            as_tuples: (list[str] or dict[str, Any]):  Don't parse Events, but
-                extract the values at certain JSON paths as tuples; If the
-                path is not defined in a result, None is used; Specify a
-                dictionary to define proper default values for each path.
+            as_tuples: (*str|tuple):  Don't parse objects, but directly extract
+                the values at certain JSON paths as tuples; If the path is not
+                defined in a result, None is used; Specify a tuple to define
+                a proper default value for each path.
 
         Returns:
             Generator for Event objects
@@ -344,7 +344,8 @@ def select(self,
             base_query,
             page_number,
             limit,
-            Event.from_json if not as_tuples else (lambda x: get_all_by_path(x, as_tuples)))
+            Event.from_json if not as_tuples else
+            lambda x: parse_as_tuples(x, *([as_tuples] if isinstance(as_tuples, str) else as_tuples)))
 
     def get_all(
             self,
@@ -361,7 +362,7 @@ def get_all(
             reverse: bool = False, limit: int = None,
             with_source_assets: bool = None, with_source_devices: bool = None,
             page_size: int = 1000, page_number: int = None,
-            as_tuples: list[str] | dict[str, Any] = None,
+            as_tuples: str | tuple | list[str|tuple] = None,
             **kwargs) -> List[Event]:
         """Query the database for events and return the results as list.
 

c8y_api/model/inventory.py

@@ -5,7 +5,8 @@
 
 from typing import Any, Generator, List
 
-from c8y_api.model._base import CumulocityResource, get_all_by_path
+from c8y_api.model._base import CumulocityResource
+from c8y_api.model._parser import as_tuples as parse_as_tuples
 from c8y_api.model._util import _QueryUtil
 from c8y_api.model.managedobjects import ManagedObjectUtil, ManagedObject, Device, Availability, DeviceGroup
 
@@ -61,7 +62,7 @@ def get_all(
             reverse: bool = None,
             limit: int = None,
             page_size: int = 1000,
-            as_tuples: list[str] | dict[str, Any] = None,
+            as_tuples: str | tuple | list[str|tuple] = None,
             **kwargs) -> List[ManagedObject]:
         """ Query the database for managed objects and return the results
         as list.
@@ -156,7 +157,7 @@ def select(
             limit: int = None,
             page_size: int = 1000,
             page_number: int = None,
-            as_tuples: list[str] | dict[str, Any] = None,
+            as_tuples: str | tuple | list[str|tuple] = None,
             **kwargs) -> Generator[ManagedObject]:
         """ Query the database for managed objects and iterate over the
         results.
@@ -208,10 +209,10 @@ def select(
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
                 automatic follow-up page retrieval.
-            as_tuples: (list[str] or dict[str, Any]):  Don't parse ManagedObjects,
-                but extract the values at certain JSON paths as tuples; If the path
-                is not defined in a result, None is used; Specify a dictionary to
-                define proper default values for each path.
+            as_tuples: (*str|tuple):  Don't parse objects, but directly extract
+                the values at certain JSON paths as tuples; If the path is not
+                defined in a result, None is used; Specify a tuple to define
+                a proper default value for each path.
 
         Returns:
             Generator for ManagedObject instances
@@ -351,7 +352,8 @@ def _select(self, parse_fun, device_mode: bool, page_number, limit, as_tuples, *
             base_query,
             page_number,
             limit,
-            parse_fun if not as_tuples else (lambda x: get_all_by_path(x, as_tuples)))
+            parse_fun if not as_tuples else
+            lambda x: parse_as_tuples(x, *([as_tuples] if isinstance(as_tuples, str) else as_tuples)))
 
     def create(self, *objects: ManagedObject):
         """Create managed objects within the database.
@@ -501,7 +503,7 @@ def select(  # noqa (order)
             limit: int = None,
             page_size: int = 100,
             page_number: int = None,
-            as_tuples: list[str] | dict[str, Any] = None,
+            as_tuples: str | tuple | list[str|tuple] = None,
             **kwargs,) -> Generator[Device]:
         # pylint: disable=arguments-differ, arguments-renamed
         """ Query the database for devices and iterate over the results.
@@ -556,10 +558,10 @@ def select(  # noqa (order)
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
                 automatic follow-up page retrieval.
-            as_tuples: (list[str] or dict[str, Any]):  Don't parse Device objects,
-                but extract the values at certain JSON paths as tuples; If the path
-                is not defined in a result, None is used; Specify a dictionary to
-                define proper default values for each path.
+            as_tuples: (*str|tuple):  Don't parse objects, but directly extract
+                the values at certain JSON paths as tuples; If the path is not
+                defined in a result, None is used; Specify a tuple to define
+                a proper default value for each path.
 
         Returns:
             Generator for Device objects
@@ -614,7 +616,7 @@ def get_all(  # noqa (changed signature)
             limit: int = None,
             page_size: int = 100,
             page_number: int = None,
-            as_tuples: list[str] | dict[str, Any] = None,
+            as_tuples: str | tuple | list[str|tuple] = None,
             **kwargs) -> List[Device]:
         # pylint: disable=arguments-differ, arguments-renamed
         """ Query the database for devices and return the results as list.
@@ -754,7 +756,7 @@ def select(  # noqa (changed signature)
             limit: int = None,
             page_size: int = 100,
             page_number: int = None,
-            as_tuples: list[str] | dict[str, Any] = None,
+            as_tuples: str | tuple | list[str|tuple] = None,
             **kwargs) -> Generator[DeviceGroup]:
         # pylint: disable=arguments-differ, arguments-renamed
         """ Select device groups by various parameters.
@@ -812,11 +814,10 @@ def select(  # noqa (changed signature)
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
                 automatic follow-up page retrieval.
-            as_tuples: (list[str] or dict[str, Any]):  Don't parse DeviceGroup
-                objects, but extract the values at certain JSON paths as
-                tuples; If the path is not defined in a result, None is used;
-                Specify a dictionary to define proper default values for each
-                path.
+            as_tuples: (*str|tuple):  Don't parse objects, but directly extract
+                the values at certain JSON paths as tuples; If the path is not
+                defined in a result, None is used; Specify a tuple to define
+                a proper default value for each path.
 
         Returns:
             Generator of DeviceGroup instances
@@ -923,7 +924,7 @@ def get_all(  # noqa (changed signature)
             limit: int = None,
             page_size: int = 100,
             page_number: int = None,
-            as_tuples: list[str] | dict[str, Any] = None,
+            as_tuples: str | tuple | list[str|tuple] = None,
             **kwargs ) -> List[DeviceGroup]:
         # pylint: disable=arguments-differ, arguments-renamed
         """ Select managed objects by various parameters.

c8y_api/model/measurements.py

@@ -9,7 +9,7 @@
 from c8y_api._base_api import CumulocityRestApi
 
 from c8y_api.model._base import CumulocityResource, ComplexObject
-from c8y_api.model._parser import ComplexObjectParser
+from c8y_api.model._parser import as_tuples as parse_as_tuples, ComplexObjectParser
 from c8y_api.model._base import _DictWrapper
 from c8y_api.model._util import _DateUtil
 
@@ -495,6 +495,7 @@ def select(
             limit: int = None,
             page_size: int = 1000,
             page_number: int = None,
+            as_tuples: str | tuple | list[str|tuple] = None,
             **kwargs
     ) -> Generator[Measurement]:
         """ Query the database for measurements and iterate over the results.
@@ -538,6 +539,10 @@ def select(
                 related setting.
             page_number (int): Pull a specific page; this effectively disables
                 automatic follow-up page retrieval.
+            as_tuples: (*str|tuple):  Don't parse objects, but directly extract
+                the values at certain JSON paths as tuples; If the path is not
+                defined in a result, None is used; Specify a tuple to define
+                a proper default value for each path.
 
         Returns:
             Generator[Measurement]: Iterable of matching Measurement objects
@@ -559,7 +564,12 @@ def select(
             page_size=page_size,
             **kwargs
         )
-        return super()._iterate(base_query, page_number, limit, Measurement.from_json)
+        return super()._iterate(
+            base_query,
+            page_number,
+            limit,
+            Measurement.from_json if not as_tuples else
+            lambda x: parse_as_tuples(x, *([as_tuples] if isinstance(as_tuples, str) else as_tuples)))
 
     def get_all(
             self,
@@ -579,6 +589,7 @@ def get_all(
             limit: int = None,
             page_size: int = 1000,
             page_number: int = None,
+            as_tuples: str | tuple | list[str|tuple] = None,
             **kwargs
     ) -> List[Measurement]:
         """ Query the database for measurements and return the results
@@ -607,6 +618,7 @@ def get_all(
             limit=limit,
             page_size=page_size,
             page_number=page_number,
+            as_tuples=as_tuples,
             **kwargs))
 
     def get_last(

requirements.txt

@@ -6,6 +6,7 @@ requests
 Deprecated
 pytest
 pytest-asyncio
+pytest-cov
 responses
 python-dotenv
 invoke