Merge pull request #548 from sentinel-hub/develop

Release version 1.4.0

Author: zigaLuksic

.flake8

@@ -1,5 +1,6 @@
 [flake8]
-ignore = E203, W503, C408
+# B028 is ignored because !r flags cannot be used in python < 3.8
+ignore = E203, W503, C408, B028
 exclude = .git, __pycache__, build, dist
 max-line-length= 120
 max-complexity = 15

.github/workflows/ci_action.yml

@@ -7,7 +7,8 @@ on:
       - "master"
       - "develop"
   schedule:
-    - cron: "0 0 * * *"
+    # Schedule events are triggered by whoever last changed the cron schedule
+    - cron: "5 0 * * *"
 
 env:
   # The only way to simulate if-else statement
@@ -60,7 +61,14 @@ jobs:
         run: make pylint
 
       - name: Run mypy
-        run: mypy core/eolearn/core
+        run: |
+          mypy core/eolearn/core
+          mypy coregistration/eolearn/coregistration
+          mypy geometry/eolearn/geometry
+          mypy features/eolearn/features
+          mypy mask/eolearn/mask
+          mypy ml_tools/eolearn/ml_tools
+          mypy visualization/eolearn/visualization
 
   test-on-github:
     runs-on: ubuntu-latest

.gitlab-ci.yml

@@ -12,5 +12,5 @@ run_sh_integration_tests:
   script:
     - pip install -r requirements-dev.txt --upgrade
     - python install_all.py
-    - sentinelhub.config --sh_client_id "$SH_CLIENT_ID" --sh_client_secret "$SH_CLIENT_SECRET" > /dev/null  # Gitlab can't mask SH_CLIENT_SECRET in logs
+    - sentinelhub.config --sh_client_id "$SH_CLIENT_ID" --sh_client_secret "$SH_CLIENT_SECRET" > /dev/null # Gitlab can't mask SH_CLIENT_SECRET in logs
     - pytest -m sh_integration

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: end-of-file-fixer
       - id: requirements-txt-fixer
@@ -13,19 +13,19 @@ repos:
       - id: debug-statements
 
   - repo: https://github.com/psf/black
-    rev: 22.8.0
+    rev: 22.12.0
     hooks:
       - id: black
         language_version: python3
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.11.4
     hooks:
       - id: isort
         name: isort (python)
 
   - repo: https://github.com/PyCQA/autoflake
-    rev: v1.5.3
+    rev: v2.0.0
     hooks:
       - id: autoflake
         args:
@@ -36,7 +36,7 @@ repos:
           ]
 
   - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
       - id: flake8
         additional_dependencies:
@@ -46,7 +46,7 @@ repos:
           - flake8-typing-imports
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.4.0
+    rev: 1.6.0
     hooks:
       - id: nbqa-black
       - id: nbqa-isort

CHANGELOG.md

@@ -1,3 +1,18 @@
+## [Version 1.4.0] - 2023-1-20
+
+- (**codebreaking**) Complete overhaul of `eolearn.coregistration`. See documentation for details.
+- (**codebreaking**) Removed non-working HVPlot backend for `eolearn.visualization`.
+- (**codebreaking**) The `SpatialResizeTask` had a bug when resizing w.r.t resolution. The issue was fixed and the signature of the task was redesigned to better avoid mistakes. See documentation for details.
+- (**codebreaking**) The `EOPatch` methods `get_features` and `get_feature_list` were recombined into a new `get_features` method. The method `get_time_series` was removed. See documentation for details.
+- (**codebreaking**) Removed unsound `use_int_coords` option in `eolearn.ml_tools.sampling.random_point_in_triangle`.
+- Added ability to specify query in execute method of `MeteoblueTask`.
+- `SentinelHubInputTask` no longer saves redundant data into meta-features.
+- Module `eolearn.core.utils.types` was moved to `eolearn.core.types`. Old one will be removed in the future.
+- Switched `opencv-contrib-python-headless` requirement to `opencv-python-headless`
+- Added type annotations to most of the code base.
+- Various improvements to tests and code.
+
+
 ## [Version 1.3.1] - 2022-11-23
 
 - Sentinel Hub IO tasks now support a custom timestamp filtration via `timestamp_filter` parameter.

core/eolearn/core/__init__.py

@@ -32,4 +32,4 @@
 from .utils.parallelize import execute_with_mp_lock, join_futures, join_futures_iter, parallelize
 from .utils.parsing import FeatureParser
 
-__version__ = "1.3.1"
+__version__ = "1.4.0"

core/eolearn/core/core_tasks.py

@@ -26,8 +26,8 @@
 from .constants import FeatureType
 from .eodata import EOPatch
 from .eotask import EOTask
+from .types import FeatureSpec, FeaturesSpecification, SingleFeatureSpec
 from .utils.fs import get_filesystem, pickle_fs, unpickle_fs
-from .utils.parsing import FeatureSpec, FeaturesSpecification
 
 
 class CopyTask(EOTask):
@@ -313,7 +313,7 @@ def execute(self, eopatch: EOPatch) -> EOPatch:
         """
         shape = eopatch[self.shape_feature].shape if self.shape_feature else self.shape
 
-        add_features = set(self.features) - set(self.parse_features(eopatch.get_feature_list()))
+        add_features = set(self.features) - set(self.parse_features(eopatch.get_features()))
 
         for feature in add_features:
             eopatch[feature] = np.ones(shape, dtype=self.dtype) * self.init_value
@@ -472,7 +472,7 @@ def zip_method(self, *f):
     def __init__(
         self,
         input_features: FeaturesSpecification,
-        output_feature: FeaturesSpecification,
+        output_feature: SingleFeatureSpec,
         zip_function: Optional[Callable] = None,
         **kwargs: Any,
     ):

core/eolearn/core/eodata.py

@@ -30,10 +30,10 @@
 from .constants import FeatureType, OverwritePermission
 from .eodata_io import FeatureIO, load_eopatch, save_eopatch
 from .eodata_merge import merge_eopatches
+from .types import EllipsisType, FeatureSpec, FeaturesSpecification, Literal
 from .utils.common import deep_eq, is_discrete_type
 from .utils.fs import get_filesystem
-from .utils.parsing import FeatureSpec, FeaturesSpecification, parse_features
-from .utils.types import EllipsisType, Literal
+from .utils.parsing import parse_features
 
 _T = TypeVar("_T")
 _Self = TypeVar("_Self")
@@ -352,19 +352,22 @@ def __setitem__(
 
         return self.__setattr__(FeatureType(feature_type).value, value, feature_name=feature_name)
 
-    def __delitem__(self, feature: Tuple[FeatureType, str]) -> None:
+    def __delitem__(self, feature: FeatureSpec) -> None:
         """Deletes the selected feature.
 
         :param feature: EOPatch feature
         """
         self._check_tuple_key(feature)
         feature_type, feature_name = feature
-        del self[feature_type][feature_name]
+        if feature_type in [FeatureType.BBOX, FeatureType.TIMESTAMP]:
+            self.reset_feature_type(feature_type)
+        else:
+            del self[feature_type][feature_name]
 
     @staticmethod
     def _check_tuple_key(key: tuple) -> None:
         """A helper function that checks a tuple, which should hold (feature_type, feature_name)."""
-        if len(key) != 2:
+        if not isinstance(key, (tuple, list)) or len(key) != 2:
             raise ValueError(f"Given element should be a tuple of (feature_type, feature_name), but {key} found.")
 
     def __eq__(self, other: object) -> bool:
@@ -520,21 +523,6 @@ def reset_feature_type(self, feature_type: FeatureType) -> None:
         else:
             self[feature_type] = []
 
-    def get_features(self) -> Dict[FeatureType, Union[Set[str], Literal[True]]]:
-        """Returns a dictionary of all non-empty features of EOPatch.
-
-        The elements are either sets of feature names or a boolean `True` in case feature type has no dictionary of
-        feature names.
-
-        :return: A dictionary of features
-        """
-        feature_dict: Dict[FeatureType, Union[Set[str], Literal[True]]] = {}
-        for feature_type in FeatureType:
-            if self[feature_type]:
-                feature_dict[feature_type] = set(self[feature_type]) if feature_type.has_dict() else True
-
-        return feature_dict
-
     def get_spatial_dimension(self, feature_type: FeatureType, feature_name: str) -> Tuple[int, int]:
         """
         Returns a tuple of spatial dimension (height, width) of a feature.
@@ -552,20 +540,19 @@ def get_spatial_dimension(self, feature_type: FeatureType, feature_name: str) ->
             "FeatureType used to determine the width and height of raster must be time dependent or spatial."
         )
 
-    def get_feature_list(self) -> List[Union[FeatureType, Tuple[FeatureType, str]]]:
+    def get_features(self) -> List[FeatureSpec]:
         """Returns a list of all non-empty features of EOPatch.
 
-        The elements are either only FeatureType or a pair of FeatureType and feature name.
-
-        :return: list of features
+        :return: List of non-empty features
         """
-        feature_list: List[Union[FeatureType, Tuple[FeatureType, str]]] = []
+        feature_list: List[FeatureSpec] = []
         for feature_type in FeatureType:
-            if feature_type.has_dict():
+            if feature_type is FeatureType.BBOX or feature_type is FeatureType.TIMESTAMP:
+                if feature_type in self:
+                    feature_list.append((feature_type, None))
+            else:
                 for feature_name in self[feature_type]:
                     feature_list.append((feature_type, feature_name))
-            elif self[feature_type]:
-                feature_list.append(feature_type)
         return feature_list
 
     def save(
@@ -661,27 +648,6 @@ def merge(
 
         return merged_eopatch
 
-    def get_time_series(self, ref_date: Optional[dt.datetime] = None, scale_time: int = 1) -> np.ndarray:
-        """Returns a numpy array with seconds passed between the reference date and the timestamp of each image.
-
-        An array is constructed as time_series[i] = (timestamp[i] - ref_date).total_seconds().
-        If reference date is None the first date in the EOPatch's timestamp is taken.
-        If EOPatch timestamp attribute is empty the method returns None.
-
-        :param ref_date: reference date relative to which the time is measured
-        :param scale_time: scale seconds by factor. If `60`, time will be in minutes, if `3600` hours
-        """
-
-        if not self.timestamp:
-            return np.zeros(0, dtype=np.int64)
-
-        if ref_date is None:
-            ref_date = self.timestamp[0]
-
-        return np.asarray(
-            [round((timestamp - ref_date).total_seconds() / scale_time) for timestamp in self.timestamp], dtype=np.int64
-        )
-
     def consolidate_timestamps(self, timestamps: List[dt.datetime]) -> Set[dt.datetime]:
         """Removes all frames from the EOPatch with a date not found in the provided timestamps list.
 

core/eolearn/core/eodata_io.py

@@ -53,8 +53,8 @@
 from sentinelhub.exceptions import SHUserWarning
 
 from .constants import FeatureType, FeatureTypeSet, OverwritePermission
-from .utils.parsing import FeatureParser, FeaturesSpecification
-from .utils.types import EllipsisType
+from .types import EllipsisType, FeaturesSpecification
+from .utils.parsing import FeatureParser
 from .utils.vector_io import infer_schema
 
 _T = TypeVar("_T")

core/eolearn/core/eodata_merge.py

@@ -25,8 +25,8 @@
 
 from .constants import FeatureType
 from .exceptions import EORuntimeWarning
-from .utils.parsing import FeatureParser, FeatureSpec, FeaturesSpecification
-from .utils.types import Literal
+from .types import FeatureSpec, FeaturesSpecification, Literal
+from .utils.parsing import FeatureParser
 
 if TYPE_CHECKING:
     from .eodata import EOPatch