Merge pull request #190 from sentinel-hub/feat/sh-py-proc-api-update

Use the SentinelHubRequest for issuing Processing API requests

Author: AleksMat

.travis.yml

@@ -5,14 +5,11 @@ notifications:
   email: false
 
 python:
-- "3.5"
 - "3.6"
 - "3.7"
 - "3.8"
 
 before_install:
-- sudo add-apt-repository -y ppa:jonathonf/backports
-- sudo apt-get -qq update
 - sudo apt-get install -y libgdal-dev
 - sudo apt-get install graphviz
 - sudo apt-get install proj-bin
@@ -26,6 +23,7 @@ install:
   else
     python install_all.py;
   fi
+- pip install geopandas==0.6.3  # This is temporal fix
 
 script:
 - sentinelhub.config --instance_id $INSTANCE_ID

CHANGELOG.md

@@ -1,4 +1,8 @@
-## [Unreleased]
+## [Version 0.7.2] - 2019-02-17
+- Support additional data in the Processing API input task (such as sunAzimuthAngles, sunZenithAngles, viewAzimuthMean, viewZenithMean)
+- Compatibility with the `sentinelhub-py` 3.0
+- Removed support for python 3.5
+- Multiprocessing Log filtering
 
 ## [Version 0.7.1] - 2019-02-05
 ### Fixed

core/eolearn/core/__init__.py

@@ -18,4 +18,4 @@
 from .utilities import deep_eq, negate_mask, constant_pad, get_common_timestamps, bgr_to_rgb, FeatureParser
 
 
-__version__ = '0.7.1'
+__version__ = '0.7.2'

core/requirements.txt

@@ -2,7 +2,7 @@ attrs>=18.2.0
 python-dateutil
 numpy>=1.16
 geopandas
-sentinelhub==3.0.0b1
+sentinelhub>=3.0.0
 tqdm>=4.27
 boto3
 fs

geometry/eolearn/geometry/__init__.py

@@ -7,4 +7,4 @@
 from .superpixel import SuperpixelSegmentation, FelzenszwalbSegmentation, SlicSegmentation, MarkSegmentationBoundaries
 from .transformations import VectorToRaster, RasterToVector
 
-__version__ = '0.7.0'
+__version__ = '0.7.2'

geometry/eolearn/geometry/transformations.py

@@ -384,7 +384,7 @@ def execute(self, eopatch):
 
             affine_transform = rasterio.transform.from_bounds(*eopatch.bbox, width=width, height=height)
 
-            crs = eopatch.bbox.get_crs()
+            crs = eopatch.bbox.crs
 
             if raster_ft.is_timeless():
                 eopatch[vector_ft][vector_fn] = self._vectorize_single_raster(raster, affine_transform, crs)

geometry/setup.py

@@ -28,7 +28,7 @@ def get_version():
 
 setup(
     name='eo-learn-geometry',
-    python_requires='>=3.5',
+    python_requires='>=3.6',
     version=get_version(),
     description='A collection of geometry utilities and EOTasks',
     long_description=get_long_description(),

io/eolearn/io/__init__.py

@@ -9,4 +9,4 @@
 from .local_io import ExportToTiff, ImportFromTiff
 from .processing_api import SentinelHubInputTask, SentinelHubDemTask
 
-__version__ = '0.7.1'
+__version__ = '0.7.2'

io/eolearn/io/geopedia.py

@@ -16,7 +16,7 @@
 import rasterio.transform
 import rasterio.warp
 
-from sentinelhub import MimeType, CustomUrlParam, CRS, GeopediaWmsRequest, transform_bbox
+from sentinelhub import MimeType, CustomUrlParam, CRS, GeopediaWmsRequest
 
 from eolearn.core import EOTask, FeatureType
 
@@ -55,7 +55,7 @@ def _get_wms_request(self, bbox, size_x, size_y):
         """
         Returns WMS request.
         """
-        bbox_3857 = transform_bbox(bbox, CRS.POP_WEB)
+        bbox_3857 = bbox.transform(CRS.POP_WEB)
 
         return GeopediaWmsRequest(layer=self.layer,
                                   theme=self.theme,
@@ -79,7 +79,7 @@ def _reproject(self, eopatch, src_raster):
 
         dst_raster = np.ones((height, width), dtype=self.raster_dtype)
 
-        src_bbox = transform_bbox(eopatch.bbox, CRS.POP_WEB)
+        src_bbox = eopatch.bbox.transform(CRS.POP_WEB)
         src_transform = rasterio.transform.from_bounds(*src_bbox, width=width, height=height)
 
         dst_bbox = eopatch.bbox

io/eolearn/io/processing_api.py

@@ -6,9 +6,8 @@
 import datetime as dt
 import numpy as np
 
-from sentinelhub import WebFeatureService, MimeType, SentinelHubDownloadClient, DownloadRequest, SHConfig,\
+from sentinelhub import SentinelHubRequest, WebFeatureService, MimeType, SentinelHubDownloadClient, SHConfig, \
     bbox_to_dimensions, parse_time_interval, DataSource
-import sentinelhub.sentinelhub_request as shr
 from sentinelhub.time_utils import iso_to_datetime
 
 from eolearn.core import EOPatch, EOTask, FeatureType
@@ -43,15 +42,7 @@ def __init__(self, data_source, size=None, resolution=None, cache_folder=None, c
         self.config = config or SHConfig()
         self.max_threads = max_threads
         self.data_source = data_source
-
-        self.request_args = dict(
-            url=self.config.get_sh_processing_api_url(),
-            headers={"accept": "application/tar", 'content-type': 'application/json'},
-            data_folder=cache_folder,
-            hash_save=bool(cache_folder),
-            request_type='POST',
-            data_type=MimeType.TAR
-        )
+        self.cache_folder = cache_folder
 
     def execute(self, eopatch=None, bbox=None, time_interval=None):
         """ Main execute method for the Processing API tasks
@@ -80,8 +71,8 @@ def execute(self, eopatch=None, bbox=None, time_interval=None):
         elif timestamp:
             eopatch.timestamp = timestamp
 
-        payloads = self._build_payloads(bbox, size_x, size_y, timestamp, time_interval)
-        requests = [DownloadRequest(post_values=payload, **self.request_args) for payload in payloads]
+        requests = self._build_requests(bbox, size_x, size_y, timestamp, time_interval)
+        requests = [request.download_list[0] for request in requests]
 
         LOGGER.debug('Downloading %d requests of type %s', len(requests), str(self.data_source))
         client = SentinelHubDownloadClient(config=self.config)
@@ -118,10 +109,10 @@ def _extract_data(self, eopatch, images, shape):
         """
         raise NotImplementedError("The _extract_data method should be implemented by the subclass.")
 
-    def _build_payloads(self, bbox, size_x, size_y, timestamp, time_interval):
-        """ Build payloads for the requests to the service
+    def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval):
+        """ Build requests
         """
-        raise NotImplementedError("The _build_payloads method should be implemented by the subclass.")
+        raise NotImplementedError("The _build_requests method should be implemented by the subclass.")
 
     def _get_timestamp(self, time_interval, bbox):
         """ Get the timestamp array needed as a parameter for downloading the images
@@ -318,39 +309,36 @@ def _get_timestamp(self, time_interval, bbox):
 
         return [dates[0]] + [d2 for d1, d2 in zip(dates[:-1], dates[1:]) if d2 - d1 > self.time_difference]
 
-    def _build_payloads(self, bbox, size_x, size_y, timestamp, time_interval):
-        """ Build payloads for the requests to the service
+    def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval):
+        """ Build requests
         """
         if self.single_scene:
             dates = [(iso_to_datetime(time_interval[0]), iso_to_datetime(time_interval[1]))]
         else:
             dates = [(date - self.time_difference, date + self.time_difference) for date in timestamp]
 
-        return [self._request_payload(date1, date2, bbox, size_x, size_y) for date1, date2 in dates]
+        return [self._create_sh_request(date1, date2, bbox, size_x, size_y) for date1, date2 in dates]
 
-    def _request_payload(self, date_from, date_to, bbox, size_x, size_y):
-        """ Build the payload dictionary for the request
+    def _create_sh_request(self, date_from, date_to, bbox, size_x, size_y):
+        """ Create an instance of SentinelHubRequest
         """
-        time_from, time_to = date_from.isoformat() + 'Z', date_to.isoformat() + 'Z'
-
-        responses = [shr.response(btype.id, 'image/tiff') for btype in self.requested_bands]
-
-        responses.append(shr.response('userdata', 'application/json'))
-
-        data_type = 'CUSTOM' if self.data_source.is_custom() else self.data_source.api_identifier()
-
-        data = shr.data(time_from=time_from, time_to=time_to, data_type=data_type)
-        data['dataFilter']['maxCloudCoverage'] = int(self.maxcc * 100)
-        data['dataFilter']['mosaickingOrder'] = self.mosaicking_order
-
-        if data_type == 'CUSTOM':
-            data['dataFilter']['collectionId'] = self.data_source.value
-
-        return shr.body(
-            request_bounds=shr.bounds(crs=bbox.crs.opengis_string, bbox=list(bbox)),
-            request_data=[data],
-            request_output=shr.output(size_x=size_x, size_y=size_y, responses=responses),
-            evalscript=self.generate_evalscript()
+        responses = [SentinelHubRequest.output_response(btype.id, MimeType.TIFF) for btype in self.requested_bands]
+        responses.append(SentinelHubRequest.output_response('userdata', MimeType.JSON))
+
+        return SentinelHubRequest(
+            evalscript=self.generate_evalscript(),
+            input_data=[
+                SentinelHubRequest.input_data(
+                    data_source=self.data_source,
+                    time_interval=(date_from, date_to),
+                    mosaicking_order=self.mosaicking_order,
+                    maxcc=self.maxcc
+                )
+            ],
+            responses=responses,
+            bbox=bbox,
+            size=(size_x, size_y),
+            data_folder=self.cache_folder
         )
 
     def _extract_data(self, eopatch, images, shape):
@@ -440,8 +428,8 @@ def __init__(self, dem_feature, size=None, resolution=None, cache_folder=None, c
 
         self.dem_feature = next(feature_parser())
 
-    def _build_payloads(self, bbox, size_x, size_y, timestamp, time_interval):
-        """ Build payloads for the requests to the service
+    def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval):
+        """ Build requests
         """
         evalscript = """
             //VERSION=3
@@ -462,19 +450,19 @@ def _build_payloads(self, bbox, size_x, size_y, timestamp, time_interval):
             }
         """
 
-        responses = [shr.response('default', 'image/tiff'), shr.response('userdata', 'application/json')]
-        request_body = shr.body(
-            request_bounds=shr.bounds(crs=bbox.crs.opengis_string, bbox=list(bbox)),
-            request_data=[{"type": "DEM"}],
-            request_output=shr.output(size_x=size_x, size_y=size_y, responses=responses),
-            evalscript=evalscript
+        request = SentinelHubRequest(
+            evalscript=evalscript,
+            input_data=[SentinelHubRequest.input_data(data_source=self.data_source)],
+            responses=[SentinelHubRequest.output_response('default', MimeType.TIFF)],
+            bbox=bbox,
+            size=(size_x, size_y),
+            data_folder=self.cache_folder
         )
 
-        return [request_body]
+        return [request]
 
     def _extract_data(self, eopatch, images, shape):
         """ Extract data from the received images and assign them to eopatch features
         """
-        tif = images[0]['default.tif']
-
+        tif = images[0]
         eopatch[self.dem_feature] = tif[..., np.newaxis].astype(np.int16)