Drone-pipeline related and some Python2-to-Python3 changes

terrautils/extractors.py

@@ -26,8 +26,15 @@
 
 logging.basicConfig(format='%(asctime)s %(message)s')
 
+# Experiment YAML file name
 DEFAULT_EXPERIMENT_JSON_FILENAME = 'experiment.yaml'
 
+# The name of the BETYdb URL environment variable
+BETYDB_URL_ENV_NAME = 'BETYDB_URL'
+
+# The name of the BETYDB key environment variable
+BETYDB_KEY_ENV_NAME = 'BETYDB_KEY'
+
 class __internal__(object):
     """Class for functions intended for internal use only for this file
     """
@@ -152,6 +159,60 @@ def prep_string_for_filename(source):
 
         return formatted
 
+    @staticmethod
+    def find_betydb_config(metadata, key):
+        """Performs a shalow search for a key in the metadata and
+           returns it
+        Args:
+            metadata(dict): the metadata to search
+            key(str): the name of the key to find
+        Return:
+            The found value or None
+        """
+        if 'betydb' in metadata:
+            if key in metadata['betydb']:
+                return metadata['betydb'][key]
+        return None
+
+    @staticmethod
+    def setup_env_var(env_name, value):
+        """Sets up the environemt variable. Returns the old
+           value if it had been set
+        Args:
+            env_name(str): the name of the environment variable to set
+            value(str): the value to set the environment variable to
+        Return:
+            The current environment value or None if the variable was not
+            set. If the variable is currently not set an empty string is
+            returned.
+        Exceptions:
+            ValueError is thrown if a parameter is not a string
+        """
+        if not env_name or not value:
+            return None
+
+        if not isinstance(env_name, str):
+            if sys.version_info[0] < 3:
+                if isinstance(env_name, unicode):
+                    env_name = env_name.encode('ascii', 'ignore')
+                else:
+                    ValueError("Environment variable name is not a string")
+            else:
+                ValueError("Environment variable name is not a string")
+        if not isinstance(value, str):
+            if sys.version_info[0] < 3:
+                if isinstance(value, unicode):
+                    value = value.encode('ascii', 'ignore')
+                else:
+                    raise ValueError("Environment variable value is not a string")
+            else:
+                raise ValueError("Environment variable value is not a string")
+
+        old_value = os.environ.get(env_name, '')
+        os.environ[env_name] = value
+
+        return old_value
+
 def add_arguments(parser):
 
     # TODO: Move defaults into a level-based dict
@@ -469,6 +530,18 @@ def setup_overrides(self, host, secret_key, resource):
         sensor_old_base = self.sensors.base
         self.sensors.base = new_base
 
+        # Check if a new BETYdb URL/KEY has been specified and set it
+        if self.experiment_metadata:
+            old_betydb_url = __internal__.setup_env_var(BETYDB_URL_ENV_NAME,
+                                                        __internal__.find_betydb_config(self.experiment_metadata,
+                                                                                        'url'))
+            old_betydb_key = __internal__.setup_env_var(BETYDB_KEY_ENV_NAME,
+                                                        __internal__.find_betydb_config(self.experiment_metadata,
+                                                                                        'key'))
+        else:
+            old_betydb_url = None
+            old_betydb_key = None
+
         def restore_func():
             """Internal function for restoring class variables that's returned to caller
             """
@@ -478,6 +551,10 @@ def restore_func():
                 self.sensors.station = old_station
                 if added_station and sitename and sitename in STATIONS:
                     del STATIONS[sitename]
+                if old_betydb_url:
+                    __internal__.setup_env_var(BETYDB_URL_ENV_NAME, old_betydb_url)
+                if old_betydb_key:
+                    __internal__.setup_env_var(BETYDB_KEY_ENV_NAME, old_betydb_key)
             except Exception as ex:
                 logging.warning("Restoring Extractor class variables failed: " + str(ex))
 
@@ -646,8 +723,22 @@ def find_sitename(self):
             Returns a found sitename or None
         """
         if self.experiment_metadata:
-            return __internal__.case_insensitive_find(self.experiment_metadata, 'site')
+            return __internal__.case_insensitive_find(self.experiment_metadata, 'collectingSite')
+
+        return None
 
+    def find_extractor_json(self, extractor=None):
+        """Finds extractor specific configuration in experiment data
+        Keyword Arguments:
+            extractor(str): the name of the extractor to find experiment data on. If not specified
+                the configured sensor name is used.
+        Return:
+            Returns the extractor JSON when found. None is returned otherwise
+        """
+        if self.experiment_metadata:
+            config_json = __internal__.case_insensitive_find(self.experiment_metadata, 'extractors')
+            if config_json:
+                return __internal__.case_insensitive_find(config_json, extractor if extractor else self.sensor_name)
         return None
 
     def get_username_with_base_path(self, host, key, dataset_id, base_path=None):
@@ -783,7 +874,12 @@ def get_clowder_context(self, host, key):
                     result.raise_for_status()
 
                     if not len(result.json()) == 0:
-                        ret_space = result.json()[0]['id']
+                        # Even though we ask for an exact match, it's possible to get more than one
+                        # space back if they start with the same characters
+                        for one_space in result.json():
+                            if one_space['name'] == cur_space:
+                                ret_space = one_space['id']
+                                break
                     else:
                         ret_space = cur_space
 
@@ -796,15 +892,14 @@ def get_file_filters(self):
         """
         file_filters = None
         if self.get_terraref_metadata is None and self.experiment_metadata:
-            if 'extractors' in self.experiment_metadata:
-                extractor_json = self.experiment_metadata['extractors']
-                if self.sensor_name in extractor_json:
-                    if 'filters' in extractor_json[self.sensor_name]:
-                        file_filters = extractor_json[self.sensor_name]['filters']
-                        if ',' in file_filters:
-                            file_filters = file_filters.split(',')
-                        elif file_filters:
-                            file_filters = [file_filters]
+            extractor_json = self.find_extractor_json()
+            if extractor_json:
+                file_filters = __internal__.case_insensitive_find(extractor_json, 'filters')
+                if file_filters:
+                    if ',' in file_filters:
+                        file_filters = file_filters.split(',')
+                    elif file_filters:
+                        file_filters = [file_filters]
         return file_filters
 
 
@@ -831,6 +926,37 @@ def timestamp_to_terraref(timestamp):
 
     return return_ts
 
+def terraref_timestamp_to_iso(timestamp):
+    """Converts a timestamp to ISO
+    Args:
+        timestamp(str): the terraref timestamp to convert
+    Return:
+        Returns the timestamp formatted to ISO standard. If the timestamp is in
+        TERRA REF format and a time is specified, the Maricopa, AZ time offset is
+        automatically added to the returned string
+    Exceptions:
+        RuntimeError is raised if a TERRA REF formatted timestamp can't be converted
+    Note:
+        Does a simple check for the TERRA REF date/time separator ('__') to determine
+        if a reformatting is needed. If that separator isn't found, the original string
+        is returned. If that separator is found and a time isn't specified, only the date
+        is returned.
+        No checks are made on the validity of the date & time
+    """
+    return_ts = timestamp
+
+    if '__' in timestamp:
+        (ts_date, ts_time) = timestamp.split('__')
+        if ts_date and ts_time:
+            # We have a TERRA REF date and a time, add Maricopa, AZ time adjustment
+            return_ts = ts_date + 'T' + ts_time.replace('-', ':') + '-07:00'
+        elif ts_date:
+            return_ts = ts_date
+        else:
+            raise RuntimeError('Unable to convert invalid date to ISO format: "' + timestamp + '"')
+
+    return return_ts
+
 def build_metadata(clowderhost, extractorinfo, target_id, content, target_type='file', context=None):
     """Construct extractor metadata object ready for submission to a Clowder file/dataset.
 

terrautils/lemnatec.py

@@ -94,7 +94,7 @@ def _get_spatial_metadata(cleaned_md, sensorId):
     gps_bounds = calculate_gps_bounds(cleaned_md, sensorId) 
 
     spatial_metadata = {}
-    for label, bounds in gps_bounds.iteritems():
+    for label, bounds in gps_bounds.items():
         spatial_metadata[label] = {}
         spatial_metadata[label]["bounding_box"] = tuples_to_geojson(bounds)
         spatial_metadata[label]["centroid"] = calculate_centroid(bounds)
@@ -109,7 +109,7 @@ def _get_sites(cleaned_md, date, sensorId):
     gps_bounds = calculate_gps_bounds(cleaned_md, sensorId)
 
     sites = {}
-    for label, bounds in gps_bounds.iteritems():
+    for label, bounds in gps_bounds.items():
         centroid = calculate_centroid(bounds)
         bety_sites = terrautils.betydb.get_sites_by_latlon(centroid, date)
         for bety_site in bety_sites:
@@ -122,7 +122,7 @@ def _get_sites(cleaned_md, date, sensorId):
                 sites[site_id]["url"] = ""
             sites['notes'] = "sitename is the plot that contains the image centroid"
 
-    return sites.values()
+    return list(sites.values())
 
 
 def _get_experiment_metadata(date, sensorId): 
@@ -900,7 +900,7 @@ def read_scan_program_map():
 
     if not scan_programs:
         scan_path = os.path.join(os.path.dirname(__file__), "data/scan_programs.csv")
-        with open(scan_path, 'rb') as file:
+        with open(scan_path, 'r') as file:
             reader = csv.DictReader(file)
             for row in reader:
                 scan_programs[row["program_name"]] = {

terrautils/sensors.py

@@ -632,3 +632,13 @@ def get_display_name(self, sensor=''):
                 return self.stations[self.station][self.sensor]['display']
             else:
                 return self.sensor
+
+    def get_pattern(self, sensor=''):
+        """Gets the pattern associated with the sensor"""
+        if not sensor:
+            sensor = self.sensor
+
+        if 'pattern' in self.stations[self.station][sensor]:
+            return self.stations[self.station][sensor]['pattern']
+
+        return ''

terrautils/spatial.py

@@ -318,11 +318,6 @@ def find_plots_intersect_boundingbox(bounding_box, all_plots, fullmac=True):
             if poly_sr and not bb_sr.IsSame(poly_sr):
                 # We need to convert to the same coordinate system before an intersection
                 check_poly = convert_geometry(current_poly, bb_sr)
-                transform = osr.CreateCoordinateTransformation(poly_sr, bb_sr)
-                new_poly = current_poly.Clone()
-                if new_poly:
-                    new_poly.Transform(transform)
-                    check_poly = new_poly
 
         intersection_with_bounding_box = bbox_poly.Intersection(check_poly)