diff --git a/terrautils/extractors.py b/terrautils/extractors.py index 2db7670..f246d92 100644 --- a/terrautils/extractors.py +++ b/terrautils/extractors.py @@ -26,8 +26,15 @@ logging.basicConfig(format='%(asctime)s %(message)s') +# Experiment YAML file name DEFAULT_EXPERIMENT_JSON_FILENAME = 'experiment.yaml' +# The name of the BETYdb URL environment variable +BETYDB_URL_ENV_NAME = 'BETYDB_URL' + +# The name of the BETYDB key environment variable +BETYDB_KEY_ENV_NAME = 'BETYDB_KEY' + class __internal__(object): """Class for functions intended for internal use only for this file """ @@ -152,6 +159,60 @@ def prep_string_for_filename(source): return formatted + @staticmethod + def find_betydb_config(metadata, key): + """Performs a shalow search for a key in the metadata and + returns it + Args: + metadata(dict): the metadata to search + key(str): the name of the key to find + Return: + The found value or None + """ + if 'betydb' in metadata: + if key in metadata['betydb']: + return metadata['betydb'][key] + return None + + @staticmethod + def setup_env_var(env_name, value): + """Sets up the environemt variable. Returns the old + value if it had been set + Args: + env_name(str): the name of the environment variable to set + value(str): the value to set the environment variable to + Return: + The current environment value or None if the variable was not + set. If the variable is currently not set an empty string is + returned. + Exceptions: + ValueError is thrown if a parameter is not a string + """ + if not env_name or not value: + return None + + if not isinstance(env_name, str): + if sys.version_info[0] < 3: + if isinstance(env_name, unicode): + env_name = env_name.encode('ascii', 'ignore') + else: + ValueError("Environment variable name is not a string") + else: + ValueError("Environment variable name is not a string") + if not isinstance(value, str): + if sys.version_info[0] < 3: + if isinstance(value, unicode): + value = value.encode('ascii', 'ignore') + else: + raise ValueError("Environment variable value is not a string") + else: + raise ValueError("Environment variable value is not a string") + + old_value = os.environ.get(env_name, '') + os.environ[env_name] = value + + return old_value + def add_arguments(parser): # TODO: Move defaults into a level-based dict @@ -469,6 +530,18 @@ def setup_overrides(self, host, secret_key, resource): sensor_old_base = self.sensors.base self.sensors.base = new_base + # Check if a new BETYdb URL/KEY has been specified and set it + if self.experiment_metadata: + old_betydb_url = __internal__.setup_env_var(BETYDB_URL_ENV_NAME, + __internal__.find_betydb_config(self.experiment_metadata, + 'url')) + old_betydb_key = __internal__.setup_env_var(BETYDB_KEY_ENV_NAME, + __internal__.find_betydb_config(self.experiment_metadata, + 'key')) + else: + old_betydb_url = None + old_betydb_key = None + def restore_func(): """Internal function for restoring class variables that's returned to caller """ @@ -478,6 +551,10 @@ def restore_func(): self.sensors.station = old_station if added_station and sitename and sitename in STATIONS: del STATIONS[sitename] + if old_betydb_url: + __internal__.setup_env_var(BETYDB_URL_ENV_NAME, old_betydb_url) + if old_betydb_key: + __internal__.setup_env_var(BETYDB_KEY_ENV_NAME, old_betydb_key) except Exception as ex: logging.warning("Restoring Extractor class variables failed: " + str(ex)) @@ -646,8 +723,22 @@ def find_sitename(self): Returns a found sitename or None """ if self.experiment_metadata: - return __internal__.case_insensitive_find(self.experiment_metadata, 'site') + return __internal__.case_insensitive_find(self.experiment_metadata, 'collectingSite') + + return None + def find_extractor_json(self, extractor=None): + """Finds extractor specific configuration in experiment data + Keyword Arguments: + extractor(str): the name of the extractor to find experiment data on. If not specified + the configured sensor name is used. + Return: + Returns the extractor JSON when found. None is returned otherwise + """ + if self.experiment_metadata: + config_json = __internal__.case_insensitive_find(self.experiment_metadata, 'extractors') + if config_json: + return __internal__.case_insensitive_find(config_json, extractor if extractor else self.sensor_name) return None def get_username_with_base_path(self, host, key, dataset_id, base_path=None): @@ -783,7 +874,12 @@ def get_clowder_context(self, host, key): result.raise_for_status() if not len(result.json()) == 0: - ret_space = result.json()[0]['id'] + # Even though we ask for an exact match, it's possible to get more than one + # space back if they start with the same characters + for one_space in result.json(): + if one_space['name'] == cur_space: + ret_space = one_space['id'] + break else: ret_space = cur_space @@ -796,15 +892,14 @@ def get_file_filters(self): """ file_filters = None if self.get_terraref_metadata is None and self.experiment_metadata: - if 'extractors' in self.experiment_metadata: - extractor_json = self.experiment_metadata['extractors'] - if self.sensor_name in extractor_json: - if 'filters' in extractor_json[self.sensor_name]: - file_filters = extractor_json[self.sensor_name]['filters'] - if ',' in file_filters: - file_filters = file_filters.split(',') - elif file_filters: - file_filters = [file_filters] + extractor_json = self.find_extractor_json() + if extractor_json: + file_filters = __internal__.case_insensitive_find(extractor_json, 'filters') + if file_filters: + if ',' in file_filters: + file_filters = file_filters.split(',') + elif file_filters: + file_filters = [file_filters] return file_filters @@ -831,6 +926,37 @@ def timestamp_to_terraref(timestamp): return return_ts +def terraref_timestamp_to_iso(timestamp): + """Converts a timestamp to ISO + Args: + timestamp(str): the terraref timestamp to convert + Return: + Returns the timestamp formatted to ISO standard. If the timestamp is in + TERRA REF format and a time is specified, the Maricopa, AZ time offset is + automatically added to the returned string + Exceptions: + RuntimeError is raised if a TERRA REF formatted timestamp can't be converted + Note: + Does a simple check for the TERRA REF date/time separator ('__') to determine + if a reformatting is needed. If that separator isn't found, the original string + is returned. If that separator is found and a time isn't specified, only the date + is returned. + No checks are made on the validity of the date & time + """ + return_ts = timestamp + + if '__' in timestamp: + (ts_date, ts_time) = timestamp.split('__') + if ts_date and ts_time: + # We have a TERRA REF date and a time, add Maricopa, AZ time adjustment + return_ts = ts_date + 'T' + ts_time.replace('-', ':') + '-07:00' + elif ts_date: + return_ts = ts_date + else: + raise RuntimeError('Unable to convert invalid date to ISO format: "' + timestamp + '"') + + return return_ts + def build_metadata(clowderhost, extractorinfo, target_id, content, target_type='file', context=None): """Construct extractor metadata object ready for submission to a Clowder file/dataset. diff --git a/terrautils/lemnatec.py b/terrautils/lemnatec.py index 74c85b7..6868131 100644 --- a/terrautils/lemnatec.py +++ b/terrautils/lemnatec.py @@ -94,7 +94,7 @@ def _get_spatial_metadata(cleaned_md, sensorId): gps_bounds = calculate_gps_bounds(cleaned_md, sensorId) spatial_metadata = {} - for label, bounds in gps_bounds.iteritems(): + for label, bounds in gps_bounds.items(): spatial_metadata[label] = {} spatial_metadata[label]["bounding_box"] = tuples_to_geojson(bounds) spatial_metadata[label]["centroid"] = calculate_centroid(bounds) @@ -109,7 +109,7 @@ def _get_sites(cleaned_md, date, sensorId): gps_bounds = calculate_gps_bounds(cleaned_md, sensorId) sites = {} - for label, bounds in gps_bounds.iteritems(): + for label, bounds in gps_bounds.items(): centroid = calculate_centroid(bounds) bety_sites = terrautils.betydb.get_sites_by_latlon(centroid, date) for bety_site in bety_sites: @@ -121,8 +121,7 @@ def _get_sites(cleaned_md, date, sensorId): else: sites["url"] = "" - return sites - + return list(sites.values()) def _get_experiment_metadata(date, sensorId): sensors = Sensors(base="", station=STATION_NAME, sensor=sensorId) @@ -899,7 +898,7 @@ def read_scan_program_map(): if not scan_programs: scan_path = os.path.join(os.path.dirname(__file__), "data/scan_programs.csv") - with open(scan_path, 'rb') as file: + with open(scan_path, 'r') as file: reader = csv.DictReader(file) for row in reader: scan_programs[row["program_name"]] = { diff --git a/terrautils/sensors.py b/terrautils/sensors.py index 67b8773..68be255 100644 --- a/terrautils/sensors.py +++ b/terrautils/sensors.py @@ -635,3 +635,13 @@ def get_display_name(self, sensor=''): return self.stations[self.station][self.sensor]['display'] else: return self.sensor + + def get_pattern(self, sensor=''): + """Gets the pattern associated with the sensor""" + if not sensor: + sensor = self.sensor + + if 'pattern' in self.stations[self.station][sensor]: + return self.stations[self.station][sensor]['pattern'] + + return '' \ No newline at end of file diff --git a/terrautils/spatial.py b/terrautils/spatial.py index 29f191f..9b23259 100644 --- a/terrautils/spatial.py +++ b/terrautils/spatial.py @@ -318,11 +318,6 @@ def find_plots_intersect_boundingbox(bounding_box, all_plots, fullmac=True): if poly_sr and not bb_sr.IsSame(poly_sr): # We need to convert to the same coordinate system before an intersection check_poly = convert_geometry(current_poly, bb_sr) - transform = osr.CreateCoordinateTransformation(poly_sr, bb_sr) - new_poly = current_poly.Clone() - if new_poly: - new_poly.Transform(transform) - check_poly = new_poly intersection_with_bounding_box = bbox_poly.Intersection(check_poly)