Joerd data

Dockerfile

@@ -6,6 +6,9 @@ MAINTAINER Nils Nolde <nils@openrouteservice.org>
 
 RUN apt-get update
 RUN apt-get install -y locales git python3-venv
+RUN apt-get install -y build-essential libssl-dev libffi-dev python3-dev
+RUN apt-get install -y libpq-dev
+RUN apt-get install -y git curl
 
 # Set the locale
 RUN locale-gen en_US.UTF-8
@@ -32,13 +35,24 @@ RUN mkdir -p /deploy/app
 COPY gunicorn_config.py /deploy/gunicorn_config.py
 COPY manage.py /deploy/app/manage.py
 
-COPY requirements.txt /deploy/app/requirements.txt
-
 RUN python3 -m venv /oes_venv
 
 RUN /bin/bash -c "source /oes_venv/bin/activate"
 
-RUN /oes_venv/bin/pip3 install -r /deploy/app/requirements.txt
+# install poetry
+RUN curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python3
+ENV PATH "/root/.poetry/bin:/oes_venv/bin:${PATH}"
+
+# install dependencies via poetry
+# RUN poetry config settings.virtualenvs.create false
+# RUN poetry self:update --preview
+RUN poetry config virtualenvs.create false
+COPY pyproject.toml poetry.lock README.rst /
+RUN poetry install
+
+RUN apt-get install -y python-gdal python-bs4 python-numpy gdal-bin python-setuptools python-shapely
+RUN apt-get install -y libgdal-dev python3-dev
+RUN /oes_venv/bin/pip3 install GDAL==$(gdal-config --version) --global-option=build_ext --global-option="-I/usr/include/gdal"
 
 COPY openelevationservice /deploy/app/openelevationservice
 COPY ops_settings_docker.yml /deploy/app/openelevationservice/server/ops_settings.yml

docker-compose.yml

@@ -1,10 +1,11 @@
 version: '2.2'
 services:
  gunicorn_flask:
-  #network_mode: "host"
+  network_mode: "host"
   build: .
   volumes:
-  - ./tiles:/deploy/app/tiles
+   - ./tiles:/deploy/app/tiles
+   - ./ops_settings_docker.yml:/deploy/app/openelevationservice/server/ops_settings.yml:rw
   ports:
    - "5020:5000"
   mem_limit: 28g

manage.py

@@ -6,64 +6,51 @@
 from openelevationservice.server.db_import.models import db
 from openelevationservice.server.db_import import filestreams
 
-import click
-
 log = get_logger(__name__)
 
 app = create_app()
 
+
 @app.cli.command()
-@click.option('--xyrange', default='0,73,0,25')
-def download(xyrange):
-    """
-    Downloads SRTM tiles to disk. Can be specified over minx, maxx, miny, maxy.
-
-    :param xyrange: A comma-separated list of x_min, x_max, y_min, y_max
-        in that order. For reference grid, see http://srtm.csi.cgiar.org/SELECTION/inputCoord.asp
-    :type xyrange: comma-separated integers
-    """
-
-    filestreams.downloadsrtm(_arg_format(xyrange))
+def prepare():
+    """Downloads SRTM tiles to disk. Can be specified over extent values in ops_settings.yml"""
+
+    filestreams.download()
     log.info("Downloaded all files")
+
+
+@app.cli.command()
+def merge():
+    """Merges downloaded single SRTM and GMTED tiles to one raster"""
+
+    filestreams.merge_data()
+    log.info("Merged downloaded files")
 
 
 @app.cli.command()
 def create():
     """Creates all tables defined in models.py"""
-    
+
     db.create_all()
-    log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name']))
-
+    for table in SETTINGS['provider_parameters']['tables'].items():
+        if table[1]:
+            log.info("Table {} was created.".format(table[1]))
+
 
 @app.cli.command()
 def drop():
     """Drops all tables defined in models.py"""
 
     db.drop_all()
-    log.info("Table {} was dropped.".format(SETTINGS['provider_parameters']['table_name']))
-
+    for table in SETTINGS['provider_parameters']['tables'].items():
+        if table[1]:
+            log.info("Table {} was dropped.".format(table[1]))
+
 
 @app.cli.command()
 def importdata(): 
-    """
-    Imports all data found in ./tiles
-
-    :param xyrange: A comma-separated list of x_min, x_max, y_min, y_max
-        in that order. For reference grid, see http://srtm.csi.cgiar.org/SELECTION/inputCoord.asp
-    :type xyrange: comma-separated integers
-    """
+    """Imports all '_raster.tif' files found in ./tiles"""
+
     log.info("Starting to import data...")
-
     filestreams.raster2pgsql()
-
     log.info("Imported data successfully!")
-
-
-def _arg_format(xy_range_txt):
-
-    str_split = [int(s.strip()) for s in xy_range_txt.split(',')]
-
-    xy_range = [[str_split[0], str_split[2]],
-                [str_split[1], str_split[3]]]
-
-    return xy_range

openelevationservice/__init__.py

@@ -9,13 +9,11 @@
 TILES_DIR = path.join(getcwd(), 'tiles')
 
 if "TESTING" in environ:
-    SETTINGS['provider_parameters']['table_name'] = SETTINGS['provider_parameters']['table_name'] + '_test'
+    SETTINGS['provider_parameters']['tables']['terrestrial'] = SETTINGS['provider_parameters']['tables'][
+                                                                    'terrestrial'] + '_test'
     TILES_DIR = path.join(basedir, 'tests', 'tile')
-    # if "CI" in environ:
-    #     SETTINGS['provider_parameters']['port'] = 5433
 
 if not path.exists(TILES_DIR):
     makedirs(TILES_DIR)
-
 
-__version__ = "0.2.1"
+__version__ = "0.2.1"

openelevationservice/server/__init__.py

@@ -12,6 +12,7 @@
 
 log = logger.get_logger(__name__)
 
+
 def create_app(script_info=None):
     # instantiate the app
 
@@ -32,13 +33,15 @@ def create_app(script_info=None):
 
     # set up extensions
     db.init_app(app)
-    
+
     provider_details = SETTINGS['provider_parameters']
-    log.info("Following provider parameters are active:\n"
-              "Host:\t{host}\n"
-              "DB:\t{db_name}\n"
-              "Table:\t{table_name}\n"
-              "User:\t{user_name}".format(**provider_details))
+
+    for table in provider_details['tables']:
+        log.info("Following provider parameters are active:\n"
+                  "Host:\t{host}\n"
+                  "DB:\t{db_name}\n"
+                  "Table:\t{table_name}\n"
+                  "User:\t{user_name}".format(**provider_details, table_name=table))
 
     # register blueprints
     from openelevationservice.server.api.views import main_blueprint
@@ -94,4 +97,4 @@ def handle_invalid_usage(error):
         'db': db}
     )
 
-    return app
+    return app

openelevationservice/server/api/querybuilder.py

@@ -2,7 +2,8 @@
 
 from openelevationservice import SETTINGS
 from openelevationservice.server.utils.logger import get_logger
-from openelevationservice.server.db_import.models import db, Cgiar
+from openelevationservice.server.db_import.models import db
+from openelevationservice.server.db_import import models
 from openelevationservice.server.utils.custom_func import ST_SnapToGrid
 from openelevationservice.server.api.api_exceptions import InvalidUsage
 
@@ -26,7 +27,13 @@ def _getModel(dataset):
     :rtype: SQLAlchemy model
     """
     if dataset == 'srtm':
-        model = Cgiar
+        model = models.Terrestrial
+
+    elif dataset == 'etopo1':
+        model = models.Bathymetry
+
+    elif dataset == 'gv_at':
+        model = models.At
 
     return model
 

openelevationservice/server/config.py

@@ -2,6 +2,7 @@
 
 from openelevationservice import SETTINGS
 
+
 class BaseConfig(object):
     """Base configuration."""