pull in hotosm updates

.ebignore

@@ -0,0 +1 @@
+yarn.lock

.gitignore

@@ -49,3 +49,8 @@ frontend/build/asset-*
 
 ############## Jekyll Stuff ##################
 .sass-cache/
+
+# Elastic Beanstalk Files
+.elasticbeanstalk/*
+!.elasticbeanstalk/*.cfg.yml
+!.elasticbeanstalk/*.global.yml

.nvmrc

@@ -1 +1 @@
-6
+8

Dockerfile

@@ -11,7 +11,7 @@ RUN \
     curl \
     software-properties-common \
   && curl -sf https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
-  && add-apt-repository -y -u -s "deb https://deb.nodesource.com/node_6.x $(lsb_release -c -s) main" \
+  && add-apt-repository -y -u -s "deb https://deb.nodesource.com/node_8.x $(lsb_release -c -s) main" \
   && apt install -y --no-install-recommends \
     build-essential \
     default-jre-headless \
@@ -49,8 +49,7 @@ RUN \
   && yarn build
 
 RUN \
-  git submodule update --init \
-  && useradd omkserver -m \
+  useradd omkserver -m \
   && chown -R omkserver:omkserver /app/data
 
 USER omkserver

README.md

@@ -21,14 +21,14 @@ If you want to run OpenMapKit Server for your field mapping survey, use the
 
 ## Project Structure
 
-[__deployments__](deployments) is the OpenMapKit
+[__deployments__](api/deployments) is the OpenMapKit
 deployment API. This services deployment data in the
 [deployments data directory](data/deployments).
 Manifest files in this directory are read by OpenMapKit Android as metadata regarding the deployment
 data provided. Deployments are created by [posm-admin](https://github.com/AmericanRedCross/posm-admin).
 to provision deployment data that is fetched by OpenMapKit Android.
 
-[__odk__](odk) is the OpenMapKit ODK API.
+[__odk__](api/odk) is the OpenMapKit ODK API.
 This is a REST API that handles all of the ODK business logic, including communicating with ODK Collect,
 as well as serving and ingesting ODK XForm data.
 
@@ -44,7 +44,7 @@ useful to OpenMapKit Server as a whole.
 
 #### After your installation is done, you can see if the server is alive at:
 
-    http://{{your_host_url}}/info
+    http://{{your_host_url}}/omk/info
 
 #### To get forms and send submissions in ODK Collect set:
 
@@ -54,11 +54,11 @@ __ODK Collect__ > __General Settings__ > __Configure platform settings__ > __URL
 
 #### See all of the pages on your server:
 
-    http://{{your_host_url}}/pages/
+    http://{{your_host_url}}/
 
 #### Upload an XLS Form:
 
-    http://{{your_host_url}}/pages/upload-form/
+    http://{{your_host_url}}/omk/pages/#/upload-form
 
 
 ## API

alarms.md

@@ -0,0 +1,23 @@
+# Alarms
+
+## Server crashes: Accessing the website returns a `504 Gateway Timeout` or `502 Bad Gateway`
+
+This means that the request was routed to an unhealthy instance behind the OpenMapKitServer load balancer, or that there are no healthy instances behind the OpenMapKitServer load balancer. In order to fix this issue:
+
+* Log onto the EC2 console. Find `Target Groups` from the left panel.
+* Search using the full OMK stack name as a search string. For example: `OpenMapKitServer-tanzania` (Not `tanzania`). Select the target group that shows up in the search results.
+* Select the `Targets` tab. At least one target must be unhealthy. Navigate to the EC2 console of the unhealthy target by clicking on the instance ID.
+* SSH into the EC2 instance by using its public DNS and its key file: `ssh -i ../<path-to-keyfile> ubuntu@<Public DNS>`
+* `sudo su`
+* Run `lsof -i:3210`. If this shows a result, the server should become healthy, and no further actions are required. If it doesn't show a result:
+  * Check if there is a file called `screenlog.0` in `/home/ubuntu`. If yes, open this file, and check the last few lines. `tail -n 100 /home/ubuntu/screenlog.0 | less`.
+  * If this file does not exist, check the last few lines on `/var/log/cloud-init-output.log`, by running: `tail -n 100 /var/log/cloud-init-output.log | less`. Note down the last request that may have caused the server to crash on https://github.com/hotosm/OpenMapKitServer/issues/108. There are a few examples of such requests on that ticket, if you need help identifying a faulty request.
+
+To resolve:
+  * `screen -S restart`
+  * `cd /`
+  * `sh /var/log/cloud/instances/<instance-ID>/user-data.txt`
+  * Run `Ctrl + a`, and then press `H`. This creates a new logfile for the application.
+  * Run `Ctrl + a` and press `d` to exit the application.
+
+Check the target group again to see if the instance has become healthy.

api/deployments/deployment-routes.js

@@ -1,10 +1,12 @@
 var router = require('express').Router({ mergeParams: true });
+
 var getDeployments = require('./controllers/deployments');
 
 /**
  * Retrieve the list of deployment areas of interest.
  */
-router.route('/').get(getDeployments.getAll);
+router.route('/')
+  .get(getDeployments.getAll);
 
 /**
  * Retrieve meta-data for a given deployment.

api/odk/README.md

@@ -8,7 +8,3 @@ This API persists data [directly on the filesystem](https://github.com/AmericanR
 
 This API is inspired from and directly uses some of [SimpleODK](https://github.com/digidem/simple-odk)
 as well as several assocaited node modules.
-
-[pyxform](https://github.com/spatialdev/pyxform) is installed in this directory. This python tool is used to 
-convert XLS Forms into XForm XML - the format understood by ODK Collect. The SpatialDev fork is used in a submodule
-which fixes a bug preventing it from working on OS X.

api/odk/controllers/archive-form.js

@@ -7,7 +7,7 @@ var settings = require('../../../settings.js');
 
 
 const moveFiles = (archiveDir, formName) => {
-  // move forms file to archive
+  // move forms files to archive/forms/ dir
   fs.readdir(path.join(settings.dataDir, 'forms'), (err, items) => {
     items.filter(
       i => [`${formName}.xls`, `${formName}.xlsx`, `${formName}.xml`].includes(i)
@@ -19,7 +19,8 @@ const moveFiles = (archiveDir, formName) => {
           if (renameError) {
             console.log(`Error when moving file ${i}.`);
           } else {
-            syncDataDir();
+            syncDataDir('forms');
+            syncDataDir('archive/forms');
           };
         }
       )
@@ -28,22 +29,6 @@ const moveFiles = (archiveDir, formName) => {
 }
 
 
-const moveSubmissions = (submissionDir, archiveDir, formName) => {
-  // move submissions dir to archive
-  fse.move(
-    submissionDir,
-    path.join(archiveDir, 'submissions', formName),
-    renameError => {
-      if (renameError) {
-        console.log('It was not possible to move the submissions directory.');
-      } else {
-        syncDataDir();
-      };
-    }
-  );
-}
-
-
 module.exports = (req, res, next) => {
   const formName = req.params.formName;
   const submissionDir = path.join(settings.dataDir, 'submissions', formName);
@@ -59,9 +44,6 @@ module.exports = (req, res, next) => {
           fs.mkdir(path.join(archiveDir, 'forms'), formsDirError => {
             if (!formsDirError) moveFiles(archiveDir, formName);
           });
-          fs.mkdir(path.join(archiveDir, 'submissions'), subDirError => {
-            if (!subDirError) moveSubmissions(submissionDir, archiveDir, formName);
-          });
           return res.status(200).json({detail: "Form archived successfully."});
         }
       });
@@ -80,20 +62,6 @@ module.exports = (req, res, next) => {
       } else {
         moveFiles(archiveDir, formName);
       }
-      if (items.filter(i => i === 'submissions').length === 0) {
-        fs.mkdir(path.join(archiveDir, 'submissions'), err => {
-          if (!err) {
-            moveSubmissions(submissionDir, archiveDir, formName);
-          } else {
-            return res.status(500).json(
-              {detail: "It wasn't possible to create submissions archive dir."}
-            );
-          }
-        }
-      );
-      } else {
-        moveSubmissions(submissionDir, archiveDir, formName);
-      }
       return res.status(200).json({detail: "Form archived successfully."});
     }
   });

api/odk/controllers/create-deployment-folder.js

@@ -0,0 +1,23 @@
+const path = require('path');
+var fs = require('fs');
+
+var settings = require('../../../settings.js');
+
+
+module.exports = (req, res, next) => {
+  if (req.body && req.body.name && !/[^a-zA-Z0-9\-\/]/.test(req.body.name)) {
+    const newDir = path.join(settings.dataDir, 'deployments', req.body.name);
+
+    fs.mkdir(newDir, {recursive: true}, mkdirError => {
+      if (mkdirError) {
+        return res.status(500).json({detail: "It wasn't possible to create new deployment folder."});
+      } else {
+        return res.status(200).json({detail: "New deployment folder created successfully."});
+      }
+    });
+  } else {
+    return res.status(500).json(
+      {detail: "Missing deployment name in the request body or name contains invalid characters."}
+    );
+  }
+}

api/odk/controllers/delete-form.js

@@ -11,6 +11,7 @@ module.exports = (req, res, next) => {
   const submissionDir = path.join(settings.dataDir, 'submissions', formName);
   let errors = [];
 
+  // check if the form has submissions
   fs.readdir(submissionDir, (err, items) => {
     if (!err && items.length > 0) {
       return res.status(403).json(
@@ -40,7 +41,7 @@ module.exports = (req, res, next) => {
           {detail: "It was not possible to find or delete the form files"}
         );
       } else {
-        syncDataDir();
+        syncDataDir('forms');
         res.status(200).json({detail: "Form deleted successfully"});
       }
     }

api/odk/controllers/get-csv-features-submissions.js

@@ -0,0 +1,94 @@
+var _ = require('lodash')
+var aggregateOsm = require('../osm/aggregate-osm-to-geojson');
+var getOsmSubmissionsDirs = require('../helpers/get-osm-submissions-dirs');
+var osmtogeojson = require('osmtogeojson');
+var DOMParser = require("xmldom").DOMParser;
+/**
+ * Aggregates together all of the OSM submissions
+ * from ODK Collect / OpenMapKit Android to the
+ * file system for the given form.
+ */
+module.exports = function(req, res, next) {
+  var formName = req.params.formName;
+  var filters = {
+    deviceId: req.query.deviceId,
+    username: req.query.username,
+    startDate: req.query.start_date,
+    endDate: req.query.end_date,
+    offset: req.query.offset,
+    limit: req.query.limit
+  };
+
+  getOsmSubmissionsDirs(formName, {
+    filters: filters
+  }, function(err, osmDirs) {
+    if (err) {
+      res.status(err.status || 500).json(err);
+      return;
+    }
+    aggregate(osmDirs, req, res);
+  });
+};
+
+/**
+ * Calls aggregate-osm middleware to read OSM edit files
+ * and concatenate into a single OSM XML aggregation.
+ *
+ * @param osmDirs  - submission dirs with array of osm files
+ * @param req       - the http request
+ * @param res       - the http response
+ */
+function aggregate(osmDirs, req, res) {
+  var osmFiles = [];
+  for (var i in osmDirs) {
+    osmFiles = osmFiles.concat(osmDirs[i].files);
+  }
+  if (req.query.offset != null) {
+    var offset = parseInt(req.query.offset);
+    var limit = parseInt(req.query.limit);
+    osmFiles = osmFiles.slice(offset, offset + limit);
+  }
+  //We filter by the query parameters of the request
+  aggregateOsm(osmFiles, req.query, function(err, osmXml) {
+    if (err) {
+      if (!res._headerSent) { // prevents trying to send multiple error responses on a single request
+        res.status(500).json({
+          status: 500,
+          msg: 'There was a problem with aggregating OSM JOSM editor files in the submissions directory.',
+          err: err
+        });
+      }
+      return;
+    }
+    var xmlObj = (new DOMParser()).parseFromString(osmXml, 'text/xml');
+    var geojson = osmtogeojson(xmlObj);
+    var csv = '';
+
+    // add headers (list all properties keys of the geojson file)
+    var properties = geojson.features.reduce(
+      (props, item) =>
+        props.concat(
+          Object.keys(item.properties).filter(i => props.indexOf(i) === -1)
+        ),
+      []
+    );
+
+    _.each(properties, function(property) {
+      csv += `${property},`;
+    });
+    csv += '\n';
+
+    // add data (if a feature has not a property, add a empty space)
+    _.each(geojson.features, function(feature){
+      _.each(properties, function(key, i) {
+        let value = feature.properties[key] ? feature.properties[key] : ' ';
+        csv += `"${value}",`;
+      });
+      csv += '\n';
+    });
+
+    res.status(200)
+      .set('Content-Type', 'text/csv')
+      .send(csv);
+  });
+}