From d39853b0da6792584c0ebe2bb60fda671818e415 Mon Sep 17 00:00:00 2001 From: dlutz2 Date: Wed, 29 Jul 2020 15:15:58 -0400 Subject: [PATCH] Added exception handling for geoJSON mixed property types Now scan all (non-null) properties to find properties not in first Feature. --- .../pentaho/di/gis/io/GeoJSONReader.java | 25 +- .../di/gis/io/features/FeatureConverter.java | 34 +- .../geojson/geojsonReadWriteTestIDs.ktr | 526 ++++++++++++++++++ .../geojson/geojsonReadWriteTestNulls.ktr | 526 ++++++++++++++++++ .../geojson/geojsonReadWriteTestNumbers.ktr | 526 ++++++++++++++++++ .../geojson/geojsonReadWriteTestProps.ktr | 526 ++++++++++++++++++ sample-data/geojson/geojson_test_ids.geojson | 13 + .../geojson/geojson_test_nulls.geojson | 11 + .../geojson/geojson_test_numbers.geojson | 9 + .../geojson/geojson_test_props.geojson | 12 + 10 files changed, 2197 insertions(+), 11 deletions(-) create mode 100644 sample-data/geojson/geojsonReadWriteTestIDs.ktr create mode 100644 sample-data/geojson/geojsonReadWriteTestNulls.ktr create mode 100644 sample-data/geojson/geojsonReadWriteTestNumbers.ktr create mode 100644 sample-data/geojson/geojsonReadWriteTestProps.ktr create mode 100644 sample-data/geojson/geojson_test_ids.geojson create mode 100644 sample-data/geojson/geojson_test_nulls.geojson create mode 100644 sample-data/geojson/geojson_test_numbers.geojson create mode 100644 sample-data/geojson/geojson_test_props.geojson diff --git a/pentaho-gis-plugins/src/main/java/com/atolcd/pentaho/di/gis/io/GeoJSONReader.java b/pentaho-gis-plugins/src/main/java/com/atolcd/pentaho/di/gis/io/GeoJSONReader.java index 04d5087..7075e4a 100644 --- a/pentaho-gis-plugins/src/main/java/com/atolcd/pentaho/di/gis/io/GeoJSONReader.java +++ b/pentaho-gis-plugins/src/main/java/com/atolcd/pentaho/di/gis/io/GeoJSONReader.java @@ -26,6 +26,7 @@ import java.io.File; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -65,8 +66,7 @@ public GeoJSONReader(String fileName, String geometryFieldName, String charsetNa if (this.json instanceof FeatureCollection) { - org.wololo.geojson.Feature geoJsonfeature = ((FeatureCollection) json).getFeatures()[0]; - for (Map.Entry entry : geoJsonfeature.getProperties().entrySet()) { + for (Map.Entry entry : selectPropertyTypes(((FeatureCollection) json)).entrySet()) { Field field = null; String fieldName = entry.getKey(); @@ -92,6 +92,14 @@ public GeoJSONReader(String fileName, String geometryFieldName, String charsetNa field = new Field(fieldName, FieldType.DATE, null, null); + }else if (value == null) { + // Should log this + field = new Field(fieldName, FieldType.STRING, null, null); + + }else { + // Should log this + field = new Field(fieldName, FieldType.STRING, null, null); + } this.fields.add(field); @@ -105,6 +113,19 @@ public GeoJSONReader(String fileName, String geometryFieldName, String charsetNa } } + private Map selectPropertyTypes(FeatureCollection collection) { + Map props = new HashMap(); + + for (org.wololo.geojson.Feature f : collection.getFeatures()) { + for (Map.Entry entry : f.getProperties().entrySet()) { + if (entry.getValue() != null) { + props.put(entry.getKey(), entry.getValue()); + } + } + } + return props; + } + public List getFeatures() { List features = new ArrayList(); diff --git a/pentaho-gis-plugins/src/main/java/com/atolcd/pentaho/di/gis/io/features/FeatureConverter.java b/pentaho-gis-plugins/src/main/java/com/atolcd/pentaho/di/gis/io/features/FeatureConverter.java index c2075e6..d9eec97 100644 --- a/pentaho-gis-plugins/src/main/java/com/atolcd/pentaho/di/gis/io/features/FeatureConverter.java +++ b/pentaho-gis-plugins/src/main/java/com/atolcd/pentaho/di/gis/io/features/FeatureConverter.java @@ -198,7 +198,8 @@ public static Object[] getRow(RowMetaInterface rowMeta, Feature feature) throws Object value = null; Object featureValue = feature.getValue(field); - if (featureValue != null) { + try { + if (featureValue != null) { if (field.getType().equals(FieldType.GEOMETRY)) { @@ -212,19 +213,34 @@ public static Object[] getRow(RowMetaInterface rowMeta, Feature feature) throws value = valueMeta.getDate(featureValue); - } else if (field.getType().equals(FieldType.DOUBLE)) { + } else if (field.getType().equals(FieldType.DOUBLE)) { - value = valueMeta.getNumber(featureValue); + try { - } else if (field.getType().equals(FieldType.LONG)) { + value = valueMeta.getNumber(Double.parseDouble(String.valueOf(featureValue))); - value = valueMeta.getInteger(Long.parseLong(String.valueOf(featureValue))); + } catch (NumberFormatException e) { - } else { - value = valueMeta.getString(String.valueOf(featureValue)); - } + value = valueMeta.getNumber(Long.valueOf((String.valueOf(featureValue))).doubleValue()); - } + } + + } else if (field.getType().equals(FieldType.LONG)) { + + value = valueMeta.getInteger(Long.parseLong(String.valueOf(featureValue))); + + } else { + + value = valueMeta.getString(String.valueOf(featureValue)); + + } + + } + } catch (NumberFormatException e) { + value = null; + } catch (KettleValueException e) { + value = null; + } row[fieldIndex] = value; diff --git a/sample-data/geojson/geojsonReadWriteTestIDs.ktr b/sample-data/geojson/geojsonReadWriteTestIDs.ktr new file mode 100644 index 0000000..efe3574 --- /dev/null +++ b/sample-data/geojson/geojsonReadWriteTestIDs.ktr @@ -0,0 +1,526 @@ + + + + geojsonReadWriteTest + + + + Normal + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2020/07/26 17:25:37.877 + - + 2020/07/26 17:25:37.877 + + N + + + + + + GIS File input + GIS File output + Y + + + + GIS File input + GisFileInput + + Y + + 1 + + none + + + GEOJSON + + + FORCE_TO_MULTIGEOMETRY + FALSE + + + ${Internal.Entry.Current.Directory}\geojson_test_ids.geojson + geometry + UTF-8 + 0 + + + + + + + + + + 96 + 80 + Y + + + + GIS File output + GisFileOutput + + Y + + 1 + + none + + + GEOJSON + + + GEOJSON_FEATURE_ID + + + + + + ${Internal.Entry.Current.Directory}\geojson_test_ids_out.geojson + geometry + UTF-8 + N + N + + + + + + + + + + 272 + 80 + Y + + + + + + + N + + diff --git a/sample-data/geojson/geojsonReadWriteTestNulls.ktr b/sample-data/geojson/geojsonReadWriteTestNulls.ktr new file mode 100644 index 0000000..148a477 --- /dev/null +++ b/sample-data/geojson/geojsonReadWriteTestNulls.ktr @@ -0,0 +1,526 @@ + + + + geojsonReadWriteTest + + + + Normal + / + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2020/07/26 17:25:37.877 + - + 2020/07/26 17:25:37.877 + + N + + + + + + GIS File input + GIS File output + Y + + + + GIS File input + GisFileInput + + Y + + 1 + + none + + + GEOJSON + + + FORCE_TO_MULTIGEOMETRY + FALSE + + + ${Internal.Entry.Current.Directory}\geojson_test_nulls.geojson + geometry + UTF-8 + 0 + + + + + + + + + + 96 + 80 + Y + + + + GIS File output + GisFileOutput + + Y + + 1 + + none + + + GEOJSON + + + GEOJSON_FEATURE_ID + + + + + + ${Internal.Entry.Current.Directory}\geojson_test_nulls_out.geojson + geometry + UTF-8 + N + N + + + + + + + + + + 272 + 80 + Y + + + + + + + N + + diff --git a/sample-data/geojson/geojsonReadWriteTestNumbers.ktr b/sample-data/geojson/geojsonReadWriteTestNumbers.ktr new file mode 100644 index 0000000..0a8d5f2 --- /dev/null +++ b/sample-data/geojson/geojsonReadWriteTestNumbers.ktr @@ -0,0 +1,526 @@ + + + + geojsonReadWriteTest + + + + Normal + / + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2020/07/26 17:25:37.877 + - + 2020/07/26 17:25:37.877 + + N + + + + + + GIS File input + GIS File output + Y + + + + GIS File input + GisFileInput + + Y + + 1 + + none + + + GEOJSON + + + FORCE_TO_MULTIGEOMETRY + FALSE + + + ${Internal.Entry.Current.Directory}\geojson_test_numbers.geojson + geometry + UTF-8 + 0 + + + + + + + + + + 96 + 80 + Y + + + + GIS File output + GisFileOutput + + Y + + 1 + + none + + + GEOJSON + + + GEOJSON_FEATURE_ID + + + + + + ${Internal.Entry.Current.Directory}\geojson_test_numbers_out.geojson + geometry + UTF-8 + N + N + + + + + + + + + + 272 + 80 + Y + + + + + + + N + + diff --git a/sample-data/geojson/geojsonReadWriteTestProps.ktr b/sample-data/geojson/geojsonReadWriteTestProps.ktr new file mode 100644 index 0000000..4fcb336 --- /dev/null +++ b/sample-data/geojson/geojsonReadWriteTestProps.ktr @@ -0,0 +1,526 @@ + + + + geojsonReadWriteTest + + + + Normal + / + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2020/07/26 17:25:37.877 + - + 2020/07/26 17:25:37.877 + + N + + + + + + GIS File input + GIS File output + Y + + + + GIS File input + GisFileInput + + Y + + 1 + + none + + + GEOJSON + + + FORCE_TO_MULTIGEOMETRY + FALSE + + + ${Internal.Entry.Current.Directory}\geojson_test_props.geojson + geometry + UTF-8 + 0 + + + + + + + + + + 96 + 80 + Y + + + + GIS File output + GisFileOutput + + Y + + 1 + + none + + + GEOJSON + + + GEOJSON_FEATURE_ID + + + + + + ${Internal.Entry.Current.Directory}\geojson_test_props_out.geojson + geometry + UTF-8 + N + N + + + + + + + + + + 272 + 80 + Y + + + + + + + N + + diff --git a/sample-data/geojson/geojson_test_ids.geojson b/sample-data/geojson/geojson_test_ids.geojson new file mode 100644 index 0000000..4b9159a --- /dev/null +++ b/sample-data/geojson/geojson_test_ids.geojson @@ -0,0 +1,13 @@ +{ +"type": "FeatureCollection", +"name": "GeoJSON ID Test", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "id": 1, "properties": {"numeric_prop": 1, "cause": "integer ID, OK" }, "geometry": { "type": "Point", "coordinates": [ 2.357265851625572, 48.879543984750278 ] } }, +{ "type": "Feature", "properties": {"numeric_prop": 2, "cause": "missing ID" }, "geometry": { "type": "Point", "coordinates": [ 2.344267096066682, 48.853824832535921 ] } }, +{ "type": "Feature", "id": 1, "properties": {"numeric_prop": 3, "cause": "duplicate ID" }, "geometry": { "type": "Point", "coordinates": [ 2.334098578028437, 48.861485103910752 ] } }, +{ "type": "Feature", "id": null, "properties": {"numeric_prop": 4, "cause": "null ID" }, "geometry": { "type": "Point", "coordinates": [ 2.334098578028437, 48.861485103910752 ] } }, +{ "type": "Feature", "id": "abcd","properties": {"numeric_prop": 5, "cause": "string ID" }, "geometry": { "type": "Point", "coordinates": [ 2.334098578028437, 48.861485103910752 ] } }, +{ "type": "Feature", "id": "", "properties": {"numeric_prop": 6, "cause": "empty string ID" }, "geometry": { "type": "Point", "coordinates": [ 2.334098578028437, 48.861485103910752 ] } } +] +} diff --git a/sample-data/geojson/geojson_test_nulls.geojson b/sample-data/geojson/geojson_test_nulls.geojson new file mode 100644 index 0000000..46ce186 --- /dev/null +++ b/sample-data/geojson/geojson_test_nulls.geojson @@ -0,0 +1,11 @@ +{ +"type": "FeatureCollection", +"name": "GeoJSON Null Properties Test", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "numeric_prop": 1, "cause": "an integer,OK" }, "geometry": { "type": "Point", "coordinates": [ 2.357265851625572, 48.879543984750278 ] } }, +{ "type": "Feature", "properties": { "numeric_prop": "{}", "cause": "an empty object" }, "geometry": { "type": "Point", "coordinates": [ 2.344267096066682, 48.853824832535921 ] } }, +{ "type": "Feature", "properties": { "numeric_prop": null, "cause": "a null property" }, "geometry": { "type": "Point", "coordinates": [ 2.334098578028437, 48.861485103910752 ] } }, +{ "type": "Feature", "properties": { "numeric_prop": "", "cause": "an empty string" }, "geometry": { "type": "Point", "coordinates": [ 2.334098578028437, 48.861485103910752 ] } } +] +} diff --git a/sample-data/geojson/geojson_test_numbers.geojson b/sample-data/geojson/geojson_test_numbers.geojson new file mode 100644 index 0000000..42d4360 --- /dev/null +++ b/sample-data/geojson/geojson_test_numbers.geojson @@ -0,0 +1,9 @@ +{ +"type": "FeatureCollection", +"name": "GeoJSON Numbers Test", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "numeric_prop": 1, "cause": "an integer" }, "geometry": { "type": "Point", "coordinates": [ 2.357265851625572, 48.879543984750278 ] } }, +{ "type": "Feature", "properties": { "numeric_prop": 2.0, "cause": "a decimal" }, "geometry": { "type": "Point", "coordinates": [ 2.344267096066682, 48.853824832535921 ] } } +] +} diff --git a/sample-data/geojson/geojson_test_props.geojson b/sample-data/geojson/geojson_test_props.geojson new file mode 100644 index 0000000..26f2055 --- /dev/null +++ b/sample-data/geojson/geojson_test_props.geojson @@ -0,0 +1,12 @@ +{ +"type": "FeatureCollection", +"name": "GeoJSON Inconsistent Properties Test", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "numeric_prop": 1, "cause": "OK" }, "geometry": { "type": "Point", "coordinates": [ 2.357265851625572, 48.879543984750278 ] } }, +{ "type": "Feature", "properties": { "numeric_prop": 2, "cause": "extra property", "extra_prop":"extra value"}, "geometry": { "type": "Point", "coordinates": [ 2.314871605171158, 48.893936256589363 ] } }, +{ "type": "Feature", "properties": { "numeric_prop": 3, "cause": "another extra property", "extra_prop2":"extra value"}, "geometry": { "type": "Point", "coordinates": [ 2.314871605171158, 48.893936256589363 ] } }, +{ "type": "Feature", "properties": { "cause": "missing property", "extra_prop2":"extra value"}, "geometry": { "type": "Point", "coordinates": [ 2.314871605171158, 48.893936256589363 ] } }, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 2.314871605171158, 48.893936256589363 ] } } +] +}