Skip to content

Commit 515b729

Browse files
joker234joker234
authored
Merge pull request #281 from GIScience/oshdb-1.0.0-SNAPSHOT
update to OSHDB 1.0.0-RC1
2 parents 7e358a5 + baee9f0 commit 515b729

File tree

12 files changed

+227
-197
lines changed

12 files changed

+227
-197
lines changed

‎CHANGELOG.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,15 @@ Changelog
33

44
## 1.8.0-SNAPSHOT (current master)
55

6+
### Breaking Changes
7+
* remove caching command line parameter ([#281])
8+
* update to OSHDB 1.0.0-SNAPSHOT ([#281])
9+
10+
### New Features
11+
* tag translator parameter are now configurable via CLI parameter: `tt.maxbytesvalue`, `tt.maxnumroles` ([#281])
12+
13+
[#281]: https://github.com/GIScience/ohsome-api/pull/281
14+
615

716
## 1.7.0
817

‎README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,6 @@ Now you should have a running local API, which is ready for receiving requests u
131131
*Note:*
132132
* additionally you can add optional run-parameters:
133133
* to disable multithreading: `--database.multithreading=false`
134-
* to enable in-memory-caching: `--database.caching=true` (caution.. enabling this option requires quite some memory, but makes processing much faster)
135134
* if you want to run the maven project in your IDE, you need to set the paths to your data in the run configurations
136135
* in Eclipse: *Run As --> Run Configurations --> (x)= Arguments --> Program arguments: 'enter the parameters here'*
137136
* if you want to get information about the code directly, you can access the [Javadoc](https://docs.ohsome.org/java/ohsome-api/), which gets updated daily.

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
<jts2geojson.version>0.17.0</jts2geojson.version>
3636
<mavenjar.version>3.2.0</mavenjar.version>
3737
<opencsv.version>5.6</opencsv.version>
38-
<oshdb.version>1.0.0-beta-1</oshdb.version>
38+
<oshdb.version>1.0.0-RC1</oshdb.version>
3939
<projectlombok.version>1.18.16</projectlombok.version>
4040
<sonar.sources>src/main/lombok</sonar.sources>
4141
<springboot.version>2.5.14</springboot.version>

‎src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java‎

Lines changed: 2 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,11 @@
11
package org.heigit.ohsome.ohsomeapi;
22

3-
import com.zaxxer.hikari.HikariConfig;
43
import java.io.IOException;
5-
import java.sql.DriverManager;
64
import java.sql.SQLException;
75
import java.util.Properties;
86
import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException;
9-
import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages;
10-
import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData;
11-
import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData;
12-
import org.heigit.ohsome.ohsomeapi.oshdb.RemoteTagTranslator;
13-
import org.heigit.ohsome.ohsomeapi.utils.RequestUtils;
14-
import org.heigit.ohsome.oshdb.api.db.OSHDBH2;
15-
import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite;
16-
import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc;
7+
import org.heigit.ohsome.ohsomeapi.utils.ConfigureApplication;
178
import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException;
18-
import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator;
199
import org.springframework.boot.ApplicationArguments;
2010
import org.springframework.boot.ApplicationRunner;
2111
import org.springframework.boot.DefaultApplicationArguments;
@@ -84,125 +74,7 @@ public static void main(String[] args) {
8474
*/
8575
public static void preRun(ApplicationArguments args)
8676
throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException {
87-
final String dbProperty = "database.db";
88-
boolean multithreading = true;
89-
boolean caching = false;
90-
String dbPrefix = null;
91-
long timeoutInMilliseconds = DEFAULT_TIMEOUT_IN_MILLISECONDS;
92-
int numberOfClusterNodes = DEFAULT_NUMBER_OF_CLUSTER_NODES;
93-
int numberOfDataExtractionThreads = DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS;
94-
// only used when tests are executed directly in Eclipse
95-
if (System.getProperty(dbProperty) != null) {
96-
DbConnData.db = new OSHDBH2(System.getProperty(dbProperty));
97-
}
98-
for (String paramName : args.getOptionNames()) {
99-
switch (paramName) {
100-
case dbProperty:
101-
DbConnData.db = new OSHDBH2(args.getOptionValues(paramName).get(0));
102-
break;
103-
case "database.jdbc":
104-
String[] jdbcParam = args.getOptionValues(paramName).get(0).split(";");
105-
DbConnData.db = new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]);
106-
break;
107-
case "database.ignite":
108-
if (DbConnData.db != null) {
109-
break;
110-
}
111-
DbConnData.db = new OSHDBIgnite(args.getOptionValues(paramName).get(0));
112-
break;
113-
case "database.keytables":
114-
DbConnData.keytables = new OSHDBH2(args.getOptionValues(paramName).get(0));
115-
break;
116-
case "database.keytables.jdbc":
117-
jdbcParam = args.getOptionValues(paramName).get(0).split(";");
118-
DbConnData.keytables =
119-
new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]);
120-
DbConnData.mapTagTranslator = new RemoteTagTranslator(() -> {
121-
try {
122-
Class.forName(jdbcParam[0]);
123-
return new TagTranslator(
124-
DriverManager.getConnection(jdbcParam[1], jdbcParam[2], jdbcParam[3]));
125-
} catch (ClassNotFoundException e) {
126-
throw new RuntimeException("A class with this specific name could not be found");
127-
} catch (OSHDBKeytablesNotFoundException | SQLException e) {
128-
throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS);
129-
}
130-
});
131-
HikariConfig hikariConfig = new HikariConfig();
132-
hikariConfig.setJdbcUrl(jdbcParam[1]);
133-
hikariConfig.setUsername(jdbcParam[2]);
134-
hikariConfig.setPassword(jdbcParam[3]);
135-
hikariConfig.setMaximumPoolSize(numberOfDataExtractionThreads);
136-
DbConnData.keytablesDbPoolConfig = hikariConfig;
137-
break;
138-
case "database.multithreading":
139-
if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("false")) {
140-
multithreading = false;
141-
}
142-
break;
143-
case "database.caching":
144-
if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("true")) {
145-
caching = true;
146-
}
147-
break;
148-
case "database.prefix":
149-
dbPrefix = args.getOptionValues(paramName).get(0);
150-
break;
151-
case "database.timeout":
152-
timeoutInMilliseconds = Long.parseLong(args.getOptionValues(paramName).get(0));
153-
break;
154-
case "cluster.servernodes.count":
155-
numberOfClusterNodes = Integer.parseInt(args.getOptionValues(paramName).get(0));
156-
break;
157-
case "cluster.dataextraction.threadcount":
158-
numberOfDataExtractionThreads = Integer.parseInt(args.getOptionValues(paramName).get(0));
159-
break;
160-
default:
161-
break;
162-
}
163-
}
164-
if (DbConnData.db == null) {
165-
throw new RuntimeException(
166-
"You have to define one of the following three database parameters: '--database.db', "
167-
+ "'--database.ignite', or '--database.jdbc'.");
168-
}
169-
ProcessingData.setTimeout(timeoutInMilliseconds / 1000.0);
170-
DbConnData.db.timeoutInMilliseconds(timeoutInMilliseconds);
171-
ProcessingData.setNumberOfClusterNodes(numberOfClusterNodes);
172-
ProcessingData.setNumberOfDataExtractionThreads(numberOfDataExtractionThreads);
173-
if (DbConnData.db instanceof OSHDBJdbc) {
174-
DbConnData.db = ((OSHDBJdbc) DbConnData.db).multithreading(multithreading);
175-
}
176-
if (DbConnData.db instanceof OSHDBH2) {
177-
DbConnData.db = ((OSHDBH2) DbConnData.db).inMemory(caching);
178-
}
179-
if (DbConnData.keytables != null) {
180-
DbConnData.tagTranslator = new TagTranslator(DbConnData.keytables.getConnection());
181-
} else {
182-
if (!(DbConnData.db instanceof OSHDBJdbc)) {
183-
throw new DatabaseAccessException("Missing keytables.");
184-
}
185-
DbConnData.tagTranslator = new TagTranslator(((OSHDBJdbc) DbConnData.db).getConnection());
186-
}
187-
RequestUtils.extractOSHDBMetadata();
188-
if (DbConnData.mapTagTranslator == null) {
189-
DbConnData.mapTagTranslator = new RemoteTagTranslator(DbConnData.tagTranslator);
190-
}
191-
if (DbConnData.db instanceof OSHDBIgnite) {
192-
RemoteTagTranslator mtt = DbConnData.mapTagTranslator;
193-
((OSHDBIgnite) DbConnData.db).onClose(() -> {
194-
try {
195-
if (mtt.wasEvaluated()) {
196-
mtt.get().getConnection().close();
197-
}
198-
} catch (SQLException e) {
199-
throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS);
200-
}
201-
});
202-
}
203-
if (dbPrefix != null) {
204-
DbConnData.db = DbConnData.db.prefix(dbPrefix);
205-
}
77+
ConfigureApplication.parseArguments(args);
20678
}
20779

20880
@Override

‎src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.heigit.ohsome.oshdb.filter.FilterParser;
5555
import org.heigit.ohsome.oshdb.osm.OSMEntity;
5656
import org.heigit.ohsome.oshdb.osm.OSMType;
57+
import org.heigit.ohsome.oshdb.util.OSHDBTagKey;
5758
import org.heigit.ohsome.oshdb.util.celliterator.ContributionType;
5859
import org.heigit.ohsome.oshdb.util.function.SerializableFunction;
5960
import org.heigit.ohsome.oshdb.util.geometry.Geo;
@@ -180,10 +181,10 @@ public static <P extends Geometry & Polygonal> Response aggregateGroupByBoundary
180181
TagTranslator tt = DbConnData.tagTranslator;
181182
Integer[] valuesInt = new Integer[groupByValues.length];
182183
ArrayList<Pair<Integer, Integer>> zeroFill = new ArrayList<>();
183-
int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).toInt();
184+
int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1);
184185
if (groupByValues.length != 0) {
185186
for (int j = 0; j < groupByValues.length; j++) {
186-
valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).getValue();
187+
valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j);
187188
zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j]));
188189
}
189190
}
@@ -217,7 +218,7 @@ public static <P extends Geometry & Polygonal> Response aggregateGroupByBoundary
217218
String tagIdentifier;
218219
// check for non-remainder objects (which do have the defined key and value)
219220
if (entry.getKey().getSecondIndex().getKey() != -1 && tagValue != -1) {
220-
tagIdentifier = tt.getOSMTagOf(keysInt, tagValue).toString();
221+
tagIdentifier = tt.lookupTag(keysInt, tagValue).toString();
221222
} else {
222223
tagIdentifier = "remainder";
223224
}
@@ -287,10 +288,10 @@ public static Response aggregateGroupByTag(RequestResource requestResource,
287288
TagTranslator tt = DbConnData.tagTranslator;
288289
Integer[] valuesInt = new Integer[groupByValues.length];
289290
ArrayList<Pair<Integer, Integer>> zeroFill = new ArrayList<>();
290-
int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).toInt();
291+
int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1);
291292
if (groupByValues.length != 0) {
292293
for (int j = 0; j < groupByValues.length; j++) {
293-
valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).getValue();
294+
valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j);
294295
zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j]));
295296
}
296297
}
@@ -307,7 +308,7 @@ public static Response aggregateGroupByTag(RequestResource requestResource,
307308
entry.getValue(), requestParameters.isDensity(), df, geom);
308309
// check for non-remainder objects (which do have the defined key and value)
309310
if (entry.getKey().getKey() != -1 && entry.getKey().getValue() != -1) {
310-
groupByName = tt.getOSMTagOf(keysInt, entry.getKey().getValue()).toString();
311+
groupByName = tt.lookupTag(keysInt, entry.getKey().getValue()).toString();
311312
} else {
312313
groupByName = "remainder";
313314
}
@@ -430,7 +431,7 @@ public static Response aggregateGroupByKey(RequestResource requestResource,
430431
TagTranslator tt = DbConnData.tagTranslator;
431432
Integer[] keysInt = new Integer[groupByKeys.length];
432433
for (int i = 0; i < groupByKeys.length; i++) {
433-
keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).toInt();
434+
keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).map(OSHDBTagKey::toInt).orElse(-i);
434435
}
435436
MapAggregator<OSHDBCombinedIndex<OSHDBTimestamp, Integer>, OSMEntitySnapshot> preResult =
436437
mapRed.flatMap(f -> {
@@ -460,7 +461,7 @@ public static Response aggregateGroupByKey(RequestResource requestResource,
460461
entry.getValue(), requestParameters.isDensity(), df, null);
461462
// check for non-remainder objects (which do have the defined key)
462463
if (entry.getKey() != -1) {
463-
groupByName = tt.getOSMTagKeyOf(entry.getKey().intValue()).toString();
464+
groupByName = tt.lookupTag(entry.getKey(), 0).getKey();
464465
} else {
465466
groupByName = "remainder";
466467
}
@@ -530,17 +531,18 @@ public static Response aggregateBasicFiltersRatio(RequestResource requestResourc
530531
Integer[] keysInt2 = new Integer[keys2.length];
531532
Integer[] valuesInt2 = new Integer[values2.length];
532533
for (int i = 0; i < requestParameters.getKeys().length; i++) {
533-
keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).toInt();
534+
keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).map(OSHDBTagKey::toInt)
535+
.orElse(-i);
534536
if (requestParameters.getValues() != null && i < requestParameters.getValues().length) {
535537
valuesInt1[i] =
536538
tt.getOSHDBTagOf(requestParameters.getKeys()[i], requestParameters.getValues()[i])
537-
.getValue();
539+
.map(OSHDBTag::getValue).orElse(-i);
538540
}
539541
}
540542
for (int i = 0; i < keys2.length; i++) {
541-
keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).toInt();
543+
keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).map(OSHDBTagKey::toInt).orElse(-i);
542544
if (i < values2.length) {
543-
valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).getValue();
545+
valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).map(OSHDBTag::getValue).orElse(-i);
544546
}
545547
}
546548
EnumSet<OSMType> osmTypes1 =
@@ -780,17 +782,18 @@ public static <P extends Geometry & Polygonal> Response aggregateBasicFiltersRat
780782
Integer[] valuesInt2 = new Integer[values2.length];
781783
TagTranslator tt = DbConnData.tagTranslator;
782784
for (int i = 0; i < requestParameters.getKeys().length; i++) {
783-
keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).toInt();
785+
keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).map(OSHDBTagKey::toInt)
786+
.orElse(-i);
784787
if (requestParameters.getValues() != null && i < requestParameters.getValues().length) {
785788
valuesInt1[i] =
786789
tt.getOSHDBTagOf(requestParameters.getKeys()[i], requestParameters.getValues()[i])
787-
.getValue();
790+
.map(OSHDBTag::getValue).orElse(-i);
788791
}
789792
}
790793
for (int i = 0; i < keys2.length; i++) {
791-
keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).toInt();
794+
keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).map(OSHDBTagKey::toInt).orElse(-i);
792795
if (i < values2.length) {
793-
valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).getValue();
796+
valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).map(OSHDBTag::getValue).orElse(-i);
794797
}
795798
}
796799
EnumSet<OSMType> osmTypes1 = (EnumSet<OSMType>) processingData.getOsmTypes();

0 commit comments

Comments
 (0)