diff --git a/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java b/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java index 910cd0d946..a51180e7b5 100644 --- a/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java +++ b/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java @@ -31,6 +31,7 @@ public class BuiltinStorages { public static final String SEQUENCE_FILE = "SEQUENCEFILE"; public static final String AVRO = "AVRO"; public static final String HBASE = "HBASE"; + public static final String MONGODB = "MONGODB"; public static final String SYSTEM = "SYSTEM"; public static final String EX_HTTP_JSON = "EX_HTTP_JSON"; public static final String KAFKA = "KAFKA"; diff --git a/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst b/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst new file mode 100644 index 0000000000..642b225f8c --- /dev/null +++ b/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst @@ -0,0 +1,49 @@ +******************* +MongoDB Integration +******************* + +Overview +======== + +Apache Tajo™ storage modules contains a storage plugin for MongoDB which allows Apache Tajo to access databases and collections in MongoDB. In order to use MongoDB storage plugin user needs to follow following steps. + +* Download Mongo Java Driver 3.2.2 and add to the class path. +* Setup a MongoDB server in a localhost or another. +* Configure the cluster using storage-site.json + +Download MongoDB Java Driver +========================== + +You can download latest version of MongoDB java driver from https://mongodb.github.io/mongo-java-driver/ +Download the .jar file and place it in the class path. You can place the file in TAJO_HOME or you can place the .jar file somewhere else and configure the CLASS_PATH variable from conf/tajo-env.sh (or tajo-env.cmd). + +Setup a Mongo Server +==================== +In order to proceed you need to have a running mongo server instance. If you are new to MongoDB just follow the Getting Started Guide(https://docs.mongodb.com/getting-started/shell/). After setting up the server, find the host and port to connect to the particular server. If it is a local installation by default, the port is 27017. + + +Configure the cluster +===================== + +MongoDB storage handler configuration will be like this. +First you have to register the StorageHandler.Then register the space. Update the storage-site.json as following. + +.. code-block:: json + + { + "storages": { + "mongodb": { + "handler": "org.apache.tajo.storage.mongodb.MongoDBTableSpace", + "default-format": "text" + } + }, + "spaces": { + "space_name": { + "uri": "mongodb://:/?user=&password=", + "configs": { + "mapped_database": "" + } + } + } + } + diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 27df7f8184..66eb834d01 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -255,6 +255,7 @@ public Expr visitCreateTable(Context context, Stack stack, CreateTable exp if (expr.hasSelfDescSchema()) { // TODO: support other types like Parquet and ORC. if (!expr.getStorageType().equalsIgnoreCase(BuiltinStorages.JSON) && + !expr.getStorageType().equalsIgnoreCase(BuiltinStorages.MONGODB) && !expr.getStorageType().equalsIgnoreCase(BuiltinStorages.EX_HTTP_JSON)) { if (expr.getStorageType().equalsIgnoreCase(BuiltinStorages.PARQUET) || expr.getStorageType().equalsIgnoreCase(BuiltinStorages.ORC)) { diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml index 396484f08e..7a9652dad3 100644 --- a/tajo-storage/pom.xml +++ b/tajo-storage/pom.xml @@ -40,6 +40,7 @@ tajo-storage-jdbc tajo-storage-pgsql tajo-storage-s3 + tajo-storage-mongodb tajo-storage-kafka @@ -138,8 +139,8 @@ run cp -r ${basedir}/tajo-storage-hdfs/target/tajo-storage-hdfs-${project.version}*.jar . run cp -r ${basedir}/tajo-storage-hbase/target/tajo-storage-hbase-${project.version}*.jar . run cp -r ${basedir}/tajo-storage-s3/target/tajo-storage-s3-${project.version}*.jar . + run cp -r ${basedir}/tajo-storage-mongodb/target/tajo-storage-mongodb-${project.version}*.jar . run cp -r ${basedir}/tajo-storage-kafka/target/tajo-storage-kafka-${project.version}*.jar . - echo echo "Tajo Storage dist layout available at: ${project.build.directory}/tajo-storage-${project.version}" echo diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json index a0b428049c..fdc962e089 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json @@ -32,6 +32,10 @@ "handler": "org.apache.tajo.storage.s3.S3TableSpace", "default-format": "text" }, + "mongodb": { + "handler": "org.apache.tajo.storage.mongodb.MongoDBTableSpace", + "default-format": "text" + }, "kafka": { "handler": "org.apache.tajo.storage.kafka.KafkaTablespace", "default-format": "kafka" diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml index 7fd57cd751..774f8c05f2 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml @@ -39,7 +39,7 @@ tajo.storage.scanner-handler - text,json,regex,raw,draw,rcfile,row,parquet,orc,sequencefile,avro,hbase,ex_http_json,kafka + text,json,regex,raw,draw,rcfile,row,parquet,orc,sequencefile,avro,hbase,mongodb,ex_http_json,kafka @@ -55,6 +55,10 @@ tajo.storage.fragment.kind.jdbc org.apache.tajo.storage.jdbc.JdbcFragment + + tajo.storage.fragment.kind.mongodb + org.apache.tajo.storage.mongodb.MongoDBFragment + tajo.storage.fragment.kind.example-http org.apache.tajo.storage.http.ExampleHttpFileFragment @@ -75,6 +79,10 @@ tajo.storage.fragment.serde.jdbc org.apache.tajo.storage.jdbc.JdbcFragmentSerde + + tajo.storage.fragment.serde.mongodb + org.apache.tajo.storage.mongodb.MongoDBFragmentSerde + tajo.storage.fragment.serde.example-http org.apache.tajo.storage.http.ExampleHttpFileFragmentSerde @@ -144,7 +152,10 @@ tajo.storage.scanner-handler.hbase.class org.apache.tajo.storage.hbase.HBaseScanner - + + tajo.storage.scanner-handler.mongodb.class + org.apache.tajo.storage.mongodb.MongoDBScanner + tajo.storage.scanner-handler.ex_http_json.class org.apache.tajo.storage.http.ExampleHttpJsonScanner diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml new file mode 100644 index 0000000000..9d6ae363e1 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -0,0 +1,269 @@ + + + + + tajo-project + org.apache.tajo + 0.12.0-SNAPSHOT + ../../tajo-project + + 4.0.0 + + tajo-storage-mongodb + jar + Tajo MongoDB Storage + + UTF-8 + UTF-8 + + + + + + + org.apache.rat + apache-rat-plugin + + + derby.log + src/test/resources/** + + + + + verify + + check + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + TRUE + + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-protobuf-generated-sources-directory + initialize + + + + + + + run + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2 + + + generate-sources + generate-sources + + protoc + + -Isrc/main/proto/ + --proto_path=../../tajo-common/src/main/proto + --proto_path=../../tajo-catalog/tajo-catalog-common/src/main/proto + --java_out=target/generated-sources/proto + src/main/proto/MongoDBFragmentProtos.proto + + + + exec + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + add-source + generate-sources + + add-source + + + + target/generated-sources/proto + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + + + + + + + + + + + org.apache.tajo + tajo-common + provided + + + org.apache.tajo + tajo-catalog-common + provided + + + org.apache.tajo + tajo-plan + provided + + + org.apache.tajo + tajo-storage-common + provided + + + org.apache.tajo + tajo-storage-hdfs + provided + + + org.apache.tajo + tajo-rpc-common + + + org.apache.tajo + tajo-cluster-tests + test-jar + test + + + junit + junit + test + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + netty-all + io.netty + + + + + org.apache.hadoop + hadoop-minicluster + test + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + netty-all + io.netty + + + + + org.mongodb + mongo-java-driver + 3.3.0 + + + de.flapdoodle.embed + de.flapdoodle.embed.mongo + 1.50.5 + + + + + + \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java new file mode 100644 index 0000000000..441b006a43 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + + +import com.mongodb.MongoClientURI; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.exception.TajoInternalError; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +/* +ConnectionInfo keeps the details about the mongodb server connection. +MongoURI, db credentials and URI specific details such as schema, host, port + */ + +public class ConnectionInfo { + + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); + + private MongoClientURI mongoDBURI; + private String scheme; + private String host; + private String dbName; + private String tableName; + private String user; + private String password; + private int port; + private Map params; + + //To create an instance using provided string of a URI + public static ConnectionInfo fromURI(String originalUri) { + return fromURI(URI.create(originalUri)); + } + + //creates a instance using provided URI + public static ConnectionInfo fromURI(URI originalUri) { + final String uriStr = originalUri.toASCIIString(); + URI uri = originalUri; + + final ConnectionInfo connInfo = new ConnectionInfo(); + connInfo.scheme = uriStr.substring(0, uriStr.indexOf("://")); + + connInfo.host = uri.getHost(); + connInfo.port = uri.getPort(); + + //Set the db name + String path = uri.getPath(); + if (path != null && !path.isEmpty()) { + String[] pathElements = path.substring(1).split("/"); + if (pathElements.length != 1) { + throw new TajoInternalError("Invalid JDBC path: " + path); + } + connInfo.dbName = pathElements[0]; + } + + //Convert parms into a Map + Map params = new HashMap<>(); + int paramIndex = uriStr.indexOf("?"); + if (paramIndex > 0) { + String parameterPart = uriStr.substring(paramIndex + 1, uriStr.length()); + + String[] eachParam = parameterPart.split("&"); + + for (String each : eachParam) { + String[] keyValues = each.split("="); + if (keyValues.length != 2) { + throw new TajoInternalError("Invalid URI Parameters: " + parameterPart); + } + params.put(keyValues[0], keyValues[1]); + } + } + + if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_TABLE)) { + connInfo.tableName = params.remove(MongoDBTableSpace.CONFIG_KEY_TABLE); + } + + if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_USERNAME)) { + connInfo.user = params.remove(MongoDBTableSpace.CONFIG_KEY_USERNAME); + } + if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_PASSWORD)) { + connInfo.password = params.remove(MongoDBTableSpace.CONFIG_KEY_PASSWORD); + } + + connInfo.params = params; + + String mongoDbURIStr = ""; + + //Generate the MongoURI + mongoDbURIStr += connInfo.getScheme(); + mongoDbURIStr += "://"; + if (connInfo.getUser() != null) { + mongoDbURIStr += connInfo.getUser(); + if (connInfo.getPassword() != null) + mongoDbURIStr += ":" + connInfo.getPassword(); + mongoDbURIStr += "@"; + } + mongoDbURIStr += connInfo.getHost(); + mongoDbURIStr += ":"; + mongoDbURIStr += connInfo.getPort(); + mongoDbURIStr += "/"; + mongoDbURIStr += connInfo.getDbName(); + + LOG.info(mongoDbURIStr); + connInfo.mongoDBURI = new MongoClientURI(mongoDbURIStr); + return connInfo; + } + + public MongoClientURI getMongoDBURI() { + return mongoDBURI; + } + + public String getScheme() { + return scheme; + } + + public String getHost() { + return host; + } + + public String getDbName() { + return dbName; + } + + public String getTableName() { + return tableName; + } + + public String getUser() { + return user; + } + + public String getPassword() { + return password; + } + + public int getPort() { + return port; + } + + public Map getParams() { + return params; + } +} + + diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java new file mode 100644 index 0000000000..7eb0f563c3 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import com.mongodb.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import io.netty.util.CharsetUtil; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.VTuple; +import org.apache.tajo.storage.text.TextLineParsingError; +import org.bson.Document; + +import java.io.IOException; +import java.nio.charset.CharsetEncoder; +import java.util.ArrayList; +import java.util.List; + +/* + Reads data from a mongodb collection, line by line. + Used within the MongoScanner to read tuples. + */ +public class MongoDBCollectionReader { + private final CharsetEncoder encoder = CharsetUtil.getEncoder(CharsetUtil.UTF_8); + List documentList; + private ConnectionInfo connectionInfo; + private MongoDBDocumentDeserializer deserializer; + private int targetLength; + private int currentIndex; + + + public MongoDBCollectionReader(ConnectionInfo connectionInfo, MongoDBDocumentDeserializer deserializer, int targetLength) { + this.connectionInfo = connectionInfo; + this.deserializer = deserializer; + this.targetLength = targetLength; + } + + public void init() throws IOException { + currentIndex = 0; + MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); + MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); + + MongoCollection collection = db.getCollection(connectionInfo.getTableName()); + documentList = collection.find().into( + new ArrayList<>()); + + deserializer.init(); + } + + public Tuple readTuple() throws IOException, TextLineParsingError { + if (currentIndex >= documentList.size()) return null; + + Tuple outTuple = new VTuple(targetLength); + + deserializer.deserialize(documentList.get(currentIndex), outTuple); + currentIndex++; + return outTuple; + + } + + public float getProgress() { + return ((float) currentIndex) / documentList.size(); + } + +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java new file mode 100644 index 0000000000..61a4bf651f --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java @@ -0,0 +1,243 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mongodb; + + +import com.google.common.collect.Lists; +import io.netty.util.CharsetUtil; +import net.minidev.json.JSONObject; +import net.minidev.json.parser.JSONParser; +import net.minidev.json.parser.ParseException; +import org.apache.commons.net.util.Base64; +import org.apache.tajo.catalog.*; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.storage.StorageUtil; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.text.TextLineParsingError; +import org.bson.Document; + +import java.io.IOException; +import java.nio.charset.CharsetDecoder; +import java.util.Map; +import java.util.TimeZone; + +/* +* Used inside the Collection Reader to map mongo documents to tuples. +* Use org.apache.tajo.storage.json.JsonLineDeserializer to understand the structure +* */ + +public class MongoDBDocumentDeserializer { + protected final Schema schema; + protected final TableMeta meta; + // Full Path -> Type + private final Map types; + private final String[] projectedPaths; + private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8); + private final TimeZone timezone; + private JSONParser parser; + + public MongoDBDocumentDeserializer(Schema schema, TableMeta meta, Column[] projected) { + this.schema = schema; + this.meta = meta; + + projectedPaths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(projected), true); + types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths); + + timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, + StorageUtil.TAJO_CONF.getSystemTimezone().getID())); + } + + + //Initialize and setup parser + public void init() { + parser = new JSONParser(JSONParser.MODE_JSON_SIMPLE | JSONParser.IGNORE_CONTROL_CHAR); + } + + /** + * @param object + * @param pathElements + * @param depth + * @param fieldIndex + * @param output + * @throws IOException + */ + private void getValue(JSONObject object, + String fullPath, + String[] pathElements, + int depth, + int fieldIndex, + Tuple output) throws IOException { + String fieldName = pathElements[depth]; + + if (!object.containsKey(fieldName)) { + output.put(fieldIndex, NullDatum.get()); + } + + switch (types.get(fullPath)) { + case BOOLEAN: + String boolStr = object.getAsString(fieldName); + if (boolStr != null) { + output.put(fieldIndex, DatumFactory.createBool(boolStr.equals("true"))); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case CHAR: + String charStr = object.getAsString(fieldName); + if (charStr != null) { + output.put(fieldIndex, DatumFactory.createChar(charStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case INT1: + case INT2: + Number int2Num = object.getAsNumber(fieldName); + if (int2Num != null) { + output.put(fieldIndex, DatumFactory.createInt2(int2Num.shortValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case INT4: + Number int4Num = object.getAsNumber(fieldName); + if (int4Num != null) { + output.put(fieldIndex, DatumFactory.createInt4(int4Num.intValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case INT8: + Number int8Num = object.getAsNumber(fieldName); + if (int8Num != null) { + output.put(fieldIndex, DatumFactory.createInt8(int8Num.longValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case FLOAT4: + Number float4Num = object.getAsNumber(fieldName); + if (float4Num != null) { + output.put(fieldIndex, DatumFactory.createFloat4(float4Num.floatValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case FLOAT8: + Number float8Num = object.getAsNumber(fieldName); + if (float8Num != null) { + output.put(fieldIndex, DatumFactory.createFloat8(float8Num.doubleValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case TEXT: + String textStr = object.getAsString(fieldName); + if (textStr != null) { + output.put(fieldIndex, DatumFactory.createText(textStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case TIMESTAMP: + String timestampStr = object.getAsString(fieldName); + if (timestampStr != null) { + output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr, timezone)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case TIME: + String timeStr = object.getAsString(fieldName); + if (timeStr != null) { + output.put(fieldIndex, DatumFactory.createTime(timeStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case DATE: + String dateStr = object.getAsString(fieldName); + if (dateStr != null) { + output.put(fieldIndex, DatumFactory.createDate(dateStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case BIT: + case BINARY: + case VARBINARY: + case BLOB: { + Object jsonObject = object.getAsString(fieldName); + + if (jsonObject == null) { + output.put(fieldIndex, NullDatum.get()); + break; + } + + output.put(fieldIndex, DatumFactory.createBlob(Base64.decodeBase64((String) jsonObject))); + break; + } + + case RECORD: + JSONObject nestedObject = (JSONObject) object.get(fieldName); + if (nestedObject != null) { + getValue(nestedObject, fullPath + "/" + pathElements[depth + 1], pathElements, depth + 1, fieldIndex, output); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + + case NULL_TYPE: + output.put(fieldIndex, NullDatum.get()); + break; + + default: + throw new TajoRuntimeException( + new NotImplementedException("" + types.get(fullPath).name() + " for json")); + } + } + + + //Map the given documents into the given tuple and fill with calues + public void deserialize(Document doc, Tuple output) throws IOException, TextLineParsingError { + + String line = doc.toJson(); + JSONObject object; + try { + object = (JSONObject) parser.parse(line); + } catch (ParseException pe) { + throw new TextLineParsingError(line, pe); + } catch (ArrayIndexOutOfBoundsException ae) { + // truncated value + throw new TextLineParsingError(line, ae); + } + + for (int i = 0; i < projectedPaths.length; i++) { + String[] paths = projectedPaths[i].split(NestedPathUtil.PATH_DELIMITER); + getValue(object, paths[0], paths, 0, i, output); + + //output.put(i, DatumFactory.createText( doc.get(projectedPaths[i]).toString() )); + } + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java new file mode 100644 index 0000000000..3775f32ed7 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import org.apache.tajo.storage.fragment.Fragment; + +import java.net.URI; + +/* + Fragment is similar to splits in Map Reduce. + Which contains enough details to access, a relevant section from the storage. + This will be serialized and passed to the cluster. + */ + +public class MongoDBFragment extends Fragment { + + + protected MongoDBFragment(URI uri, + String inputSourceId, + long startKey, + long endKey) { + super("MONGODB", uri, inputSourceId, startKey, endKey, endKey - startKey, null); + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java new file mode 100644 index 0000000000..20a7c3c282 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mongodb; + +import com.google.protobuf.GeneratedMessage.Builder; +import org.apache.tajo.storage.fragment.FragmentSerde; +import org.apache.tajo.storage.mongodb.MongoDBFragmentProtos.MongoDBFragmentProto; + +import java.net.URI; + +/** + * Fragment Serde for MongoFragment. Which is used to serialize and deserialize + * MongoDBFragment objects using ProtocolBuffer. + */ +public class MongoDBFragmentSerde implements FragmentSerde { + + @Override + public Builder newBuilder() { + return MongoDBFragmentProto.newBuilder(); + } + + @Override + public MongoDBFragmentProto serialize(MongoDBFragment fragment) { + return MongoDBFragmentProto.newBuilder() + .setUri(fragment.getUri().toASCIIString()) + .setTableName(fragment.getInputSourceId()) + .setStartKey(fragment.getStartKey()) + .setEndKey(fragment.getEndKey()) + .build(); + } + + @Override + public MongoDBFragment deserialize(MongoDBFragmentProto proto) { + return new MongoDBFragment( + URI.create(proto.getUri()), + proto.getTableName(), + proto.getStartKey(), + proto.getEndKey() + ); + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java new file mode 100644 index 0000000000..3908e0701f --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mongodb; + +import com.mongodb.MongoClient; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.MongoIterable; +import org.apache.tajo.catalog.MetadataProvider; +import org.apache.tajo.catalog.SchemaBuilder; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.exception.UndefinedTablespaceException; +import org.apache.tajo.schema.IdentifierUtil; +import org.apache.tajo.util.KeyValueSet; + +import javax.annotation.Nullable; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; + +/* +* Used to provide MetaData about a particular mongodb tablespace. +* Provides table details, list of tables and statistics of tables +* * */ +public class MongoDBMetadataProvider implements MetadataProvider { + + MongoDatabase db; + private MongoDBTableSpace tableSpace; + private String mappedDbName; + private ConnectionInfo connectionInfo; + + public MongoDBMetadataProvider(MongoDBTableSpace tableSpace, String dbName) { + this.tableSpace = tableSpace; + this.mappedDbName = dbName; + + connectionInfo = tableSpace.getConnectionInfo(); + MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); + db = mongoClient.getDatabase(connectionInfo.getDbName()); + } + + @Override + public String getTablespaceName() { + return tableSpace.getName(); + } + + @Override + public URI getTablespaceUri() { + return tableSpace.getUri(); + } + + @Override + public String getDatabaseName() { + return mappedDbName; + } + + @Override + public Collection getSchemas() { + return Collections.EMPTY_SET; + } + + @Override + public Collection getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { + + //Get a list of table(=collection) names + MongoIterable collectionList = db.listCollectionNames(); + + //Map to a string list and return + Collection list = new ArrayList(); + for (String item : collectionList) { + list.add(item); + } + return list; + } + + @Override + public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException { + + //Create table description and meta fro a specific table + TableMeta tbMeta = new TableMeta("mongodb", new KeyValueSet()); + TableDesc tbDesc = new TableDesc( + IdentifierUtil.buildFQName(mappedDbName, tableName), + SchemaBuilder.builder() +// .add(new Column("title", TajoDataTypes.Type.TEXT)) +// .add(new Column("first_name", TajoDataTypes.Type.TEXT)) +// .add(new Column("last_name", TajoDataTypes.Type.TEXT)) + .build(), + tbMeta, + tableSpace.getTableUri(null, null, tableName)); + + final TableStats stats = new TableStats(); + stats.setNumRows(-1); // unknown + + //Set the raw count + stats.setNumRows(db.getCollection(tableName).count()); + + + tbDesc.setStats(stats); + return tbDesc; + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java new file mode 100644 index 0000000000..f5264481c2 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import org.apache.hadoop.conf.Configuration; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.storage.Scanner; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.storage.text.TextLineParsingError; + +import java.io.IOException; + +/* +* Reads data from a MongoDB table +* Uses MongoDBCollectionReader to read tuples +* */ + +//Todo Remove FileScanner +public class MongoDBScanner implements Scanner { + + private final TableMeta meta; + private final Schema schema; + private final MongoDBFragment fragment; + private MongoDBCollectionReader collectionReader; + private Column[] targets; + + private boolean inited = false; + + public MongoDBScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { + this.schema = schema; + this.meta = meta; + this.fragment = (MongoDBFragment) fragment; + } + + @Override + public void init() throws IOException { + if (targets == null) { + targets = schema.toArray(); + } + inited = true; + reset(); + } + + @Override + public Tuple next() throws IOException { + try { + Tuple t = collectionReader.readTuple(); + return t; + } catch (TextLineParsingError textLineParsingError) { + textLineParsingError.printStackTrace(); + return null; + } + } + + @Override + public void reset() throws IOException { + MongoDBDocumentDeserializer deserializer = new MongoDBDocumentDeserializer(schema, meta, targets); + collectionReader = new MongoDBCollectionReader(ConnectionInfo.fromURI(fragment.getUri()), deserializer, targets.length); + + collectionReader.init(); + } + + @Override + public void close() throws IOException { + return; + } + + @Override + public void pushOperators(LogicalNode planPart) { + throw new TajoRuntimeException(new UnsupportedException()); + } + + @Override + public boolean isProjectable() { + return false; + } + + @Override + public void setTarget(Column[] targets) { + if (inited) { + throw new IllegalStateException("Should be called before init()"); + } + this.targets = targets; + } + + @Override + public boolean isSelectable() { + return false; + } + + @Override + public void setFilter(EvalNode filter) { + + } + + @Override + public void setLimit(long num) { + + } + + @Override + public boolean isSplittable() { + return false; + } + + @Override + public float getProgress() { + return collectionReader.getProgress(); + } + + @Override + public TableStats getInputStats() { + return null; + } + + + @Override + public Schema getSchema() { + return schema; + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java new file mode 100644 index 0000000000..3c882e601d --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mongodb; + +import com.google.common.collect.Lists; +import com.mongodb.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import net.minidev.json.JSONObject; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.ExecutionBlockId; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.catalog.*; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.schema.IdentifierUtil; +import org.apache.tajo.storage.FormatProperty; +import org.apache.tajo.storage.StorageProperty; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TupleRange; +import org.apache.tajo.storage.fragment.Fragment; +import org.bson.Document; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Optional; + +/* +* TableSpace for MongoDB +* */ + +public class MongoDBTableSpace extends Tablespace { + + //Config Keys + public static final String CONFIG_KEY_MAPPED_DATABASE = "mapped_database"; + public static final String CONFIG_KEY_CONN_PROPERTIES = "connection_properties"; + public static final String CONFIG_KEY_USERNAME = "user"; + public static final String CONFIG_KEY_PASSWORD = "password"; + public static final String CONFIG_KEY_TABLE = "table"; + //Table Space Properties + static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", // type is to be defined + false, //not movable + true, //writable at the moment + true, // Absolute path + true); // Meta data will be provided + static final FormatProperty FORMAT_PROPERTY = new FormatProperty( + true, // Insert + true, //direct insert + true);// result staging + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); + protected MongoClient mongoClient; + protected MongoDatabase db; + protected String mappedDBName; + //Mongo Client object + private ConnectionInfo connectionInfo; + + public MongoDBTableSpace(String name, URI uri, JSONObject config) { + + super(name, uri, config); + connectionInfo = ConnectionInfo.fromURI(uri); + + //set Connection Properties + if (config.containsKey(CONFIG_KEY_MAPPED_DATABASE)) { + mappedDBName = this.config.getAsString(CONFIG_KEY_MAPPED_DATABASE); + } else { + mappedDBName = getConnectionInfo().getDbName(); + } + } + + @Override + protected void storageInit() throws IOException { + //Todo Extract User Details from Configuration + try { + connectionInfo = ConnectionInfo.fromURI(uri); + mongoClient = new MongoClient(getConnectionInfo().getMongoDBURI()); + db = mongoClient.getDatabase(getConnectionInfo().getDbName()); + } catch (Exception e) { + throw new TajoInternalError(e); + } + } + + @Override + public long getTableVolume(TableDesc table, Optional filter) { + long count = 0; + try { + String[] nameSplited = IdentifierUtil.splitFQTableName(table.getName()); + count = db.getCollection(nameSplited[1]).count(); + } catch (Exception e) { + throw new TajoInternalError(e); + } + return count; + } + + + @Override + public List getSplits(String inputSourceId, TableDesc tableDesc, boolean requireSort, @Nullable EvalNode filterCondition) throws IOException, TajoException { + long tableVolume = getTableVolume(tableDesc, Optional.empty()); + MongoDBFragment mongoDBFragment = new MongoDBFragment(tableDesc.getUri(), inputSourceId, 0, tableVolume); + return Lists.newArrayList(mongoDBFragment); + } + + + @Override + public StorageProperty getProperty() { + return STORAGE_PROPERTY; + } + + @Override + public FormatProperty getFormatProperty(TableMeta meta) { + return FORMAT_PROPERTY; + } + + @Override + public void close() { + + } + + @Override + public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) throws IOException { + return new TupleRange[0]; + } + + @Override + public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) throws TajoException { + + } + + @Override + public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoException, IOException { + if (tableDesc == null) + throw new TajoRuntimeException(new NotImplementedException()); + MongoCollection table = db.getCollection(tableDesc.getName()); + + //If meta data provides. Create a table + if (STORAGE_PROPERTY.isMetadataProvided()) + db.createCollection(IdentifierUtil.extractSimpleName(tableDesc.getName())); + } + + @Override + public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { + if (STORAGE_PROPERTY.isMetadataProvided()) + db.getCollection(IdentifierUtil.extractSimpleName(tableDesc.getName())).drop(); + } + + @Override + public void prepareTable(LogicalNode node) throws IOException, TajoException { + return; + } + + @Override + public Path commitTable(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, TableDesc tableDesc) throws IOException { + return null; + } + + @Override + public void rollbackTable(LogicalNode node) throws IOException, TajoException { + + } + + @Override + public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { + return null; + } + + @Override + public URI getRootUri() { + return uri; + } + + @Override + public URI getTableUri(TableMeta meta, String databaseName, String tableName) { + //ToDo Find a better way this + String tableURI = ""; + if (this.getUri().toASCIIString().contains("?")) + tableURI = this.getUri().toASCIIString() + "&" + CONFIG_KEY_TABLE + "=" + tableName; + else + tableURI = this.getUri().toASCIIString() + "?" + CONFIG_KEY_TABLE + "=" + tableName; + + return URI.create(tableURI); + } + + //@Override + public URI getTableUri(String databaseName, String tableName) { + //ToDo set the TableURI properly + return URI.create(this.getUri() + "&" + CONFIG_KEY_TABLE + "=" + tableName); + } + + // Metadata + public MetadataProvider getMetadataProvider() { + return new MongoDBMetadataProvider(this, mappedDBName); + } + + //Return Connection info for the tablespace + public ConnectionInfo getConnectionInfo() { + return connectionInfo; + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto b/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto new file mode 100644 index 0000000000..68a7db3978 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.tajo.storage.mongodb"; +option java_outer_classname = "MongoDBFragmentProtos"; +option optimize_for = SPEED; +option java_generic_services = false; +option java_generate_equals_and_hash = true; + +import "CatalogProtos.proto"; + +message MongoDBFragmentProto { + required string uri = 1; + required string table_name = 2; + required int64 start_key = 3; + required int64 end_key = 4; +} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java new file mode 100644 index 0000000000..e47b3d4411 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + + +import com.mongodb.Block; +import com.mongodb.MongoClient; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import de.flapdoodle.embed.mongo.*; +import de.flapdoodle.embed.mongo.config.*; +import de.flapdoodle.embed.mongo.distribution.Version; +import de.flapdoodle.embed.process.runtime.Network; +import net.minidev.json.JSONObject; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.TablespaceManager; +import org.bson.Document; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; + +public class MongoDBTestServer { + + //mongo server details + public static final int PORT = 12345; + public static final String HOST = "localhost"; + public static final String DBNAME = "test_dbname"; + public static final String MAPPEDDBNAME = "test_mapped_dbname"; + //tajo tableSpace name + public static final String SPACE_NAME = "test_spacename"; + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); + //Mongo Server componenets + private static final MongodStarter starter = MongodStarter.getDefaultInstance(); + // instance for singleton server + private static MongoDBTestServer instance; + //Collection names (table names) to be created inside mongodb + public String[] collectionNames = {"github", "got"}; + private MongodExecutable _mongodExe; + private MongodProcess _mongod; + private MongoClient _mongo; + //File names to load data + private String[] filenames = {"file1.json", "file2.json"}; + + + // private constructor + private MongoDBTestServer() throws IOException, URISyntaxException { + _mongodExe = starter.prepare(new MongodConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(PORT, Network.localhostIsIPv6())) + + .cmdOptions(new MongoCmdOptionsBuilder(). + useStorageEngine("mmapv1"). + build()) + .build()); + _mongod = _mongodExe.start(); + _mongo = new MongoClient(HOST, PORT); + registerTablespace(); + + loadData(); + } + + //server object can be created using this method + public static MongoDBTestServer getInstance() { + if (instance == null) { + try { + instance = new MongoDBTestServer(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return instance; + } + + //To load files + private static File getRequestedFile(String path) throws FileNotFoundException, URISyntaxException { + + URL url = ClassLoader.getSystemResource("dataset/" + path); + + if (url == null) { + throw new FileNotFoundException(path); + } + return new File(url.toURI()); + } + + //To stop the server + public void stop() { + _mongod.stop(); + _mongodExe.stop(); + instance = null; + } + + //Returns the url which can be used to connet to the server instance + public URI getURI() { + try { + return new URI("mongodb://" + HOST + ":" + PORT + "/" + DBNAME); + } catch (Exception e) { + return null; + } + } + + //Start mongo process + private MongosProcess startMongos(int port, int defaultConfigPort, String defaultHost) throws UnknownHostException, + IOException { + IMongosConfig mongosConfig = new MongosConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(port, Network.localhostIsIPv6())) + .configDB(defaultHost + ":" + defaultConfigPort) + .build(); + + MongosExecutable mongosExecutable = MongosStarter.getDefaultInstance().prepare(mongosConfig); + MongosProcess mongos = mongosExecutable.start(); + return mongos; + } + + private MongodProcess startMongod(int defaultConfigPort) throws UnknownHostException, IOException { + IMongodConfig mongoConfigConfig = new MongodConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(defaultConfigPort, Network.localhostIsIPv6())) + .configServer(true) + .build(); + + MongodExecutable mongodExecutable = MongodStarter.getDefaultInstance().prepare(mongoConfigConfig); + MongodProcess mongod = mongodExecutable.start(); + return mongod; + } + + //Register the new table space for testing + private void registerTablespace() throws IOException { + JSONObject configElements = new JSONObject(); + configElements.put(MongoDBTableSpace.CONFIG_KEY_MAPPED_DATABASE, MAPPEDDBNAME); + + MongoDBTableSpace tablespace = new MongoDBTableSpace(SPACE_NAME, getURI(), configElements); + tablespace.init(new TajoConf()); + + + TablespaceManager.addTableSpaceForTest(tablespace); + } + + //Create tables and Load data into the mongo server instance + private void loadData() throws IOException, URISyntaxException { + MongoDatabase db = _mongo.getDatabase(DBNAME); + for (int i = 0; i < filenames.length; i++) { + + db.createCollection(collectionNames[i]); + MongoCollection coll = db.getCollection(collectionNames[i]); + + String fileContent = new Scanner(getRequestedFile(filenames[i])).useDelimiter("\\Z").next(); + + //Document list + List documentList = new ArrayList(); + try { + JSONArray jsonarray = new JSONArray(fileContent); + for (int j = 0; j < jsonarray.length(); j++) { + String jsonStr = jsonarray.getJSONObject(j).toString(); + documentList.add(Document.parse(jsonStr)); + } + + } catch (JSONException e) { + e.printStackTrace(); + } + + coll.insertMany(documentList); + + FindIterable docs = coll.find(); + + docs.forEach(new Block() { + @Override + public void apply(final Document document) { + LOG.info(document.toJson()); + } + }); + } + } + + //Return a mongo client which directly connect to the mongo database. + public MongoClient getMongoClient() { + return _mongo; + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java new file mode 100644 index 0000000000..96922865cf --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestConnectionInfo { + + @Test + public final void testGetConnectionInfoType1() { + + ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass"); + + assertEquals(connInfo.getMongoDBURI().getURI(), "mongodb://testuser:testpass@localhost:1336/db1"); + assertEquals(connInfo.getScheme(), "mongodb"); + assertEquals(connInfo.getHost(), "localhost"); + assertEquals(connInfo.getPort(), 1336); + assertEquals(connInfo.getDbName(), "db1"); + assertEquals(connInfo.getUser(), "testuser"); + assertEquals(connInfo.getPassword(), "testpass"); + assertEquals(connInfo.getTableName(), "tb1"); + } + + @Test + public final void testGetConnectionInfoType2() { + + ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); + + assertEquals(connInfo.getScheme(), "mongodb"); + assertEquals(connInfo.getHost(), "localhost"); + assertEquals(connInfo.getPort(), 1336); + assertEquals(connInfo.getDbName(), "db1"); + assertEquals(connInfo.getUser(), "testuser"); + assertEquals(connInfo.getPassword(), "testpass"); + assertEquals(connInfo.getTableName(), "tb1"); + assertEquals(1, connInfo.getParams().size()); + assertEquals("GMT+9", connInfo.getParams().get("TZ")); + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java new file mode 100644 index 0000000000..684f76786a --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import com.google.common.collect.Sets; +import org.apache.tajo.catalog.MetadataProvider; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TablespaceManager; +import org.junit.AfterClass; +import org.junit.Test; + +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestMetadataProvider { + static MongoDBTestServer server = MongoDBTestServer.getInstance(); + + + @AfterClass + public static void tearDownClass() throws Exception { + // server.stop(); + } + + @Test + public void testGetTablespaceName() throws Exception { + //check for Space Name validity + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertEquals(server.SPACE_NAME, provider.getTablespaceName()); + } + + @Test + public void testGetDatabaseName() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertEquals(MongoDBTestServer.MAPPEDDBNAME, provider.getDatabaseName()); + } + + + @Test + public void testGetSchemas() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertTrue(provider.getSchemas().isEmpty()); + } + + @Test + public void testGetTables() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + + final Set expected = Sets.newHashSet(server.collectionNames); + expected.add("system.indexes"); + final Set found = Sets.newHashSet(provider.getTables(null, null)); + + assertEquals(expected, found); + } + + @Test + public void testGetTableDescription() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + + for (String tableName : server.collectionNames) { + TableDesc table = provider.getTableDesc(null, tableName); + assertEquals(server.MAPPEDDBNAME + "." + tableName, table.getName()); + assertEquals(server.getURI() + "?table=" + tableName, table.getUri().toASCIIString()); + } + } + + +} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java new file mode 100644 index 0000000000..0cc670d8f9 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.exception.TajoException; +import org.junit.*; + +import java.net.URI; + +public class TestMongoDBQueryTest extends QueryTestCaseBase { + + static MongoDBTestServer server = MongoDBTestServer.getInstance(); + static URI uri = server.getURI(); + + public TestMongoDBQueryTest() { + super(server.MAPPEDDBNAME); + } + + @BeforeClass + public static void setup() throws Exception { + QueryTestCaseBase.testingCluster.getMaster().refresh(); + } + + @AfterClass + public static void tearDownClass() throws Exception { + server.stop(); + } + + @Before + public void prepareTables() throws TajoException { + if (!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + executeString("create table got (title,first_name,last_name) tablespace test_spacename using mongodb"); + executeString("create table github (*) tablespace test_spacename using mongodb"); + } + } + + @After + public void dropTables() throws TajoException { + if (!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + executeString("drop table got"); + executeString("drop table github"); + } + } + + + @SimpleTest + @Test + public void testSelect() throws Exception { + runSimpleTests(); + } + + + @SimpleTest + @Test + public void testSort() throws Exception { + runSimpleTests(); + } + + @SimpleTest + @Test + public void testGroupby() throws Exception { + runSimpleTests(); + } + + @SimpleTest + @Test + public void testJoin() throws Exception { + runSimpleTests(); + } + + + @SimpleTest + @Test + public void testInsert() throws Exception { + //Todo Enable when insert is supported for Metadata provided tableSpaces. + //runSimpleTests(); + } + + @SimpleTest + @Test + public void testCTAS() throws Exception { + runSimpleTests(); + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java new file mode 100644 index 0000000000..aba0186215 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import com.google.common.collect.Sets; +import com.mongodb.MongoClient; +import com.mongodb.client.MongoIterable; +import org.apache.tajo.catalog.SchemaBuilder; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.schema.IdentifierUtil; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TablespaceManager; +import org.junit.AfterClass; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.junit.Assert.*; + + +public class TestMongoDBTableSpace { + //mongodb://:@ds017231.mlab.com:17231/tajo_test + static MongoDBTestServer server = MongoDBTestServer.getInstance(); + static URI uri = server.getURI(); + + + @AfterClass + public static void tearDownClass() throws Exception { + // server.stop(); + } + + @Test + public void testTablespaceHandler() { + assertTrue((TablespaceManager.getByName(server.SPACE_NAME)) instanceof MongoDBTableSpace); + assertEquals(server.SPACE_NAME, (TablespaceManager.getByName(server.SPACE_NAME).getName())); + + assertTrue((TablespaceManager.get(uri.toASCIIString() + "&table=tb1")) instanceof MongoDBTableSpace); + assertTrue((TablespaceManager.get(uri)) instanceof MongoDBTableSpace); + + + //Test the URI same + assertEquals(uri.toASCIIString(), TablespaceManager.get(uri).getUri().toASCIIString()); + } + + @Test(timeout = 1000, expected = TajoRuntimeException.class) + public void testCreateTable() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); + space.createTable(null, false); + } + + @Test(timeout = 1000) + public void testCreateTable_and_Purg() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); + + TableDesc tableDesc = new TableDesc( + IdentifierUtil.buildFQName(server.MAPPEDDBNAME, "Table1"), + SchemaBuilder.builder() + .build(), + null, + server.getURI()); + + //Test create and delete if meta data provided + if (MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + space.createTable(tableDesc, false); + + //Check whether the created table is in the collection + final Set found = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); + assertTrue(found.contains("Table1")); + + //Check whether the created table is in the mongo database + MongoClient mongoClient = server.getMongoClient(); + Boolean foundInMongoDB = false; + MongoIterable collectionNames = mongoClient.getDatabase(server.DBNAME).listCollectionNames(); + for (final String name : collectionNames) { + if (name.equalsIgnoreCase("Table1")) { + foundInMongoDB = true; + } + } + assertTrue(foundInMongoDB); + + //Purg the table and check whether it exists + space.purgeTable(tableDesc); + final Set found_after = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); + assertFalse(found_after.contains("Table1")); + + } + } + + @Test + public void testTableVolume() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); + Map tableSizes = new HashMap(); + tableSizes.put("github", 4); + tableSizes.put("got", 5); + + //Check whether the volumes of tables are correct + for (String tbl : server.collectionNames) { + TableDesc tbDesc = new TableDesc( + IdentifierUtil.buildFQName(server.MAPPEDDBNAME, tbl), + SchemaBuilder.builder() + .build(), + null, + space.getTableUri(null, null, tbl)); + + assertEquals((int) tableSizes.get(tbl), space.getTableVolume(tbDesc, Optional.empty())); + + } + } + +} \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json new file mode 100644 index 0000000000..b6434eaeab --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json @@ -0,0 +1,194 @@ +[ + { + "id": "2937257753", + "type": "PushEvent", + "actor": { + "id": 5266949, + "login": "hardrubic", + "gravatar_id": "", + "url": "https://api.github.com/users/hardrubic", + "avatar_url": "https://avatars.githubusercontent.com/u/5266949?" + }, + "repo": { + "id": 38299397, + "name": "hardrubic/rxJavaTest", + "url": "https://api.github.com/repos/hardrubic/rxJavaTest" + }, + "payload": { + "push_id": 712081726, + "size": 1, + "distinct_size": 1, + "ref": "refs/heads/master", + "head": "ea79d7a424f2693b70b9496726f315a5711b6fe7", + "before": "613f05557ad353f4bedc6df54128f8091ed1f1e9", + "commits": [ + { + "sha": "ea79d7a424f2693b70b9496726f315a5711b6fe7", + "author": { + "email": "dgzx106@163.com", + "name": "hardrubic" + }, + "message": "增加rxJava例子", + "distinct": true, + "url": "https://api.github.com/repos/hardrubic/rxJavaTest/commits/ea79d7a424f2693b70b9496726f315a5711b6fe7" + } + ] + }, + "public": true, + "created_at": "2015-07-01T00:00:01Z" + }, + { + "id": "2937257758", + "type": "WatchEvent", + "actor": { + "id": 11455393, + "login": "chrischjh", + "gravatar_id": "", + "url": "https://api.github.com/users/chrischjh", + "avatar_url": "https://avatars.githubusercontent.com/u/11455393?" + }, + "repo": { + "id": 18218031, + "name": "dead-horse/co-and-koa-talk", + "url": "https://api.github.com/repos/dead-horse/co-and-koa-talk" + }, + "payload": { + "action": "started" + }, + "public": true, + "created_at": "2015-07-01T00:00:01Z" + }, + { + "id": "2937257759", + "type": "CreateEvent", + "actor": { + "id": 206379, + "login": "gvn", + "gravatar_id": "", + "url": "https://api.github.com/users/gvn", + "avatar_url": "https://avatars.githubusercontent.com/u/206379?" + }, + "repo": { + "id": 24345476, + "name": "gvn/webmaker-android", + "url": "https://api.github.com/repos/gvn/webmaker-android" + }, + "payload": { + "ref": "use-self-building", + "ref_type": "branch", + "master_branch": "master", + "description": "Webmaker for Firefox OS & Android", + "pusher_type": "user" + }, + "public": true, + "created_at": "2015-07-01T00:00:01Z" + }, + { + "id": "2937257761", + "type": "ForkEvent", + "actor": { + "id": 1088854, + "login": "CAOakleyII", + "gravatar_id": "", + "url": "https://api.github.com/users/CAOakleyII", + "avatar_url": "https://avatars.githubusercontent.com/u/1088854?" + }, + "repo": { + "id": 11909954, + "name": "skycocker/chromebrew", + "url": "https://api.github.com/repos/skycocker/chromebrew" + }, + "payload": { + "forkee": { + "id": 38339291, + "name": "chromebrew", + "full_name": "CAOakleyII/chromebrew", + "owner": { + "login": "CAOakleyII", + "id": 1088854, + "avatar_url": "https://avatars.githubusercontent.com/u/1088854?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/CAOakleyII", + "html_url": "https://github.com/CAOakleyII", + "followers_url": "https://api.github.com/users/CAOakleyII/followers", + "following_url": "https://api.github.com/users/CAOakleyII/following{/other_user}", + "gists_url": "https://api.github.com/users/CAOakleyII/gists{/gist_id}", + "starred_url": "https://api.github.com/users/CAOakleyII/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/CAOakleyII/subscriptions", + "organizations_url": "https://api.github.com/users/CAOakleyII/orgs", + "repos_url": "https://api.github.com/users/CAOakleyII/repos", + "events_url": "https://api.github.com/users/CAOakleyII/events{/privacy}", + "received_events_url": "https://api.github.com/users/CAOakleyII/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/CAOakleyII/chromebrew", + "description": "Package manager for Chrome OS", + "fork": true, + "url": "https://api.github.com/repos/CAOakleyII/chromebrew", + "forks_url": "https://api.github.com/repos/CAOakleyII/chromebrew/forks", + "keys_url": "https://api.github.com/repos/CAOakleyII/chromebrew/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/CAOakleyII/chromebrew/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/CAOakleyII/chromebrew/teams", + "hooks_url": "https://api.github.com/repos/CAOakleyII/chromebrew/hooks", + "issue_events_url": "https://api.github.com/repos/CAOakleyII/chromebrew/issues/events{/number}", + "events_url": "https://api.github.com/repos/CAOakleyII/chromebrew/events", + "assignees_url": "https://api.github.com/repos/CAOakleyII/chromebrew/assignees{/user}", + "branches_url": "https://api.github.com/repos/CAOakleyII/chromebrew/branches{/branch}", + "tags_url": "https://api.github.com/repos/CAOakleyII/chromebrew/tags", + "blobs_url": "https://api.github.com/repos/CAOakleyII/chromebrew/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/CAOakleyII/chromebrew/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/CAOakleyII/chromebrew/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/CAOakleyII/chromebrew/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/CAOakleyII/chromebrew/statuses/{sha}", + "languages_url": "https://api.github.com/repos/CAOakleyII/chromebrew/languages", + "stargazers_url": "https://api.github.com/repos/CAOakleyII/chromebrew/stargazers", + "contributors_url": "https://api.github.com/repos/CAOakleyII/chromebrew/contributors", + "subscribers_url": "https://api.github.com/repos/CAOakleyII/chromebrew/subscribers", + "subscription_url": "https://api.github.com/repos/CAOakleyII/chromebrew/subscription", + "commits_url": "https://api.github.com/repos/CAOakleyII/chromebrew/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/CAOakleyII/chromebrew/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/CAOakleyII/chromebrew/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/CAOakleyII/chromebrew/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/CAOakleyII/chromebrew/contents/{+path}", + "compare_url": "https://api.github.com/repos/CAOakleyII/chromebrew/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/CAOakleyII/chromebrew/merges", + "archive_url": "https://api.github.com/repos/CAOakleyII/chromebrew/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/CAOakleyII/chromebrew/downloads", + "issues_url": "https://api.github.com/repos/CAOakleyII/chromebrew/issues{/number}", + "pulls_url": "https://api.github.com/repos/CAOakleyII/chromebrew/pulls{/number}", + "milestones_url": "https://api.github.com/repos/CAOakleyII/chromebrew/milestones{/number}", + "notifications_url": "https://api.github.com/repos/CAOakleyII/chromebrew/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/CAOakleyII/chromebrew/labels{/name}", + "releases_url": "https://api.github.com/repos/CAOakleyII/chromebrew/releases{/id}", + "created_at": "2015-07-01T00:00:00Z", + "updated_at": "2015-06-28T10:11:09Z", + "pushed_at": "2015-06-09T07:46:57Z", + "git_url": "git://github.com/CAOakleyII/chromebrew.git", + "ssh_url": "git@github.com:CAOakleyII/chromebrew.git", + "clone_url": "https://github.com/CAOakleyII/chromebrew.git", + "svn_url": "https://github.com/CAOakleyII/chromebrew", + "homepage": "http://skycocker.github.io/chromebrew/", + "size": 846, + "stargazers_count": 0, + "watchers_count": 0, + "language": null, + "has_issues": false, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "open_issues_count": 0, + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "master", + "public": true + } + }, + "public": true, + "created_at": "2015-07-01T00:00:01Z" + } +] \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json new file mode 100644 index 0000000000..180bd32670 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json @@ -0,0 +1,27 @@ +[ + { + "title": "Hand of the King", + "first_name": "Eddard", + "last_name": "Stark" + }, + { + "title": "Assassin", + "first_name": "Arya", + "last_name": "Stark" + }, + { + "title": "Lady", + "first_name": "Sansa", + "last_name": "Stark" + }, + { + "title": "Dancing Master", + "first_name": "Syrio", + "last_name": "Forel" + }, + { + "title": "Dancing Master's Father", + "first_name": "Nyrio", + "last_name": "Forel" + } +] \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testCTAS.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testCTAS.sql new file mode 100644 index 0000000000..ec1605a886 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testCTAS.sql @@ -0,0 +1 @@ +CREATE TABLE got2(first_name text,last_name text) AS SELECT 'a','b'; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testGroupby.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testGroupby.sql new file mode 100644 index 0000000000..26fd5b5358 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testGroupby.sql @@ -0,0 +1 @@ +select last_name, count(*) from got group by last_name; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql new file mode 100644 index 0000000000..1bebd5008e --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql @@ -0,0 +1 @@ +INSERT INTO got(first_name,last_name) SELECT 'a','b'; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testJoin.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testJoin.sql new file mode 100644 index 0000000000..b0915358c0 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testJoin.sql @@ -0,0 +1 @@ +select g1.title from got g1, got g2 where g1.first_name = g2.first_name \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql new file mode 100644 index 0000000000..9cfc3a1aa5 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql @@ -0,0 +1 @@ +select title from got; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSort.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSort.sql new file mode 100644 index 0000000000..38d00e83bc --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSort.sql @@ -0,0 +1 @@ +select title,first_name from got order by first_name; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/testBetweenDates.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/testBetweenDates.sql new file mode 100644 index 0000000000..e6b20a6046 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/testBetweenDates.sql @@ -0,0 +1 @@ +SELECT * FROM LINEITEM WHERE L_SHIPDATE BETWEEN DATE '1996-01-1' AND DATE '1997-12-31'; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testGroupby.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testGroupby.result new file mode 100644 index 0000000000..f14620d207 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testGroupby.result @@ -0,0 +1,4 @@ +last_name,?count +------------------------------- +Stark,3 +Forel,2 diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testInsert.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testInsert.result new file mode 100644 index 0000000000..d1794ef189 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testInsert.result @@ -0,0 +1,7 @@ +title,first_name +------------------------------- +Assassin,Arya +Hand of the King,Eddard +Dancing Master's Father,Nyrio +Lady,Sansa +Dancing Master,Syrio diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testJoin.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testJoin.result new file mode 100644 index 0000000000..0888ed5321 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testJoin.result @@ -0,0 +1,7 @@ +title +------------------------------- +Hand of the King +Assassin +Lady +Dancing Master +Dancing Master's Father diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result new file mode 100644 index 0000000000..0888ed5321 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result @@ -0,0 +1,7 @@ +title +------------------------------- +Hand of the King +Assassin +Lady +Dancing Master +Dancing Master's Father diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSort.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSort.result new file mode 100644 index 0000000000..d1794ef189 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSort.result @@ -0,0 +1,7 @@ +title,first_name +------------------------------- +Assassin,Arya +Hand of the King,Eddard +Dancing Master's Father,Nyrio +Lady,Sansa +Dancing Master,Syrio diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/testBetweenDates.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/testBetweenDates.result new file mode 100644 index 0000000000..2f6e7a145b --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/testBetweenDates.result @@ -0,0 +1,5 @@ +l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment +------------------------------- +1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold +1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the +2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a