TAJO-2068: MySQL JDBC Tablespace support

.gitignore

@@ -17,3 +17,6 @@ atlassian-ide-plugin.xml
 
 # Patch files
 *.patch
+
+# Antlr files
+*.tokens

.travis.yml

@@ -32,6 +32,10 @@ matrix:
     - jdk: oraclejdk8
       env: PATH=$PATH:$HOME/local/bin HADOOP_FLAG="" TEST_FLAG="test -Ptest-storage-pgsql -DLOG_LEVEL=WARN -pl org.apache.tajo:tajo-storage-pgsql"
 
+    # Test tajo-storage-mysql
+    - jdk: oraclejdk8
+      env: PATH=$PATH:$HOME/local/bin HADOOP_FLAG="" TEST_FLAG="test -Ptest-storage-mysql -DLOG_LEVEL=WARN -pl org.apache.tajo:tajo-storage-mysql"
+
     # Test tajo-storage-hdfs for hadoop-2.7.1
     - jdk: oraclejdk8
       env: PATH=$PATH:$HOME/local/bin HADOOP_FLAG="-Dhadoop.version=2.7.1" TEST_FLAG="test -pl org.apache.tajo:tajo-storage-hdfs"

CHANGES

@@ -4,7 +4,14 @@ Release 0.12.0 - unreleased
 
   NEW FEATURES
 
-    TAJO-1686: Allow Tajo to use Hive UDF. (jihoon)
+    TAJO-2165: Add 'ALTER TABLE UNSET PROPERTY' statement to Tajo DDL.
+    (Lee Dongjin via jihoon)
+
+    TAJO-2160: Implement string_agg function. (Byunghwa Yun via jinho)
+
+    TAJO-2148: Implement an example HTTP tablespace. (jihoon)
+
+    TAJO-1686: Allow Tajo to use Hive UDF. (Jongyoung Park via jihoon)
 
     TAJO-2122: PullServer as an Auxiliary service of Yarn. (jihoon)
 
@@ -14,6 +21,10 @@ Release 0.12.0 - unreleased
 
   IMPROVEMENT
 
+    TAJO-2146: Fragment interface cleanup. (jihoon)
+
+    TAJO-2129: Apply new type implementation to Schema and Catalog. (hyunsik)
+
     TAJO-2071: Supporting DATE type in Parquet format. 
     (Jongyoung Park via hyunsik)
 
@@ -147,6 +158,22 @@ Release 0.12.0 - unreleased
 
   BUG FIXES
 
+    TAJO-2168: NullPointerException occurs when a simple query contains a python udf. 
+    (jihoon)
+
+    TAJO-2164: SequenceFile print wrong values with TextSerializerDeserializer.
+    (jaehwa)
+
+    TAJO-2163: No error message is showed when query execution failed. 
+    (Wonjun Hong via jihoon)
+
+    TAJO-2150: Fix invalid implementation of date time in jdbc. (jinho)
+
+    TAJO-2151: Fix broken CI. (jihoon)
+
+    TAJO-2158: The concat_ws function can't support a tab separator.
+    (Byunghwa Yun via jihoon)
+
     TAJO-2143: Fix race condition in task history writer. (jinho)
 
     TAJO-2140: TajoInternalError does not contains reason stack trace. (jinho)
@@ -266,6 +293,13 @@ Release 0.12.0 - unreleased
 
   TASKS
 
+    TAJO-2154: Refactor Datum to use new Type implementation. (hyunsik)
+
+    TAJO-2027: Writing Hive UDF integration document. (Jongyoung via hyunsik)
+
+    TAJO-2157: EvalNodeSerializer/Deserializer should use new Type 
+    implementation. (hyunsik)
+
     TAJO-2145: Error codes based on errno.h need prefix. 
     (Jongyoung Park via hyunsik)
 
@@ -363,6 +397,11 @@ Release 0.12.0 - unreleased
 
   SUB TASKS
 
+    TAJO-2156: Create GeoIP functions taking various types instead of INET4 type. (jongyoung)
+
+    TAJO-2130: Remove inet4 data type.
+    (Contributed by Jongyoung Park. Committed by jihoon)
+
     TAJO-1997: Registering UDF, it needs to check duplication.
     (Contributed by Jongyoung Park. Committed by jihoon)
 

pom.xml

@@ -98,6 +98,7 @@
     <module>tajo-metrics</module>
     <module>tajo-core-tests</module>
     <module>tajo-cluster-tests</module>
+    <module>tajo-tablespace-example</module>
     <module>tajo-dist</module>
   </modules>
 
@@ -144,7 +145,7 @@
         </plugin>
         <plugin>
           <artifactId>maven-assembly-plugin</artifactId>
-          <version>2.3</version>
+          <version>2.4.1</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>

tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java

@@ -25,6 +25,7 @@
 import org.apache.tajo.util.TUtil;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public class AlterTable extends Expr {
@@ -43,6 +44,8 @@ public class AlterTable extends Expr {
   private AlterTableOpType alterTableOpType;
   @Expose @SerializedName("TableProperties")
   private Map<String, String> params;
+  @Expose @SerializedName("UnsetPropertyKeys")
+  private List<String> unsetPropertyKeys;
 
   @Expose @SerializedName("Columns")
   ColumnReferenceExpr [] columns;
@@ -137,6 +140,14 @@ public void setParams(Map<String, String> params) {
     this.params = params;
   }
 
+  public List<String> getUnsetPropertyKeys() {
+    return unsetPropertyKeys;
+  }
+
+  public void setUnsetPropertyKeys(List<String> unsetPropertyKeys) {
+    this.unsetPropertyKeys = unsetPropertyKeys;
+  }
+
   public boolean isPurge() {
     return purge;
   }

tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTableOpType.java

@@ -18,5 +18,5 @@
 package org.apache.tajo.algebra;
 
 public enum AlterTableOpType {
-  RENAME_TABLE, RENAME_COLUMN, ADD_COLUMN, ADD_PARTITION, DROP_PARTITION, SET_PROPERTY, REPAIR_PARTITION
+  RENAME_TABLE, RENAME_COLUMN, ADD_COLUMN, ADD_PARTITION, DROP_PARTITION, SET_PROPERTY, UNSET_PROPERTY, REPAIR_PARTITION
 }

tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java

@@ -24,11 +24,6 @@ public  class ColumnDefinition extends DataTypeExpr {
   @Expose @SerializedName("ColumnDefName")
   String columnName;
 
-  public ColumnDefinition(String columnName, String dataType) {
-    super(dataType);
-    this.columnName = columnName;
-  }
-
   public ColumnDefinition(String columnName, DataTypeExpr dataType) {
     super(dataType.getTypeName());
 
@@ -42,15 +37,17 @@ public ColumnDefinition(String columnName, DataTypeExpr dataType) {
       }
     }
 
-    // nested records
     if (dataType.isRecordType()) {
       this.recordType = dataType.recordType;
     }
 
-    // map type
     if (dataType.isMapType()) {
       this.mapType = dataType.mapType;
     }
+
+    if (dataType.isArrayType()) {
+      this.arrayType = dataType.arrayType;
+    }
   }
 
   public String getColumnName() {

tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java

@@ -19,37 +19,52 @@
 package org.apache.tajo.algebra;
 
 import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
+import org.apache.tajo.Assert;
 import org.apache.tajo.common.TajoDataTypes.Type;
 import org.apache.tajo.util.TUtil;
 
+import static org.apache.tajo.Assert.assertNotNull;
+
 public class DataTypeExpr extends Expr {
   @Expose @SerializedName("DataTypeName")
   String typeName;
   @Expose @SerializedName("LengthOrPrecision")
   Integer lengthOrPrecision;
   @Expose @SerializedName("Scale")
   Integer scale;
+  @Expose @SerializedName("Array")
+  ArrayType arrayType; // not null if the type is ARRAY
   @Expose @SerializedName("Record")
   RecordType recordType; // not null if the type is RECORD
   @Expose @SerializedName("Map")
   MapType mapType;
 
   public DataTypeExpr(String typeName) {
     super(OpType.DataType);
+    assertNotNull(typeName);
     this.typeName = typeName;
   }
 
+  public DataTypeExpr(ArrayType array) {
+    super(OpType.DataType);
+    assertNotNull(array);
+    this.typeName = Type.ARRAY.name();
+    this.arrayType = array;
+  }
+
   public DataTypeExpr(RecordType record) {
     super(OpType.DataType);
+    assertNotNull(record);
     this.typeName = Type.RECORD.name();
     this.recordType = record;
   }
 
   public DataTypeExpr(MapType map) {
     super(OpType.DataType);
-    // RECORD = 51 in DataTypes.proto
+    assertNotNull(map);
     this.typeName = Type.MAP.name();
     this.mapType = map;
   }
@@ -59,15 +74,34 @@ public String getTypeName() {
   }
 
   public boolean isPrimitiveType() {
-    return !this.isRecordType() && !isMapType();
+    return !isArrayType()&& !isRecordType() && !isMapType();
+  }
+
+  public boolean isArrayType() {
+    return arrayType != null;
   }
 
   public boolean isRecordType() {
-    return this.typeName.equals(Type.RECORD.name());
+    return recordType != null;
   }
 
   public boolean isMapType() {
-    return this.typeName.equals(Type.MAP.name());
+    return mapType != null;
+  }
+
+  public DataTypeExpr getElementType() {
+    Preconditions.checkState(isArrayType());
+    return arrayType.type;
+  }
+
+  public DataTypeExpr getKeyType() {
+    Preconditions.checkState(isMapType());
+    return mapType.keyType;
+  }
+
+  public DataTypeExpr getValueType() {
+    Preconditions.checkState(isMapType());
+    return mapType.valueType;
   }
 
   public ColumnDefinition [] getNestedRecordTypes() {
@@ -125,6 +159,27 @@ public Object clone() throws CloneNotSupportedException {
     return dataType;
   }
 
+  public static class ArrayType implements JsonSerializable, Cloneable {
+    @Expose
+    @SerializedName("type")
+    DataTypeExpr type;
+
+    public ArrayType(DataTypeExpr elementType) {
+      this.type = elementType;
+    }
+
+    @Override
+    public String toJson() {
+      return JsonHelper.toJson(this);
+    }
+
+    public Object clone() throws CloneNotSupportedException {
+      ArrayType newMap = (ArrayType) super.clone();
+      newMap.type = type;
+      return newMap;
+    }
+  }
+
   public static class RecordType implements JsonSerializable, Cloneable {
     @Expose @SerializedName("Schema")
     ColumnDefinition [] schema; // not null if the type is RECORD

tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java

@@ -32,6 +32,7 @@
 import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto;
 import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState;
 import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringListResponse;
+import org.apache.tajo.schema.IdentifierUtil;
 import org.apache.tajo.util.ProtoUtil;
 
 import java.io.Closeable;
@@ -281,7 +282,7 @@ public final TableDesc getTableDesc(final String databaseName, final String tabl
 
   @Override
   public TableDesc getTableDesc(String qualifiedName) throws UndefinedTableException {
-    String[] splitted = CatalogUtil.splitFQTableName(qualifiedName);
+    String[] splitted = IdentifierUtil.splitFQTableName(qualifiedName);
     return getTableDesc(splitted[0], splitted[1]);
   }
 
@@ -596,7 +597,7 @@ public final void createTable(final TableDesc desc)
   public void dropTable(String tableName)
       throws UndefinedDatabaseException, UndefinedTableException, InsufficientPrivilegeException {
 
-    String[] splitted = CatalogUtil.splitFQTableName(tableName);
+    String[] splitted = IdentifierUtil.splitFQTableName(tableName);
     final String databaseName = splitted[0];
     final String simpleName = splitted[1];
 
@@ -617,7 +618,7 @@ public void dropTable(String tableName)
 
   @Override
   public final boolean existsTable(final String databaseName, final String tableName) {
-    if (CatalogUtil.isFQTableName(tableName)) {
+    if (IdentifierUtil.isFQTableName(tableName)) {
       throw new IllegalArgumentException(
           "tableName cannot be composed of multiple parts, but it is \"" + tableName + "\"");
     }
@@ -640,7 +641,7 @@ public final boolean existsTable(final String databaseName, final String tableNa
 
   @Override
   public final boolean existsTable(final String tableName) {
-    String[] splitted = CatalogUtil.splitFQTableName(tableName);
+    String[] splitted = IdentifierUtil.splitFQTableName(tableName);
     return existsTable(splitted[0], splitted[1]);
   }
 

tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java

@@ -70,8 +70,6 @@ private static void putAcceptableType(TajoDataTypes.Type given, TajoDataTypes.Ty
     putAcceptableType(DATE, DATE);
 
     putAcceptableType(TEXT, TEXT);
-
-    putAcceptableType(INET4, INET4);
   }
 
   public static boolean isUpperCastable(TajoDataTypes.Type define, TajoDataTypes.Type given) {
@@ -87,8 +85,8 @@ public static boolean isUpperCastable(TajoDataTypes.Type define, TajoDataTypes.T
    */
   public static org.apache.tajo.type.Type determineType(org.apache.tajo.type.Type left,
                                                         org.apache.tajo.type.Type right) {
-    TajoDataTypes.Type rhsBaseType = right.baseType();
-    switch (left.baseType()) {
+    TajoDataTypes.Type rhsBaseType = right.kind();
+    switch (left.kind()) {
 
     case INT1:
     case INT2: