TAJO-1891: Allow arbitrary partition path

tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java

@@ -104,6 +104,11 @@ public static TableDesc newTableDesc(String tableName, Schema schema, TableMeta
     return new TableDesc(tableName, schema, meta, path.toUri());
   }
 
+  public static TableDesc newTableDesc(String tableName, Schema schema, TableMeta meta, Path path
+    , PartitionMethodDesc partitionMethodDesc) {
+    return new TableDesc(tableName, schema, meta, path.toUri(), partitionMethodDesc);
+  }
+
   public static TableDesc newTableDesc(TableDescProto proto) {
     return new TableDesc(proto);
   }

tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java

@@ -51,32 +51,46 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
 	public TableDesc() {
 	}
 
+  public TableDesc(String tableName, @Nullable Schema schema, TableMeta meta, @Nullable URI uri, boolean external) {
+    this(tableName, schema, meta, uri, null, external);
+  }
+
   public TableDesc(String tableName, @Nullable Schema schema, TableMeta meta,
-                   @Nullable URI uri, boolean external) {
+                   @Nullable URI uri, @Nullable PartitionMethodDesc partitionMethodDesc, boolean external) {
     this.tableName = tableName;
     this.schema = schema;
     this.meta = meta;
     this.uri = uri;
+    this.partitionMethodDesc = partitionMethodDesc;
     this.external = external;
   }
 
 	public TableDesc(String tableName, @Nullable Schema schema, TableMeta meta, @Nullable URI path) {
-		this(tableName, schema, meta, path, true);
+		this(tableName, schema, meta, path, null, true);
 	}
-
+
+  public TableDesc(String tableName, @Nullable Schema schema, TableMeta meta, @Nullable URI path,
+                   @Nullable PartitionMethodDesc partitionMethodDesc) {
+    this(tableName, schema, meta, path, partitionMethodDesc, true);
+  }
+
 	public TableDesc(String tableName, @Nullable Schema schema, String dataFormat, KeyValueSet options,
                    @Nullable URI path) {
 	  this(tableName, schema, new TableMeta(dataFormat, options), path);
 	}
-
-	public TableDesc(TableDescProto proto) {
+
+  public TableDesc(String tableName, @Nullable Schema schema, String dataFormat, KeyValueSet options,
+                   @Nullable URI path, @Nullable PartitionMethodDesc partitionMethodDesc) {
+    this(tableName, schema, new TableMeta(dataFormat, options), path, partitionMethodDesc);
+  }
+
+  public TableDesc(TableDescProto proto) {
 	  this(proto.getTableName(), proto.hasSchema() ? SchemaFactory.newV1(proto.getSchema()) : null,
-        new TableMeta(proto.getMeta()), proto.hasPath() ? URI.create(proto.getPath()) : null, proto.getIsExternal());
+        new TableMeta(proto.getMeta()), proto.hasPath() ? URI.create(proto.getPath()) : null,
+      proto.hasPartition() ? new PartitionMethodDesc(proto.getPartition()) : null,
+      proto.getIsExternal());
     if(proto.hasStats()) {
       this.stats = new TableStats(proto.getStats());
-    }
-    if (proto.hasPartition()) {
-      this.partitionMethodDesc = new PartitionMethodDesc(proto.getPartition());
     }
 	}
 

tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml

@@ -33,6 +33,7 @@
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+    <hive.antlr.version>3.4</hive.antlr.version>
   </properties>
 
   <build>
@@ -163,7 +164,7 @@
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-metastore</artifactId>
-      <version>${hive.version}</version>
+        <version>${hive.version}</version>
       <scope>provided</scope>
       <exclusions>
         <exclusion>
@@ -369,6 +370,12 @@
       <artifactId>parquet-hadoop-bundle</artifactId>
       <version>${parquet.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.antlr</groupId>
+      <artifactId>antlr-runtime</artifactId>
+      <version>${hive.antlr.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <profiles>

tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java

@@ -253,7 +253,7 @@ public void testTableWithNullValue() throws Exception {
 
   }
 
-  // TODO: This should be added at TAJO-1891
+  @Test
   public void testAddTableByPartition() throws Exception {
     TableMeta meta = new TableMeta("TEXT", new KeyValueSet());
 

tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java

@@ -452,11 +452,6 @@ public ReturnState alterTable(RpcController controller, AlterTableDescProto prot
         return errInsufficientPrivilege("alter a table in database '" + split[0] + "'");
       }
 
-      // TODO: This should be removed at TAJO-1891
-      if (proto.getAlterTableType() == CatalogProtos.AlterTableType.ADD_PARTITION) {
-        return errFeatureNotImplemented("ADD PARTTIION");
-      }
-
       wlock.lock();
 
       try {

tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java

@@ -784,7 +784,7 @@ public final void testAddAndDeleteTablePartitionByRange() throws Exception {
     assertFalse(catalog.existsTable(tableName));
   }
 
-  // TODO: This should be added at TAJO-1891
+  @Test
   public final void testAddAndDeleteTablePartitionByColumn() throws Exception {
     Schema schema = SchemaBuilder.builder()
         .add("id", Type.INT4)

tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java

@@ -165,7 +165,7 @@ public void testTable() throws Exception {
     }
   }
 
-  // TODO: This should be added at TAJO-1891
+  @Test
   public void testTablePartition() throws Exception {
     //////////////////////////////////////////////////////////////////////////////
     // Test add partition

tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java

@@ -159,7 +159,7 @@ public void testUpdateTableStatsOfUndefinedTable() throws Exception {
         build());
   }
 
-  // TODO: This should be added at TAJO-1891
+  @Test
   public void testAddPartitionWithWrongUri() throws Exception {
     // TODO: currently, wrong uri does not occur any exception.
     String partitionName = "DaTe=/=AaA";
@@ -173,7 +173,7 @@ public void testAddPartitionWithWrongUri() throws Exception {
     catalog.alterTable(alterTableDesc);
   }
 
-  // TODO: This should be added at TAJO-1891
+  @Test(expected = DuplicatePartitionException.class)
   public void testAddDuplicatePartition() throws Exception {
     String partitionName = "DaTe=bBb/dAtE=AaA";
     PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName);
@@ -196,7 +196,7 @@ public void testAddDuplicatePartition() throws Exception {
     catalog.alterTable(alterTableDesc);
   }
 
-  // TODO: This should be added at TAJO-1891
+  @Test(expected = UndefinedTableException.class)
   public void testAddPartitionToUndefinedTable() throws Exception {
     String partitionName = "DaTe=bBb/dAtE=AaA";
     PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName);
@@ -209,19 +209,6 @@ public void testAddPartitionToUndefinedTable() throws Exception {
     catalog.alterTable(alterTableDesc);
   }
 
-  // TODO: This should be removed at TAJO-1891
-  @Test(expected = NotImplementedException.class)
-  public void testAddPartitionNotimplementedException() throws Exception {
-    String partitionName = "DaTe=/=AaA";
-    PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName);
-
-    AlterTableDesc alterTableDesc = new AlterTableDesc();
-    alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestPartition1"));
-    alterTableDesc.setPartitionDesc(partitionDesc);
-    alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION);
-
-    catalog.alterTable(alterTableDesc);
-  }
 
   @Test(expected = UndefinedPartitionException.class)
   public void testDropUndefinedPartition() throws Exception {

tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java

@@ -193,14 +193,10 @@ private static void dumpDatabase(TajoClient client, String databaseName, PrintWr
           writer.write("--\n");
           writer.write(String.format("-- Table Partitions: %s%n", tableName));
           writer.write("--\n");
-          // TODO: This should be improved at TAJO-1891
-//          List<PartitionDescProto> partitionProtos = client.getPartitionsOfTable(fqName);
-//          for (PartitionDescProto eachPartitionProto : partitionProtos) {
-//            writer.write(DDLBuilder.buildDDLForAddPartition(table, eachPartitionProto));
-//          }
-          writer.write(String.format("ALTER TABLE %s REPAIR PARTITION;",
-            IdentifierUtil.denormalizeIdentifier(databaseName) + "." + IdentifierUtil.denormalizeIdentifier(tableName)));
-
+          List<CatalogProtos.PartitionDescProto> partitionProtos = client.getPartitionsOfTable(fqName);
+          for (CatalogProtos.PartitionDescProto eachPartitionProto : partitionProtos) {
+            writer.write(DDLBuilder.buildDDLForAddPartition(table, eachPartitionProto));
+          }
           writer.write("\n\n");
         }
 

tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java

@@ -119,16 +119,17 @@ public void testPartitionsDump() throws Exception {
           + " partition by column(\"col3\" int4, \"col4\" int4)"
       );
 
-      // TODO: This should be improved at TAJO-1891
-//      executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName3\"" +
-//        " ADD PARTITION (\"col3\" = 1 , \"col4\" = 2)");
-//      executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName4\"" +
-//        " ADD PARTITION (\"col3\" = 'tajo' , \"col4\" = '2015-09-01')");
+      executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName3\"" +
+        " ADD PARTITION (\"col3\" = 1 , \"col4\" = 2)");
+
       executeString("create table \"" + getCurrentDatabase() + "\".\"TableName4\""
           + " (\"col1\" int4, \"col2\" int4) "
           + " partition by column(\"col3\" TEXT, \"col4\" date)"
       );
 
+      executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName4\"" +
+        " ADD PARTITION (\"col3\" = 'tajo' , \"col4\" = '2015-09-01')");
+
       try {
         UserRoleInfo userInfo = UserRoleInfo.getCurrentUser();
         ByteArrayOutputStream bos = new ByteArrayOutputStream();

tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java

@@ -547,19 +547,7 @@ public void testResultRowNumWhenSelectingOnPartitionedTable() throws Exception {
     }
   }
 
-  // TODO: This should be removed at TAJO-1891
   @Test
-  public void testAddPartitionNotimplementedException() throws Exception {
-    String tableName = IdentifierUtil.normalizeIdentifier("testAddPartitionNotimplementedException");
-    tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)");
-    tajoCli.executeScript("alter table " + tableName + " add partition (key2 = 0.1)");
-
-    String consoleResult;
-    consoleResult = new String(out.toByteArray());
-    assertOutputResult(consoleResult);
-  }
-
-  // TODO: This should be added at TAJO-1891
   public void testAlterTableAddDropPartition() throws Exception {
     String tableName = IdentifierUtil.normalizeIdentifier("testAlterTableAddPartition");
 
@@ -586,8 +574,10 @@ public void testAlterTableAddDropPartition() throws Exception {
 
     tajoCli.executeScript("alter table " + tableName + " drop partition (col3 = 0.1, col4 = 10)");
 
-    String consoleResult = new String(out.toByteArray());
-    assertOutputResult(consoleResult);
+    String stdoutResult = new String(out.toByteArray());
+    assertOutputResult(stdoutResult);
+    String stdErrResult = new String(err.toByteArray());
+    assertErrorResult(stdErrResult, false);
   }
 
   public static class TajoCliOutputTestFormatter extends DefaultTajoCliOutputFormatter {

tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java

@@ -1233,7 +1233,7 @@ public final void testAlterTableRepairPartiton() throws TajoException {
     "ALTER TABLE partitioned_table DROP PARTITION (col1 = '2015' , col2 = '01', col3 = '11' )", //1
   };
 
-  // TODO: This should be added at TAJO-1891
+  @Test
   public final void testAddPartitionAndDropPartition() throws TajoException {
     String tableName = IdentifierUtil.normalizeIdentifier("partitioned_table");
     String qualifiedTableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName);