From 657fc0273ce704fa29edc6133502bba1fca42183 Mon Sep 17 00:00:00 2001 From: ciuccioj Date: Mon, 8 May 2017 16:01:30 -0400 Subject: [PATCH 1/2] Added a se=impole method to verify syntax against a table definition --- .../org/finra/hiveqlunit/syntax/Check.java | 63 ++++++++++++++++ .../finra/hiveqlunit/syntax/CheckTest.java | 73 +++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 src/main/java/org/finra/hiveqlunit/syntax/Check.java create mode 100644 src/test/java/org/finra/hiveqlunit/syntax/CheckTest.java diff --git a/src/main/java/org/finra/hiveqlunit/syntax/Check.java b/src/main/java/org/finra/hiveqlunit/syntax/Check.java new file mode 100644 index 0000000..7268e6e --- /dev/null +++ b/src/main/java/org/finra/hiveqlunit/syntax/Check.java @@ -0,0 +1,63 @@ +package org.finra.hiveqlunit.syntax; + +import org.apache.commons.io.FileUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.hive.HiveContext; + +import java.io.File; +import java.io.IOException; + +/** + * With a column heder defintion and a sql statement this class can validate the syntax of the sql statement will parse. + * It takes approximately 20-30 seconds since it starts an intance of hiverserver through spark context. + */ +public class Check { + + HiveContext hc; + + public Check() { + + String header; + String sql; + + SparkConf sparkConf = new SparkConf().setAppName("HiveQLUnit").setMaster("local[1]"); + JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); + + + hc = new HiveContext(sparkContext) ; + + + //Blow away hive meta store before execution + try { + FileUtils.deleteDirectory(new File("/tmp/foo")); + } catch (IOException e) { + e.printStackTrace(); + } + + hc.setConf("hive.metastore.warehouse.dir","/tmp/foo"); + + + } + + public static boolean verify(String createTableStatement, String sqlStatement ) { + + Boolean isGood = true; + + Check check = new Check(); + + check.hc.runSqlHive(createTableStatement); + + check.hc.runSqlHive(sqlStatement); + + try { + check.hc.runSqlHive(sqlStatement); + } catch (Exception e) { + e.printStackTrace(); + isGood = false; + } + + return isGood; + } + +} diff --git a/src/test/java/org/finra/hiveqlunit/syntax/CheckTest.java b/src/test/java/org/finra/hiveqlunit/syntax/CheckTest.java new file mode 100644 index 0000000..3f57715 --- /dev/null +++ b/src/test/java/org/finra/hiveqlunit/syntax/CheckTest.java @@ -0,0 +1,73 @@ +package org.finra.hiveqlunit.syntax; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.finra.hiveqlunit.syntax.Check.verify; + +/** + * Created by ciuccioj on 5/8/17. + */ +public class CheckTest { + + /** + * Check Tester. + * + * @author + * @since 
May 8, 2017
+ * @version 1.0 + */ + + + @Before + public void before() throws Exception { + } + + @After + public void after() throws Exception { + } + + /** + * + * Method: verify(String createTableStatement, String sqlStatement) + * + */ + @Test + public void testVerify() throws Exception { + + Boolean isValid = verify("Create table if not exists trial (cola string, colb string) " + + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' ", + "select * from trial"); + + assert isValid; + + + + } + + //failure case with bad syntax + @Test + public void testVerify2() throws Exception { + + Boolean isValid = null; + try { + isValid = verify("Create table if not exists trial (cola string, colb string) " + + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' ", + "select z from trial"); + } catch (Exception e) { + System.out.println("this should Fail"); + e.printStackTrace(); + isValid = false; + } + + assert !isValid; + + + + } + + + } + + From 5a7f0a780ea94bf2a11c73901fdb92aebf4705be Mon Sep 17 00:00:00 2001 From: ciuccioj Date: Wed, 10 May 2017 12:03:23 -0400 Subject: [PATCH 2/2] Added a contstructor that allow you to pass in an existing hive context Removed unit test as github doesnt allocate enough memeory to test --- src/main/java/org/finra/hiveqlunit/syntax/Check.java | 11 +++++++++++ .../java/org/finra/hiveqlunit/syntax/CheckTest.java | 7 ++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/finra/hiveqlunit/syntax/Check.java b/src/main/java/org/finra/hiveqlunit/syntax/Check.java index 7268e6e..a2a9558 100644 --- a/src/main/java/org/finra/hiveqlunit/syntax/Check.java +++ b/src/main/java/org/finra/hiveqlunit/syntax/Check.java @@ -16,6 +16,7 @@ public class Check { HiveContext hc; + public Check() { String header; @@ -40,6 +41,16 @@ public Check() { } + public Check(HiveContext hiveContext) { + + String header; + String sql; + + hc = hiveContext; + + } + + public static boolean verify(String createTableStatement, String sqlStatement ) { Boolean isGood = true; diff --git a/src/test/java/org/finra/hiveqlunit/syntax/CheckTest.java b/src/test/java/org/finra/hiveqlunit/syntax/CheckTest.java index 3f57715..709442b 100644 --- a/src/test/java/org/finra/hiveqlunit/syntax/CheckTest.java +++ b/src/test/java/org/finra/hiveqlunit/syntax/CheckTest.java @@ -2,6 +2,7 @@ import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import static org.finra.hiveqlunit.syntax.Check.verify; @@ -31,9 +32,9 @@ public void after() throws Exception { /** * * Method: verify(String createTableStatement, String sqlStatement) - * + * dont run becuase no memeory on github build server */ - @Test + @Ignore public void testVerify() throws Exception { Boolean isValid = verify("Create table if not exists trial (cola string, colb string) " + @@ -47,7 +48,7 @@ public void testVerify() throws Exception { } //failure case with bad syntax - @Test + @Ignore public void testVerify2() throws Exception { Boolean isValid = null;