From c34a03cd5f82a53390ca156a52190174820afaff Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 19 Feb 2016 14:04:39 +0900 Subject: [PATCH 01/13] TAJO-1866: TestHAServiceHDFSImpl:: testAutoFailOver fails occasionally in the Travis CI build. --- .../org/apache/tajo/TajoTestingCluster.java | 14 +- .../apache/tajo/ha/TestHAServiceHDFSImpl.java | 166 ++++++++++++------ 2 files changed, 127 insertions(+), 53 deletions(-) diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 4e7d236e87..32fbe57f16 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -306,13 +306,16 @@ public HBaseTestClusterUtil getHBaseUtil() { //////////////////////////////////////////////////////// // Catalog Section //////////////////////////////////////////////////////// - public CatalogServer startCatalogCluster() throws Exception { + private void initCatalogCluster() throws Exception { if(isCatalogServerRunning) throw new IOException("Catalog Cluster already running"); CatalogTestingUtil.configureCatalog(conf, clusterTestBuildDir.getAbsolutePath()); LOG.info("Apache Derby repository is set to " + conf.get(CatalogConstants.CATALOG_URI)); conf.setVar(ConfVars.CATALOG_ADDRESS, "localhost:0"); + } + public CatalogServer startCatalogCluster() throws Exception { + initCatalogCluster(); catalogServer = new CatalogServer(); catalogServer.init(conf); catalogServer.start(); @@ -465,6 +468,15 @@ private void startTajoWorkers(int numSlaves) throws Exception { } } + public void startMaster() throws Exception { + initCatalogCluster(); + tajoMaster = new TajoMaster(); + tajoMaster.init(conf); + tajoMaster.start(); + isCatalogServerRunning = true; + catalogServer = tajoMaster.getCatalogServer(); + } + public TajoMaster getMaster() { return this.tajoMaster; } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index 97c3b7d396..2354c897e7 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -18,96 +18,157 @@ package org.apache.tajo.ha; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.*; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; -import org.apache.tajo.TajoConstants; -import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.TpchTestBase; +import org.apache.tajo.*; +import org.apache.tajo.catalog.*; import org.apache.tajo.client.TajoClient; import org.apache.tajo.client.TajoClientImpl; +import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; +import org.apache.tajo.storage.*; +import org.apache.tajo.util.CommonTestingUtil; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import java.util.Random; + +import static junit.framework.Assert.assertNotNull; import static junit.framework.Assert.assertTrue; import static junit.framework.TestCase.assertEquals; +import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.junit.Assert.assertNotEquals; public class TestHAServiceHDFSImpl { - private TajoTestingCluster cluster; + private static final Log LOG = LogFactory.getLog(TestHAServiceHDFSImpl.class); + + private TajoConf conf; + private TajoTestingCluster util; + private FileTablespace sm; + private final String TEST_PATH = TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestHAServiceHDFSImpl"; + private CatalogService catalog; + private Path testDir; - private TajoMaster primaryMaster; - private TajoMaster backupMaster; + private final int numTuple = 1000; + private Random rnd = new Random(System.currentTimeMillis()); + + private TableDesc employee; private Path haPath, activePath, backupPath; - //@Test TODO: enable this test after TAJO-1866 fixed - public final void testAutoFailOver() throws Exception { - cluster = TpchTestBase.getInstance().getTestingCluster(); + @Before + public void setUp() throws Exception { + this.conf = new TajoConf(); + util = new TajoTestingCluster(true); + + util.startMaster(); + catalog = util.getCatalogService(); + + sm = TablespaceManager.getLocalFs(); + + testDir = CommonTestingUtil.getTestDir(TEST_PATH); + conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); + + Schema schema = new Schema(); + schema.addColumn("managerid", TajoDataTypes.Type.INT4); + schema.addColumn("empid", TajoDataTypes.Type.INT4); + schema.addColumn("deptname", TajoDataTypes.Type.TEXT); + + TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); + Path employeePath = new Path(testDir, "employee.csv"); + Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) + .getAppender(employeeMeta, schema, employeePath); + appender.enableStats(); + appender.init(); + VTuple tuple = new VTuple(schema.size()); + for (int i = 0; i < numTuple; i++) { + tuple.put(new Datum[] { + DatumFactory.createInt4(rnd.nextInt(50)), + DatumFactory.createInt4(rnd.nextInt(100)), + DatumFactory.createText("dept_" + i), + }); + appender.addTuple(tuple); + } + appender.flush(); + appender.close(); - try { - FileSystem fs = cluster.getDefaultFileSystem(); + employee = new TableDesc("default.employee", schema, employeeMeta, employeePath.toUri()); + catalog.createTable(employee); + } - TajoConf primaryConf = setConfigForHAMaster(); - primaryMaster = new TajoMaster(); - primaryMaster.init(primaryConf); - primaryMaster.start(); + @After + public void tearDown() throws Exception { + CommonTestingUtil.cleanupTestDir(TEST_PATH); + util.shutdownCatalogCluster(); + } - TajoConf backupConf = setConfigForHAMaster(); - backupMaster = new TajoMaster(); - backupMaster.init(backupConf); - backupMaster.start(); + @Test + public final void testAutoFailOver() throws Exception { + FileSystem fs = sm.getFileSystem(); + TajoConf primaryConf = util.getConfiguration(); + TajoMaster primaryMaster = util.getMaster(); + assertNotNull(primaryMaster); - ServiceTracker tracker = ServiceTrackerFactory.get(primaryConf); + TajoConf backupConf = getBackupMasterConfiguration(); + TajoMaster backupMaster = new TajoMaster(); + backupMaster.init(backupConf); + backupMaster.start(); + Assert.assertNotNull(backupMaster); - assertNotEquals(primaryMaster.getMasterName(), backupMaster.getMasterName()); - verifySystemDirectories(fs); + ServiceTracker tracker = ServiceTrackerFactory.get(primaryConf); + assertNotEquals(primaryMaster.getMasterName(), backupMaster.getMasterName()); - assertEquals(2, fs.listStatus(activePath).length); - assertEquals(1, fs.listStatus(backupPath).length); + verifySystemDirectories(fs, primaryConf); - assertTrue(fs.exists(new Path(activePath, HAConstants.ACTIVE_LOCK_FILE))); - assertTrue(fs.exists(new Path(activePath, primaryMaster.getMasterName().replaceAll(":", "_")))); - assertTrue(fs.exists(new Path(backupPath, backupMaster.getMasterName().replaceAll(":", "_")))); + assertEquals(2, fs.listStatus(activePath).length); + assertEquals(1, fs.listStatus(backupPath).length); - createDatabaseAndTable(tracker); - verifyDataBaseAndTable(tracker); + assertTrue(fs.exists(new Path(activePath, HAConstants.ACTIVE_LOCK_FILE))); + assertTrue(fs.exists(new Path(activePath, primaryMaster.getMasterName().replaceAll(":", "_")))); + assertTrue(fs.exists(new Path(backupPath, backupMaster.getMasterName().replaceAll(":", "_")))); - primaryMaster.stop(); + createDatabaseAndTable(tracker); + verifyDataBaseAndTable(tracker); - verifyDataBaseAndTable(tracker); + primaryMaster.stop(); - assertTrue(fs.exists(new Path(activePath, HAConstants.ACTIVE_LOCK_FILE))); - assertTrue(fs.exists(new Path(activePath, backupMaster.getMasterName().replaceAll(":", "_")))); + verifyDataBaseAndTable(tracker); - assertEquals(2, fs.listStatus(activePath).length); - assertEquals(0, fs.listStatus(backupPath).length); - } finally { - if (backupMaster != null) { - backupMaster.stop(); - } - } + assertTrue(fs.exists(new Path(activePath, HAConstants.ACTIVE_LOCK_FILE))); + assertTrue(fs.exists(new Path(activePath, backupMaster.getMasterName().replaceAll(":", "_")))); + + assertEquals(2, fs.listStatus(activePath).length); + assertEquals(0, fs.listStatus(backupPath).length); + + assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, "employee")); } - private TajoConf setConfigForHAMaster() { - TajoConf conf = new TajoConf(cluster.getConfiguration()); + + private TajoConf getBackupMasterConfiguration() { + TajoConf conf = util.getConfiguration(); conf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); + "localhost:" + NetUtils.getFreeSocketPort()); conf.setVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); + "localhost:" + NetUtils.getFreeSocketPort()); conf.setVar(TajoConf.ConfVars.RESOURCE_TRACKER_RPC_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); + "localhost:" + NetUtils.getFreeSocketPort()); conf.setVar(TajoConf.ConfVars.CATALOG_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); + "localhost:" + NetUtils.getFreeSocketPort()); conf.setVar(TajoConf.ConfVars.TAJO_MASTER_INFO_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); + "localhost:" + NetUtils.getFreeSocketPort()); conf.setVar(TajoConf.ConfVars.REST_SERVICE_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); + "localhost:" + NetUtils.getFreeSocketPort()); conf.setBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE, true); conf.setIntVar(TajoConf.ConfVars.TAJO_MASTER_HA_MONITOR_INTERVAL, 1000); @@ -126,8 +187,8 @@ private TajoConf setConfigForHAMaster() { return conf; } - private void verifySystemDirectories(FileSystem fs) throws Exception { - haPath = TajoConf.getSystemHADir(cluster.getConfiguration()); + private void verifySystemDirectories(FileSystem fs, TajoConf conf) throws Exception { + haPath = TajoConf.getSystemHADir(conf); assertTrue(fs.exists(haPath)); activePath = new Path(haPath, TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME); @@ -159,4 +220,5 @@ private void verifyDataBaseAndTable(ServiceTracker tracker) throws Exception { IOUtils.cleanup(null, client); } } + } From d0cd5b1935b5d6a67cafbc7e03c46f3efc681a30 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 19 Feb 2016 14:41:51 +0900 Subject: [PATCH 02/13] Remove unused codes --- .../test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index 2354c897e7..50bb618020 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -18,8 +18,6 @@ package org.apache.tajo.ha; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.*; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; @@ -50,8 +48,6 @@ import static org.junit.Assert.assertNotEquals; public class TestHAServiceHDFSImpl { - private static final Log LOG = LogFactory.getLog(TestHAServiceHDFSImpl.class); - private TajoConf conf; private TajoTestingCluster util; private FileTablespace sm; From 6937257e2d258d3fdf1444d1c547957ca9208145 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 19 Feb 2016 14:50:40 +0900 Subject: [PATCH 03/13] Clean up codes --- .../apache/tajo/ha/TestHAServiceHDFSImpl.java | 75 +++++-------------- 1 file changed, 18 insertions(+), 57 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index 50bb618020..d2cecfeba5 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -27,8 +27,6 @@ import org.apache.tajo.client.TajoClientImpl; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; @@ -39,8 +37,6 @@ import org.junit.Before; import org.junit.Test; -import java.util.Random; - import static junit.framework.Assert.assertNotNull; import static junit.framework.Assert.assertTrue; import static junit.framework.TestCase.assertEquals; @@ -48,23 +44,15 @@ import static org.junit.Assert.assertNotEquals; public class TestHAServiceHDFSImpl { - private TajoConf conf; + private final String TEST_PATH = TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestHAServiceHDFSImpl"; private TajoTestingCluster util; private FileTablespace sm; - private final String TEST_PATH = TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestHAServiceHDFSImpl"; private CatalogService catalog; private Path testDir; - - private final int numTuple = 1000; - private Random rnd = new Random(System.currentTimeMillis()); - private TableDesc employee; - private Path haPath, activePath, backupPath; - @Before public void setUp() throws Exception { - this.conf = new TajoConf(); util = new TajoTestingCluster(true); util.startMaster(); @@ -73,7 +61,6 @@ public void setUp() throws Exception { sm = TablespaceManager.getLocalFs(); testDir = CommonTestingUtil.getTestDir(TEST_PATH); - conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); Schema schema = new Schema(); schema.addColumn("managerid", TajoDataTypes.Type.INT4); @@ -86,15 +73,6 @@ public void setUp() throws Exception { .getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); - VTuple tuple = new VTuple(schema.size()); - for (int i = 0; i < numTuple; i++) { - tuple.put(new Datum[] { - DatumFactory.createInt4(rnd.nextInt(50)), - DatumFactory.createInt4(rnd.nextInt(100)), - DatumFactory.createText("dept_" + i), - }); - appender.addTuple(tuple); - } appender.flush(); appender.close(); @@ -110,21 +88,27 @@ public void tearDown() throws Exception { @Test public final void testAutoFailOver() throws Exception { - FileSystem fs = sm.getFileSystem(); - TajoConf primaryConf = util.getConfiguration(); TajoMaster primaryMaster = util.getMaster(); assertNotNull(primaryMaster); - TajoConf backupConf = getBackupMasterConfiguration(); + TajoConf conf = getBackupMasterConfiguration(); TajoMaster backupMaster = new TajoMaster(); - backupMaster.init(backupConf); + backupMaster.init(conf); backupMaster.start(); Assert.assertNotNull(backupMaster); - ServiceTracker tracker = ServiceTrackerFactory.get(primaryConf); + ServiceTracker tracker = ServiceTrackerFactory.get(util.getConfiguration()); assertNotEquals(primaryMaster.getMasterName(), backupMaster.getMasterName()); - verifySystemDirectories(fs, primaryConf); + FileSystem fs = sm.getFileSystem(); + Path haPath = TajoConf.getSystemHADir(util.getConfiguration()); + assertTrue(fs.exists(haPath)); + + Path activePath = new Path(haPath, TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME); + assertTrue(fs.exists(activePath)); + + Path backupPath = new Path(haPath, TajoConstants.SYSTEM_HA_BACKUP_DIR_NAME); + assertTrue(fs.exists(backupPath)); assertEquals(2, fs.listStatus(activePath).length); assertEquals(1, fs.listStatus(backupPath).length); @@ -134,11 +118,11 @@ public final void testAutoFailOver() throws Exception { assertTrue(fs.exists(new Path(backupPath, backupMaster.getMasterName().replaceAll(":", "_")))); createDatabaseAndTable(tracker); - verifyDataBaseAndTable(tracker); + existDataBaseAndTable(tracker); primaryMaster.stop(); - verifyDataBaseAndTable(tracker); + existDataBaseAndTable(tracker); assertTrue(fs.exists(new Path(activePath, HAConstants.ACTIVE_LOCK_FILE))); assertTrue(fs.exists(new Path(activePath, backupMaster.getMasterName().replaceAll(":", "_")))); @@ -149,9 +133,8 @@ public final void testAutoFailOver() throws Exception { assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, "employee")); } - private TajoConf getBackupMasterConfiguration() { - TajoConf conf = util.getConfiguration(); + TajoConf conf = new TajoConf(util.getConfiguration()); conf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, "localhost:" + NetUtils.getFreeSocketPort()); @@ -169,31 +152,9 @@ private TajoConf getBackupMasterConfiguration() { conf.setBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE, true); conf.setIntVar(TajoConf.ConfVars.TAJO_MASTER_HA_MONITOR_INTERVAL, 1000); - //Client API service RPC Server - conf.setIntVar(TajoConf.ConfVars.MASTER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2); - conf.setIntVar(TajoConf.ConfVars.WORKER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2); - - // Internal RPC Server - conf.setIntVar(TajoConf.ConfVars.MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2); - conf.setIntVar(TajoConf.ConfVars.QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2); - conf.setIntVar(TajoConf.ConfVars.WORKER_RPC_SERVER_WORKER_THREAD_NUM, 2); - conf.setIntVar(TajoConf.ConfVars.CATALOG_RPC_SERVER_WORKER_THREAD_NUM, 2); - conf.setIntVar(TajoConf.ConfVars.SHUFFLE_RPC_SERVER_WORKER_THREAD_NUM, 2); - return conf; } - private void verifySystemDirectories(FileSystem fs, TajoConf conf) throws Exception { - haPath = TajoConf.getSystemHADir(conf); - assertTrue(fs.exists(haPath)); - - activePath = new Path(haPath, TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME); - assertTrue(fs.exists(activePath)); - - backupPath = new Path(haPath, TajoConstants.SYSTEM_HA_BACKUP_DIR_NAME); - assertTrue(fs.exists(backupPath)); - } - private void createDatabaseAndTable(ServiceTracker tracker) throws Exception { TajoClient client = null; try { @@ -205,7 +166,7 @@ private void createDatabaseAndTable(ServiceTracker tracker) throws Exception { } } - private void verifyDataBaseAndTable(ServiceTracker tracker) throws Exception { + private void existDataBaseAndTable(ServiceTracker tracker) throws Exception { TajoClient client = null; try { client = new TajoClientImpl(tracker); @@ -217,4 +178,4 @@ private void verifyDataBaseAndTable(ServiceTracker tracker) throws Exception { } } -} +} \ No newline at end of file From 2020db4c710a6e0051b6c2d0fc1ef4ae517bfbc3 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 19 Feb 2016 14:51:54 +0900 Subject: [PATCH 04/13] Remove unnecessary lines --- .../apache/tajo/ha/TestHAServiceHDFSImpl.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index d2cecfeba5..290551ddda 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -136,18 +136,12 @@ public final void testAutoFailOver() throws Exception { private TajoConf getBackupMasterConfiguration() { TajoConf conf = new TajoConf(util.getConfiguration()); - conf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); - conf.setVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); - conf.setVar(TajoConf.ConfVars.RESOURCE_TRACKER_RPC_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); - conf.setVar(TajoConf.ConfVars.CATALOG_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); - conf.setVar(TajoConf.ConfVars.TAJO_MASTER_INFO_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); - conf.setVar(TajoConf.ConfVars.REST_SERVICE_ADDRESS, - "localhost:" + NetUtils.getFreeSocketPort()); + conf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, "localhost:" + NetUtils.getFreeSocketPort()); + conf.setVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, "localhost:" + NetUtils.getFreeSocketPort()); + conf.setVar(TajoConf.ConfVars.RESOURCE_TRACKER_RPC_ADDRESS, "localhost:" + NetUtils.getFreeSocketPort()); + conf.setVar(TajoConf.ConfVars.CATALOG_ADDRESS, "localhost:" + NetUtils.getFreeSocketPort()); + conf.setVar(TajoConf.ConfVars.TAJO_MASTER_INFO_ADDRESS, "localhost:" + NetUtils.getFreeSocketPort()); + conf.setVar(TajoConf.ConfVars.REST_SERVICE_ADDRESS, "localhost:" + NetUtils.getFreeSocketPort()); conf.setBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE, true); conf.setIntVar(TajoConf.ConfVars.TAJO_MASTER_HA_MONITOR_INTERVAL, 1000); From 522267fc9e55a43a8ac823a2cb48a9d13d4b504d Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 19 Feb 2016 14:54:17 +0900 Subject: [PATCH 05/13] Shutdown backup TajoMaster --- .../apache/tajo/ha/TestHAServiceHDFSImpl.java | 69 +++++++++++-------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index 290551ddda..27959fdc23 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -88,49 +88,58 @@ public void tearDown() throws Exception { @Test public final void testAutoFailOver() throws Exception { - TajoMaster primaryMaster = util.getMaster(); - assertNotNull(primaryMaster); + TajoMaster backupMaster = null; - TajoConf conf = getBackupMasterConfiguration(); - TajoMaster backupMaster = new TajoMaster(); - backupMaster.init(conf); - backupMaster.start(); - Assert.assertNotNull(backupMaster); + try { + TajoMaster primaryMaster = util.getMaster(); + assertNotNull(primaryMaster); + + TajoConf conf = getBackupMasterConfiguration(); + backupMaster = new TajoMaster(); + backupMaster.init(conf); + backupMaster.start(); + Assert.assertNotNull(backupMaster); + + ServiceTracker tracker = ServiceTrackerFactory.get(util.getConfiguration()); + assertNotEquals(primaryMaster.getMasterName(), backupMaster.getMasterName()); - ServiceTracker tracker = ServiceTrackerFactory.get(util.getConfiguration()); - assertNotEquals(primaryMaster.getMasterName(), backupMaster.getMasterName()); + FileSystem fs = sm.getFileSystem(); + Path haPath = TajoConf.getSystemHADir(util.getConfiguration()); + assertTrue(fs.exists(haPath)); - FileSystem fs = sm.getFileSystem(); - Path haPath = TajoConf.getSystemHADir(util.getConfiguration()); - assertTrue(fs.exists(haPath)); + Path activePath = new Path(haPath, TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME); + assertTrue(fs.exists(activePath)); - Path activePath = new Path(haPath, TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME); - assertTrue(fs.exists(activePath)); + Path backupPath = new Path(haPath, TajoConstants.SYSTEM_HA_BACKUP_DIR_NAME); + assertTrue(fs.exists(backupPath)); - Path backupPath = new Path(haPath, TajoConstants.SYSTEM_HA_BACKUP_DIR_NAME); - assertTrue(fs.exists(backupPath)); + assertEquals(2, fs.listStatus(activePath).length); + assertEquals(1, fs.listStatus(backupPath).length); - assertEquals(2, fs.listStatus(activePath).length); - assertEquals(1, fs.listStatus(backupPath).length); + assertTrue(fs.exists(new Path(activePath, HAConstants.ACTIVE_LOCK_FILE))); + assertTrue(fs.exists(new Path(activePath, primaryMaster.getMasterName().replaceAll(":", "_")))); + assertTrue(fs.exists(new Path(backupPath, backupMaster.getMasterName().replaceAll(":", "_")))); - assertTrue(fs.exists(new Path(activePath, HAConstants.ACTIVE_LOCK_FILE))); - assertTrue(fs.exists(new Path(activePath, primaryMaster.getMasterName().replaceAll(":", "_")))); - assertTrue(fs.exists(new Path(backupPath, backupMaster.getMasterName().replaceAll(":", "_")))); + createDatabaseAndTable(tracker); + existDataBaseAndTable(tracker); - createDatabaseAndTable(tracker); - existDataBaseAndTable(tracker); + primaryMaster.stop(); - primaryMaster.stop(); + existDataBaseAndTable(tracker); - existDataBaseAndTable(tracker); + assertTrue(fs.exists(new Path(activePath, HAConstants.ACTIVE_LOCK_FILE))); + assertTrue(fs.exists(new Path(activePath, backupMaster.getMasterName().replaceAll(":", "_")))); - assertTrue(fs.exists(new Path(activePath, HAConstants.ACTIVE_LOCK_FILE))); - assertTrue(fs.exists(new Path(activePath, backupMaster.getMasterName().replaceAll(":", "_")))); + assertEquals(2, fs.listStatus(activePath).length); + assertEquals(0, fs.listStatus(backupPath).length); - assertEquals(2, fs.listStatus(activePath).length); - assertEquals(0, fs.listStatus(backupPath).length); + assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, "employee")); + } finally { + if (backupMaster != null) { + backupMaster.close(); + } + } - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, "employee")); } private TajoConf getBackupMasterConfiguration() { From a2ceadcab24e18759e7a81d7f3a3d7095a95c4e9 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 22 Feb 2016 15:34:56 +0900 Subject: [PATCH 06/13] Trigger for travis CI build --- .../src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index 27959fdc23..0c38c602cd 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -44,6 +44,7 @@ import static org.junit.Assert.assertNotEquals; public class TestHAServiceHDFSImpl { + private final String TEST_PATH = TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestHAServiceHDFSImpl"; private TajoTestingCluster util; private FileTablespace sm; From e04c65eae63a8e24a1870a6ca7eefacf315465b3 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 22 Feb 2016 15:40:25 +0900 Subject: [PATCH 07/13] Trigger for travis CI build --- .../src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index 0c38c602cd..27959fdc23 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -44,7 +44,6 @@ import static org.junit.Assert.assertNotEquals; public class TestHAServiceHDFSImpl { - private final String TEST_PATH = TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestHAServiceHDFSImpl"; private TajoTestingCluster util; private FileTablespace sm; From 323f7a49c10c4b86d941d98853d8f043a7595ace Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 22 Feb 2016 15:41:11 +0900 Subject: [PATCH 08/13] Trigger for travis CI build --- .../src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index 27959fdc23..0c38c602cd 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -44,6 +44,7 @@ import static org.junit.Assert.assertNotEquals; public class TestHAServiceHDFSImpl { + private final String TEST_PATH = TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestHAServiceHDFSImpl"; private TajoTestingCluster util; private FileTablespace sm; From 9e5f8463dfdd40cebe23e283c7d91e42016b2924 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 22 Feb 2016 16:23:13 +0900 Subject: [PATCH 09/13] Trigger for travis CI build --- .../src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index 0c38c602cd..27959fdc23 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -44,7 +44,6 @@ import static org.junit.Assert.assertNotEquals; public class TestHAServiceHDFSImpl { - private final String TEST_PATH = TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestHAServiceHDFSImpl"; private TajoTestingCluster util; private FileTablespace sm; From 62895c02f627d2ae71aec41341402bfc343393ad Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 7 Mar 2016 11:30:06 +0900 Subject: [PATCH 10/13] Trigger for travis CI build --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 592bfe3e20..3aea02e006 100644 --- a/CHANGES +++ b/CHANGES @@ -2,6 +2,7 @@ Tajo Change Log Release 0.12.0 - unreleased + NEW FEATURES TAJO-1955: Add a feature to strip quotes from CSV file. (hyunsik) From 68d48c682a27a74d5a41888248770c7bb4dd32a1 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 30 Mar 2016 10:32:58 +0900 Subject: [PATCH 11/13] Fix a compile bug --- .../src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index 27959fdc23..d6182760b2 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -62,7 +62,7 @@ public void setUp() throws Exception { testDir = CommonTestingUtil.getTestDir(TEST_PATH); - Schema schema = new Schema(); + Schema schema = SchemaFactory.newV1(); schema.addColumn("managerid", TajoDataTypes.Type.INT4); schema.addColumn("empid", TajoDataTypes.Type.INT4); schema.addColumn("deptname", TajoDataTypes.Type.TEXT); From 1e34ff55e9150e9499328d80b658576381bdc67a Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 21 Apr 2016 11:16:21 +0900 Subject: [PATCH 12/13] Fix a compile bug --- .../java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index d6182760b2..636fa6769a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -62,10 +62,11 @@ public void setUp() throws Exception { testDir = CommonTestingUtil.getTestDir(TEST_PATH); - Schema schema = SchemaFactory.newV1(); - schema.addColumn("managerid", TajoDataTypes.Type.INT4); - schema.addColumn("empid", TajoDataTypes.Type.INT4); - schema.addColumn("deptname", TajoDataTypes.Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("managerid", TajoDataTypes.Type.INT4) + .add("empid", TajoDataTypes.Type.INT4) + .add("deptname", TajoDataTypes.Type.TEXT) + .build(); TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); Path employeePath = new Path(testDir, "employee.csv"); From 9113442b3f28eaf9ff0b9583a0a382639dc65839 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 9 May 2016 00:32:07 +0900 Subject: [PATCH 13/13] Update parameters of CatalogUtil::newTableMeta --- .../test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index 636fa6769a..27bcae666b 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -68,7 +68,8 @@ public void setUp() throws Exception { .add("deptname", TajoDataTypes.Type.TEXT) .build(); - TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); + TableMeta employeeMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration()); + Path employeePath = new Path(testDir, "employee.csv"); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(employeeMeta, schema, employeePath);