http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java index 1be8352..463021f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java @@ -17,19 +17,15 @@ */ package org.apache.drill.exec.store.parquet; -import com.google.common.io.Resources; import mockit.Mock; import mockit.MockUp; -import mockit.integration.junit4.JMockit; import com.google.common.collect.Lists; +import mockit.integration.junit4.JMockit; import org.apache.drill.PlanTestBase; import org.apache.drill.categories.UnlikelyTest; -import org.apache.drill.common.util.TestTools; import org.apache.commons.io.FileUtils; import org.apache.drill.exec.store.dfs.MetadataContext; import org.apache.drill.exec.planner.physical.PlannerSettings; -import org.apache.hadoop.fs.Path; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -37,7 +33,10 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import java.io.File; +import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; import static org.junit.Assert.assertEquals; @@ -46,42 +45,23 @@ import static org.junit.Assert.assertTrue; @RunWith(JMockit.class) public class TestParquetMetadataCache extends PlanTestBase { - private static final String WORKING_PATH = TestTools.getWorkingPath(); - private static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources"; - private static final String tableName1 = "parquetTable1"; - private static final String tableName2 = "parquetTable2"; - private static File dataDir1; - private static File dataDir2; - + private static final String TABLE_NAME_1 = "parquetTable1"; + private static final String TABLE_NAME_2 = "parquetTable2"; @BeforeClass public static void copyData() throws Exception { - // copy the data into the temporary location - String tmpLocation = getDfsTestTmpSchemaLocation(); - dataDir1 = new File(tmpLocation, tableName1); - dataDir1.mkdir(); - FileUtils.copyDirectory(new File(String.format(String.format("%s/multilevel/parquet", TEST_RES_PATH))), - dataDir1); - - dataDir2 = new File(tmpLocation, tableName2); - dataDir2.mkdir(); - FileUtils.copyDirectory(new File(String.format(String.format("%s/multilevel/parquet2", TEST_RES_PATH))), - dataDir2); - } - - @AfterClass - public static void cleanupTestData() throws Exception { - FileUtils.deleteQuietly(dataDir1); - FileUtils.deleteQuietly(dataDir2); + dirTestWatcher.copyResourceToRoot(Paths.get("multilevel")); + dirTestWatcher.copyResourceToRoot(Paths.get("multilevel/parquet"), Paths.get(TABLE_NAME_1)); + dirTestWatcher.copyResourceToRoot(Paths.get("multilevel/parquet2"), Paths.get(TABLE_NAME_2)); + dirTestWatcher.copyResourceToRoot(Paths.get("parquet")); } @Test public void testPartitionPruningWithMetadataCache_1() throws Exception { - test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName1)); - checkForMetadataFile(tableName1); - String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " + - " where dir0=1994 and dir1 in ('Q1', 'Q2')", - getDfsTestTmpSchemaLocation(), tableName1); + test("refresh table metadata dfs.`%s`", TABLE_NAME_1); + checkForMetadataFile(TABLE_NAME_1); + String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " + + " where dir0=1994 and dir1 in ('Q1', 'Q2')", TABLE_NAME_1); int expectedRowCount = 20; int expectedNumFiles = 2; @@ -89,7 +69,7 @@ public class TestParquetMetadataCache extends PlanTestBase { assertEquals(expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1994", getDfsTestTmpSchemaLocation(), tableName1); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1994", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_1); PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[] {}); } @@ -97,11 +77,10 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test // DRILL-3917, positive test case for DRILL-4530 @Category(UnlikelyTest.class) public void testPartitionPruningWithMetadataCache_2() throws Exception { - test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName1)); - checkForMetadataFile(tableName1); - String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " + - " where dir0=1994", - getDfsTestTmpSchemaLocation(), tableName1); + test("refresh table metadata dfs.`%s`", TABLE_NAME_1); + checkForMetadataFile(TABLE_NAME_1); + String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " + + " where dir0=1994", TABLE_NAME_1); int expectedRowCount = 40; int expectedNumFiles = 4; @@ -109,7 +88,7 @@ public class TestParquetMetadataCache extends PlanTestBase { assertEquals(expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1994", getDfsTestTmpSchemaLocation(), tableName1); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1994", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_1); PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[] {"Filter"}); } @@ -118,11 +97,10 @@ public class TestParquetMetadataCache extends PlanTestBase { @Category(UnlikelyTest.class) public void testPartitionPruningWithMetadataCache_3() throws Exception { String tableName = "orders_ctas_varchar"; - test("use dfs_test.tmp"); - - test(String.format("create table %s (o_orderdate, o_orderpriority) partition by (o_orderpriority) " - + "as select o_orderdate, o_orderpriority from dfs_test.`%s/multilevel/parquet/1994/Q1`", tableName, TEST_RES_PATH)); - test(String.format("refresh table metadata %s", tableName)); + test("use dfs"); + test("create table %s (o_orderdate, o_orderpriority) partition by (o_orderpriority) " + + "as select o_orderdate, o_orderpriority from dfs.`multilevel/parquet/1994/Q1`", tableName); + test("refresh table metadata %s", tableName); checkForMetadataFile(tableName); String query = String.format("select * from %s where o_orderpriority = '1-URGENT'", tableName); int expectedRowCount = 3; @@ -140,12 +118,12 @@ public class TestParquetMetadataCache extends PlanTestBase { @Category(UnlikelyTest.class) public void testPartitionPruningWithMetadataCache_4() throws Exception { String tableName = "orders_ctas_binary"; - test("use dfs_test.tmp"); + test("use dfs"); - test(String.format("create table %s (o_orderdate, o_orderpriority) partition by (o_orderpriority) " + test("create table %s (o_orderdate, o_orderpriority) partition by (o_orderpriority) " + "as select o_orderdate, convert_to(o_orderpriority, 'UTF8') as o_orderpriority " - + "from dfs_test.`%s/multilevel/parquet/1994/Q1`", tableName, TEST_RES_PATH)); - test(String.format("refresh table metadata %s", tableName)); + + "from dfs.`multilevel/parquet/1994/Q1`", tableName); + test("refresh table metadata %s", tableName); checkForMetadataFile(tableName); String query = String.format("select * from %s where o_orderpriority = '1-URGENT'", tableName); int expectedRowCount = 3; @@ -162,10 +140,10 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test public void testCache() throws Exception { String tableName = "nation_ctas"; - test("use dfs_test.tmp"); - test(String.format("create table `%s/t1` as select * from cp.`tpch/nation.parquet`", tableName)); - test(String.format("create table `%s/t2` as select * from cp.`tpch/nation.parquet`", tableName)); - test(String.format("refresh table metadata %s", tableName)); + test("use dfs"); + test("create table `%s/t1` as select * from cp.`tpch/nation.parquet`", tableName); + test("create table `%s/t2` as select * from cp.`tpch/nation.parquet`", tableName); + test("refresh table metadata %s", tableName); checkForMetadataFile(tableName); String query = String.format("select * from %s", tableName); int rowCount = testSql(query); @@ -176,12 +154,12 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test public void testUpdate() throws Exception { String tableName = "nation_ctas_update"; - test("use dfs_test.tmp"); - test(String.format("create table `%s/t1` as select * from cp.`tpch/nation.parquet`", tableName)); - test(String.format("refresh table metadata %s", tableName)); + test("use dfs"); + test("create table `%s/t1` as select * from cp.`tpch/nation.parquet`", tableName); + test("refresh table metadata %s", tableName); checkForMetadataFile(tableName); Thread.sleep(1000); - test(String.format("create table `%s/t2` as select * from cp.`tpch/nation.parquet`", tableName)); + test("create table `%s/t2` as select * from cp.`tpch/nation.parquet`", tableName); int rowCount = testSql(String.format("select * from %s", tableName)); Assert.assertEquals(50, rowCount); } @@ -189,23 +167,23 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test public void testCacheWithSubschema() throws Exception { String tableName = "nation_ctas_subschema"; - test(String.format("create table dfs_test.tmp.`%s/t1` as select * from cp.`tpch/nation.parquet`", tableName)); - test(String.format("refresh table metadata dfs_test.tmp.%s", tableName)); + test("create table dfs.`%s/t1` as select * from cp.`tpch/nation.parquet`", tableName); + test("refresh table metadata dfs.%s", tableName); checkForMetadataFile(tableName); - int rowCount = testSql(String.format("select * from dfs_test.tmp.%s", tableName)); + int rowCount = testSql(String.format("select * from dfs.%s", tableName)); Assert.assertEquals(25, rowCount); } @Test public void testFix4449() throws Exception { - runSQL("CREATE TABLE dfs_test.tmp.`4449` PARTITION BY(l_discount) AS SELECT l_orderkey, l_discount FROM cp.`tpch/lineitem.parquet`"); - runSQL("REFRESH TABLE METADATA dfs_test.tmp.`4449`"); + runSQL("CREATE TABLE dfs.tmp.`4449` PARTITION BY(l_discount) AS SELECT l_orderkey, l_discount FROM cp.`tpch/lineitem.parquet`"); + runSQL("REFRESH TABLE METADATA dfs.tmp.`4449`"); testBuilder() .sqlQuery("SELECT COUNT(*) cnt FROM (" + - "SELECT l_orderkey FROM dfs_test.tmp.`4449` WHERE l_discount < 0.05" + + "SELECT l_orderkey FROM dfs.tmp.`4449` WHERE l_discount < 0.05" + " UNION ALL" + - " SELECT l_orderkey FROM dfs_test.tmp.`4449` WHERE l_discount > 0.02)") + " SELECT l_orderkey FROM dfs.tmp.`4449` WHERE l_discount > 0.02)") .unOrdered() .baselineColumns("cnt") .baselineValues(71159L) @@ -215,10 +193,10 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test public void testAbsentPluginOrWorkspaceError() throws Exception { testBuilder() - .sqlQuery("refresh table metadata dfs_test.incorrect.table_name") + .sqlQuery("refresh table metadata dfs.incorrect.table_name") .unOrdered() .baselineColumns("ok", "summary") - .baselineValues(false, "Storage plugin or workspace does not exist [dfs_test.incorrect]") + .baselineValues(false, "Storage plugin or workspace does not exist [dfs.incorrect]") .go(); testBuilder() @@ -232,15 +210,17 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test //DRILL-4511 @Category(UnlikelyTest.class) public void testTableDoesNotExistWithEmptyDirectory() throws Exception { - File path = new File(getTempDir("empty_directory")); - String pathString = path.toURI().getPath(); + String dirName = "empty_directory"; + File path = new File(dirTestWatcher.getRootDir(), dirName); + path.mkdirs(); + try { path.mkdir(); testBuilder() - .sqlQuery("refresh table metadata dfs.`%s`", pathString) + .sqlQuery("refresh table metadata dfs.`%s`", dirName) .unOrdered() .baselineColumns("ok", "summary") - .baselineValues(false, String.format("Table %s does not exist.", pathString)) + .baselineValues(false, String.format("Table %s does not exist.", dirName)) .go(); } finally { FileUtils.deleteQuietly(path); @@ -274,13 +254,12 @@ public class TestParquetMetadataCache extends PlanTestBase { @Category(UnlikelyTest.class) public void testDrill4530_1() throws Exception { // create metadata cache - test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2)); - checkForMetadataFile(tableName2); + test("refresh table metadata dfs.`%s`", TABLE_NAME_2); + checkForMetadataFile(TABLE_NAME_2); // run query and check correctness - String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " + - " where dir0=1995 and dir1='Q3'", - getDfsTestTmpSchemaLocation(), tableName2); + String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " + + " where dir0=1995 and dir1='Q3'", TABLE_NAME_2); int expectedRowCount = 20; int expectedNumFiles = 2; @@ -288,7 +267,7 @@ public class TestParquetMetadataCache extends PlanTestBase { assertEquals(expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995/Q3", getDfsTestTmpSchemaLocation(), tableName2); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995/Q3", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2); PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[] {"Filter"}); } @@ -297,13 +276,12 @@ public class TestParquetMetadataCache extends PlanTestBase { @Category(UnlikelyTest.class) public void testDrill4530_2() throws Exception { // create metadata cache - test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2)); - checkForMetadataFile(tableName2); + test("refresh table metadata dfs.`%s`", TABLE_NAME_2); + checkForMetadataFile(TABLE_NAME_2); // run query and check correctness - String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " + - " where dir0=1995", - getDfsTestTmpSchemaLocation(), tableName2); + String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " + + " where dir0=1995", TABLE_NAME_2); int expectedRowCount = 80; int expectedNumFiles = 8; @@ -311,7 +289,7 @@ public class TestParquetMetadataCache extends PlanTestBase { assertEquals(expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995", getDfsTestTmpSchemaLocation(), tableName2); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2); PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[] {"Filter"}); } @@ -320,13 +298,12 @@ public class TestParquetMetadataCache extends PlanTestBase { @Category(UnlikelyTest.class) public void testDrill4530_3() throws Exception { // create metadata cache - test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2)); - checkForMetadataFile(tableName2); + test("refresh table metadata dfs.`%s`", TABLE_NAME_2); + checkForMetadataFile(TABLE_NAME_2); // run query and check correctness - String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " + - " where dir1='Q3'", - getDfsTestTmpSchemaLocation(), tableName2); + String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " + + " where dir1='Q3'", TABLE_NAME_2); int expectedRowCount = 40; int expectedNumFiles = 4; @@ -334,7 +311,7 @@ public class TestParquetMetadataCache extends PlanTestBase { assertEquals(expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tableName2); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2); PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[] {}); } @@ -343,13 +320,12 @@ public class TestParquetMetadataCache extends PlanTestBase { @Category(UnlikelyTest.class) public void testDrill4530_4() throws Exception { // create metadata cache - test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2)); - checkForMetadataFile(tableName2); + test("refresh table metadata dfs.`%s`", TABLE_NAME_2); + checkForMetadataFile(TABLE_NAME_2); // run query and check correctness - String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " + - " where dir0=1995 and dir1='Q6'", - getDfsTestTmpSchemaLocation(), tableName2); + String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " + + " where dir0=1995 and dir1='Q6'", TABLE_NAME_2); int expectedRowCount = 0; int expectedNumFiles = 1; @@ -357,7 +333,7 @@ public class TestParquetMetadataCache extends PlanTestBase { assertEquals(expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/*/*", getDfsTestTmpSchemaLocation(), tableName2); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/*/*", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2); PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[] {}); } @@ -365,11 +341,10 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test // DRILL-4794 @Category(UnlikelyTest.class) public void testDrill4794() throws Exception { - test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName1)); - checkForMetadataFile(tableName1); - String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " + - " where dir0=1994 or dir1='Q3'", - getDfsTestTmpSchemaLocation(), tableName1); + test("refresh table metadata dfs.`%s`", TABLE_NAME_1); + checkForMetadataFile(TABLE_NAME_1); + String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " + + " where dir0=1994 or dir1='Q3'", TABLE_NAME_1); int expectedRowCount = 60; int expectedNumFiles = 6; @@ -378,7 +353,7 @@ public class TestParquetMetadataCache extends PlanTestBase { assertEquals(expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tableName1); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_1); PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[] {}); } @@ -387,13 +362,12 @@ public class TestParquetMetadataCache extends PlanTestBase { @Category(UnlikelyTest.class) public void testDrill4786_1() throws Exception { // create metadata cache - test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2)); - checkForMetadataFile(tableName2); + test("refresh table metadata dfs.`%s`", TABLE_NAME_2); + checkForMetadataFile(TABLE_NAME_2); // run query and check correctness - String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " + - " where dir0=1995 and dir1 in ('Q1', 'Q2')", - getDfsTestTmpSchemaLocation(), tableName2); + String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " + + " where dir0=1995 and dir1 in ('Q1', 'Q2')", TABLE_NAME_2); int expectedRowCount = 40; int expectedNumFiles = 4; @@ -402,7 +376,7 @@ public class TestParquetMetadataCache extends PlanTestBase { assertEquals(expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995", getDfsTestTmpSchemaLocation(), tableName2); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2); PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[] {}); @@ -412,13 +386,12 @@ public class TestParquetMetadataCache extends PlanTestBase { @Category(UnlikelyTest.class) public void testDrill4786_2() throws Exception { // create metadata cache - test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2)); - checkForMetadataFile(tableName2); + test("refresh table metadata dfs.`%s`", TABLE_NAME_2); + checkForMetadataFile(TABLE_NAME_2); // run query and check correctness - String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " + - " where dir0 in (1994, 1995) and dir1 = 'Q3'", - getDfsTestTmpSchemaLocation(), tableName2); + String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " + + " where dir0 in (1994, 1995) and dir1 = 'Q3'", TABLE_NAME_2); int expectedRowCount = 40; int expectedNumFiles = 4; @@ -427,22 +400,20 @@ public class TestParquetMetadataCache extends PlanTestBase { assertEquals(expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tableName2); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2); PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[] {}); - } @Test // DRILL-4877 @Category(UnlikelyTest.class) public void testDrill4877() throws Exception { // create metadata cache - test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2)); - checkForMetadataFile(tableName2); + test("refresh table metadata dfs.`%s`", TABLE_NAME_2); + checkForMetadataFile(TABLE_NAME_2); // run query and check correctness - String query1 = String.format("select max(dir0) as max0, max(dir1) as max1 from dfs_test.`%s/%s` ", - getDfsTestTmpSchemaLocation(), tableName2); + String query1 = String.format("select max(dir0) as max0, max(dir1) as max1 from dfs.`%s` ", TABLE_NAME_2); testBuilder() .sqlQuery(query1) @@ -455,7 +426,7 @@ public class TestParquetMetadataCache extends PlanTestBase { String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tableName2); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2); PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[] {}); @@ -466,13 +437,13 @@ public class TestParquetMetadataCache extends PlanTestBase { final String tableName = "nation_move"; final String newTableName = "nation_moved"; try { - test("use dfs_test.tmp"); + test("use dfs"); test("create table `%s/t1` as select * from cp.`tpch/nation.parquet`", tableName); test("create table `%s/t2` as select * from cp.`tpch/nation.parquet`", tableName); test("refresh table metadata %s", tableName); checkForMetadataFile(tableName); - File srcFile = new File(getDfsTestTmpSchemaLocation(), tableName); - File dstFile = new File(getDfsTestTmpSchemaLocation(), newTableName); + File srcFile = new File(dirTestWatcher.getRootDir(), tableName); + File dstFile = new File(dirTestWatcher.getRootDir(), newTableName); FileUtils.moveDirectory(srcFile, dstFile); assertFalse("Cache file was not moved successfully", srcFile.exists()); int rowCount = testSql(String.format("select * from %s", newTableName)); @@ -484,28 +455,49 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test public void testOldMetadataVersions() throws Exception { - final String tablePath = "absolute_paths_metadata"; - String rootMetadataPath = new Path("parquet", "metadata_files_with_old_versions").toUri().getPath(); + final Path tablePath = Paths.get("absolute_paths_metadata"); + final Path rootMetadataPath = Paths.get("parquet", "metadata_files_with_old_versions"); // gets folders with different metadata cache versions - String[] metadataPaths = new File(Resources.getResource(rootMetadataPath).getFile()).list(); + String[] metadataPaths = dirTestWatcher.getRootDir() + .toPath() + .resolve(rootMetadataPath) + .toFile() + .list(); + for (String metadataPath : metadataPaths) { try { - test("use dfs_test.tmp"); + test("use dfs.tmp"); // creating two inner directories to leverage METADATA_DIRECTORIES_FILENAME metadata file as well - final String absolutePathsMetadataT1 = new Path(tablePath, "t1").toUri().getPath(); - final String absolutePathsMetadataT2 = new Path(tablePath, "t2").toUri().getPath(); + final Path absolutePathsMetadataT1 = tablePath.resolve("t1"); + final Path absolutePathsMetadataT2 = tablePath.resolve("t2"); + String createQuery = "create table `%s` as select * from cp.`tpch/nation.parquet`"; + test(createQuery, absolutePathsMetadataT1); test(createQuery, absolutePathsMetadataT2); - Path relativePath = new Path(rootMetadataPath, metadataPath); - copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_directories.requires_replace.txt"), - tablePath, Metadata.METADATA_DIRECTORIES_FILENAME); - copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table.requires_replace.txt"), - tablePath, Metadata.METADATA_FILENAME); - copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table_t1.requires_replace.txt"), - absolutePathsMetadataT1, Metadata.METADATA_FILENAME); - copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table_t2.requires_replace.txt"), - absolutePathsMetadataT2, Metadata.METADATA_FILENAME); + + Path relativePath = rootMetadataPath.resolve(metadataPath); + + File metaFile = dirTestWatcher.copyResourceToTestTmp( + relativePath.resolve("metadata_directories.requires_replace.txt"), + tablePath.resolve(Metadata.METADATA_DIRECTORIES_FILENAME)); + dirTestWatcher.replaceMetaDataContents(metaFile, dirTestWatcher.getDfsTestTmpDir(), + null); + metaFile = dirTestWatcher.copyResourceToTestTmp( + relativePath.resolve("metadata_table.requires_replace.txt"), + tablePath.resolve(Metadata.METADATA_FILENAME)); + dirTestWatcher.replaceMetaDataContents(metaFile, dirTestWatcher.getDfsTestTmpDir(), + null); + metaFile = dirTestWatcher.copyResourceToTestTmp( + relativePath.resolve("metadata_table_t1.requires_replace.txt"), + absolutePathsMetadataT1.resolve(Metadata.METADATA_FILENAME)); + dirTestWatcher.replaceMetaDataContents(metaFile, dirTestWatcher.getDfsTestTmpDir(), + null); + metaFile = dirTestWatcher.copyResourceToTestTmp( + relativePath.resolve("metadata_table_t2.requires_replace.txt"), + absolutePathsMetadataT2.resolve(Metadata.METADATA_FILENAME)); + dirTestWatcher.replaceMetaDataContents(metaFile, dirTestWatcher.getDfsTestTmpDir(), + null); String query = String.format("select * from %s", tablePath); int expectedRowCount = 50; int expectedNumFiles = 1; // point to selectionRoot since no pruning is done in this query @@ -514,7 +506,7 @@ public class TestParquetMetadataCache extends PlanTestBase { expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tablePath); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getDfsTestTmpDir().getCanonicalPath(), tablePath); PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[]{"Filter"}); } finally { @@ -527,7 +519,7 @@ public class TestParquetMetadataCache extends PlanTestBase { public void testSpacesInMetadataCachePath() throws Exception { final String pathWithSpaces = "path with spaces"; try { - test("use dfs_test.tmp"); + test("use dfs"); // creating multilevel table to store path with spaces in both metadata files (METADATA and METADATA_DIRECTORIES) test("create table `%s` as select * from cp.`tpch/nation.parquet`", pathWithSpaces); test("create table `%1$s/%1$s` as select * from cp.`tpch/nation.parquet`", pathWithSpaces); @@ -541,7 +533,7 @@ public class TestParquetMetadataCache extends PlanTestBase { expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), pathWithSpaces); + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getRootDir().getCanonicalPath(), pathWithSpaces); PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, new String[] {"Filter"}); } finally { @@ -553,13 +545,15 @@ public class TestParquetMetadataCache extends PlanTestBase { public void testFutureUnsupportedMetadataVersion() throws Exception { final String unsupportedMetadataVersion = "unsupported_metadata_version"; try { - test("use dfs_test.tmp"); + test("use dfs.tmp"); test("create table `%s` as select * from cp.`tpch/nation.parquet`", unsupportedMetadataVersion); MetadataVersion lastVersion = MetadataVersion.Constants.SUPPORTED_VERSIONS.last(); // Get the future version, which is absent in MetadataVersions.SUPPORTED_VERSIONS set String futureVersion = new MetadataVersion(lastVersion.getMajor() + 1, 0).toString(); - copyMetaDataCacheToTempWithReplacements("parquet/unsupported_metadata/unsupported_metadata_version.requires_replace.txt", - unsupportedMetadataVersion, Metadata.METADATA_FILENAME, futureVersion); + File metaDataFile = dirTestWatcher.copyResourceToTestTmp( + Paths.get("parquet", "unsupported_metadata", "unsupported_metadata_version.requires_replace.txt"), + Paths.get(unsupportedMetadataVersion, Metadata.METADATA_FILENAME)); + dirTestWatcher.replaceMetaDataContents(metaDataFile, dirTestWatcher.getDfsTestTmpDir(), futureVersion); String query = String.format("select * from %s", unsupportedMetadataVersion); int expectedRowCount = 25; int expectedNumFiles = 1; @@ -579,10 +573,11 @@ public class TestParquetMetadataCache extends PlanTestBase { public void testCorruptedMetadataFile() throws Exception { final String corruptedMetadata = "corrupted_metadata"; try { - test("use dfs_test.tmp"); + test("use dfs.tmp"); test("create table `%s` as select * from cp.`tpch/nation.parquet`", corruptedMetadata); - copyMetaDataCacheToTempReplacingInternalPaths("parquet/unsupported_metadata/" + - "corrupted_metadata.requires_replace.txt", corruptedMetadata, Metadata.METADATA_FILENAME); + dirTestWatcher.copyResourceToTestTmp( + Paths.get("parquet", "unsupported_metadata", "corrupted_metadata.requires_replace.txt"), + Paths.get(corruptedMetadata, Metadata.METADATA_FILENAME)); String query = String.format("select * from %s", corruptedMetadata); int expectedRowCount = 25; int expectedNumFiles = 1; @@ -602,10 +597,11 @@ public class TestParquetMetadataCache extends PlanTestBase { public void testEmptyMetadataFile() throws Exception { final String emptyMetadataFile = "empty_metadata_file"; try { - test("use dfs_test.tmp"); + test("use dfs.tmp"); test("create table `%s` as select * from cp.`tpch/nation.parquet`", emptyMetadataFile); - copyMetaDataCacheToTempReplacingInternalPaths("parquet/unsupported_metadata/" + - "empty_metadata_file.requires_replace.txt", emptyMetadataFile, Metadata.METADATA_FILENAME); + dirTestWatcher.copyResourceToTestTmp( + Paths.get("parquet", "unsupported_metadata", "empty_metadata_file.requires_replace.txt"), + Paths.get(emptyMetadataFile, Metadata.METADATA_FILENAME)); String query = String.format("select * from %s", emptyMetadataFile); int expectedRowCount = 25; int expectedNumFiles = 1; @@ -623,40 +619,35 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test public void testRootMetadataFileIsAbsent() throws Exception { - final String tmpDir = getDfsTestTmpSchemaLocation(); final String rootMetaCorruptedTable = "root_meta_corrupted_table"; - File dataDir = new File(tmpDir, rootMetaCorruptedTable); - try { - // copy the data into the temporary location, delete root metadata file - dataDir.mkdir(); - FileUtils.copyDirectory(new File(String.format(String.format("%s/multilevel/parquet", TEST_RES_PATH))), dataDir); + File dataDir = dirTestWatcher.copyResourceToRoot( + Paths.get("multilevel", "parquet"), + Paths.get(rootMetaCorruptedTable)); - test("use dfs_test.tmp"); + try { + test("use dfs"); test("refresh table metadata `%s`", rootMetaCorruptedTable); checkForMetadataFile(rootMetaCorruptedTable); - File rootMetadataFile = FileUtils.getFile(tmpDir, rootMetaCorruptedTable, Metadata.METADATA_FILENAME); - assertTrue(String.format("Metadata cache file '%s' isn't deleted", rootMetadataFile.getPath()), - rootMetadataFile.delete()); + File rootMetadataFile = FileUtils.getFile(dataDir, Metadata.METADATA_FILENAME); + assertTrue(String.format("Metadata cache file '%s' isn't deleted", rootMetadataFile.getPath()), rootMetadataFile.delete()); // mock Metadata tableModified method to avoid occasional metadata files updating new MockUp<Metadata>() { @Mock - boolean tableModified(List<String> directories, Path metaFilePath, Path parentDir, MetadataContext metaContext) { + boolean tableModified(List<String> directories, org.apache.hadoop.fs.Path metaFilePath, + org.apache.hadoop.fs.Path parentDir, MetadataContext metaContext) { return false; } }; - String query = String.format("select dir0, dir1, o_custkey, o_orderdate from `%s` " + - " where dir0=1994 or dir1='Q3'", rootMetaCorruptedTable); + String query = String.format("select dir0, dir1, o_custkey, o_orderdate from `%s` " + " where dir0=1994 or dir1='Q3'", rootMetaCorruptedTable); int expectedRowCount = 60; int expectedNumFiles = 6; int actualRowCount = testSql(query); - assertEquals("An incorrect result was obtained while querying a table with metadata cache files", - expectedRowCount, actualRowCount); + assertEquals("An incorrect result was obtained while querying a table with metadata cache files", expectedRowCount, actualRowCount); String numFilesPattern = "numFiles=" + expectedNumFiles; String usedMetaPattern = "usedMetadataFile=false"; - PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern}, - new String[] {"cacheFileRoot", "Filter"}); + PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern}, new String[]{"cacheFileRoot", "Filter"}); } finally { FileUtils.deleteQuietly(dataDir); } @@ -664,19 +655,17 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test public void testInnerMetadataFilesAreAbsent() throws Exception { - final String tmpDir = getDfsTestTmpSchemaLocation(); final String innerMetaCorruptedTable = "inner_meta_corrupted_table"; - File dataDir = new File(tmpDir, innerMetaCorruptedTable); - try { - // copy the data into the temporary location, delete a few inner metadata files - dataDir.mkdir(); - FileUtils.copyDirectory(new File(String.format(String.format("%s/multilevel/parquet", TEST_RES_PATH))), dataDir); + File dataDir = dirTestWatcher.copyResourceToRoot( + Paths.get("multilevel", "parquet"), + Paths.get(innerMetaCorruptedTable)); - test("use dfs_test.tmp"); + try { + test("use dfs"); test("refresh table metadata `%s`", innerMetaCorruptedTable); checkForMetadataFile(innerMetaCorruptedTable); - File firstInnerMetadataFile = FileUtils.getFile(tmpDir, innerMetaCorruptedTable, "1994", Metadata.METADATA_FILENAME); - File secondInnerMetadataFile = FileUtils.getFile(tmpDir, innerMetaCorruptedTable, "1994", "Q3", Metadata.METADATA_FILENAME); + File firstInnerMetadataFile = FileUtils.getFile(dataDir, "1994", Metadata.METADATA_FILENAME); + File secondInnerMetadataFile = FileUtils.getFile(dataDir, "1994", "Q3", Metadata.METADATA_FILENAME); assertTrue(String.format("Metadata cache file '%s' isn't deleted", firstInnerMetadataFile.getPath()), firstInnerMetadataFile.delete()); assertTrue(String.format("Metadata cache file '%s' isn't deleted", secondInnerMetadataFile.getPath()), @@ -685,7 +674,8 @@ public class TestParquetMetadataCache extends PlanTestBase { // mock Metadata tableModified method to avoid occasional metadata files updating new MockUp<Metadata>() { @Mock - boolean tableModified(List<String> directories, Path metaFilePath, Path parentDir, MetadataContext metaContext) { + boolean tableModified(List<String> directories, org.apache.hadoop.fs.Path metaFilePath, + org.apache.hadoop.fs.Path parentDir, MetadataContext metaContext) { return false; } }; @@ -709,7 +699,7 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test // DRILL-4264 @Category(UnlikelyTest.class) public void testMetadataCacheFieldWithDots() throws Exception { - final String tableWithDots = "dfs_test.tmp.`complex_table`"; + final String tableWithDots = "dfs.tmp.`complex_table`"; try { test("create table %s as\n" + "select cast(1 as int) as `column.with.dots`, t.`column`.`with.dots`\n" + @@ -734,7 +724,7 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test // DRILL-4139 public void testBooleanPartitionPruning() throws Exception { - final String boolPartitionTable = "dfs_test.tmp.`interval_bool_partition`"; + final String boolPartitionTable = "dfs.tmp.`interval_bool_partition`"; try { test("create table %s partition by (col_bln) as " + "select * from cp.`parquet/alltypes_required.parquet`", boolPartitionTable); @@ -758,7 +748,7 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test // DRILL-4139 public void testIntervalDayPartitionPruning() throws Exception { - final String intervalDayPartitionTable = "dfs_test.tmp.`interval_day_partition`"; + final String intervalDayPartitionTable = "dfs.tmp.`interval_day_partition`"; try { test("create table %s partition by (col_intrvl_day) as " + "select * from cp.`parquet/alltypes_optional.parquet`", intervalDayPartitionTable); @@ -783,7 +773,7 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test // DRILL-4139 public void testIntervalYearPartitionPruning() throws Exception { - final String intervalYearPartitionTable = "dfs_test.tmp.`interval_yr_partition`"; + final String intervalYearPartitionTable = "dfs.tmp.`interval_yr_partition`"; try { test("create table %s partition by (col_intrvl_yr) as " + "select * from cp.`parquet/alltypes_optional.parquet`", intervalYearPartitionTable); @@ -808,7 +798,7 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test // DRILL-4139 public void testVarCharWithNullsPartitionPruning() throws Exception { - final String intervalYearPartitionTable = "dfs_test.tmp.`varchar_optional_partition`"; + final String intervalYearPartitionTable = "dfs.tmp.`varchar_optional_partition`"; try { test("create table %s partition by (col_vrchr) as " + "select * from cp.`parquet/alltypes_optional.parquet`", intervalYearPartitionTable); @@ -845,7 +835,7 @@ public class TestParquetMetadataCache extends PlanTestBase { ctasQueries.add("create table %s partition by (manager_id) as " + "select cast(manager_id as decimal(18, 6)) as manager_id, EMPLOYEE_ID, FIRST_NAME, LAST_NAME " + "from cp.`parquet/fixedlenDecimal.parquet`"); - final String decimalPartitionTable = "dfs_test.tmp.`decimal_optional_partition`"; + final String decimalPartitionTable = "dfs.tmp.`decimal_optional_partition`"; for (String ctasQuery : ctasQueries) { try { test("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY); @@ -873,76 +863,76 @@ public class TestParquetMetadataCache extends PlanTestBase { @Test // DRILL-4139 public void testIntWithNullsPartitionPruning() throws Exception { try { - test("create table dfs_test.tmp.`t5/a` as\n" + + test("create table dfs.tmp.`t5/a` as\n" + "select 100 as mykey from cp.`tpch/nation.parquet`\n" + "union all\n" + "select col_notexist from cp.`tpch/region.parquet`"); - test("create table dfs_test.tmp.`t5/b` as\n" + + test("create table dfs.tmp.`t5/b` as\n" + "select 200 as mykey from cp.`tpch/nation.parquet`\n" + "union all\n" + "select col_notexist from cp.`tpch/region.parquet`"); - String query = "select mykey from dfs_test.tmp.`t5` where mykey = 100"; + String query = "select mykey from dfs.tmp.`t5` where mykey = 100"; int actualRowCount = testSql(query); assertEquals("Row count does not match the expected value", 25, actualRowCount); - test("refresh table metadata dfs_test.tmp.`t5`"); + test("refresh table metadata dfs.tmp.`t5`"); actualRowCount = testSql(query); assertEquals("Row count does not match the expected value", 25, actualRowCount); } finally { - test("drop table if exists dfs_test.tmp.`t5`"); + test("drop table if exists dfs.tmp.`t5`"); } } @Test // DRILL-4139 public void testPartitionPruningWithIsNull() throws Exception { try { - test("create table dfs_test.tmp.`t6/a` as\n" + + test("create table dfs.tmp.`t6/a` as\n" + "select col_notexist as mykey from cp.`tpch/region.parquet`"); - test("create table dfs_test.tmp.`t6/b` as\n" + + test("create table dfs.tmp.`t6/b` as\n" + "select 100 as mykey from cp.`tpch/region.parquet`"); - String query = "select mykey from dfs_test.tmp.t6 where mykey is null"; + String query = "select mykey from dfs.tmp.t6 where mykey is null"; int actualRowCount = testSql(query); assertEquals("Row count does not match the expected value", 5, actualRowCount); PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=false"}, new String[]{"Filter"}); - test("refresh table metadata dfs_test.tmp.`t6`"); + test("refresh table metadata dfs.tmp.`t6`"); actualRowCount = testSql(query); assertEquals("Row count does not match the expected value", 5, actualRowCount); PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=true"}, new String[]{"Filter"}); } finally { - test("drop table if exists dfs_test.tmp.`t6`"); + test("drop table if exists dfs.tmp.`t6`"); } } @Test // DRILL-4139 public void testPartitionPruningWithIsNotNull() throws Exception { try { - test("create table dfs_test.tmp.`t7/a` as\n" + + test("create table dfs.tmp.`t7/a` as\n" + "select col_notexist as mykey from cp.`tpch/region.parquet`"); - test("create table dfs_test.tmp.`t7/b` as\n" + + test("create table dfs.tmp.`t7/b` as\n" + "select 100 as mykey from cp.`tpch/region.parquet`"); - String query = "select mykey from dfs_test.tmp.t7 where mykey is null"; + String query = "select mykey from dfs.tmp.t7 where mykey is null"; int actualRowCount = testSql(query); assertEquals("Row count does not match the expected value", 5, actualRowCount); PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=false"}, new String[]{"Filter"}); - test("refresh table metadata dfs_test.tmp.`t7`"); + test("refresh table metadata dfs.tmp.`t7`"); actualRowCount = testSql(query); assertEquals("Row count does not match the expected value", 5, actualRowCount); PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=true"}, new String[]{"Filter"}); } finally { - test("drop table if exists dfs_test.tmp.`t7`"); + test("drop table if exists dfs.tmp.`t7`"); } } @@ -952,7 +942,14 @@ public class TestParquetMetadataCache extends PlanTestBase { * @param table table name or table path */ private void checkForMetadataFile(String table) { - String tmpDir = getDfsTestTmpSchemaLocation(); + final String tmpDir; + + try { + tmpDir = dirTestWatcher.getRootDir().getCanonicalPath(); + } catch (IOException e) { + throw new RuntimeException(e); + } + File metaFile = table.startsWith(tmpDir) ? FileUtils.getFile(table, Metadata.METADATA_FILENAME) : FileUtils.getFile(tmpDir, table, Metadata.METADATA_FILENAME); assertTrue(String.format("There is no metadata cache file for the %s table", table),
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java index 39df4e0..c95b201 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java @@ -18,41 +18,29 @@ package org.apache.drill.exec.store.parquet; import com.google.common.io.Resources; -import org.apache.drill.BaseTestQuery; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.junit.BeforeClass; +import org.apache.commons.io.FileUtils; +import org.apache.drill.test.BaseTestQuery; import org.junit.Test; -public class TestParquetScan extends BaseTestQuery { - - static FileSystem fs; - - @BeforeClass - public static void initFs() throws Exception { - fs = getLocalFileSystem(); - } +import java.io.File; +import java.nio.file.Path; +public class TestParquetScan extends BaseTestQuery { @Test public void testSuccessFile() throws Exception { - Path p = new Path("/tmp/nation_test_parquet_scan"); - if (fs.exists(p)) { - fs.delete(p, true); - } - - fs.mkdirs(p); + final byte[] bytes = Resources.toByteArray(Resources.getResource("tpch/nation.parquet")); - byte[] bytes = Resources.toByteArray(Resources.getResource("tpch/nation.parquet")); + final Path rootPath = dirTestWatcher.getRootDir().toPath(); + final File scanFile = rootPath.resolve("nation_test_parquet_scan").toFile(); + final File successFile = rootPath.resolve("_SUCCESS").toFile(); + final File logsFile = rootPath.resolve("_logs").toFile(); - FSDataOutputStream os = fs.create(new Path(p, "nation.parquet")); - os.write(bytes); - os.close(); - fs.create(new Path(p, "_SUCCESS")).close(); - fs.create(new Path(p, "_logs")).close(); + FileUtils.writeByteArrayToFile(scanFile, bytes); + successFile.createNewFile(); + logsFile.createNewFile(); testBuilder() - .sqlQuery("select count(*) c from dfs.tmp.nation_test_parquet_scan where 1 = 1") + .sqlQuery("select count(*) c from dfs.nation_test_parquet_scan where 1 = 1") .unOrdered() .baselineColumns("c") .baselineValues(25L) http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java index bfd894d..cf11d5f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.store.parquet.columnreaders; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.junit.AfterClass; import org.junit.BeforeClass; http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java index e95842c..b641707 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.store.parquet.columnreaders; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.junit.Test; public class TestDateReader extends BaseTestQuery { http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java index 477b825..ec2651d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.store.parquet2; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -75,12 +75,12 @@ public class TestDrillParquetReader extends BaseTestQuery { @Test public void test4349() throws Exception { // start by creating a parquet file from the input csv file - runSQL("CREATE TABLE dfs_test.tmp.`4349` AS SELECT columns[0] id, CAST(NULLIF(columns[1], '') AS DOUBLE) val FROM cp.`parquet2/4349.csv.gz`"); + runSQL("CREATE TABLE dfs.tmp.`4349` AS SELECT columns[0] id, CAST(NULLIF(columns[1], '') AS DOUBLE) val FROM cp.`parquet2/4349.csv.gz`"); // querying the parquet file should return the same results found in the csv file testBuilder() .unOrdered() - .sqlQuery("SELECT * FROM dfs_test.tmp.`4349` WHERE id = 'b'") + .sqlQuery("SELECT * FROM dfs.tmp.`4349` WHERE id = 'b'") .sqlBaselineQuery("SELECT columns[0] id, CAST(NULLIF(columns[1], '') AS DOUBLE) val FROM cp.`parquet2/4349.csv.gz` WHERE columns[0] = 'b'") .go(); } http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoder.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoder.java index 56c2dbe..6e58ccf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoder.java @@ -17,10 +17,9 @@ package org.apache.drill.exec.store.pcap; import com.google.common.io.Resources; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.exec.store.pcap.decoder.Packet; import org.apache.drill.exec.store.pcap.decoder.PacketDecoder; -import org.junit.BeforeClass; import org.junit.Test; import java.io.BufferedInputStream; http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java index ce2a4f8..706694d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java @@ -16,33 +16,39 @@ */ package org.apache.drill.exec.store.pcap; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; +import java.nio.file.Paths; import java.util.List; public class TestPcapRecordReader extends BaseTestQuery { + @BeforeClass + public static void setupTestFiles() { + dirTestWatcher.copyResourceToRoot(Paths.get("store", "pcap")); + } @Test public void testStarQuery() throws Exception { - runSQLVerifyCount("select * from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 16); - runSQLVerifyCount("select distinct DST_IP from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 1); - runSQLVerifyCount("select distinct DsT_IP from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 1); - runSQLVerifyCount("select distinct dst_ip from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 1); + runSQLVerifyCount("select * from dfs.`store/pcap/tcp-1.pcap`", 16); + runSQLVerifyCount("select distinct DST_IP from dfs.`store/pcap/tcp-1.pcap`", 1); + runSQLVerifyCount("select distinct DsT_IP from dfs.`store/pcap/tcp-1.pcap`", 1); + runSQLVerifyCount("select distinct dst_ip from dfs.`store/pcap/tcp-1.pcap`", 1); } @Test public void testCountQuery() throws Exception { - runSQLVerifyCount("select count(*) from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 1); - runSQLVerifyCount("select count(*) from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-2.pcap`", 1); + runSQLVerifyCount("select count(*) from dfs.`store/pcap/tcp-1.pcap`", 1); + runSQLVerifyCount("select count(*) from dfs.`store/pcap/tcp-2.pcap`", 1); } @Test public void testDistinctQuery() throws Exception { - runSQLVerifyCount("select distinct * from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 1); + runSQLVerifyCount("select distinct * from dfs.`store/pcap/tcp-1.pcap`", 1); } private void runSQLVerifyCount(String sql, int expectedRowCount) throws Exception { http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/sequencefile/TestSequenceFileReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sequencefile/TestSequenceFileReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sequencefile/TestSequenceFileReader.java index f48eb2c..6d3ce64 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sequencefile/TestSequenceFileReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sequencefile/TestSequenceFileReader.java @@ -21,8 +21,7 @@ import java.io.DataOutputStream; import java.io.ByteArrayOutputStream; import org.junit.Test; -import org.apache.drill.BaseTestQuery; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.test.BaseTestQuery; import org.apache.hadoop.io.BytesWritable; public class TestSequenceFileReader extends BaseTestQuery { @@ -37,11 +36,9 @@ public class TestSequenceFileReader extends BaseTestQuery { @Test public void testSequenceFileReader() throws Exception { - String root = FileUtils.getResourceAsFile("/sequencefiles/simple.seq").toURI().toString(); - final String query = String.format("select convert_from(t.binary_key, 'UTF8') as k, convert_from(t.binary_value, 'UTF8') as v " + - "from dfs_test.`%s` t", root); testBuilder() - .sqlQuery(query) + .sqlQuery("select convert_from(t.binary_key, 'UTF8') as k, convert_from(t.binary_value, 'UTF8') as v " + + "from cp.`sequencefiles/simple.seq` t") .ordered() .baselineColumns("k", "v") .baselineValues(byteWritableString("key0"), byteWritableString("value0")) http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java index bcd2f5e..d3116c7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java @@ -41,7 +41,10 @@ public class PStoreTestUtil { expectedMap.put(keys[i], values[i]); store.put(keys[i], values[i]); } - // allow one second for puts to propagate back to cache + + // Wait for store caches to update, this is necessary because ZookeeperClient caches can update asynchronously in some cases. + waitForNumProps(store, keys.length); + { Iterator<Map.Entry<String, String>> iter = store.getAll(); for(int i =0; i < keys.length; i++){ @@ -61,8 +64,29 @@ public class PStoreTestUtil { } } - // allow one second for deletes to propagate back to cache - + // Wait for store caches to update, this is necessary because ZookeeperClient caches can update asynchronously in some cases. + waitForNumProps(store, 0); assertFalse(store.getAll().hasNext()); } + + private static void waitForNumProps(PersistentStore store, int expected) throws InterruptedException { + for (int numProps = getNumProps(store); + numProps < expected; + numProps = getNumProps(store)) { + Thread.sleep(100L); + } + } + + private static int getNumProps(PersistentStore store) { + Iterator<Map.Entry<String, String>> iter = store.getAll(); + + int numProps = 0; + + while (iter.hasNext()) { + iter.next(); + numProps++; + } + + return numProps; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java index e7b150c..7325fd7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java @@ -22,8 +22,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.RetryNTimes; +import org.apache.drill.categories.SlowTest; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.TestWithZookeeper; import org.apache.drill.exec.coord.zk.PathUtils; @@ -32,22 +33,22 @@ import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.server.options.PersistedOptionValue; import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider; import org.apache.drill.exec.store.sys.store.provider.ZookeeperPersistentStoreProvider; +import org.apache.drill.test.BaseDirTestWatcher; import org.apache.drill.test.ClientFixture; import org.apache.drill.test.ClusterFixture; import org.apache.drill.test.ClusterFixtureBuilder; -import org.apache.drill.testutils.DirTestWatcher; import org.apache.zookeeper.CreateMode; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.io.File; +@Category(SlowTest.class) public class TestPStoreProviders extends TestWithZookeeper { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestPStoreProviders.class); - @Rule - public DirTestWatcher dirTestWatcher = new DirTestWatcher(); + public BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher(); @Test public void verifyLocalStore() throws Exception { @@ -79,18 +80,14 @@ public class TestPStoreProviders extends TestWithZookeeper { private void localLoadTestHelper(String propertiesDir) throws Exception { File localOptionsResources = new File(propertiesDir); - String optionsDirPath = String.format("%s/sys.options", dirTestWatcher.getDirPath()); - File optionsDir = new File(optionsDirPath); - optionsDir.mkdirs(); - - org.apache.commons.io.FileUtils.copyDirectory(localOptionsResources, optionsDir); - - ClusterFixtureBuilder builder = ClusterFixture.builder(). - configProperty(ExecConstants.HTTP_ENABLE, false). + ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher). configProperty(ExecConstants.SYS_STORE_PROVIDER_CLASS, LocalPersistentStoreProvider.class.getCanonicalName()). - configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH, String.format("file://%s", dirTestWatcher.getDirPath())). configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true); + File optionsDir = new File(dirTestWatcher.getStoreDir(), "sys.options"); + optionsDir.mkdirs(); + org.apache.commons.io.FileUtils.copyDirectory(localOptionsResources, optionsDir); + try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { String parquetPushdown = client.queryBuilder(). @@ -129,7 +126,7 @@ public class TestPStoreProviders extends TestWithZookeeper { try (ZookeeperClient zkClient = new ZookeeperClient(curator, PathUtils.join("/", storeConfig.getName()), CreateMode.PERSISTENT)) { zkClient.start(); - String oldOptionJson = FileUtils.getResourceAsString("/options/old_booleanopt.json"); + String oldOptionJson = DrillFileUtils.getResourceAsString("/options/old_booleanopt.json"); zkClient.put(oldName, oldOptionJson.getBytes(), null); } http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java index dff1504..c6c5a05 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.store.sys; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.categories.UnlikelyTest; import org.apache.drill.exec.ExecConstants; import org.junit.BeforeClass; http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestCsvHeader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestCsvHeader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestCsvHeader.java index cf54bb0..2685496 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestCsvHeader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestCsvHeader.java @@ -18,13 +18,13 @@ package org.apache.drill.exec.store.text; import com.google.common.collect.Lists; -import org.apache.drill.BaseTestQuery; -import org.apache.drill.TestBuilder; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.test.BaseTestQuery; +import org.apache.drill.test.TestBuilder; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; +import java.nio.file.Paths; import java.util.List; import org.junit.Before; @@ -32,20 +32,20 @@ import org.junit.Test; public class TestCsvHeader extends BaseTestQuery{ - private String root; + private static final String ROOT = "store/text/data/cars.csvh"; @Before public void initialize() throws Exception { - root = FileUtils.getResourceAsFile("/store/text/data/cars.csvh").toURI().toString(); + dirTestWatcher.copyResourceToRoot(Paths.get("store", "text", "data")); + test("alter session set `exec.errors.verbose` = true "); } @Test //DRILL-951 public void testCsvWithHeader() throws Exception { - //Pre DRILL-951: Qry: select * from dfs_test.`%s` LIMIT 2 - String query = String.format("select * from dfs_test.`%s` LIMIT 2", root); + //Pre DRILL-951: Qry: select * from cp.`%s` LIMIT 2 testBuilder() - .sqlQuery(query) + .sqlQuery("select * from cp.`%s` LIMIT 2", ROOT) .unOrdered() .baselineColumns("Year", "Make", "Model", "Description", "Price") .baselineValues("1997", "Ford", "E350", "ac, abs, moon", "3000.00") @@ -55,11 +55,9 @@ public class TestCsvHeader extends BaseTestQuery{ @Test //DRILL-951 public void testCsvWhereWithHeader() throws Exception { - //Pre DRILL-951: Qry: select * from dfs_test.`%s` where columns[1] = 'Chevy' - String query = String.format("select * from dfs_test.`%s` where Make = 'Chevy'", root); - + //Pre DRILL-951: Qry: select * from cp.`%s` where columns[1] = 'Chevy' testBuilder() - .sqlQuery(query) + .sqlQuery("select * from cp.`%s` where Make = 'Chevy'", ROOT) .unOrdered() .baselineColumns("Year", "Make", "Model", "Description", "Price") .baselineValues("1999", "Chevy", "Venture \"Extended Edition\"", "", "4900.00") @@ -69,11 +67,9 @@ public class TestCsvHeader extends BaseTestQuery{ @Test //DRILL-951 public void testCsvStarPlusWithHeader() throws Exception { - //Pre DRILL-951: Qry: select *,columns[1] from dfs_test.`%s` where columns[1] = 'Chevy' - String query = String.format("select *, Make from dfs_test.`%s` where Make = 'Chevy'", root); - + //Pre DRILL-951: Qry: select *,columns[1] from cp.`%s` where columns[1] = 'Chevy' testBuilder() - .sqlQuery(query) + .sqlQuery("select *, Make from cp.`%s` where Make = 'Chevy'", ROOT) .unOrdered() .baselineColumns("Year", "Make", "Model", "Description", "Price", "Make0") .baselineValues("1999", "Chevy", "Venture \"Extended Edition\"", "", "4900.00", "Chevy") @@ -83,10 +79,9 @@ public class TestCsvHeader extends BaseTestQuery{ @Test //DRILL-951 public void testCsvWhereColumnsWithHeader() throws Exception { - //Pre DRILL-951: Qry: select columns[1] from dfs_test.`%s` where columns[1] = 'Chevy' - String query = String.format("select Make from dfs_test.`%s` where Make = 'Chevy'", root); + //Pre DRILL-951: Qry: select columns[1] from cp.`%s` where columns[1] = 'Chevy' testBuilder() - .sqlQuery(query) + .sqlQuery("select Make from cp.`%s` where Make = 'Chevy'", ROOT) .unOrdered() .baselineColumns("Make") .baselineValues("Chevy") @@ -96,11 +91,9 @@ public class TestCsvHeader extends BaseTestQuery{ @Test //DRILL-951 public void testCsvColumnsWithHeader() throws Exception { - //Pre DRILL-951: Qry: select columns[0],columns[2],columns[4] from dfs_test.`%s` where columns[1] = 'Chevy' - String query = String.format("select `Year`, Model, Price from dfs_test.`%s` where Make = 'Chevy'", root); - + //Pre DRILL-951: Qry: select columns[0],columns[2],columns[4] from cp.`%s` where columns[1] = 'Chevy' testBuilder() - .sqlQuery(query) + .sqlQuery("select `Year`, Model, Price from cp.`%s` where Make = 'Chevy'", ROOT) .unOrdered() .baselineColumns("Year", "Model", "Price") .baselineValues("1999", "Venture \"Extended Edition\"", "4900.00") @@ -110,10 +103,8 @@ public class TestCsvHeader extends BaseTestQuery{ @Test //DRILL-951 public void testCsvHeaderShortCircuitReads() throws Exception { - String query = String.format("select `Year`, Model from dfs_test.`%s` where Make = 'Chevy'", root); - testBuilder() - .sqlQuery(query) + .sqlQuery("select `Year`, Model from cp.`%s` where Make = 'Chevy'", ROOT) .unOrdered() .baselineColumns("Year", "Model") .baselineValues("1999", "Venture \"Extended Edition\"") @@ -123,10 +114,8 @@ public class TestCsvHeader extends BaseTestQuery{ @Test //DRILL-4108 public void testCsvHeaderNonExistingColumn() throws Exception { - String query = String.format("select `Year`, Model, Category from dfs_test.`%s` where Make = 'Chevy'", root); - testBuilder() - .sqlQuery(query) + .sqlQuery("select `Year`, Model, Category from cp.`%s` where Make = 'Chevy'", ROOT) .unOrdered() .baselineColumns("Year", "Model", "Category") .baselineValues("1999", "Venture \"Extended Edition\"", "") @@ -136,10 +125,8 @@ public class TestCsvHeader extends BaseTestQuery{ @Test //DRILL-4108 public void testCsvHeaderMismatch() throws Exception { - String ddir = FileUtils.getResourceAsFile("/store/text/data/d2").toURI().toString(); - String query = String.format("select `Year`, Model, Category from dfs_test.`%s` where Make = 'Chevy'", ddir); testBuilder() - .sqlQuery(query) + .sqlQuery("select `Year`, Model, Category from dfs.`store/text/data/d2` where Make = 'Chevy'") .unOrdered() .baselineColumns("Year", "Model", "Category") .baselineValues("1999", "", "Venture \"Extended Edition\"") @@ -151,12 +138,8 @@ public class TestCsvHeader extends BaseTestQuery{ @Test //DRILL-4108 public void testCsvHeaderSkipFirstLine() throws Exception { - // test that header is not skipped when skipFirstLine is true - // testing by defining new format plugin with skipFirstLine set to true and diff file extension - String dfile = FileUtils.getResourceAsFile("/store/text/data/cars.csvh-test").toURI().toString(); - String query = String.format("select `Year`, Model from dfs_test.`%s` where Make = 'Chevy'", dfile); testBuilder() - .sqlQuery(query) + .sqlQuery("select `Year`, Model from cp.`store/text/data/cars.csvh-test` where Make = 'Chevy'") .unOrdered() .baselineColumns("Year", "Model") .baselineValues("1999", "Venture \"Extended Edition\"") @@ -166,9 +149,7 @@ public class TestCsvHeader extends BaseTestQuery{ @Test public void testEmptyFinalColumn() throws Exception { - String dfs_temp = getDfsTestTmpSchemaLocation(); - File table_dir = new File(dfs_temp, "emptyFinalColumn"); - table_dir.mkdir(); + File table_dir = dirTestWatcher.makeTestTmpSubDir(Paths.get("emptyFinalColumn")); BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "a.csvh"))); os.write("field1,field2\n".getBytes()); for (int i = 0; i < 10000; i++) { @@ -176,7 +157,7 @@ public class TestCsvHeader extends BaseTestQuery{ } os.flush(); os.close(); - String query = "select * from dfs_test.tmp.emptyFinalColumn"; + String query = "select * from dfs.tmp.emptyFinalColumn"; TestBuilder builder = testBuilder() .sqlQuery(query) .ordered() @@ -189,12 +170,12 @@ public class TestCsvHeader extends BaseTestQuery{ @Test public void testCountOnCsvWithHeader() throws Exception { - final String query = "select count(%s) as cnt from %s.`%s`"; + final String query = "select count(%s) as cnt from cp.`%s`"; final List<Object> options = Lists.<Object>newArrayList("*", 1, "'A'"); for (Object option : options) { testBuilder() - .sqlQuery(query, option, TEMP_SCHEMA, root) + .sqlQuery(query, option, ROOT) .unOrdered() .baselineColumns("cnt") .baselineValues(4L) @@ -202,5 +183,4 @@ public class TestCsvHeader extends BaseTestQuery{ .run(); } } - } http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java index bfc3cc2..8aa939e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java @@ -21,10 +21,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.categories.UnlikelyTest; import org.apache.drill.common.exceptions.UserRemoteException; -import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.proto.UserBitShared.DrillPBError.ErrorType; import org.junit.Ignore; import org.junit.Test; @@ -69,12 +68,9 @@ public class TestNewTextReader extends BaseTestQuery { @Test // see DRILL-3718 @Category(UnlikelyTest.class) public void testTabSeparatedWithQuote() throws Exception { - final String root = FileUtils.getResourceAsFile("/store/text/WithQuote.tsv").toURI().toString(); - final String query = String.format("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" + - "from dfs_test.`%s` ", root); - testBuilder() - .sqlQuery(query) + .sqlQuery("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" + + "from cp.`store/text/WithQuote.tsv`") .unOrdered() .baselineColumns("c0", "c1", "c2") .baselineValues("a", "a", "a") @@ -87,12 +83,9 @@ public class TestNewTextReader extends BaseTestQuery { @Test // see DRILL-3718 @Category(UnlikelyTest.class) public void testSpaceSeparatedWithQuote() throws Exception { - final String root = FileUtils.getResourceAsFile("/store/text/WithQuote.ssv").toURI().toString(); - final String query = String.format("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" + - "from dfs_test.`%s` ", root); - testBuilder() - .sqlQuery(query) + .sqlQuery("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" + + "from cp.`store/text/WithQuote.ssv`") .unOrdered() .baselineColumns("c0", "c1", "c2") .baselineValues("a", "a", "a") @@ -105,12 +98,9 @@ public class TestNewTextReader extends BaseTestQuery { @Test // see DRILL-3718 @Category(UnlikelyTest.class) public void testPipSeparatedWithQuote() throws Exception { - final String root = FileUtils.getResourceAsFile("/store/text/WithQuote.tbl").toURI().toString(); - final String query = String.format("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" + - "from dfs_test.`%s` ", root); - testBuilder() - .sqlQuery(query) + .sqlQuery("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" + + "from cp.`store/text/WithQuote.tbl`") .unOrdered() .baselineColumns("c0", "c1", "c2") .baselineValues("a", "a", "a") @@ -123,12 +113,9 @@ public class TestNewTextReader extends BaseTestQuery { @Test // see DRILL-3718 @Category(UnlikelyTest.class) public void testCrLfSeparatedWithQuote() throws Exception { - final String root = FileUtils.getResourceAsFile("/store/text/WithQuotedCrLf.tbl").toURI().toString(); - final String query = String.format("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" + - "from dfs_test.`%s` ", root); - testBuilder() - .sqlQuery(query) + .sqlQuery("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" + + "from cp.`store/text/WithQuotedCrLf.tbl`") .unOrdered() .baselineColumns("c0", "c1", "c2") .baselineValues("a\n1", "a", "a") http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java index 1ff6818..f106cca 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java @@ -14,24 +14,21 @@ package org.apache.drill.exec.store.text; import java.util.List; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.junit.Test; import static org.junit.Assert.*; public class TestTextColumn extends BaseTestQuery { - - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTextColumn.class); - @Test public void testCsvColumnSelection() throws Exception { - test("select columns[0] as region_id, columns[1] as country from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/regions.csv`"); + test("select columns[0] as region_id, columns[1] as country from cp.`store/text/data/regions.csv`"); } @Test public void testDefaultDelimiterColumnSelection() throws Exception { List<QueryDataBatch> actualResults = testSqlWithResults("SELECT columns[0] as entire_row " - + "from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/letters.txt`"); + + "from cp.`store/text/data/letters.txt`"); final TestResultSet expectedResultSet = new TestResultSet(); expectedResultSet.addRow("a, b,\",\"c\",\"d,, \\n e"); @@ -45,7 +42,7 @@ public class TestTextColumn extends BaseTestQuery { @Test public void testCsvColumnSelectionCommasInsideQuotes() throws Exception { List<QueryDataBatch> actualResults = testSqlWithResults("SELECT columns[0] as col1, columns[1] as col2, columns[2] as col3," - + "columns[3] as col4 from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/letters.csv`"); + + "columns[3] as col4 from cp.`store/text/data/letters.csv`"); final TestResultSet expectedResultSet = new TestResultSet(); expectedResultSet.addRow("a, b,", "c", "d,, \\n e", "f\\\"g"); http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TextRecordReaderTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TextRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TextRecordReaderTest.java index 5e781d2..d4098f9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TextRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TextRecordReaderTest.java @@ -21,7 +21,7 @@ import static org.junit.Assert.assertEquals; import java.util.List; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.record.RecordBatchLoader; @@ -47,8 +47,8 @@ public class TextRecordReaderTest extends PopUnitTestBase { client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Files.toString( - FileUtils.getResourceAsFile("/store/text/test.json"), Charsets.UTF_8) - .replace("#{DATA_FILE}", FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString())); + DrillFileUtils.getResourceAsFile("/store/text/test.json"), Charsets.UTF_8) + .replace("#{DATA_FILE}", DrillFileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString())); int count = 0; RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator()); for(QueryDataBatch b : results) { http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java index 407cb7c..6404e27 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java @@ -22,7 +22,7 @@ import static org.junit.Assert.fail; import java.util.concurrent.CountDownLatch; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.common.concurrent.ExtendedLatch; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.proto.UserBitShared.QueryId; http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java index 22da20b..44f8727 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java @@ -22,7 +22,7 @@ import static org.junit.Assert.fail; import java.io.IOException; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ZookeeperHelper; import org.apache.drill.exec.exception.DrillbitStartupException; http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java index c618739..2e8aa02 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java @@ -22,7 +22,7 @@ import static org.junit.Assert.fail; import java.util.concurrent.CountDownLatch; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.common.concurrent.ExtendedLatch; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ZookeeperHelper;