http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestBroadcastExchange.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestBroadcastExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestBroadcastExchange.java index 3ef5651..38a01d5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestBroadcastExchange.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestBroadcastExchange.java @@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals; import java.util.List; import org.apache.drill.categories.OperatorTest; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.proto.UserBitShared.QueryType; @@ -50,9 +50,9 @@ public class TestBroadcastExchange extends PopUnitTestBase { client.connect(); String physicalPlan = Files.toString( - FileUtils.getResourceAsFile("/sender/broadcast_exchange.json"), Charsets.UTF_8) - .replace("#{LEFT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.left.json").toURI().toString()) - .replace("#{RIGHT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.right.json").toURI().toString()); + DrillFileUtils.getResourceAsFile("/sender/broadcast_exchange.json"), Charsets.UTF_8) + .replace("#{LEFT_FILE}", DrillFileUtils.getResourceAsFile("/join/merge_single_batch.left.json").toURI().toString()) + .replace("#{RIGHT_FILE}", DrillFileUtils.getResourceAsFile("/join/merge_single_batch.right.json").toURI().toString()); List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, physicalPlan); int count = 0; for (QueryDataBatch b : results) { @@ -78,7 +78,7 @@ public class TestBroadcastExchange extends PopUnitTestBase { client.connect(); String physicalPlan = Files.toString( - FileUtils.getResourceAsFile("/sender/broadcast_exchange_long_run.json"), Charsets.UTF_8); + DrillFileUtils.getResourceAsFile("/sender/broadcast_exchange_long_run.json"), Charsets.UTF_8); List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, physicalPlan); int count = 0; for (QueryDataBatch b : results) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java index 3cd293e..11f739d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java @@ -24,7 +24,7 @@ import java.util.List; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers; @@ -62,9 +62,7 @@ import com.google.common.io.Files; import mockit.Injectable; -public class TestCastFunctions extends PopUnitTestBase{ - //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class); - +public class TestCastFunctions extends PopUnitTestBase { @Test // cast to bigint. public void testCastBigInt(@Injectable final DrillbitContext bitContext, @@ -73,7 +71,7 @@ public class TestCastFunctions extends PopUnitTestBase{ mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastBigInt.json"), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastBigInt.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); @@ -111,7 +109,7 @@ public class TestCastFunctions extends PopUnitTestBase{ mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastInt.json"), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastInt.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); @@ -147,7 +145,7 @@ public class TestCastFunctions extends PopUnitTestBase{ mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastFloat4.json"), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastFloat4.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); @@ -184,7 +182,7 @@ public class TestCastFunctions extends PopUnitTestBase{ mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastFloat8.json"), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastFloat8.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); @@ -221,7 +219,7 @@ public class TestCastFunctions extends PopUnitTestBase{ mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarChar.json"), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastVarChar.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); @@ -257,7 +255,7 @@ public class TestCastFunctions extends PopUnitTestBase{ mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarBinary.json"), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastVarBinary.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); @@ -294,7 +292,7 @@ public class TestCastFunctions extends PopUnitTestBase{ mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastNested.json"), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastNested.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); @@ -332,7 +330,7 @@ public class TestCastFunctions extends PopUnitTestBase{ mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastNumException.json"), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastNumException.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); @@ -360,7 +358,7 @@ public class TestCastFunctions extends PopUnitTestBase{ client.connect(); final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarCharNull.json"), Charsets.UTF_8).replace("#{TEST_FILE}", "/jsoninput/input1.json")); + Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastVarCharNull.json"), Charsets.UTF_8).replace("#{TEST_FILE}", "/jsoninput/input1.json")); final QueryDataBatch batch = results.get(0); http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java index 8d904eb..5bf5711 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java @@ -22,7 +22,7 @@ import static org.junit.Assert.assertTrue; import java.util.List; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.record.RecordBatchLoader; @@ -50,7 +50,7 @@ public class TestCastVarCharToBigInt extends PopUnitTestBase { bit.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/functions/cast/test_cast_varchar_to_bigint.json"), Charsets.UTF_8) + Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/test_cast_varchar_to_bigint.json"), Charsets.UTF_8) .replace("#{TEST_FILE}", "/scan_json_test_cast.json") ); http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java index 55dc130..c1951a3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java @@ -17,8 +17,8 @@ */ package org.apache.drill.exec.physical.impl; -import static org.apache.drill.TestBuilder.listOf; -import static org.apache.drill.TestBuilder.mapOf; +import static org.apache.drill.test.TestBuilder.listOf; +import static org.apache.drill.test.TestBuilder.mapOf; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -28,8 +28,8 @@ import static org.junit.Assert.fail; import java.util.ArrayList; import java.util.List; -import org.apache.drill.BaseTestQuery; -import org.apache.drill.QueryTestUtil; +import org.apache.drill.test.BaseTestQuery; +import org.apache.drill.test.QueryTestUtil; import org.apache.drill.categories.UnlikelyTest; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.compile.ClassTransformer.ScalarReplacementOption; @@ -60,8 +60,6 @@ import org.junit.experimental.categories.Category; @Category(UnlikelyTest.class) public class TestConvertFunctions extends BaseTestQuery { -// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestConvertFunctions.class); - private static final String CONVERSION_TEST_LOGICAL_PLAN = "functions/conv/conversionTestWithLogicalPlan.json"; private static final String CONVERSION_TEST_PHYSICAL_PLAN = "functions/conv/conversionTestWithPhysicalPlan.json"; @@ -98,19 +96,15 @@ public class TestConvertFunctions extends BaseTestQuery { final OptionValue srOption = QueryTestUtil.setupScalarReplacementOption(bits[0], ScalarReplacementOption.OFF); try { final String newTblName = "testConvertFromConvertToInt_tbl"; - final String ctasQuery = String.format("CREATE TABLE %s.%s as \n" + - "SELECT convert_to(r_regionkey, 'INT') as ct \n" + - "FROM cp.`tpch/region.parquet`", - TEMP_SCHEMA, newTblName); - final String query = String.format("SELECT convert_from(ct, 'INT') as cf \n" + - "FROM %s.%s \n" + - "ORDER BY ct", - TEMP_SCHEMA, newTblName); test("alter session set `planner.slice_target` = 1"); - test(ctasQuery); + test("CREATE TABLE dfs.%s as \n" + + "SELECT convert_to(r_regionkey, 'INT') as ct \n" + + "FROM cp.`tpch/region.parquet`", newTblName); testBuilder() - .sqlQuery(query) + .sqlQuery("SELECT convert_from(ct, 'INT') as cf \n" + + "FROM dfs.%s \n" + + "ORDER BY ct", newTblName) .ordered() .baselineColumns("cf") .baselineValues(0) @@ -132,7 +126,7 @@ public class TestConvertFunctions extends BaseTestQuery { String listStr = "[ 4, 6 ]"; testBuilder() - .sqlQuery("select cast(convert_to(rl[1], 'JSON') as varchar(100)) as json_str from cp.`/store/json/input2.json`") + .sqlQuery("select cast(convert_to(rl[1], 'JSON') as varchar(100)) as json_str from cp.`store/json/input2.json`") .unOrdered() .baselineColumns("json_str") .baselineValues(listStr) @@ -143,7 +137,7 @@ public class TestConvertFunctions extends BaseTestQuery { Object listVal = listOf(4l, 6l); testBuilder() - .sqlQuery("select convert_from(convert_to(rl[1], 'JSON'), 'JSON') list_col from cp.`/store/json/input2.json`") + .sqlQuery("select convert_from(convert_to(rl[1], 'JSON'), 'JSON') list_col from cp.`store/json/input2.json`") .unOrdered() .baselineColumns("list_col") .baselineValues(listVal) @@ -155,7 +149,7 @@ public class TestConvertFunctions extends BaseTestQuery { Object mapVal1 = mapOf("f1", 4l, "f2", 6l); Object mapVal2 = mapOf("f1", 11l); testBuilder() - .sqlQuery("select convert_from(convert_to(rl[1], 'JSON'), 'JSON') as map_col from cp.`/store/json/json_project_null_object_from_list.json`") + .sqlQuery("select convert_from(convert_to(rl[1], 'JSON'), 'JSON') as map_col from cp.`store/json/json_project_null_object_from_list.json`") .unOrdered() .baselineColumns("map_col") .baselineValues(mapVal1) @@ -171,10 +165,10 @@ public class TestConvertFunctions extends BaseTestQuery { Object mapVal2 = mapOf("y", "bill", "z", "peter"); // right side of union-all produces 0 rows due to FALSE filter, column t.x is a map - String query1 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t " + String query1 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t " + " where t.`integer` = 2010 " + " union all " - + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t" + + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t" + " where 1 = 0"); testBuilder() @@ -185,10 +179,10 @@ public class TestConvertFunctions extends BaseTestQuery { .go(); // left side of union-all produces 0 rows due to FALSE filter, column t.x is a map - String query2 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t " + String query2 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t " + " where 1 = 0 " + " union all " - + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t " + + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t " + " where t.`integer` = 2010"); testBuilder() @@ -199,10 +193,10 @@ public class TestConvertFunctions extends BaseTestQuery { .go(); // sanity test where neither side produces 0 rows - String query3 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t " + String query3 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t " + " where t.`integer` = 2010 " + " union all " - + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t " + + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t " + " where t.`integer` = 2001"); testBuilder() @@ -217,9 +211,9 @@ public class TestConvertFunctions extends BaseTestQuery { Object listVal1 = listOf(listOf(2l, 1l), listOf(4l, 6l)); Object listVal2 = listOf(); // empty - String query4 = String.format("select 'abc' as col1, convert_from(convert_to(t.rl, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t " + String query4 = String.format("select 'abc' as col1, convert_from(convert_to(t.rl, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t " + " union all " - + " select 'abc' as col1, convert_from(convert_to(t.rl, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t" + + " select 'abc' as col1, convert_from(convert_to(t.rl, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t" + " where 1 = 0"); testBuilder() @@ -238,12 +232,10 @@ public class TestConvertFunctions extends BaseTestQuery { public void testConvertFromJson_drill4693() throws Exception { Object mapVal1 = mapOf("x", "y"); - String query = String.format("select 'abc' as col1, convert_from('{\"x\" : \"y\"}', 'json') as col2, 'xyz' as col3 " - + " from cp.`/store/json/input2.json` t" - + " where t.`integer` = 2001"); - testBuilder() - .sqlQuery(query) + .sqlQuery("select 'abc' as col1, convert_from('{\"x\" : \"y\"}', 'json') as col2, 'xyz' as col3 " + + " from cp.`store/json/input2.json` t" + + " where t.`integer` = 2001") .unOrdered() .baselineColumns("col1", "col2", "col3") .baselineValues("abc", mapVal1, "xyz") @@ -263,7 +255,7 @@ public class TestConvertFunctions extends BaseTestQuery { String result2 = "[ ]"; testBuilder() - .sqlQuery("select cast(convert_to(rl[1], 'EXTENDEDJSON') as varchar(100)) as json_str from cp.`/store/json/input2.json`") + .sqlQuery("select cast(convert_to(rl[1], 'EXTENDEDJSON') as varchar(100)) as json_str from cp.`store/json/input2.json`") .unOrdered() .baselineColumns("json_str") .baselineValues(result1) http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java index a24e511..8ba7e57 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java @@ -24,7 +24,7 @@ import java.util.Iterator; import java.util.List; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.record.RecordBatchLoader; @@ -58,7 +58,7 @@ public class TestDecimal extends PopUnitTestBase{ bit.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/decimal/cast_simple_decimal.json"), Charsets.UTF_8) + Files.toString(DrillFileUtils.getResourceAsFile("/decimal/cast_simple_decimal.json"), Charsets.UTF_8) .replace("#{TEST_FILE}", "/input_simple_decimal.json") ); @@ -103,7 +103,7 @@ public class TestDecimal extends PopUnitTestBase{ bit.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/decimal/cast_float_decimal.json"), Charsets.UTF_8) + Files.toString(DrillFileUtils.getResourceAsFile("/decimal/cast_float_decimal.json"), Charsets.UTF_8) .replace("#{TEST_FILE}", "/input_simple_decimal.json") ); @@ -148,7 +148,7 @@ public class TestDecimal extends PopUnitTestBase{ bit.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/decimal/simple_decimal_arithmetic.json"), Charsets.UTF_8) + Files.toString(DrillFileUtils.getResourceAsFile("/decimal/simple_decimal_arithmetic.json"), Charsets.UTF_8) .replace("#{TEST_FILE}", "/input_simple_decimal.json") ); @@ -199,7 +199,7 @@ public class TestDecimal extends PopUnitTestBase{ bit.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/decimal/test_decimal_complex.json"), Charsets.UTF_8) + Files.toString(DrillFileUtils.getResourceAsFile("/decimal/test_decimal_complex.json"), Charsets.UTF_8) .replace("#{TEST_FILE}", "/input_complex_decimal.json") ); @@ -242,7 +242,7 @@ public class TestDecimal extends PopUnitTestBase{ bit.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/decimal/test_decimal_sort_complex.json"), Charsets.UTF_8) + Files.toString(DrillFileUtils.getResourceAsFile("/decimal/test_decimal_sort_complex.json"), Charsets.UTF_8) .replace("#{TEST_FILE}", "/input_sort_complex_decimal.json") ); @@ -291,7 +291,7 @@ public class TestDecimal extends PopUnitTestBase{ bit.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/decimal/simple_decimal_math.json"), Charsets.UTF_8) + Files.toString(DrillFileUtils.getResourceAsFile("/decimal/simple_decimal_math.json"), Charsets.UTF_8) .replace("#{TEST_FILE}", "/input_simple_decimal.json") ); http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDistributedFragmentRun.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDistributedFragmentRun.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDistributedFragmentRun.java index 0349352..4ae4f40 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDistributedFragmentRun.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDistributedFragmentRun.java @@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals; import java.util.List; import org.apache.drill.categories.SlowTest; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.proto.UserBitShared.QueryType; @@ -46,7 +46,7 @@ public class TestDistributedFragmentRun extends PopUnitTestBase{ try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet); DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());){ bit1.run(); client.connect(); - List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/physical_single_exchange.json"), Charsets.UTF_8)); + List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(DrillFileUtils.getResourceAsFile("/physical_single_exchange.json"), Charsets.UTF_8)); int count = 0; for(QueryDataBatch b : results){ count += b.getHeader().getRowCount(); @@ -66,7 +66,7 @@ public class TestDistributedFragmentRun extends PopUnitTestBase{ try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet); DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());){ bit1.run(); client.connect(); - List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/physical_single_exchange_double_entry.json"), Charsets.UTF_8)); + List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(DrillFileUtils.getResourceAsFile("/physical_single_exchange_double_entry.json"), Charsets.UTF_8)); int count = 0; for(QueryDataBatch b : results){ count += b.getHeader().getRowCount(); @@ -85,7 +85,7 @@ public class TestDistributedFragmentRun extends PopUnitTestBase{ try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet); DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());){ bit1.run(); client.connect(); - List<QueryDataBatch> results = client.runQuery(QueryType.LOGICAL, Files.toString(FileUtils.getResourceAsFile("/scan_screen_logical.json"), Charsets.UTF_8)); + List<QueryDataBatch> results = client.runQuery(QueryType.LOGICAL, Files.toString(DrillFileUtils.getResourceAsFile("/scan_screen_logical.json"), Charsets.UTF_8)); int count = 0; for(QueryDataBatch b : results){ count += b.getHeader().getRowCount(); @@ -105,7 +105,7 @@ public class TestDistributedFragmentRun extends PopUnitTestBase{ bit1.run(); bit2.run(); client.connect(); - List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/physical_single_exchange_double_entry.json"), Charsets.UTF_8)); + List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(DrillFileUtils.getResourceAsFile("/physical_single_exchange_double_entry.json"), Charsets.UTF_8)); int count = 0; for(QueryDataBatch b : results){ count += b.getHeader().getRowCount(); http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java index 937faf8..87bc182 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue; import java.util.List; import org.apache.drill.categories.SlowTest; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.record.RecordBatchLoader; @@ -114,7 +114,7 @@ public class TestExtractFunctions extends PopUnitTestBase { bit.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/functions/extractFrom.json"), Charsets.UTF_8) + Files.toString(DrillFileUtils.getResourceAsFile("/functions/extractFrom.json"), Charsets.UTF_8) .replace("#{TEST_TYPE}", fromType) .replace("#{TEST_FILE}", testDataFile) .replace("#{COLUMN_NAME}", columnName)); http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java index c9954de..50e0b54 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java @@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals; import java.util.List; import org.apache.drill.categories.OperatorTest; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.rpc.user.QueryDataBatch; @@ -50,7 +50,7 @@ public class TestHashToRandomExchange extends PopUnitTestBase { bit2.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/sender/hash_exchange.json"), + Files.toString(DrillFileUtils.getResourceAsFile("/sender/hash_exchange.json"), Charsets.UTF_8)); int count = 0; for (QueryDataBatch b : results) { http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java index f712930..3c10884 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java @@ -23,7 +23,7 @@ import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.drill.PlanTestBase; -import org.apache.drill.TestBuilder; +import org.apache.drill.test.TestBuilder; import org.apache.drill.exec.physical.base.Exchange; import org.apache.drill.exec.physical.config.UnorderedDeMuxExchange; import org.apache.drill.exec.physical.config.HashToRandomExchange; @@ -54,6 +54,7 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.nio.file.Paths; import java.util.List; import static org.apache.drill.exec.planner.physical.HashPrelUtil.HASH_EXPR_NAME; @@ -71,16 +72,15 @@ import static org.junit.Assert.assertTrue; * at most one partition sender per Drillbit. */ public class TestLocalExchange extends PlanTestBase { - - public static TemporaryFolder testTempFolder = new TemporaryFolder(); - - private final static int CLUSTER_SIZE = 3; - private final static String MUX_EXCHANGE = "\"unordered-mux-exchange\""; - private final static String DEMUX_EXCHANGE = "\"unordered-demux-exchange\""; - private final static String MUX_EXCHANGE_CONST = "unordered-mux-exchange"; - private final static String DEMUX_EXCHANGE_CONST = "unordered-demux-exchange"; + private static final int CLUSTER_SIZE = 3; + private static final String MUX_EXCHANGE = "\"unordered-mux-exchange\""; + private static final String DEMUX_EXCHANGE = "\"unordered-demux-exchange\""; + private static final String MUX_EXCHANGE_CONST = "unordered-mux-exchange"; + private static final String DEMUX_EXCHANGE_CONST = "unordered-demux-exchange"; private static final String HASH_EXCHANGE = "hash-to-random-exchange"; - private final static UserSession USER_SESSION = UserSession.Builder.newBuilder() + private static final String EMPT_TABLE = "empTable"; + private static final String DEPT_TABLE = "deptTable"; + private static final UserSession USER_SESSION = UserSession.Builder.newBuilder() .withCredentials(UserBitShared.UserCredentials.newBuilder().setUserName("foo").build()) .build(); @@ -95,9 +95,6 @@ public class TestLocalExchange extends PlanTestBase { private final static int NUM_MNGRS = 1; private final static int NUM_IDS = 1; - private static String empTableLocation; - private static String deptTableLocation; - private static String groupByQuery; private static String joinQuery; @@ -112,23 +109,18 @@ public class TestLocalExchange extends PlanTestBase { updateTestCluster(CLUSTER_SIZE, null); } - @BeforeClass - public static void setupTempFolder() throws IOException { - testTempFolder.create(); - } - /** * Generate data for two tables. Each table consists of several JSON files. */ @BeforeClass public static void generateTestDataAndQueries() throws Exception { // Table 1 consists of two columns "emp_id", "emp_name" and "dept_id" - empTableLocation = testTempFolder.newFolder().getAbsolutePath(); + final File empTableLocation = dirTestWatcher.makeRootSubDir(Paths.get(EMPT_TABLE)); // Write 100 records for each new file final int empNumRecsPerFile = 100; for(int fileIndex=0; fileIndex<NUM_EMPLOYEES/empNumRecsPerFile; fileIndex++) { - File file = new File(empTableLocation + File.separator + fileIndex + ".json"); + File file = new File(empTableLocation, fileIndex + ".json"); PrintWriter printWriter = new PrintWriter(file); for (int recordIndex = fileIndex*empNumRecsPerFile; recordIndex < (fileIndex+1)*empNumRecsPerFile; recordIndex++) { String record = String.format("{ \"emp_id\" : %d, \"emp_name\" : \"Employee %d\", \"dept_id\" : %d, \"mng_id\" : %d, \"some_id\" : %d }", @@ -139,12 +131,12 @@ public class TestLocalExchange extends PlanTestBase { } // Table 2 consists of two columns "dept_id" and "dept_name" - deptTableLocation = testTempFolder.newFolder().getAbsolutePath(); + final File deptTableLocation = dirTestWatcher.makeRootSubDir(Paths.get(DEPT_TABLE)); // Write 4 records for each new file final int deptNumRecsPerFile = 4; for(int fileIndex=0; fileIndex<NUM_DEPTS/deptNumRecsPerFile; fileIndex++) { - File file = new File(deptTableLocation + File.separator + fileIndex + ".json"); + File file = new File(deptTableLocation, fileIndex + ".json"); PrintWriter printWriter = new PrintWriter(file); for (int recordIndex = fileIndex*deptNumRecsPerFile; recordIndex < (fileIndex+1)*deptNumRecsPerFile; recordIndex++) { String record = String.format("{ \"dept_id\" : %d, \"dept_name\" : \"Department %d\" }", @@ -155,9 +147,9 @@ public class TestLocalExchange extends PlanTestBase { } // Initialize test queries - groupByQuery = String.format("SELECT dept_id, count(*) as numEmployees FROM dfs.`%s` GROUP BY dept_id", empTableLocation); + groupByQuery = String.format("SELECT dept_id, count(*) as numEmployees FROM dfs.`%s` GROUP BY dept_id", EMPT_TABLE); joinQuery = String.format("SELECT e.emp_name, d.dept_name FROM dfs.`%s` e JOIN dfs.`%s` d ON e.dept_id = d.dept_id", - empTableLocation, deptTableLocation); + EMPT_TABLE, DEPT_TABLE); // Generate and store output data for test queries. Used when verifying the output of queries ran using different // configurations. @@ -200,7 +192,7 @@ public class TestLocalExchange extends PlanTestBase { test("ALTER SESSION SET `planner.enable_mux_exchange`=" + true); test("ALTER SESSION SET `planner.enable_demux_exchange`=" + false); - final String groupByMultipleQuery = String.format("SELECT dept_id, mng_id, some_id, count(*) as numEmployees FROM dfs.`%s` e GROUP BY dept_id, mng_id, some_id", empTableLocation); + final String groupByMultipleQuery = String.format("SELECT dept_id, mng_id, some_id, count(*) as numEmployees FROM dfs.`%s` e GROUP BY dept_id, mng_id, some_id", EMPT_TABLE); final String[] groupByMultipleQueryBaselineColumns = new String[] { "dept_id", "mng_id", "some_id", "numEmployees" }; final int numOccurrances = NUM_EMPLOYEES/NUM_DEPTS; @@ -478,9 +470,4 @@ public class TestLocalExchange extends PlanTestBase { assertTrue("Some endpoints have more than one fragment that has ParitionSender", ImmutableSet.copyOf(assignments).size() == assignments.size()); } } - - @AfterClass - public static void cleanupTempFolder() throws IOException { - testTempFolder.delete(); - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java index f09e569..b6bc497 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java @@ -25,7 +25,7 @@ import mockit.NonStrictExpectations; import org.apache.drill.categories.PlannerTest; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.logical.LogicalPlan; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.coord.ClusterCoordinator; @@ -119,7 +119,7 @@ public class TestOptiqPlans extends ExecTest { final QueryContext qc = new QueryContext(UserSession.Builder.newBuilder().setSupportComplexTypes(true).build(), bitContext, QueryId.getDefaultInstance()); final PhysicalPlanReader reader = bitContext.getPlanReader(); - final LogicalPlan plan = reader.readLogicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8)); + final LogicalPlan plan = reader.readLogicalPlan(Files.toString(DrillFileUtils.getResourceAsFile(file), Charsets.UTF_8)); final PhysicalPlan pp = new BasicOptimizer(qc, connection).optimize(new BasicOptimizer.BasicOptimizationContext(qc), plan); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(config); @@ -312,7 +312,7 @@ public class TestOptiqPlans extends ExecTest { final StoragePluginRegistry reg = new StoragePluginRegistryImpl(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(config, reg); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile(file), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(config); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false) http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestReverseImplicitCast.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestReverseImplicitCast.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestReverseImplicitCast.java index ae8302d..84d68f5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestReverseImplicitCast.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestReverseImplicitCast.java @@ -25,7 +25,7 @@ import java.util.List; import mockit.Injectable; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.record.RecordBatchLoader; @@ -56,7 +56,7 @@ public class TestReverseImplicitCast extends PopUnitTestBase { bit.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/functions/cast/two_way_implicit_cast.json"), Charsets.UTF_8)); + Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/two_way_implicit_cast.json"), Charsets.UTF_8)); RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator()); http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java index 3b8ab3f..6dd181b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue; import java.util.List; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.pop.PopUnitTestBase; @@ -53,7 +53,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase { client.connect(); final String path = "/physical_test2.json"; // String path = "/filter/test1.json"; - final List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile(path), Charsets.UTF_8)); + final List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(DrillFileUtils.getResourceAsFile(path), Charsets.UTF_8)); // look at records final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator()); @@ -113,8 +113,8 @@ public class TestSimpleFragmentRun extends PopUnitTestBase { bit.run(); client.connect(); final List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/physical_json_scan_test1.json"), Charsets.UTF_8) - .replace("#{TEST_FILE}", FileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString()) + Files.toString(DrillFileUtils.getResourceAsFile("/physical_json_scan_test1.json"), Charsets.UTF_8) + .replace("#{TEST_FILE}", DrillFileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString()) ); // look at records http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java index c21facb..1f83fc4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java @@ -33,7 +33,7 @@ import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.expr.fn.DrillFuncHolder; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; @@ -148,7 +148,7 @@ public class TestSimpleFunctions extends ExecTest { mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/testSubstring.json"), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/testSubstring.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); @@ -181,7 +181,7 @@ public class TestSimpleFunctions extends ExecTest { mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/testSubstringNegative.json"), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/testSubstringNegative.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); @@ -215,7 +215,7 @@ public class TestSimpleFunctions extends ExecTest { mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/testByteSubstring.json"), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/testByteSubstring.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java index b87a085..dfcbc71 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java @@ -41,7 +41,6 @@ import com.google.common.io.Resources; import mockit.Injectable; public class TestStringFunctions extends ExecTest { - //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestStringFunctions.class); private final DrillConfig c = DrillConfig.create(); private PhysicalPlanReader reader; http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestUnionExchange.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestUnionExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestUnionExchange.java index dc653c2..74b3a7e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestUnionExchange.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestUnionExchange.java @@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals; import java.util.List; import org.apache.drill.categories.OperatorTest; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.rpc.user.QueryDataBatch; @@ -50,7 +50,7 @@ public class TestUnionExchange extends PopUnitTestBase { bit2.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/sender/union_exchange.json"), + Files.toString(DrillFileUtils.getResourceAsFile("/sender/union_exchange.json"), Charsets.UTF_8)); int count = 0; for (QueryDataBatch b : results) { http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestSimpleTopN.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestSimpleTopN.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestSimpleTopN.java deleted file mode 100644 index 43c0af0..0000000 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestSimpleTopN.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.physical.impl.TopN; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.apache.drill.categories.OperatorTest; -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.expression.ExpressionPosition; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.util.FileUtils; -import org.apache.drill.exec.client.DrillClient; -import org.apache.drill.exec.pop.PopUnitTestBase; -import org.apache.drill.exec.record.RecordBatchLoader; -import org.apache.drill.exec.rpc.user.QueryDataBatch; -import org.apache.drill.exec.server.Drillbit; -import org.apache.drill.exec.server.RemoteServiceSet; -import org.apache.drill.exec.vector.BigIntVector; -import org.junit.Test; - -import com.google.common.base.Charsets; -import com.google.common.io.Files; -import org.junit.experimental.categories.Category; - -@Category(OperatorTest.class) -public class TestSimpleTopN extends PopUnitTestBase { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleTopN.class); - DrillConfig c = DrillConfig.create(); - - @Test - public void sortOneKeyAscending() throws Throwable{ - RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); - - try (Drillbit bit1 = new Drillbit(CONFIG, serviceSet); - Drillbit bit2 = new Drillbit(CONFIG, serviceSet); - DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) { - - bit1.run(); - bit2.run(); - client.connect(); - List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/topN/one_key_sort.json"), - Charsets.UTF_8)); - int count = 0; - for (QueryDataBatch b : results) { - if (b.getHeader().getRowCount() != 0) { - count += b.getHeader().getRowCount(); - } - } - assertEquals(100, count); - - long previousBigInt = Long.MIN_VALUE; - - int recordCount = 0; - int batchCount = 0; - - for (QueryDataBatch b : results) { - if (b.getHeader().getRowCount() == 0) { - continue; - } - batchCount++; - RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator()); - loader.load(b.getHeader().getDef(),b.getData()); - BigIntVector c1 = (BigIntVector) loader.getValueAccessorById(BigIntVector.class, loader.getValueVectorId(new SchemaPath("blue", ExpressionPosition.UNKNOWN)).getFieldIds()).getValueVector(); - - - BigIntVector.Accessor a1 = c1.getAccessor(); -// IntVector.Accessor a2 = c2.getAccessor(); - - for (int i =0; i < c1.getAccessor().getValueCount(); i++) { - recordCount++; - assertTrue(previousBigInt <= a1.get(i)); - previousBigInt = a1.get(i); - } - loader.clear(); - b.release(); - } - System.out.println(String.format("Sorted %,d records in %d batches.", recordCount, batchCount)); - - } - - } - -} http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestTopNSchemaChanges.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestTopNSchemaChanges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestTopNSchemaChanges.java index 066d966..031041f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestTopNSchemaChanges.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestTopNSchemaChanges.java @@ -17,39 +17,57 @@ */ package org.apache.drill.exec.physical.impl.TopN; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.categories.OperatorTest; -import org.apache.drill.TestBuilder; +import org.apache.drill.test.TestBuilder; +import org.apache.drill.test.SubDirTestWatcher; +import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; +import java.nio.file.Path; +import java.nio.file.Paths; @Category(OperatorTest.class) public class TestTopNSchemaChanges extends BaseTestQuery { + private static final Path TABLE = Paths.get("table"); + + private static File tableDir; + + @BeforeClass + public static void setupTestFiles() { + tableDir = dirTestWatcher.getRootDir() + .toPath() + .resolve(TABLE) + .toFile(); + } + + @Rule + public final SubDirTestWatcher localDirTestWatcher = + new SubDirTestWatcher.Builder(dirTestWatcher.getRootDir()) + .addSubDir(TABLE) + .build(); @Test public void testNumericTypes() throws Exception { - final File data_dir = new File(BaseTestQuery.getTempDir("topn-schemachanges")); - data_dir.mkdirs(); - // left side int and strings - BufferedWriter writer = new BufferedWriter(new FileWriter(new File(data_dir, "d1.json"))); + BufferedWriter writer = new BufferedWriter(new FileWriter(new File(tableDir, "d1.json"))); for (int i = 0; i < 10000; i+=2) { writer.write(String.format("{ \"kl\" : %d , \"vl\": %d }\n", i, i)); } writer.close(); - writer = new BufferedWriter(new FileWriter(new File(data_dir, "d2.json"))); + writer = new BufferedWriter(new FileWriter(new File(tableDir, "d2.json"))); for (int i = 1; i < 10000; i+=2) { writer.write(String.format("{ \"kl\" : %f , \"vl\": %f }\n", (float)i, (float)i)); } writer.close(); - String query = String.format("select * from dfs_test.`%s` order by kl limit 12", data_dir.toPath().toString()); TestBuilder builder = testBuilder() - .sqlQuery(query) + .sqlQuery("select * from dfs.`%s` order by kl limit 12", TABLE) .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") .ordered() .baselineColumns("kl", "vl"); @@ -66,24 +84,20 @@ public class TestTopNSchemaChanges extends BaseTestQuery { @Test public void testNumericAndStringTypes() throws Exception { - final File data_dir = new File(BaseTestQuery.getTempDir("topn-schemachanges")); - data_dir.mkdirs(); - // left side int and strings - BufferedWriter writer = new BufferedWriter(new FileWriter(new File(data_dir, "d1.json"))); + BufferedWriter writer = new BufferedWriter(new FileWriter(new File(tableDir, "d1.json"))); for (int i = 0; i < 1000; i+=2) { writer.write(String.format("{ \"kl\" : %d , \"vl\": %d }\n", i, i)); } writer.close(); - writer = new BufferedWriter(new FileWriter(new File(data_dir, "d2.json"))); + writer = new BufferedWriter(new FileWriter(new File(tableDir, "d2.json"))); for (int i = 1; i < 1000; i+=2) { writer.write(String.format("{ \"kl\" : \"%s\" , \"vl\": \"%s\" }\n", i, i)); } writer.close(); - String query = String.format("select * from dfs_test.`%s` order by kl limit 12", data_dir.toPath().toString()); TestBuilder builder = testBuilder() - .sqlQuery(query) + .sqlQuery("select * from dfs.`%s` order by kl limit 12", TABLE) .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") .ordered() .baselineColumns("kl", "vl"); @@ -92,9 +106,8 @@ public class TestTopNSchemaChanges extends BaseTestQuery { builder.baselineValues(i, i); } - query = String.format("select * from dfs_test.`%s` order by kl desc limit 12", data_dir.toPath().toString()); builder = testBuilder() - .sqlQuery(query) + .sqlQuery("select * from dfs.`%s` order by kl desc limit 12", TABLE) .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") .ordered() .baselineColumns("kl", "vl") @@ -115,11 +128,8 @@ public class TestTopNSchemaChanges extends BaseTestQuery { @Test public void testUnionTypes() throws Exception { - final File data_dir = new File(BaseTestQuery.getTempDir("topn-schemachanges")); - data_dir.mkdirs(); - // union of int and float and string. - BufferedWriter writer = new BufferedWriter(new FileWriter(new File(data_dir, "d1.json"))); + BufferedWriter writer = new BufferedWriter(new FileWriter(new File(tableDir, "d1.json"))); for (int i = 0; i <= 9; ++i) { switch (i%3) { case 0: // 0, 3, 6, 9 @@ -134,10 +144,9 @@ public class TestTopNSchemaChanges extends BaseTestQuery { } } writer.close(); - String query = String.format("select * from dfs_test.`%s` order by kl limit 8", data_dir.toPath().toString()); TestBuilder builder = testBuilder() - .sqlQuery(query) + .sqlQuery("select * from dfs.`%s` order by kl limit 8", TABLE) .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") .ordered() .baselineColumns("kl", "vl"); @@ -155,29 +164,25 @@ public class TestTopNSchemaChanges extends BaseTestQuery { @Test public void testMissingColumn() throws Exception { - final File data_dir = new File(BaseTestQuery.getTempDir("topn-schemachanges")); - data_dir.mkdirs(); - - BufferedWriter writer = new BufferedWriter(new FileWriter(new File(data_dir, "d1.json"))); + BufferedWriter writer = new BufferedWriter(new FileWriter(new File(tableDir, "d1.json"))); for (int i = 0; i < 100; i++) { writer.write(String.format("{ \"kl1\" : %d , \"vl1\": %d }\n", i, i)); } writer.close(); - writer = new BufferedWriter(new FileWriter(new File(data_dir, "d2.json"))); + writer = new BufferedWriter(new FileWriter(new File(tableDir, "d2.json"))); for (int i = 100; i < 200; i++) { writer.write(String.format("{ \"kl\" : %f , \"vl\": %f }\n", (float)i, (float)i)); } writer.close(); - writer = new BufferedWriter(new FileWriter(new File(data_dir, "d3.json"))); + writer = new BufferedWriter(new FileWriter(new File(tableDir, "d3.json"))); for (int i = 200; i < 300; i++) { writer.write(String.format("{ \"kl2\" : \"%s\" , \"vl2\": \"%s\" }\n", i, i)); } writer.close(); - String query = String.format("select kl, vl, kl1, vl1, kl2, vl2 from dfs_test.`%s` order by kl limit 3", data_dir.toPath().toString()); TestBuilder builder = testBuilder() - .sqlQuery(query) + .sqlQuery("select kl, vl, kl1, vl1, kl2, vl2 from dfs.`%s` order by kl limit 3", TABLE) .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") .ordered() .baselineColumns("kl", "vl", "kl1", "vl1", "kl2", "vl2") @@ -186,9 +191,8 @@ public class TestTopNSchemaChanges extends BaseTestQuery { .baselineValues(102.0d, 102.0d, null, null, null, null); builder.go(); - query = String.format("select kl, vl, kl1, vl1, kl2, vl2 from dfs_test.`%s` order by kl1 limit 3", data_dir.toPath().toString()); builder = testBuilder() - .sqlQuery(query) + .sqlQuery("select kl, vl, kl1, vl1, kl2, vl2 from dfs.`%s` order by kl1 limit 3", TABLE) .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") .ordered() .baselineColumns("kl", "vl", "kl1", "vl1", "kl2", "vl2") @@ -197,9 +201,8 @@ public class TestTopNSchemaChanges extends BaseTestQuery { .baselineValues(null, null, 2l, 2l, null, null); builder.go(); - query = String.format("select kl, vl, kl1, vl1, kl2, vl2 from dfs_test.`%s` order by kl2 limit 3", data_dir.toPath().toString()); builder = testBuilder() - .sqlQuery(query) + .sqlQuery("select kl, vl, kl1, vl1, kl2, vl2 from dfs.`%s` order by kl2 limit 3", TABLE) .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") .ordered() .baselineColumns("kl", "vl", "kl1", "vl1", "kl2", "vl2") @@ -210,6 +213,6 @@ public class TestTopNSchemaChanges extends BaseTestQuery { // Since client can't handle new columns which are not in first batch, we won't test output of query. // Query should run w/o any errors. - test(String.format("select * from dfs_test.`%s` order by kl limit 3", data_dir.toPath().toString())); + test("select * from dfs.`%s` order by kl limit 3", TABLE); } } http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java new file mode 100644 index 0000000..fa6e318 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.physical.impl.TopN; + +import java.util.List; +import java.util.Properties; +import java.util.Random; + +import com.google.common.collect.Lists; +import org.apache.drill.test.BaseDirTestWatcher; +import org.apache.drill.test.ClusterFixtureBuilder; +import org.apache.drill.test.TestBuilder; +import org.apache.drill.categories.OperatorTest; +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.logical.data.Order; +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.Types; +import org.apache.drill.exec.compile.ClassBuilder; +import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; +import org.apache.drill.exec.memory.RootAllocator; +import org.apache.drill.exec.physical.impl.sort.RecordBatchData; +import org.apache.drill.exec.pop.PopUnitTestBase; +import org.apache.drill.exec.record.BatchSchema; +import org.apache.drill.exec.record.ExpandableHyperContainer; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.VectorContainer; +import org.apache.drill.test.ClientFixture; +import org.apache.drill.test.ClusterFixture; +import org.apache.drill.test.rowSet.HyperRowSetImpl; +import org.apache.drill.test.rowSet.RowSet; +import org.apache.drill.test.rowSet.RowSetBuilder; +import org.apache.drill.test.rowSet.RowSetComparison; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(OperatorTest.class) +public class TopNBatchTest extends PopUnitTestBase { + @Rule + public BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher(); + + /** + * Priority queue unit test. + * @throws Exception + */ + @Test + public void priorityQueueOrderingTest() throws Exception { + Properties properties = new Properties(); + properties.setProperty(ClassBuilder.CODE_DIR_OPTION, dirTestWatcher.getDir().getAbsolutePath()); + + DrillConfig drillConfig = DrillConfig.create(properties); + + FieldReference expr = FieldReference.getWithQuotedRef("colA"); + Order.Ordering ordering = new Order.Ordering(Order.Ordering.ORDER_DESC, expr, Order.Ordering.NULLS_FIRST); + List<Order.Ordering> orderings = Lists.newArrayList(ordering); + + MaterializedField colA = MaterializedField.create("colA", Types.required(TypeProtos.MinorType.INT)); + MaterializedField colB = MaterializedField.create("colB", Types.required(TypeProtos.MinorType.INT)); + + List<MaterializedField> cols = Lists.newArrayList(colA, colB); + BatchSchema batchSchema = new BatchSchema(BatchSchema.SelectionVectorMode.NONE, cols); + RowSet expectedRowSet; + + try (RootAllocator allocator = new RootAllocator(100_000_000)) { + expectedRowSet = new RowSetBuilder(allocator, batchSchema) + .add(110, 10) + .add(109, 9) + .add(108, 8) + .add(107, 7) + .add(106, 6) + .add(105, 5) + .add(104, 4) + .add(103, 3) + .add(102, 2) + .add(101, 1) + .build(); + + PriorityQueue queue; + ExpandableHyperContainer hyperContainer; + + { + VectorContainer container = new RowSetBuilder(allocator, batchSchema) + .build() + .container(); + hyperContainer = new ExpandableHyperContainer(container); + + queue = TopNBatch.createNewPriorityQueue( + TopNBatch.createMainMappingSet(), TopNBatch.createLeftMappingSet(), + TopNBatch.createRightMappingSet(), optionManager, + new FunctionImplementationRegistry(drillConfig), + new CodeCompiler(drillConfig, optionManager), + orderings, hyperContainer, false, + true, 10, allocator, + batchSchema.getSelectionVectorMode()); + } + + List<RecordBatchData> testBatches = Lists.newArrayList(); + + try { + final Random random = new Random(); + final int bound = 100; + final int numBatches = 11; + final int numRecordsPerBatch = 100; + + for (int batchCounter = 0; batchCounter < numBatches; batchCounter++) { + RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, batchSchema); + rowSetBuilder.add((batchCounter + bound), batchCounter); + + for (int recordCounter = 0; recordCounter < numRecordsPerBatch; recordCounter++) { + rowSetBuilder.add(random.nextInt(bound), random.nextInt(bound)); + } + + VectorContainer vectorContainer = rowSetBuilder.build().container(); + queue.add(new RecordBatchData(vectorContainer, allocator)); + } + + queue.generate(); + VectorContainer resultContainer = queue.getHyperBatch(); + resultContainer.buildSchema(BatchSchema.SelectionVectorMode.NONE); + + RowSet.HyperRowSet actualHyperSet = new HyperRowSetImpl(allocator, resultContainer, queue.getFinalSv4()); + new RowSetComparison(expectedRowSet).verify(actualHyperSet); + } finally { + if (expectedRowSet != null) { + expectedRowSet.clear(); + } + + queue.cleanup(); + hyperContainer.clear(); + + for (RecordBatchData testBatch: testBatches) { + testBatch.clear(); + } + } + } + } + + /** + * End to end test of the TopN operator. + * @throws Throwable + */ + @Test + public void sortOneKeyAscending() throws Throwable { + ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher) + .configProperty(ClassBuilder.CODE_DIR_OPTION, dirTestWatcher.getDir().getAbsolutePath()) + .configProperty(CodeCompiler.ENABLE_SAVE_CODE_FOR_DEBUG_TOPN, true); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + TestBuilder testBuilder = new TestBuilder(new ClusterFixture.FixtureTestServices(client)); + testBuilder.ordered() + .physicalPlanFromFile("topN/one_key_sort.json") + .baselineColumns("blue") + .go(); + } + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java index 8dd2da0..f07352c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue; import org.apache.drill.categories.OperatorTest; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.util.FileUtils; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.ops.FragmentContext; @@ -57,7 +57,7 @@ public class TestAgg extends ExecTest { mockDrillbitContext(bitContext); final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); - final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8)); + final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile(file), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java index b7c9e92..3de5519 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java @@ -18,7 +18,7 @@ package org.apache.drill.exec.physical.impl.agg; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.test.BaseTestQuery; import org.apache.drill.categories.OperatorTest; import org.junit.Ignore; import org.junit.Test;
