Fix hash join functions for variable types to use the correct buffer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/3eaf6d63 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/3eaf6d63 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/3eaf6d63 Branch: refs/heads/master Commit: 3eaf6d6348d05e5c8a2f1d60fd3c19df5b4889b6 Parents: a694c0a Author: vkorukanti <[email protected]> Authored: Sat May 10 16:02:03 2014 -0700 Committer: vkorukanti <[email protected]> Committed: Sat May 10 16:02:03 2014 -0700 ---------------------------------------------------------------------- .../drill/exec/expr/fn/impl/HashFunctions.java | 37 +++++++++++++-- .../exec/physical/impl/writer/TestWriter.java | 49 ++++++++++++++------ 2 files changed, 68 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3eaf6d63/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java index 36300ee..766a32a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java @@ -102,7 +102,7 @@ public class HashFunctions { if (in.isSet == 0) out.value = 0; else - out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(), 0); + out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0); } } @@ -119,7 +119,24 @@ public class HashFunctions { if (in.isSet == 0) out.value = 0; else - out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(), 0); + out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0); + } + } + + @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL ) + public static class NullableVar16CharHash implements DrillSimpleFunc { + + @Param NullableVar16CharHolder in; + @Output IntHolder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + if (in.isSet == 0) + out.value = 0; + else + out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0); } } @@ -185,7 +202,21 @@ public class HashFunctions { out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0); } } - + + @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL) + public static class Var16CharHash implements DrillSimpleFunc { + + @Param Var16CharHolder in; + @Output IntHolder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0); + } + } + @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL) public static class HashBigInt implements DrillSimpleFunc { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3eaf6d63/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java index 54e6b7f..6ddb73d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java @@ -90,46 +90,65 @@ public class TestWriter extends BaseTestQuery { @Test public void simpleCTAS() throws Exception { - String testQuery = "USE dfs.tmp;" + - "CREATE TABLE simplectas AS SELECT * FROM cp.`employee.json`;"; + testSqlWithResults("Use dfs.tmp"); - ctasHelper("/tmp/drilltest/simplectas", testQuery, 1); + String testQuery = "CREATE TABLE simplectas AS SELECT * FROM cp.`employee.json`"; + + ctasHelper("/tmp/drilltest/simplectas", testQuery, 1155); } @Test public void complex1CTAS() throws Exception { - String testQuery = "USE dfs.tmp;" + - "CREATE TABLE complex1ctas AS SELECT first_name, last_name, position_id FROM cp.`employee.json`;"; + testSqlWithResults("Use dfs.tmp"); + String testQuery = "CREATE TABLE complex1ctas AS SELECT first_name, last_name, position_id FROM cp.`employee.json`"; - ctasHelper("/tmp/drilltest/complex1ctas", testQuery, 1); + ctasHelper("/tmp/drilltest/complex1ctas", testQuery, 1155); } @Test public void complex2CTAS() throws Exception { - String testQuery = "USE dfs.tmp;" + - "CREATE TABLE complex2ctas AS SELECT CAST(`birth_date` as Timestamp) FROM cp.`employee.json` GROUP BY birth_date;"; + testSqlWithResults("Use dfs.tmp"); + String testQuery = "CREATE TABLE complex2ctas AS SELECT CAST(`birth_date` as Timestamp) FROM cp.`employee.json` GROUP BY birth_date"; - ctasHelper("/tmp/drilltest/complex2ctas", testQuery, 3); + ctasHelper("/tmp/drilltest/complex2ctas", testQuery, 52); } @Test public void simpleCTASWithSchemaInTableName() throws Exception { - String testQuery = "CREATE TABLE dfs.tmp.`/test/simplectas2` AS SELECT * FROM cp.`employee.json`;"; + String testQuery = "CREATE TABLE dfs.tmp.`/test/simplectas2` AS SELECT * FROM cp.`employee.json`"; - ctasHelper("/tmp/drilltest/test/simplectas2", testQuery, 1); + ctasHelper("/tmp/drilltest/test/simplectas2", testQuery, 1155); } - private void ctasHelper(String tableDir, String testQuery, int numExpectedFiles) throws Exception { + private void ctasHelper(String tableDir, String testQuery, int expectedOutputCount) throws Exception { Path tableLocation = new Path(tableDir); if (fs.exists(tableLocation)){ fs.delete(tableLocation, true); } - test(testQuery); + List<QueryResultBatch> results = testSqlWithResults(testQuery); + + RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator()); + + int recordsWritten = 0; + for(QueryResultBatch batch : results) { + batchLoader.load(batch.getHeader().getDef(), batch.getData()); + + if (batchLoader.getRecordCount() <= 0) + continue; + + BigIntVector recordWrittenV = (BigIntVector) batchLoader.getValueAccessorById(1, BigIntVector.class).getValueVector(); + + for (int i = 0; i < batchLoader.getRecordCount(); i++) { + recordsWritten += recordWrittenV.getAccessor().get(i); + } + + batch.release(); + } + batchLoader.clear(); assertTrue(fs.exists(tableLocation)); - FileStatus[] fileStatuses = fs.globStatus(new Path(tableLocation.toString(), "*.csv")); - assertEquals(numExpectedFiles, fileStatuses.length); + assertEquals(expectedOutputCount, recordsWritten); } }
