Repository: drill Updated Branches: refs/heads/master e3fbc6a19 -> 645e43fd3
DRILL-3364: Prune scan range if the filter is on the leading field with byte comparable encoding The change adds support to perform row-key range pruning when the row-key prefix is interpreted as UINT4_BE, TIMESTAMP_EPOCH_BE, TIME_EPOCH_BE, DATE_EPOCH_BE, UINT8_BE encoded. Testing Done: Added a unit-tests for the new feature, also ran all existing unit-tests to make sure there is no regression. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/645e43fd Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/645e43fd Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/645e43fd Branch: refs/heads/master Commit: 645e43fd34ce3b70f14df4e3d21c9c04ca9314f1 Parents: e3fbc6a Author: Smidth Panchamia <spancha...@mapr.com> Authored: Mon Jul 27 14:14:20 2015 -0700 Committer: Aditya Kishore <a...@apache.org> Committed: Fri Aug 14 22:34:33 2015 -0700 ---------------------------------------------------------------------- .../store/hbase/CompareFunctionsProcessor.java | 335 +++++++++++++++---- .../exec/store/hbase/HBaseFilterBuilder.java | 21 ++ .../org/apache/drill/hbase/HBaseTestsSuite.java | 17 +- .../drill/hbase/TestHBaseFilterPushDown.java | 167 +++++++++ .../apache/drill/hbase/TestTableGenerator.java | 125 +++++++ .../impl/conv/TimeStampEpochBEConvertFrom.java | 45 +++ .../fn/impl/conv/TimeStampEpochBEConvertTo.java | 54 +++ .../fn/impl/conv/TimeStampEpochConvertFrom.java | 47 +++ .../fn/impl/conv/TimeStampEpochConvertTo.java | 55 +++ .../expr/fn/impl/conv/UInt4BEConvertFrom.java | 45 +++ .../expr/fn/impl/conv/UInt4BEConvertTo.java | 54 +++ .../expr/fn/impl/conv/UInt4ConvertFrom.java | 46 +++ .../exec/expr/fn/impl/conv/UInt4ConvertTo.java | 55 +++ 13 files changed, 1007 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java index 803f520..87eb42e 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java @@ -20,6 +20,7 @@ package org.apache.drill.exec.store.hbase; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import org.apache.drill.common.expression.CastExpression; @@ -35,7 +36,11 @@ import org.apache.drill.common.expression.ValueExpressions.IntExpression; import org.apache.drill.common.expression.ValueExpressions.LongExpression; import org.apache.drill.common.expression.ValueExpressions.QuotedString; import org.apache.drill.common.expression.ValueExpressions.TimeExpression; +import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.PrefixFilter; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; @@ -48,6 +53,14 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr private SchemaPath path; private String functionName; + // Fields for row-key prefix comparison + // If the query is on row-key prefix, we cannot use a standard template to identify startRow, stopRow and filter + // Hence, we use these local variables(set depending upon the encoding type in user query) + private boolean isRowKeyPrefixComparison; + byte[] rowKeyPrefixStartRow; + byte[] rowKeyPrefixStopRow; + Filter rowKeyPrefixFilter; + public static boolean isCompareFunction(String functionName) { return COMPARE_FUNCTIONS_TRANSPOSE_MAP.keySet().contains(functionName); } @@ -79,6 +92,7 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr this.functionName = functionName; this.isEqualityFn = COMPARE_FUNCTIONS_TRANSPOSE_MAP.containsKey(functionName) && COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName).equals(functionName); + this.isRowKeyPrefixComparison = false; } public byte[] getValue() { @@ -97,6 +111,22 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr return functionName; } + public boolean isRowKeyPrefixComparison() { + return isRowKeyPrefixComparison; + } + + public byte[] getRowKeyPrefixStartRow() { + return rowKeyPrefixStartRow; + } + + public byte[] getRowKeyPrefixStopRow() { + return rowKeyPrefixStopRow; + } + + public Filter getRowKeyPrefixFilter() { + return rowKeyPrefixFilter; + } + @Override public Boolean visitCastExpression(CastExpression e, LogicalExpression valueArg) throws RuntimeException { if (e.getInput() instanceof CastExpression || e.getInput() instanceof SchemaPath) { @@ -107,79 +137,268 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr @Override public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression valueArg) throws RuntimeException { - if (e.getConvertFunction() == ConvertExpression.CONVERT_FROM && e.getInput() instanceof SchemaPath) { - ByteBuf bb = null; + if (e.getConvertFunction() == ConvertExpression.CONVERT_FROM) { + String encodingType = e.getEncodingType(); - switch (encodingType) { - case "INT_BE": - case "INT": - case "UINT_BE": - case "UINT": - case "UINT4_BE": - case "UINT4": - if (valueArg instanceof IntExpression - && (isEqualityFn || encodingType.startsWith("U"))) { - bb = newByteBuf(4, encodingType.endsWith("_BE")); - bb.writeInt(((IntExpression)valueArg).getInt()); + int prefixLength = 0; + + // Handle scan pruning in the following scenario: + // The row-key is a composite key and the CONVERT_FROM() function has byte_substr() as input function which is + // querying for the first few bytes of the row-key(start-offset 1) + // Example WHERE clause: + // CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'DATE_EPOCH_BE') < DATE '2015-06-17' + if (e.getInput() instanceof FunctionCall) { + + // We can prune scan range only for big-endian encoded data + if (encodingType.endsWith("_BE") == false) { + return false; } - break; - case "BIGINT_BE": - case "BIGINT": - case "UINT8_BE": - case "UINT8": - if (valueArg instanceof LongExpression - && (isEqualityFn || encodingType.startsWith("U"))) { - bb = newByteBuf(8, encodingType.endsWith("_BE")); - bb.writeLong(((LongExpression)valueArg).getLong()); + + FunctionCall call = (FunctionCall)e.getInput(); + String functionName = call.getName(); + if (!functionName.equalsIgnoreCase("byte_substr")) { + return false; } - break; - case "FLOAT": - if (valueArg instanceof FloatExpression && isEqualityFn) { + + LogicalExpression nameArg = call.args.get(0); + LogicalExpression valueArg1 = call.args.size() >= 2 ? call.args.get(1) : null; + LogicalExpression valueArg2 = call.args.size() >= 3 ? call.args.get(2) : null; + + if (((nameArg instanceof SchemaPath) == false) || + (valueArg1 == null) || ((valueArg1 instanceof IntExpression) == false) || + (valueArg2 == null) || ((valueArg2 instanceof IntExpression) == false)) { + return false; + } + + boolean isRowKey = ((SchemaPath)nameArg).getAsUnescapedPath().equals(DrillHBaseConstants.ROW_KEY); + int offset = ((IntExpression)valueArg1).getInt(); + + if (!isRowKey || (offset != 1)) { + return false; + } + + this.path = (SchemaPath)nameArg; + prefixLength = ((IntExpression)valueArg2).getInt(); + this.isRowKeyPrefixComparison = true; + return visitRowKeyPrefixConvertExpression(e, prefixLength, valueArg); + } + + if (e.getInput() instanceof SchemaPath) { + ByteBuf bb = null; + switch (encodingType) { + case "INT_BE": + case "INT": + case "UINT_BE": + case "UINT": + case "UINT4_BE": + case "UINT4": + if (valueArg instanceof IntExpression + && (isEqualityFn || encodingType.startsWith("U"))) { + bb = newByteBuf(4, encodingType.endsWith("_BE")); + bb.writeInt(((IntExpression)valueArg).getInt()); + } + break; + case "BIGINT_BE": + case "BIGINT": + case "UINT8_BE": + case "UINT8": + if (valueArg instanceof LongExpression + && (isEqualityFn || encodingType.startsWith("U"))) { + bb = newByteBuf(8, encodingType.endsWith("_BE")); + bb.writeLong(((LongExpression)valueArg).getLong()); + } + break; + case "FLOAT": + if (valueArg instanceof FloatExpression && isEqualityFn) { bb = newByteBuf(4, true); - bb.writeFloat(((FloatExpression)valueArg).getFloat()); + bb.writeFloat(((FloatExpression)valueArg).getFloat()); + } + break; + case "DOUBLE": + if (valueArg instanceof DoubleExpression && isEqualityFn) { + bb = newByteBuf(8, true); + bb.writeDouble(((DoubleExpression)valueArg).getDouble()); + } + break; + case "TIME_EPOCH": + case "TIME_EPOCH_BE": + if (valueArg instanceof TimeExpression) { + bb = newByteBuf(8, encodingType.endsWith("_BE")); + bb.writeLong(((TimeExpression)valueArg).getTime()); + } + break; + case "DATE_EPOCH": + case "DATE_EPOCH_BE": + if (valueArg instanceof DateExpression) { + bb = newByteBuf(8, encodingType.endsWith("_BE")); + bb.writeLong(((DateExpression)valueArg).getDate()); + } + break; + case "BOOLEAN_BYTE": + if (valueArg instanceof BooleanExpression) { + bb = newByteBuf(1, false /* does not matter */); + bb.writeByte(((BooleanExpression)valueArg).getBoolean() ? 1 : 0); + } + break; + case "UTF8": + // let visitSchemaPath() handle this. + return e.getInput().accept(this, valueArg); } - break; - case "DOUBLE": - if (valueArg instanceof DoubleExpression && isEqualityFn) { - bb = newByteBuf(8, true); - bb.writeDouble(((DoubleExpression)valueArg).getDouble()); + + if (bb != null) { + this.value = bb.array(); + this.path = (SchemaPath)e.getInput(); + return true; } - break; - case "TIME_EPOCH": - case "TIME_EPOCH_BE": - if (valueArg instanceof TimeExpression) { - bb = newByteBuf(8, encodingType.endsWith("_BE")); - bb.writeLong(((TimeExpression)valueArg).getTime()); + } + } + return false; + } + + private Boolean visitRowKeyPrefixConvertExpression(ConvertExpression e, + int prefixLength, LogicalExpression valueArg) { + String encodingType = e.getEncodingType(); + rowKeyPrefixStartRow = HConstants.EMPTY_START_ROW; + rowKeyPrefixStopRow = HConstants.EMPTY_START_ROW; + rowKeyPrefixFilter = null; + + if ((encodingType.compareTo("UINT4_BE") == 0) || + (encodingType.compareTo("UINT_BE") == 0)) { + if (prefixLength != 4) { + throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix"); + } + + int val; + if ((valueArg instanceof IntExpression) == false) { + return false; + } + + val = ((IntExpression)valueArg).getInt(); + + // For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >= + switch (functionName) { + case "equal": + rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(4).putInt(val).array()); + rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array(); + rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array(); + return true; + case "greater_than_or_equal_to": + rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array(); + return true; + case "greater_than": + rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val + 1).array(); + return true; + case "less_than_or_equal_to": + rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array(); + return true; + case "less_than": + rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val).array(); + return true; + } + + return false; + } + + if ((encodingType.compareTo("TIMESTAMP_EPOCH_BE") == 0) || + (encodingType.compareTo("TIME_EPOCH_BE") == 0) || + (encodingType.compareTo("UINT8_BE") == 0)) { + + if (prefixLength != 8) { + throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix"); + } + + long val; + if (encodingType.compareTo("TIME_EPOCH_BE") == 0) { + if ((valueArg instanceof TimeExpression) == false) { + return false; } - break; - case "DATE_EPOCH": - case "DATE_EPOCH_BE": - if (valueArg instanceof DateExpression) { - bb = newByteBuf(8, encodingType.endsWith("_BE")); - bb.writeLong(((DateExpression)valueArg).getDate()); + + val = ((TimeExpression)valueArg).getTime(); + } else if (encodingType.compareTo("UINT8_BE") == 0){ + if ((valueArg instanceof LongExpression) == false) { + return false; } - break; - case "BOOLEAN_BYTE": - if (valueArg instanceof BooleanExpression) { - bb = newByteBuf(1, false /* does not matter */); - bb.writeByte(((BooleanExpression)valueArg).getBoolean() ? 1 : 0); + + val = ((LongExpression)valueArg).getLong(); + } else if (encodingType.compareTo("TIMESTAMP_EPOCH_BE") == 0) { + if ((valueArg instanceof TimeStampExpression) == false) { + return false; } - break; - case "UTF8": - // let visitSchemaPath() handle this. - return e.getInput().accept(this, valueArg); + + val = ((TimeStampExpression)valueArg).getTimeStamp(); + } else { + // Should not reach here. + return false; } - if (bb != null) { - this.value = bb.array(); - this.path = (SchemaPath)e.getInput(); + // For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >= + switch (functionName) { + case "equal": + rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(8).putLong(val).array()); + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array(); + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array(); + return true; + case "greater_than_or_equal_to": + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array(); + return true; + case "greater_than": + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val + 1).array(); + return true; + case "less_than_or_equal_to": + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array(); + return true; + case "less_than": + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val).array(); return true; } + + return false; } - return false; + + if (encodingType.compareTo("DATE_EPOCH_BE") == 0) { + if ((valueArg instanceof DateExpression) == false) { + return false; + } + + if (prefixLength != 8) { + throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix"); + } + + final long MILLISECONDS_IN_A_DAY = (long)1000 * 60 * 60 * 24; + long dateToSet; + // For DATE encoding, the operators that we push-down are =, <>, <, <=, >, >= + switch (functionName) { + case "equal": + long startDate = ((DateExpression)valueArg).getDate(); + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(startDate).array(); + long stopDate = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY; + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(stopDate).array(); + return true; + case "greater_than_or_equal_to": + dateToSet = ((DateExpression)valueArg).getDate(); + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); + return true; + case "greater_than": + dateToSet = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY; + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); + return true; + case "less_than_or_equal_to": + dateToSet = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY; + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); + return true; + case "less_than": + dateToSet = ((DateExpression)valueArg).getDate(); + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); + return true; + } + + return false; } - @Override + return false; +} + +@Override public Boolean visitUnknown(LogicalExpression e, LogicalExpression valueArg) throws RuntimeException { return false; } @@ -237,4 +456,4 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr .build(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java index 245d77d..623846f 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.filter.ByteArrayComparable; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.NullComparator; +import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.RegexStringComparator; import org.apache.hadoop.hbase.filter.RowFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; @@ -167,6 +168,10 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void, return null; } + if (processor.isRowKeyPrefixComparison()) { + return createRowKeyPrefixScanSpec(call, processor); + } + CompareOp compareOp = null; boolean isNullTest = false; ByteArrayComparable comparator = new BinaryComparator(fieldValue); @@ -296,4 +301,20 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void, return null; } +private HBaseScanSpec createRowKeyPrefixScanSpec(FunctionCall call, + CompareFunctionsProcessor processor) { + byte[] startRow = processor.getRowKeyPrefixStartRow(); + byte[] stopRow = processor.getRowKeyPrefixStopRow(); + Filter filter = processor.getRowKeyPrefixFilter(); + + if (startRow != HConstants.EMPTY_START_ROW || + stopRow != HConstants.EMPTY_END_ROW || + filter != null) { + return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, filter); + } + + // else + return null; +} + } http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java index a77baba..a5dbc6f 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java @@ -50,6 +50,9 @@ public class HBaseTestsSuite { protected static final String TEST_TABLE_1 = "TestTable1"; protected static final String TEST_TABLE_3 = "TestTable3"; + protected static final String TEST_TABLE_COMPOSITE_DATE = "TestTableCompositeDate"; + protected static final String TEST_TABLE_COMPOSITE_TIME = "TestTableCompositeTime"; + protected static final String TEST_TABLE_COMPOSITE_INT = "TestTableCompositeInt"; private static Configuration conf; @@ -135,7 +138,10 @@ public class HBaseTestsSuite { } private static boolean tablesExist() throws IOException { - return admin.tableExists(TEST_TABLE_1) && admin.tableExists(TEST_TABLE_3); + return admin.tableExists(TEST_TABLE_1) && admin.tableExists(TEST_TABLE_3) && + admin.tableExists(TEST_TABLE_COMPOSITE_DATE) && + admin.tableExists(TEST_TABLE_COMPOSITE_TIME) && + admin.tableExists(TEST_TABLE_COMPOSITE_INT); } private static void createTestTables() throws Exception { @@ -146,6 +152,9 @@ public class HBaseTestsSuite { */ TestTableGenerator.generateHBaseDataset1(admin, TEST_TABLE_1, 1); TestTableGenerator.generateHBaseDataset3(admin, TEST_TABLE_3, 1); + TestTableGenerator.generateHBaseDatasetCompositeKeyDate(admin, TEST_TABLE_COMPOSITE_DATE, 1); + TestTableGenerator.generateHBaseDatasetCompositeKeyTime(admin, TEST_TABLE_COMPOSITE_TIME, 1); + TestTableGenerator.generateHBaseDatasetCompositeKeyInt(admin, TEST_TABLE_COMPOSITE_INT, 1); } private static void cleanupTestTables() throws IOException { @@ -153,6 +162,12 @@ public class HBaseTestsSuite { admin.deleteTable(TEST_TABLE_1); admin.disableTable(TEST_TABLE_3); admin.deleteTable(TEST_TABLE_3); + admin.disableTable(TEST_TABLE_COMPOSITE_DATE); + admin.deleteTable(TEST_TABLE_COMPOSITE_DATE); + admin.disableTable(TEST_TABLE_COMPOSITE_TIME); + admin.deleteTable(TEST_TABLE_COMPOSITE_TIME); + admin.disableTable(TEST_TABLE_COMPOSITE_INT); + admin.deleteTable(TEST_TABLE_COMPOSITE_INT); } public static int getZookeeperPort() { http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java index 00de0ba..5c3a463 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java @@ -58,6 +58,173 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); } + + @Test + public void testFilterPushDownCompositeDateRowKey1() throws Exception { + setColumnWidths(new int[] {11, 22, 32}); + runHBaseSQLVerifyCount("SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'date_epoch_be') d\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 9, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeDate` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'date_epoch_be') < DATE '2015-06-18' AND\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'date_epoch_be') > DATE '2015-06-13'" + , 12); + } + + @Test + public void testFilterPushDownCompositeDateRowKey2() throws Exception { + setColumnWidths(new int[] {11, 22, 32}); + runHBaseSQLVerifyCount("SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'date_epoch_be') d\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 9, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeDate` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'date_epoch_be') = DATE '2015-08-22'" + , 3); + } + + @Test + public void testFilterPushDownCompositeDateRowKey3() throws Exception { + setColumnWidths(new int[] {11, 2000}); + runHBaseSQLVerifyCount("EXPLAIN PLAN FOR SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'date_epoch_be') d\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 9, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeDate` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'date_epoch_be') < DATE '2015-06-18' AND\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'date_epoch_be') > DATE '2015-06-13'" + , 1); + } + + @Test + public void testFilterPushDownCompositeDateRowKey4() throws Exception { + setColumnWidths(new int[] {30, 22, 30, 10}); + runHBaseSQLVerifyCount("SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'timestamp_epoch_be') d\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 9, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'time_epoch_be') t\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeDate` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'timestamp_epoch_be') >= TIMESTAMP '2015-06-18 08:00:00.000' AND\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'timestamp_epoch_be') < TIMESTAMP '2015-06-20 16:00:00.000'" + , 7); + } + + @Test + public void testFilterPushDownCompositeTimeRowKey1() throws Exception { + setColumnWidths(new int[] {50, 40, 32}); + runHBaseSQLVerifyCount("SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'time_epoch_be') d\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 9, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeTime` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'time_epoch_be') = TIME '23:57:15.275'"//convert_from(binary_string('\\x00\\x00\\x00\\x00\\x55\\x4D\\xBE\\x80'), 'BIGINT_BE') \n" + , 1); + } + + @Test + public void testFilterPushDownCompositeTimeRowKey2() throws Exception { + setColumnWidths(new int[] {30, 2002, 32}); + runHBaseSQLVerifyCount("EXPLAIN PLAN FOR SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'time_epoch_be') d\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 9, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeTime` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'time_epoch_be') = TIME '23:55:51.250'"//convert_from(binary_string('\\x00\\x00\\x00\\x00\\x55\\x4D\\xBE\\x80'), 'BIGINT_BE') \n" + , 1); + } + + @Test + public void testFilterPushDownCompositeTimeRowKey3() throws Exception { + setColumnWidths(new int[] {30, 22, 32}); + runHBaseSQLVerifyCount("SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'time_epoch_be') d\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 9, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeTime` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'time_epoch_be') > TIME '23:57:06' AND"//convert_from(binary_string('\\x00\\x00\\x00\\x00\\x55\\x4D\\xBE\\x80'), 'BIGINT_BE') \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'time_epoch_be') < TIME '23:59:59'" + , 8); + } + + @Test + public void testFilterPushDownCompositeBigIntRowKey1() throws Exception { + setColumnWidths(new int[] {15, 40, 32}); + runHBaseSQLVerifyCount("SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'bigint_be') d\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 9, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeDate` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'bigint_be') = cast(1409040000000 as bigint)"//convert_from(binary_string('\\x00\\x00\\x00\\x00\\x55\\x4D\\xBE\\x80'), 'BIGINT_BE') \n" + , 1); + } + + @Test + public void testFilterPushDownCompositeBigIntRowKey2() throws Exception { + setColumnWidths(new int[] {16, 22, 32}); + runHBaseSQLVerifyCount("SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'bigint_be') i\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'date_epoch_be') d\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'time_epoch_be') t\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 9, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeDate` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'uint8_be') > cast(1438300800000 as bigint) AND\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'uint8_be') < cast(1438617600000 as bigint)" + , 10); + } + + @Test + public void testFilterPushDownCompositeIntRowKey1() throws Exception { + setColumnWidths(new int[] {16, 22, 32}); + runHBaseSQLVerifyCount("SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 4), 'uint4_be') i\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 5, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeInt` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 4), 'uint4_be') >= cast(423 as int) AND" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 4), 'uint4_be') < cast(940 as int)" + , 11); + } + + @Test + public void testFilterPushDownCompositeIntRowKey2() throws Exception { + setColumnWidths(new int[] {16, 2002, 32}); + runHBaseSQLVerifyCount("EXPLAIN PLAN FOR SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 4), 'uint4_be') i\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 5, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeInt` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 4), 'uint4_be') >= cast(300 as int) AND" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 4), 'uint4_be') < cast(900 as int)" + , 1); + } + + @Test + public void testFilterPushDownCompositeIntRowKey3() throws Exception { + setColumnWidths(new int[] {16, 22, 32}); + runHBaseSQLVerifyCount("SELECT \n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 4), 'uint4_be') i\n" + + ", CONVERT_FROM(BYTE_SUBSTR(row_key, 5, 8), 'bigint_be') id\n" + + ", CONVERT_FROM(tableName.f.c, 'UTF8') \n" + + " FROM hbase.`TestTableCompositeInt` tableName\n" + + " WHERE\n" + + " CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 4), 'uint4_be') = cast(658 as int)" + , 1); + } + @Test public void testFilterPushDownRowKeyLike() throws Exception { setColumnWidths(new int[] {8, 22}); http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java index 5cb9aa5..07ae697 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java @@ -17,7 +17,9 @@ */ package org.apache.drill.hbase; +import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Date; import java.util.Random; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -206,4 +208,127 @@ public class TestTableGenerator { admin.flush(tableName); } + public static void generateHBaseDatasetCompositeKeyDate(HBaseAdmin admin, String tableName, int numberRegions) throws Exception { + if (admin.tableExists(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY_F)); + + if (numberRegions > 1) { + admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1)); + } else { + admin.createTable(desc); + } + + HTable table = new HTable(admin.getConfiguration(), tableName); + + Date startDate = new Date(1408924800000L); + long startTime = startDate.getTime(); + long MILLISECONDS_IN_A_DAY = (long)1000 * 60 * 60 * 24; + long MILLISECONDS_IN_A_YEAR = MILLISECONDS_IN_A_DAY * 365; + long endTime = startTime + MILLISECONDS_IN_A_YEAR; + long interval = MILLISECONDS_IN_A_DAY / 3; + + for (long ts = startTime, counter = 0; ts < endTime; ts += interval, counter ++) { + byte[] rowKey = ByteBuffer.allocate(16) .putLong(ts).array(); + + for(int i = 0; i < 8; ++i) { + rowKey[8 + i] = (byte)(counter >> (56 - (i * 8))); + } + + Put p = new Put(rowKey); + p.add(FAMILY_F, COLUMN_C, "dummy".getBytes()); + table.put(p); + } + + table.flushCommits(); + table.close(); + } + + public static void generateHBaseDatasetCompositeKeyTime(HBaseAdmin admin, String tableName, int numberRegions) throws Exception { + if (admin.tableExists(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY_F)); + + if (numberRegions > 1) { + admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1)); + } else { + admin.createTable(desc); + } + + HTable table = new HTable(admin.getConfiguration(), tableName); + + long startTime = 0; + long MILLISECONDS_IN_A_SEC = (long)1000; + long MILLISECONDS_IN_A_DAY = MILLISECONDS_IN_A_SEC * 60 * 60 * 24; + long endTime = startTime + MILLISECONDS_IN_A_DAY; + long smallInterval = 25; + long largeInterval = MILLISECONDS_IN_A_SEC * 42; + long interval = smallInterval; + + for (long ts = startTime, counter = 0; ts < endTime; ts += interval, counter ++) { + byte[] rowKey = ByteBuffer.allocate(16) .putLong(ts).array(); + + for(int i = 0; i < 8; ++i) { + rowKey[8 + i] = (byte)(counter >> (56 - (i * 8))); + } + + Put p = new Put(rowKey); + p.add(FAMILY_F, COLUMN_C, "dummy".getBytes()); + table.put(p); + + if (interval == smallInterval) { + interval = largeInterval; + } else { + interval = smallInterval; + } + } + + table.flushCommits(); + table.close(); + } + + public static void generateHBaseDatasetCompositeKeyInt(HBaseAdmin admin, String tableName, int numberRegions) throws Exception { + if (admin.tableExists(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY_F)); + + if (numberRegions > 1) { + admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1)); + } else { + admin.createTable(desc); + } + + HTable table = new HTable(admin.getConfiguration(), tableName); + + int startVal = 0; + int stopVal = 1000; + int interval = 47; + long counter = 0; + for (int i = startVal; i < stopVal; i += interval, counter ++) { + byte[] rowKey = ByteBuffer.allocate(12).putInt(i).array(); + + for(int j = 0; j < 8; ++j) { + rowKey[4 + j] = (byte)(counter >> (56 - (j * 8))); + } + + Put p = new Put(rowKey); + p.add(FAMILY_F, COLUMN_C, "dummy".getBytes()); + table.put(p); + } + + table.flushCommits(); + table.close(); + } } http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochBEConvertFrom.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochBEConvertFrom.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochBEConvertFrom.java new file mode 100644 index 0000000..eec7159 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochBEConvertFrom.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.impl.conv; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.TimeStampHolder; +import org.apache.drill.exec.expr.holders.VarBinaryHolder; + +@FunctionTemplate(name = "convert_fromTIMESTAMP_EPOCH_BE", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) +public class TimeStampEpochBEConvertFrom implements DrillSimpleFunc { + + @Param VarBinaryHolder in; + @Output TimeStampHolder out; + + @Override + public void setup() { } + + @Override + public void eval() { + org.apache.drill.exec.util.ByteBufUtil.checkBufferLength(in.buffer, in.start, in.end, 8); + + in.buffer.readerIndex(in.start); + out.value = Long.reverseBytes(in.buffer.readLong()); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochBEConvertTo.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochBEConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochBEConvertTo.java new file mode 100644 index 0000000..504cb45 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochBEConvertTo.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.impl.conv; + +import io.netty.buffer.DrillBuf; + +import javax.inject.Inject; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.TimeStampHolder; +import org.apache.drill.exec.expr.holders.VarBinaryHolder; + +@FunctionTemplate(name = "convert_toTIMESTAMP_EPOCH_BE", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) +public class TimeStampEpochBEConvertTo implements DrillSimpleFunc { + + @Param TimeStampHolder in; + @Output VarBinaryHolder out; + @Inject DrillBuf buffer; + + + @Override + public void setup() { + buffer = buffer.reallocIfNeeded(8); + } + + @Override + public void eval() { + buffer.clear(); + buffer.writeLong(Long.reverseBytes(in.value)); + out.buffer = buffer; + out.start = 0; + out.end = 8; + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochConvertFrom.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochConvertFrom.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochConvertFrom.java new file mode 100644 index 0000000..e68d301 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochConvertFrom.java @@ -0,0 +1,47 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.expr.fn.impl.conv; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.TimeHolder; +import org.apache.drill.exec.expr.holders.TimeStampHolder; +import org.apache.drill.exec.expr.holders.VarBinaryHolder; + +@FunctionTemplate(name = "convert_fromTIMESTAMP_EPOCH", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) +public class TimeStampEpochConvertFrom implements DrillSimpleFunc { + + @Param VarBinaryHolder in; + @Output TimeStampHolder out; + + @Override + public void setup() { } + + @Override + public void eval() { + org.apache.drill.exec.util.ByteBufUtil.checkBufferLength(in.buffer, in.start, in.end, 8); + + in.buffer.readerIndex(in.start); + out.value = in.buffer.readLong(); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochConvertTo.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochConvertTo.java new file mode 100644 index 0000000..4022487 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeStampEpochConvertTo.java @@ -0,0 +1,55 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.expr.fn.impl.conv; + +import io.netty.buffer.DrillBuf; + +import javax.inject.Inject; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.TimeStampHolder; +import org.apache.drill.exec.expr.holders.VarBinaryHolder; + +@FunctionTemplate(name = "convert_toTIMESTAMP_EPOCH", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) +public class TimeStampEpochConvertTo implements DrillSimpleFunc { + + @Param TimeStampHolder in; + @Output VarBinaryHolder out; + @Inject DrillBuf buffer; + + + @Override + public void setup() { + buffer = buffer.reallocIfNeeded(8); + } + + @Override + public void eval() { + buffer.clear(); + buffer.writeLong(in.value); + out.buffer = buffer; + out.start = 0; + out.end = 8; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4BEConvertFrom.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4BEConvertFrom.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4BEConvertFrom.java new file mode 100644 index 0000000..dd2c29c --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4BEConvertFrom.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.impl.conv; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.UInt4Holder; +import org.apache.drill.exec.expr.holders.VarBinaryHolder; + +@FunctionTemplate(name = "convert_fromUINT4_BE", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) +public class UInt4BEConvertFrom implements DrillSimpleFunc { + + @Param VarBinaryHolder in; + @Output UInt4Holder out; + + @Override + public void setup() { } + + @Override + public void eval() { + org.apache.drill.exec.util.ByteBufUtil.checkBufferLength(in.buffer, in.start, in.end, 4); + + in.buffer.readerIndex(in.start); + out.value = Integer.reverseBytes(in.buffer.readInt()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4BEConvertTo.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4BEConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4BEConvertTo.java new file mode 100644 index 0000000..302f18c --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4BEConvertTo.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.impl.conv; + +import io.netty.buffer.DrillBuf; + +import javax.inject.Inject; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.UInt4Holder; +import org.apache.drill.exec.expr.holders.VarBinaryHolder; + +@FunctionTemplate(name = "convert_toUINT4_BE", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) +public class UInt4BEConvertTo implements DrillSimpleFunc { + + @Param UInt4Holder in; + @Output VarBinaryHolder out; + @Inject DrillBuf buffer; + + + @Override + public void setup() { + buffer = buffer.reallocIfNeeded(4); + } + + @Override + public void eval() { + buffer.clear(); + buffer.writeInt(Integer.reverseBytes(in.value)); + out.buffer = buffer; + out.start = 0; + out.end = 4; + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4ConvertFrom.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4ConvertFrom.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4ConvertFrom.java new file mode 100644 index 0000000..fba2b97 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4ConvertFrom.java @@ -0,0 +1,46 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.expr.fn.impl.conv; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.UInt4Holder; +import org.apache.drill.exec.expr.holders.VarBinaryHolder; + +@FunctionTemplate(name = "convert_fromUINT4", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) +public class UInt4ConvertFrom implements DrillSimpleFunc { + + @Param VarBinaryHolder in; + @Output UInt4Holder out; + + @Override + public void setup() { } + + @Override + public void eval() { + org.apache.drill.exec.util.ByteBufUtil.checkBufferLength(in.buffer, in.start, in.end, 4); + + in.buffer.readerIndex(in.start); + out.value = in.buffer.readInt(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/645e43fd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4ConvertTo.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4ConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4ConvertTo.java new file mode 100644 index 0000000..a362bd8 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt4ConvertTo.java @@ -0,0 +1,55 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.expr.fn.impl.conv; + +import io.netty.buffer.DrillBuf; + +import javax.inject.Inject; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.UInt4Holder; +import org.apache.drill.exec.expr.holders.VarBinaryHolder; + +@FunctionTemplate(name = "convert_toUINT4", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) +public class UInt4ConvertTo implements DrillSimpleFunc { + + @Param UInt4Holder in; + @Output VarBinaryHolder out; + @Inject DrillBuf buffer; + + + @Override + public void setup() { + buffer = buffer.reallocIfNeeded(4); + } + + @Override + public void eval() { + buffer.clear(); + buffer.writeInt(in.value); + out.buffer = buffer; + out.start = 0; + out.end = 4; + } +} \ No newline at end of file
