PHOENIX-3383 Comparison between descending row keys used in RVC is reverse
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/0e612b2b Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/0e612b2b Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/0e612b2b Branch: refs/heads/4.14-cdh5.14 Commit: 0e612b2bab54751f99ec6830a0526e8efc6882a5 Parents: 48b5fe6 Author: James Taylor <jamestay...@apache.org> Authored: Fri Jul 6 05:38:28 2018 +0100 Committer: Pedro Boado <pbo...@apache.org> Committed: Wed Oct 17 20:42:09 2018 +0100 ---------------------------------------------------------------------- .../org/apache/phoenix/end2end/QueryMoreIT.java | 12 +- .../org/apache/phoenix/compile/KeyPart.java | 2 - .../org/apache/phoenix/compile/ScanRanges.java | 77 +- .../apache/phoenix/compile/WhereOptimizer.java | 1304 +++++++++++++----- .../PhoenixTxIndexMutationGenerator.java | 2 +- .../expression/function/FunctionExpression.java | 10 +- .../expression/function/InvertFunction.java | 19 +- .../expression/function/PrefixFunction.java | 6 +- .../expression/function/RTrimFunction.java | 6 +- .../function/RoundDateExpression.java | 22 +- .../function/RoundDecimalExpression.java | 7 +- .../phoenix/iterate/BaseResultIterators.java | 4 +- .../apache/phoenix/iterate/ExplainTable.java | 10 - .../java/org/apache/phoenix/query/KeyRange.java | 28 +- .../org/apache/phoenix/schema/RowKeySchema.java | 78 ++ .../phoenix/compile/QueryCompilerTest.java | 2 +- .../phoenix/compile/QueryOptimizerTest.java | 5 +- .../TenantSpecificViewIndexCompileTest.java | 8 +- .../phoenix/compile/WhereOptimizerTest.java | 359 ++++- .../RoundFloorCeilExpressionsTest.java | 59 +- .../apache/phoenix/query/KeyRangeClipTest.java | 2 +- .../org/apache/phoenix/query/QueryPlanTest.java | 8 +- .../apache/phoenix/schema/RowKeySchemaTest.java | 48 + 23 files changed, 1567 insertions(+), 511 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/0e612b2b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java index 9109c12..04272fa 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java @@ -372,9 +372,6 @@ public class QueryMoreIT extends ParallelStatsDisabledIT { } } - // FIXME: this repros PHOENIX-3382, but turned up two more issues: - // 1) PHOENIX-3383 Comparison between descending row keys used in RVC is reverse - // 2) PHOENIX-3384 Optimize RVC expressions for non leading row key columns @Test public void testRVCOnDescWithLeadingPKEquality() throws Exception { final Connection conn = DriverManager.getConnection(getUrl()); @@ -398,14 +395,11 @@ public class QueryMoreIT extends ParallelStatsDisabledIT { conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES ('org1',1,'02')"); conn.commit(); - // FIXME: PHOENIX-3383 - // This comparison is really backwards: it should be (score, entity_id) < (2, '04'), - // but because we're matching a descending key, our comparison has to be switched. try (Statement stmt = conn.createStatement()) { final ResultSet rs = stmt.executeQuery("SELECT entity_id, score\n" + "FROM " + fullTableName + "\n" + "WHERE organization_id = 'org1'\n" + - "AND (score, entity_id) > (2, '04')\n" + + "AND (score, entity_id) < (2, '04')\n" + "ORDER BY score DESC, entity_id DESC\n" + "LIMIT 3"); assertTrue(rs.next()); @@ -416,13 +410,11 @@ public class QueryMoreIT extends ParallelStatsDisabledIT { assertEquals(1.0, rs.getDouble(2), 0.001); assertFalse(rs.next()); } - // FIXME: PHOENIX-3384 - // It should not be necessary to specify organization_id in this query try (Statement stmt = conn.createStatement()) { final ResultSet rs = stmt.executeQuery("SELECT entity_id, score\n" + "FROM " + fullTableName + "\n" + "WHERE organization_id = 'org1'\n" + - "AND (organization_id, score, entity_id) > ('org1', 2, '04')\n" + + "AND (organization_id, score, entity_id) < ('org1', 2, '04')\n" + "ORDER BY score DESC, entity_id DESC\n" + "LIMIT 3"); assertTrue(rs.next()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/0e612b2b/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java index 55385f2..6cf9938 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java @@ -33,8 +33,6 @@ import org.apache.phoenix.schema.PTable; * between a built-in function and the setting of the scan key * during query compilation. * - * - * @since 0.12 */ public interface KeyPart { /** http://git-wip-us.apache.org/repos/asf/phoenix/blob/0e612b2b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java index 019f15d..1732fce 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.filter.SkipScanFilter; @@ -45,7 +44,6 @@ import org.apache.phoenix.schema.types.PDataType.PDataCodec; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.ScanUtil; -import org.apache.phoenix.util.ScanUtil.BytesComparator; import org.apache.phoenix.util.SchemaUtil; import com.google.common.base.Throwables; @@ -56,30 +54,30 @@ import com.google.common.collect.Lists; public class ScanRanges { private static final List<List<KeyRange>> EVERYTHING_RANGES = Collections.<List<KeyRange>>emptyList(); private static final List<List<KeyRange>> NOTHING_RANGES = Collections.<List<KeyRange>>singletonList(Collections.<KeyRange>singletonList(KeyRange.EMPTY_RANGE)); - public static final ScanRanges EVERYTHING = new ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,EVERYTHING_RANGES, KeyRange.EVERYTHING_RANGE, KeyRange.EVERYTHING_RANGE, false, false, null, null); - public static final ScanRanges NOTHING = new ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,NOTHING_RANGES, KeyRange.EMPTY_RANGE, KeyRange.EMPTY_RANGE, false, false, null, null); + public static final ScanRanges EVERYTHING = new ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,EVERYTHING_RANGES, KeyRange.EVERYTHING_RANGE, false, false, null, null); + public static final ScanRanges NOTHING = new ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,NOTHING_RANGES, KeyRange.EMPTY_RANGE, false, false, null, null); private static final Scan HAS_INTERSECTION = new Scan(); public static ScanRanges createPointLookup(List<KeyRange> keys) { - return ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, KeyRange.EVERYTHING_RANGE, null, true, -1); + return ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, null, true, -1); } // For testing public static ScanRanges createSingleSpan(RowKeySchema schema, List<List<KeyRange>> ranges) { - return create(schema, ranges, ScanUtil.getDefaultSlotSpans(ranges.size()), KeyRange.EVERYTHING_RANGE, null, true, -1); + return create(schema, ranges, ScanUtil.getDefaultSlotSpans(ranges.size()), null, true, -1); } // For testing public static ScanRanges createSingleSpan(RowKeySchema schema, List<List<KeyRange>> ranges, Integer nBuckets, boolean useSkipSan) { - return create(schema, ranges, ScanUtil.getDefaultSlotSpans(ranges.size()), KeyRange.EVERYTHING_RANGE, nBuckets, useSkipSan, -1); + return create(schema, ranges, ScanUtil.getDefaultSlotSpans(ranges.size()), nBuckets, useSkipSan, -1); } - public static ScanRanges create(RowKeySchema schema, List<List<KeyRange>> ranges, int[] slotSpan, KeyRange minMaxRange, Integer nBuckets, boolean useSkipScan, int rowTimestampColIndex) { + public static ScanRanges create(RowKeySchema schema, List<List<KeyRange>> ranges, int[] slotSpan, Integer nBuckets, boolean useSkipScan, int rowTimestampColIndex) { int offset = nBuckets == null ? 0 : SaltingUtil.NUM_SALTING_BYTES; int nSlots = ranges.size(); - if (nSlots == offset && minMaxRange == KeyRange.EVERYTHING_RANGE) { + if (nSlots == offset) { return EVERYTHING; - } else if (minMaxRange == KeyRange.EMPTY_RANGE || (nSlots == 1 + offset && ranges.get(offset).size() == 1 && ranges.get(offset).get(0) == KeyRange.EMPTY_RANGE)) { + } else if ((nSlots == 1 + offset && ranges.get(offset).size() == 1 && ranges.get(offset).get(0) == KeyRange.EMPTY_RANGE)) { return NOTHING; } TimeRange rowTimestampRange = getRowTimestampColumnRange(ranges, schema, rowTimestampColIndex); @@ -88,22 +86,9 @@ public class ScanRanges { // TODO: consider keeping original to use for serialization as it would be smaller? List<byte[]> keys = ScanRanges.getPointKeys(ranges, slotSpan, schema, nBuckets); List<KeyRange> keyRanges = Lists.newArrayListWithExpectedSize(keys.size()); - KeyRange unsaltedMinMaxRange = minMaxRange; - if (nBuckets != null && minMaxRange != KeyRange.EVERYTHING_RANGE) { - unsaltedMinMaxRange = KeyRange.getKeyRange( - stripPrefix(minMaxRange.getLowerRange(),offset), - minMaxRange.lowerUnbound(), - stripPrefix(minMaxRange.getUpperRange(),offset), - minMaxRange.upperUnbound()); - } // We have full keys here, so use field from our varbinary schema - BytesComparator comparator = ScanUtil.getComparator(SchemaUtil.VAR_BINARY_SCHEMA.getField(0)); for (byte[] key : keys) { - // Filter now based on unsalted minMaxRange and ignore the point key salt byte - if ( unsaltedMinMaxRange.compareLowerToUpperBound(key, offset, key.length-offset, true, comparator) <= 0 && - unsaltedMinMaxRange.compareUpperToLowerBound(key, offset, key.length-offset, true, comparator) >= 0) { - keyRanges.add(KeyRange.getKeyRange(key)); - } + keyRanges.add(KeyRange.getKeyRange(key)); } // while doing a point look up if after intersecting with the MinMaxrange there are // no more keyranges left then just return @@ -151,17 +136,11 @@ public class ScanRanges { } scanRange = KeyRange.getKeyRange(minKey, maxKey); } - if (minMaxRange != KeyRange.EVERYTHING_RANGE) { - // Intersect using modified min/max range, but keep original range to ensure it - // can still be decomposed into it's parts - KeyRange inclusiveExclusiveMinMaxRange = ScanUtil.convertToInclusiveExclusiveRange(minMaxRange, schema, new ImmutableBytesWritable()); - scanRange = scanRange.intersect(inclusiveExclusiveMinMaxRange); - } - + if (scanRange == KeyRange.EMPTY_RANGE) { return NOTHING; } - return new ScanRanges(schema, slotSpan, sortedRanges, scanRange, minMaxRange, useSkipScan, isPointLookup, nBuckets, rowTimestampRange); + return new ScanRanges(schema, slotSpan, sortedRanges, scanRange, useSkipScan, isPointLookup, nBuckets, rowTimestampRange); } private SkipScanFilter filter; @@ -172,15 +151,13 @@ public class ScanRanges { private final boolean isSalted; private final boolean useSkipScanFilter; private final KeyRange scanRange; - private final KeyRange minMaxRange; private final TimeRange rowTimestampRange; - private ScanRanges (RowKeySchema schema, int[] slotSpan, List<List<KeyRange>> ranges, KeyRange scanRange, KeyRange minMaxRange, boolean useSkipScanFilter, boolean isPointLookup, Integer bucketNum, TimeRange rowTimestampRange) { + private ScanRanges (RowKeySchema schema, int[] slotSpan, List<List<KeyRange>> ranges, KeyRange scanRange, boolean useSkipScanFilter, boolean isPointLookup, Integer bucketNum, TimeRange rowTimestampRange) { this.isPointLookup = isPointLookup; this.isSalted = bucketNum != null; this.useSkipScanFilter = useSkipScanFilter; this.scanRange = scanRange; - this.minMaxRange = minMaxRange; this.rowTimestampRange = rowTimestampRange; if (isSalted && !isPointLookup) { @@ -199,14 +176,6 @@ public class ScanRanges { } } - /** - * Get the minMaxRange that is applied in addition to the scan range. - * Only used by the ExplainTable to generate the explain plan. - */ - public KeyRange getMinMaxRange() { - return minMaxRange; - } - public void initializeScan(Scan scan) { scan.setStartRow(scanRange.getLowerRange()); scan.setStopRow(scanRange.getUpperRange()); @@ -580,29 +549,9 @@ public class ScanRanges { } public int getBoundPkColumnCount() { - return Math.max(getBoundPkSpan(ranges, slotSpan), getBoundMinMaxSlotCount()); + return getBoundPkSpan(ranges, slotSpan); } - private int getBoundMinMaxSlotCount() { - if (minMaxRange == KeyRange.EMPTY_RANGE || minMaxRange == KeyRange.EVERYTHING_RANGE) { - return 0; - } - ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - // We don't track how many slots are bound for the minMaxRange, so we need - // to traverse the upper and lower range key and count the slots. - int lowerCount = 0; - int maxOffset = schema.iterator(minMaxRange.getLowerRange(), ptr); - for (int pos = 0; Boolean.TRUE.equals(schema.next(ptr, pos, maxOffset)); pos++) { - lowerCount++; - } - int upperCount = 0; - maxOffset = schema.iterator(minMaxRange.getUpperRange(), ptr); - for (int pos = 0; Boolean.TRUE.equals(schema.next(ptr, pos, maxOffset)); pos++) { - upperCount++; - } - return Math.max(lowerCount, upperCount); - } - public int getBoundSlotCount() { int count = 0; boolean hasUnbound = false;
