PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/49f06b33 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/49f06b33 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/49f06b33 Branch: refs/heads/calcite Commit: 49f06b331c2b364f176d77548d04a4eb9d15c5c9 Parents: 3f82975 Author: James Taylor <jtay...@salesforce.com> Authored: Tue Mar 3 12:07:56 2015 -0800 Committer: James Taylor <jtay...@salesforce.com> Committed: Tue Mar 3 12:07:56 2015 -0800 ---------------------------------------------------------------------- .../apache/phoenix/filter/SkipScanFilter.java | 31 +- .../org/apache/phoenix/schema/PTableImpl.java | 9 + .../phoenix/filter/SkipScanBigFilterTest.java | 717 +++++++++++++++++++ .../filter/SkipScanFilterIntersectTest.java | 69 +- 4 files changed, 821 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/49f06b33/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java index 0aafdbb..1923856 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java @@ -195,7 +195,7 @@ public class SkipScanFilter extends FilterBase implements Writable { Arrays.fill(position, 0); } - private boolean intersect(byte[] lowerInclusiveKey, byte[] upperExclusiveKey, List<List<KeyRange>> newSlots) { + private boolean intersect(final byte[] lowerInclusiveKey, final byte[] upperExclusiveKey, List<List<KeyRange>> newSlots) { resetState(); boolean lowerUnbound = (lowerInclusiveKey.length == 0); int startPos = 0; @@ -262,6 +262,9 @@ public class SkipScanFilter extends FilterBase implements Writable { } int[] lowerPosition = Arrays.copyOf(position, position.length); // Navigate to the upperExclusiveKey, but not past it + // TODO: We're including everything between the lowerPosition and end position, which is + // more than we need. We can optimize this by tracking whether each range in each slot position + // intersects. ReturnCode endCode = navigate(upperExclusiveKey, 0, upperExclusiveKey.length, Terminate.AT); if (endCode == ReturnCode.INCLUDE) { setStartKey(); @@ -286,6 +289,11 @@ public class SkipScanFilter extends FilterBase implements Writable { position[i] = slots.get(i).size() - 1; } } + int prevRowKeyPos = -1; + ImmutableBytesWritable lowerPtr = new ImmutableBytesWritable(); + ImmutableBytesWritable upperPtr = new ImmutableBytesWritable(); + schema.iterator(lowerInclusiveKey, lowerPtr); + schema.iterator(upperExclusiveKey, upperPtr); // Copy inclusive all positions for (int i = 0; i <= lastSlot; i++) { List<KeyRange> newRanges = slots.get(i).subList(lowerPosition[i], Math.min(position[i] + 1, slots.get(i).size())); @@ -295,12 +303,33 @@ public class SkipScanFilter extends FilterBase implements Writable { if (newSlots != null) { newSlots.add(newRanges); } + // Must include all "less-significant" slot values if: + // 1) a more-significant slot was incremented if (position[i] > lowerPosition[i]) { if (newSlots != null) { newSlots.addAll(slots.subList(i+1, slots.size())); } break; } + // 2) we're at a slot containing a range and the values differ between the lower and upper range, + // since less-significant slots may be lower after traversal than where they started. + if (!slots.get(i).get(position[i]).isSingleKey()) { + int rowKeyPos = ScanUtil.getRowKeyPosition(slotSpan, i); + // Position lowerPtr/upperPtr within lowerInclusiveKey/upperExclusiveKey at value for slot i + // The reposition method will do this incrementally, where we we're initially have prevRowKeyPos = -1. + schema.reposition(lowerPtr, prevRowKeyPos, rowKeyPos, 0, lowerInclusiveKey.length, slotSpan[i]); + schema.reposition(upperPtr, prevRowKeyPos, rowKeyPos, 0, upperExclusiveKey.length, slotSpan[i]); + // If we have a range and the values differ, we must include all slots that are less significant. + // For example: [A-D][1,23], the lower/upper keys could be B5/C2, where the C is in range and the + // next slot value of 2 is less than the next corresponding slot value of the 5. + if (lowerPtr.compareTo(upperPtr) != 0) { + if (newSlots != null) { + newSlots.addAll(slots.subList(i+1, slots.size())); + } + break; + } + prevRowKeyPos = rowKeyPos; + } } return true; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/49f06b33/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index e14565d..658ff23 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -236,6 +236,15 @@ public class PTableImpl implements PTable { table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats()); } + public static PTableImpl makePTable(PTable table, PTableStats stats) throws SQLException { + return new PTableImpl( + table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), table.getTimeStamp(), + table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), getColumnsToClone(table), + table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), + table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), + table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), stats); + } + public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum, List<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes, boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant,
