Repository: phoenix Updated Branches: refs/heads/4.x-HBase-0.98 57bd7988c -> cd492aea0
PHOENIX-2774 MemStoreScanner and KeyValueStore should not be aware of KeyValueScanner (Churro Morales) Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a62cde79 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a62cde79 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a62cde79 Branch: refs/heads/4.x-HBase-0.98 Commit: a62cde79d3ef48c199aa53867475b290e8d74c30 Parents: 57bd798 Author: James Taylor <jtay...@salesforce.com> Authored: Mon Mar 28 09:04:49 2016 -0700 Committer: James Taylor <jtay...@salesforce.com> Committed: Mon Mar 28 09:04:49 2016 -0700 ---------------------------------------------------------------------- .../hbase/index/covered/KeyValueStore.java | 4 +- .../hbase/index/covered/LocalTableState.java | 27 --------- .../hbase/index/covered/data/IndexMemStore.java | 60 +++++--------------- .../index/scanner/FilteredKeyValueScanner.java | 21 ++++--- .../hbase/index/scanner/ReseekableScanner.java | 28 +++++++++ .../hbase/index/scanner/ScannerBuilder.java | 2 +- .../index/covered/data/TestIndexMemStore.java | 7 +-- 7 files changed, 59 insertions(+), 90 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/a62cde79/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/KeyValueStore.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/KeyValueStore.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/KeyValueStore.java index 39f9062..30d2904 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/KeyValueStore.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/KeyValueStore.java @@ -18,7 +18,7 @@ package org.apache.phoenix.hbase.index.covered; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.phoenix.hbase.index.scanner.ReseekableScanner; /** * Store a collection of KeyValues in memory. @@ -27,7 +27,7 @@ public interface KeyValueStore { public void add(KeyValue kv, boolean overwrite); - public KeyValueScanner getScanner(); + public ReseekableScanner getScanner(); public void rollback(KeyValue kv); } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/a62cde79/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/LocalTableState.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/LocalTableState.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/LocalTableState.java index 2739cc2..3a7a7eb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/LocalTableState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/LocalTableState.java @@ -194,33 +194,6 @@ public class LocalTableState implements TableState { return this.update.getRow(); } - public Result getCurrentRowState() { - KeyValueScanner scanner = this.memstore.getScanner(); - List<Cell> kvs = new ArrayList<Cell>(); - while (scanner.peek() != null) { - try { - kvs.add(scanner.next()); - } catch (IOException e) { - // this should never happen - something has gone terribly arwy if it has - throw new RuntimeException("Local MemStore threw IOException!"); - } - } - return Result.create(kvs); - } - - /** - * Helper to add a {@link Mutation} to the values stored for the current row - * - * @param pendingUpdate - * update to apply - */ - public void addUpdateForTesting(Mutation pendingUpdate) { - for (Map.Entry<byte[], List<Cell>> e : pendingUpdate.getFamilyCellMap().entrySet()) { - List<KeyValue> edits = KeyValueUtil.ensureKeyValues(e.getValue()); - addUpdate(edits); - } - } - /** * @param hints */ http://git-wip-us.apache.org/repos/asf/phoenix/blob/a62cde79/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/IndexMemStore.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/IndexMemStore.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/IndexMemStore.java index 89489ec..7676134 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/IndexMemStore.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/IndexMemStore.java @@ -17,24 +17,23 @@ */ package org.apache.phoenix.hbase.index.covered.data; -import java.io.IOException; import java.util.Comparator; import java.util.Iterator; import java.util.SortedSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.regionserver.IndexKeyValueSkipListSet; -import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.regionserver.MemStore; -import org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner; import org.apache.hadoop.hbase.util.Bytes; - import org.apache.phoenix.hbase.index.covered.KeyValueStore; import org.apache.phoenix.hbase.index.covered.LocalTableState; +import org.apache.phoenix.hbase.index.scanner.ReseekableScanner; /** * Like the HBase {@link MemStore}, but without all that extra work around maintaining snapshots and @@ -53,7 +52,7 @@ import org.apache.phoenix.hbase.index.covered.LocalTableState; * <li>ignoring memstore timestamps in favor of deciding when we want to overwrite keys based on how * we obtain them</li> * <li>ignoring time range updates (so - * {@link KeyValueScanner#shouldUseScanner(Scan, SortedSet, long)} isn't supported from + * {@link ReseekableScanner#shouldUseScanner(Scan, SortedSet, long)} isn't supported from * {@link #getScanner()}).</li> * </ol> * <p> @@ -156,19 +155,19 @@ public class IndexMemStore implements KeyValueStore { } @Override - public KeyValueScanner getScanner() { + public ReseekableScanner getScanner() { return new MemStoreScanner(); } /* - * MemStoreScanner implements the KeyValueScanner. It lets the caller scan the contents of a + * MemStoreScanner implements the ReseekableScanner. It lets the caller scan the contents of a * memstore -- both current map and snapshot. This behaves as if it were a real scanner but does * not maintain position. */ // This class is adapted from org.apache.hadoop.hbase.MemStore.MemStoreScanner, HBase 0.94.12 // It does basically the same thing as the MemStoreScanner, but it only keeps track of a single // set, rather than a primary and a secondary set of KeyValues. - protected class MemStoreScanner extends NonLazyKeyValueScanner { + protected class MemStoreScanner implements ReseekableScanner { // Next row information for the set private KeyValue nextRow = null; @@ -213,7 +212,7 @@ public class IndexMemStore implements KeyValueStore { * @return false if the key is null or if there is no data */ @Override - public synchronized boolean seek(KeyValue key) { + public synchronized boolean seek(Cell key) { if (key == null) { close(); return false; @@ -221,10 +220,11 @@ public class IndexMemStore implements KeyValueStore { // kvset and snapshot will never be null. // if tailSet can't find anything, SortedSet is empty (not null). - kvsetIt = kvsetAtCreation.tailSet(key).iterator(); + KeyValue kv = KeyValueUtil.ensureKeyValue(key); + kvsetIt = kvsetAtCreation.tailSet(kv).iterator(); kvsetItRow = null; - return seekInSubLists(key); + return seekInSubLists(kv); } /** @@ -241,7 +241,7 @@ public class IndexMemStore implements KeyValueStore { * @return true if there is at least one KV to read, false otherwise */ @Override - public synchronized boolean reseek(KeyValue key) { + public synchronized boolean reseek(Cell key) { /* * See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation. This * code is executed concurrently with flush and puts, without locks. Two points must be known @@ -252,8 +252,9 @@ public class IndexMemStore implements KeyValueStore { * we iterated to and restore the reseeked set to at least that point. */ - kvsetIt = kvsetAtCreation.tailSet(getHighest(key, kvsetItRow)).iterator(); - return seekInSubLists(key); + KeyValue kv = KeyValueUtil.ensureKeyValue(key); + kvsetIt = kvsetAtCreation.tailSet(getHighest(kv, kvsetItRow)).iterator(); + return seekInSubLists(kv); } /* @@ -273,7 +274,6 @@ public class IndexMemStore implements KeyValueStore { @Override public synchronized KeyValue peek() { - // DebugPrint.println(" MS@" + hashCode() + " peek = " + getLowest()); return nextRow; } @@ -297,35 +297,5 @@ public class IndexMemStore implements KeyValueStore { this.kvsetIt = null; this.kvsetItRow = null; } - - /** - * MemStoreScanner returns max value as sequence id because it will always have the latest data - * among all files. - */ - @Override - public long getSequenceID() { - return Long.MAX_VALUE; - } - - @Override - public boolean shouldUseScanner(Scan scan, SortedSet<byte[]> columns, long oldestUnexpiredTS) { - throw new UnsupportedOperationException(this.getClass().getName() - + " doesn't support checking to see if it should use a scanner!"); - } - - @Override - public boolean backwardSeek(KeyValue arg0) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public boolean seekToLastRow() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public boolean seekToPreviousRow(KeyValue arg0) throws IOException { - throw new UnsupportedOperationException(); - } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/a62cde79/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/FilteredKeyValueScanner.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/FilteredKeyValueScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/FilteredKeyValueScanner.java index 69761ba..768776b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/FilteredKeyValueScanner.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/FilteredKeyValueScanner.java @@ -22,7 +22,6 @@ import java.io.IOException; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.Filter.ReturnCode; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; @@ -33,22 +32,22 @@ import org.apache.phoenix.hbase.index.covered.KeyValueStore; * here because we are only concerned with a single MemStore for the index; we don't need to worry about multiple column * families or minimizing seeking through file - we just want to iterate the kvs quickly, in-memory. */ -public class FilteredKeyValueScanner implements Scanner { +public class FilteredKeyValueScanner implements ReseekableScanner { - private KeyValueScanner delegate; + private ReseekableScanner delegate; private Filter filter; public FilteredKeyValueScanner(Filter filter, KeyValueStore store) { this(filter, store.getScanner()); } - private FilteredKeyValueScanner(Filter filter, KeyValueScanner delegate) { + private FilteredKeyValueScanner(Filter filter, ReseekableScanner delegate) { this.delegate = delegate; this.filter = filter; } @Override - public Cell peek() { + public Cell peek() throws IOException { return delegate.peek(); } @@ -69,15 +68,14 @@ public class FilteredKeyValueScanner implements Scanner { public boolean seek(Cell key) throws IOException { if (filter.filterAllRemaining()) { return false; } // see if we can seek to the next key - if (!delegate.seek(KeyValueUtil.ensureKeyValue(key))) { return false; } + if (!delegate.seek(key)) { return false; } return seekToNextUnfilteredKeyValue(); } - @SuppressWarnings("deprecation") private boolean seekToNextUnfilteredKeyValue() throws IOException { while (true) { - KeyValue peeked = delegate.peek(); + Cell peeked = delegate.peek(); // no more key values, so we are done if (peeked == null) { return false; } @@ -96,18 +94,19 @@ public class FilteredKeyValueScanner implements Scanner { break; // use a seek hint to find out where we should go case SEEK_NEXT_USING_HINT: - delegate.seek(KeyValueUtil.ensureKeyValue(filter.getNextCellHint(peeked))); + delegate.seek(filter.getNextCellHint(peeked)); } } } + @Override public boolean reseek(Cell key) throws IOException { - this.delegate.reseek(KeyValueUtil.ensureKeyValue(key)); + this.delegate.reseek(key); return this.seekToNextUnfilteredKeyValue(); } @Override - public void close() { + public void close() throws IOException { this.delegate.close(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a62cde79/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ReseekableScanner.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ReseekableScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ReseekableScanner.java new file mode 100644 index 0000000..a5e50e6 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ReseekableScanner.java @@ -0,0 +1,28 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hbase.index.scanner; + +import java.io.IOException; + +import org.apache.hadoop.hbase.Cell; + +public interface ReseekableScanner extends Scanner { + + boolean reseek(Cell key) throws IOException; +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/a62cde79/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java index 23e64b3..b447dd9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java @@ -159,7 +159,7 @@ public class ScannerBuilder { } @Override - public void close() { + public void close() throws IOException { kvScanner.close(); } }; http://git-wip-us.apache.org/repos/asf/phoenix/blob/a62cde79/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/data/TestIndexMemStore.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/data/TestIndexMemStore.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/data/TestIndexMemStore.java index 41e7e65..7e40b72 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/data/TestIndexMemStore.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/data/TestIndexMemStore.java @@ -22,9 +22,8 @@ import static org.junit.Assert.assertTrue; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; -import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.hbase.index.covered.data.IndexMemStore; +import org.apache.phoenix.hbase.index.scanner.ReseekableScanner; import org.junit.Test; public class TestIndexMemStore { @@ -46,7 +45,7 @@ public class TestIndexMemStore { store.add(kv, true); // adding the exact same kv shouldn't change anything stored if not overwritting store.add(kv2, false); - KeyValueScanner scanner = store.getScanner(); + ReseekableScanner scanner = store.getScanner(); KeyValue first = KeyValue.createFirstOnRow(row); scanner.seek(first); assertTrue("Overwrote kv when specifically not!", kv == scanner.next()); @@ -80,7 +79,7 @@ public class TestIndexMemStore { store.add(d, true); // null qualifiers should always sort before the non-null cases - KeyValueScanner scanner = store.getScanner(); + ReseekableScanner scanner = store.getScanner(); KeyValue first = KeyValue.createFirstOnRow(row); assertTrue("Didn't have any data in the scanner", scanner.seek(first)); assertTrue("Didn't get delete family first (no qualifier == sort first)", df == scanner.next());
