Repository: hbase Updated Branches: refs/heads/branch-1 0cd89ed65 -> cdbf001d2 refs/heads/branch-1.4 1181dab14 -> e533e0fc6 refs/heads/branch-2 2f3632a2a -> 313464f00 refs/heads/master 96ebab748 -> 393ab302a
HBASE-19863 java.lang.IllegalStateException: isDelete failed when SingleColumnValueFilter is used Signed-off-by: Chia-Ping Tsai <chia7...@gmail.com> Signed-off-by: ramkrish86 <ramkrishna.s.vasude...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e533e0fc Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e533e0fc Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e533e0fc Branch: refs/heads/branch-1.4 Commit: e533e0fc6e775aac3502e902f4caef979bd047a5 Parents: 1181dab Author: Sergey Soldatov <s...@apache.org> Authored: Tue Feb 13 22:08:11 2018 -0800 Committer: Josh Elser <els...@apache.org> Committed: Wed Feb 28 13:33:47 2018 -0500 ---------------------------------------------------------------------- .../hadoop/hbase/regionserver/StoreScanner.java | 6 + .../hadoop/hbase/HBaseTestingUtility.java | 35 ++++- .../hbase/regionserver/TestIsDeleteFailure.java | 154 +++++++++++++++++++ 3 files changed, 188 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/e533e0fc/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 0b2d12f..bb761ba 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -815,6 +815,12 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner return false; } } while ((nextCell = this.heap.peek()) != null && CellUtil.matchingRowColumn(cell, nextCell)); + // We need this check because it may happen that the new scanner that we get + // during heap.next() is requiring reseek due of fake KV previously generated for + // ROWCOL bloom filter optimization. See HBASE-19863 for more details + if (nextCell != null && matcher.compareKeyForNextColumn(nextCell, cell) < 0) { + return false; + } return true; } http://git-wip-us.apache.org/repos/asf/hbase/blob/e533e0fc/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 20277aa..5838fff 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -1480,21 +1480,42 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { /** * Create a table. - * @param htd - * @param families - * @param splitKeys + * @param htd table descriptor + * @param families array of column families + * @param splitKeys array of split keys * @param c Configuration to use * @return An HTable instance for the created table. - * @throws IOException + * @throws IOException if getAdmin or createTable fails */ public HTable createTable(HTableDescriptor htd, byte[][] families, byte[][] splitKeys, Configuration c) throws IOException { + // Disable blooms (they are on by default as of 0.95) but we disable them here because + // tests have hard coded counts of what to expect in block cache, etc., and blooms being + // on is interfering. + return createTable(htd, families, splitKeys, BloomType.NONE, HConstants.DEFAULT_BLOCKSIZE, c); + } + + /** + * Create a table. + * @param htd table descriptor + * @param families array of column families + * @param splitKeys array of split keys + * @param type Bloom type + * @param blockSize block size + * @param c Configuration to use + * @return An HTable instance for the created table. + * @throws IOException if getAdmin or createTable fails + */ + + public HTable createTable(HTableDescriptor htd, byte[][] families, byte[][] splitKeys, + BloomType type, int blockSize, Configuration c) throws IOException { for (byte[] family : families) { HColumnDescriptor hcd = new HColumnDescriptor(family); // Disable blooms (they are on by default as of 0.95) but we disable them here because // tests have hard coded counts of what to expect in block cache, etc., and blooms being // on is interfering. - hcd.setBloomFilterType(BloomType.NONE); + hcd.setBloomFilterType(type); + hcd.setBlocksize(blockSize); htd.addFamily(hcd); } getHBaseAdmin().createTable(htd, splitKeys); @@ -1506,8 +1527,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { /** * Create a table. - * @param htd - * @param splitRows + * @param htd table descriptor + * @param splitRows array of split keys * @return An HTable instance for the created table. * @throws IOException */ http://git-wip-us.apache.org/repos/asf/hbase/blob/e533e0fc/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestIsDeleteFailure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestIsDeleteFailure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestIsDeleteFailure.java new file mode 100644 index 0000000..6ae31b4 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestIsDeleteFailure.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.testclassification.FilterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +/** + * Test failure in ScanDeleteTracker.isDeleted when ROWCOL bloom filter + * is used during a scan with a filter. + */ +@Category({ RegionServerTests.class, FilterTests.class, MediumTests.class }) +public class TestIsDeleteFailure { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @Rule public TestName name = new TestName(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 2); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); + TEST_UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testIsDeleteFailure() throws Exception { + final HTableDescriptor table = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + final byte[] family = Bytes.toBytes("0"); + final byte[] c1 = Bytes.toBytes("C01"); + final byte[] c2 = Bytes.toBytes("C02"); + final byte[] c3 = Bytes.toBytes("C03"); + final byte[] c4 = Bytes.toBytes("C04"); + final byte[] c5 = Bytes.toBytes("C05"); + final byte[] c6 = Bytes.toBytes("C07"); + final byte[] c7 = Bytes.toBytes("C07"); + final byte[] c8 = Bytes.toBytes("C08"); + final byte[] c9 = Bytes.toBytes("C09"); + final byte[] c10 = Bytes.toBytes("C10"); + final byte[] c11 = Bytes.toBytes("C11"); + final byte[] c12 = Bytes.toBytes("C12"); + final byte[] c13 = Bytes.toBytes("C13"); + final byte[] c14 = Bytes.toBytes("C14"); + final byte[] c15 = Bytes.toBytes("C15"); + + final byte[] val = Bytes.toBytes("foo"); + List<byte[]> fams = new ArrayList<>(1); + fams.add(family); + Table ht = TEST_UTIL + .createTable(table, fams.toArray(new byte[0][]), null, BloomType.ROWCOL, 10000, + new Configuration(TEST_UTIL.getConfiguration())); + List<Mutation> pending = new ArrayList<Mutation>(); + for (int i = 0; i < 1000; i++) { + byte[] row = Bytes.toBytes("key" + Integer.toString(i)); + Put put = new Put(row); + put.addColumn(family, c3, val); + put.addColumn(family, c4, val); + put.addColumn(family, c5, val); + put.addColumn(family, c6, val); + put.addColumn(family, c7, val); + put.addColumn(family, c8, val); + put.addColumn(family, c12, val); + put.addColumn(family, c13, val); + put.addColumn(family, c15, val); + pending.add(put); + Delete del = new Delete(row); + del.addColumns(family, c2); + del.addColumns(family, c9); + del.addColumns(family, c10); + del.addColumns(family, c14); + pending.add(del); + } + ht.batch(pending, new Object[pending.size()]); + TEST_UTIL.flush(); + TEST_UTIL.compact(true); + for (int i = 20; i < 300; i++) { + byte[] row = Bytes.toBytes("key" + Integer.toString(i)); + Put put = new Put(row); + put.addColumn(family, c3, val); + put.addColumn(family, c4, val); + put.addColumn(family, c5, val); + put.addColumn(family, c6, val); + put.addColumn(family, c7, val); + put.addColumn(family, c8, val); + put.addColumn(family, c12, val); + put.addColumn(family, c13, val); + put.addColumn(family, c15, val); + pending.add(put); + Delete del = new Delete(row); + del.addColumns(family, c2); + del.addColumns(family, c9); + del.addColumns(family, c10); + del.addColumns(family, c14); + pending.add(del); + } + ht.batch(pending, new Object[pending.size()]); + TEST_UTIL.flush(); + + Scan scan = new Scan(); + scan.addColumn(family, c9); + scan.addColumn(family, c15); + SingleColumnValueFilter filter = + new SingleColumnValueFilter(family, c15, CompareFilter.CompareOp.EQUAL, + new BinaryComparator(c15)); + scan.setFilter(filter); + //Trigger the scan for not existing row, so it will scan over all rows + for (Result result : ht.getScanner(scan)) { + result.advance(); + } + ht.close(); + } +} \ No newline at end of file