Repository: hbase Updated Branches: refs/heads/0.98 2a4fc5adf -> 6d5d0e237
HBASE-12363 Improve how KEEP_DELETED_CELLS works with MIN_VERSIONS. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6d5d0e23 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6d5d0e23 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6d5d0e23 Branch: refs/heads/0.98 Commit: 6d5d0e237b611e84871fe52bd492ee663ac56150 Parents: 2a4fc5a Author: Lars Hofhansl <la...@apache.org> Authored: Tue Nov 4 17:13:58 2014 -0800 Committer: Lars Hofhansl <la...@apache.org> Committed: Tue Nov 4 17:14:14 2014 -0800 ---------------------------------------------------------------------- .../apache/hadoop/hbase/HColumnDescriptor.java | 23 ++- .../apache/hadoop/hbase/KeepDeletedCells.java | 45 ++++++ .../hadoop/hbase/regionserver/ScanInfo.java | 7 +- .../hbase/regionserver/ScanQueryMatcher.java | 19 ++- .../org/apache/hadoop/hbase/HBaseTestCase.java | 2 +- .../hadoop/hbase/HBaseTestingUtility.java | 2 +- .../hadoop/hbase/TestHColumnDescriptor.java | 2 +- .../hbase/regionserver/TestKeepDeletes.java | 147 +++++++++++++++++-- .../hadoop/hbase/regionserver/TestMemStore.java | 8 +- .../hbase/regionserver/TestMinVersions.java | 19 ++- .../hbase/regionserver/TestQueryMatcher.java | 11 +- .../regionserver/TestReversibleScanners.java | 3 +- .../hbase/regionserver/TestStoreScanner.java | 9 +- hbase-shell/src/main/ruby/hbase/admin.rb | 2 +- 14 files changed, 251 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java index 6638779..d84022b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java @@ -152,7 +152,7 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> /** * Default setting for preventing deleted from being collected immediately. */ - public static final boolean DEFAULT_KEEP_DELETED = false; + public static final KeepDeletedCells DEFAULT_KEEP_DELETED = KeepDeletedCells.FALSE; /** * Default setting for whether to use a block cache or not. @@ -410,7 +410,7 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> */ @Deprecated public HColumnDescriptor(final byte[] familyName, final int minVersions, - final int maxVersions, final boolean keepDeletedCells, + final int maxVersions, final KeepDeletedCells keepDeletedCells, final String compression, final boolean encodeOnDisk, final String dataBlockEncoding, final boolean inMemory, final boolean blockCacheEnabled, final int blocksize, @@ -754,10 +754,11 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> return setValue(HConstants.IN_MEMORY, Boolean.toString(inMemory)); } - public boolean getKeepDeletedCells() { + public KeepDeletedCells getKeepDeletedCells() { String value = getValue(KEEP_DELETED_CELLS); if (value != null) { - return Boolean.valueOf(value).booleanValue(); + // toUpperCase for backwards compatibility + return KeepDeletedCells.valueOf(value.toUpperCase()); } return DEFAULT_KEEP_DELETED; } @@ -766,9 +767,21 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> * @param keepDeletedCells True if deleted rows should not be collected * immediately. * @return this (for chained invocation) + * @deprecated use {@link #setKeepDeletedCells(KeepDeletedCells)} */ + @Deprecated public HColumnDescriptor setKeepDeletedCells(boolean keepDeletedCells) { - return setValue(KEEP_DELETED_CELLS, Boolean.toString(keepDeletedCells)); + return setValue(KEEP_DELETED_CELLS, (keepDeletedCells ? KeepDeletedCells.TRUE + : KeepDeletedCells.FALSE).toString()); + } + + /** + * @param keepDeletedCells True if deleted rows should not be collected + * immediately. + * @return this (for chained invocation) + */ + public HColumnDescriptor setKeepDeletedCells(KeepDeletedCells keepDeletedCells) { + return setValue(KEEP_DELETED_CELLS, keepDeletedCells.toString()); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-client/src/main/java/org/apache/hadoop/hbase/KeepDeletedCells.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/KeepDeletedCells.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/KeepDeletedCells.java new file mode 100644 index 0000000..6cd52e8 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/KeepDeletedCells.java @@ -0,0 +1,45 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +/** + * Ways to keep cells marked for delete around. + */ +/* + * Don't change the TRUE/FALSE labels below, these have to be called + * this way for backwards compatibility. + */ +public enum KeepDeletedCells { + /** Deleted Cells are not retained. */ + FALSE, + /** + * Deleted Cells are retained until they are removed by other means + * such TTL or VERSIONS. + * If no TTL is specified or no new versions of delete cells are + * written, they are retained forever. + */ + TRUE, + /** + * Deleted Cells are retained until the delete marker expires due to TTL. + * This is useful when TTL is combined with MIN_VERSIONS and one + * wants to keep a minimum number of versions around but at the same + * time remove deleted cells after the TTL. + */ + TTL; +} http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java index 1188ec8..a8b314e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionserver; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; @@ -33,7 +34,7 @@ public class ScanInfo { private int minVersions; private int maxVersions; private long ttl; - private boolean keepDeletedCells; + private KeepDeletedCells keepDeletedCells; private long timeToPurgeDeletes; private KVComparator comparator; @@ -65,7 +66,7 @@ public class ScanInfo { * @param comparator The store's comparator */ public ScanInfo(final byte[] family, final int minVersions, final int maxVersions, - final long ttl, final boolean keepDeletedCells, final long timeToPurgeDeletes, + final long ttl, final KeepDeletedCells keepDeletedCells, final long timeToPurgeDeletes, final KVComparator comparator) { this.family = family; this.minVersions = minVersions; @@ -92,7 +93,7 @@ public class ScanInfo { return ttl; } - public boolean getKeepDeletedCells() { + public KeepDeletedCells getKeepDeletedCells() { return keepDeletedCells; } http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java index fae8678..98921f4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java @@ -25,6 +25,7 @@ import java.util.NavigableSet; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.Filter; @@ -70,7 +71,7 @@ public class ScanQueryMatcher { private boolean retainDeletesInOutput; /** whether to return deleted rows */ - private final boolean keepDeletedCells; + private final KeepDeletedCells keepDeletedCells; /** whether time range queries can see rows "behind" a delete */ private final boolean seePastDeleteMarkers; @@ -97,6 +98,7 @@ public class ScanQueryMatcher { * deleted KVs. */ private final long earliestPutTs; + private final long ttl; /** readPoint over which the KVs are unconditionally included */ protected long maxReadPointToTrackVersions; @@ -162,15 +164,18 @@ public class ScanQueryMatcher { this.earliestPutTs = earliestPutTs; this.maxReadPointToTrackVersions = readPointToUse; this.timeToPurgeDeletes = scanInfo.getTimeToPurgeDeletes(); + this.ttl = oldestUnexpiredTS; /* how to deal with deletes */ this.isUserScan = scanType == ScanType.USER_SCAN; // keep deleted cells: if compaction or raw scan - this.keepDeletedCells = (scanInfo.getKeepDeletedCells() && !isUserScan) || scan.isRaw(); - // retain deletes: if minor compaction or raw scan + this.keepDeletedCells = scan.isRaw() ? KeepDeletedCells.TRUE : + isUserScan ? KeepDeletedCells.FALSE : scanInfo.getKeepDeletedCells(); + // retain deletes: if minor compaction or raw scanisDone this.retainDeletesInOutput = scanType == ScanType.COMPACT_RETAIN_DELETES || scan.isRaw(); // seePastDeleteMarker: user initiated scans - this.seePastDeleteMarkers = scanInfo.getKeepDeletedCells() && isUserScan; + this.seePastDeleteMarkers = + scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE && isUserScan; int maxVersions = scan.isRaw() ? scan.getMaxVersions() : Math.min(scan.getMaxVersions(), @@ -335,7 +340,8 @@ public class ScanQueryMatcher { */ byte type = bytes[initialOffset + keyLength - 1]; if (kv.isDelete()) { - if (!keepDeletedCells) { + if (keepDeletedCells == KeepDeletedCells.FALSE + || (keepDeletedCells == KeepDeletedCells.TTL && timestamp < ttl)) { // first ignore delete markers if the scanner can do so, and the // range does not include the marker // @@ -364,7 +370,8 @@ public class ScanQueryMatcher { // otherwise (i.e. a "raw" scan) we fall through to normal version and timerange checking return MatchCode.INCLUDE; } - } else if (keepDeletedCells) { + } else if (keepDeletedCells == KeepDeletedCells.TRUE + || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= ttl)) { if (timestamp < earliestPutTs) { // keeping delete rows, but there are no puts older than // this delete in the store files. http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java index d042f1c..8c1b20c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java @@ -205,7 +205,7 @@ public abstract class HBaseTestCase extends TestCase { * @return Column descriptor. */ protected HTableDescriptor createTableDescriptor(final String name, - final int minVersions, final int versions, final int ttl, boolean keepDeleted) { + final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) { htd.addFamily(new HColumnDescriptor(cfName) http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 2bd0abc..9023e06 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -1559,7 +1559,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { * @return Column descriptor. */ public HTableDescriptor createTableDescriptor(final String name, - final int minVersions, final int versions, final int ttl, boolean keepDeleted) { + final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) { htd.addFamily(new HColumnDescriptor(cfName) http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java index 90d5842..5ae32ec 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java @@ -45,7 +45,7 @@ public class TestHColumnDescriptor { assertEquals(v, hcd.getMaxVersions()); hcd.setMinVersions(v); assertEquals(v, hcd.getMinVersions()); - hcd.setKeepDeletedCells(!HColumnDescriptor.DEFAULT_KEEP_DELETED); + hcd.setKeepDeletedCells(KeepDeletedCells.TRUE); hcd.setInMemory(!HColumnDescriptor.DEFAULT_IN_MEMORY); boolean inmemory = hcd.isInMemory(); hcd.setScope(v); http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java index 4f8794c..f8c48db 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.client.Delete; @@ -96,7 +97,7 @@ public class TestKeepDeletes { public void testBasicScenario() throws Exception { // keep 3 versions, rows do not expire HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, true); + HConstants.FOREVER, KeepDeletedCells.TRUE); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTimeMillis(); @@ -193,7 +194,7 @@ public class TestKeepDeletes { public void testRawScanWithoutKeepingDeletes() throws Exception { // KEEP_DELETED_CELLS is NOT enabled HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, false); + HConstants.FOREVER, KeepDeletedCells.FALSE); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTimeMillis(); @@ -238,7 +239,7 @@ public class TestKeepDeletes { public void testWithoutKeepingDeletes() throws Exception { // KEEP_DELETED_CELLS is NOT enabled HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, false); + HConstants.FOREVER, KeepDeletedCells.FALSE); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTimeMillis(); @@ -282,7 +283,7 @@ public class TestKeepDeletes { @Test public void testRawScanWithColumns() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, true); + HConstants.FOREVER, KeepDeletedCells.TRUE); HRegion region = hbu.createLocalHRegion(htd, null, null); Scan s = new Scan(); @@ -306,7 +307,7 @@ public class TestKeepDeletes { @Test public void testRawScan() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, true); + HConstants.FOREVER, KeepDeletedCells.TRUE); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTimeMillis(); @@ -396,7 +397,7 @@ public class TestKeepDeletes { @Test public void testDeleteMarkerExpirationEmptyStore() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 1, - HConstants.FOREVER, true); + HConstants.FOREVER, KeepDeletedCells.TRUE); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTimeMillis(); @@ -439,7 +440,7 @@ public class TestKeepDeletes { @Test public void testDeleteMarkerExpiration() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 1, - HConstants.FOREVER, true); + HConstants.FOREVER, KeepDeletedCells.TRUE); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTimeMillis(); @@ -497,12 +498,90 @@ public class TestKeepDeletes { } /** + * Test delete marker removal from store files. + */ + @Test + public void testWithOldRow() throws Exception { + HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 1, + HConstants.FOREVER, KeepDeletedCells.TRUE); + HRegion region = hbu.createLocalHRegion(htd, null, null); + + long ts = EnvironmentEdgeManager.currentTimeMillis(); + + Put p = new Put(T1, ts); + p.add(c0, c0, T1); + region.put(p); + + // a put another (older) row in the same store + p = new Put(T2, ts-10); + p.add(c0, c0, T1); + region.put(p); + + // all the following deletes affect the put + Delete d = new Delete(T1, ts); + d.deleteColumns(c0, c0, ts); + region.delete(d); + + d = new Delete(T1, ts); + d.deleteFamily(c0, ts); + region.delete(d); + + d = new Delete(T1, ts); + d.deleteColumn(c0, c0, ts+1); + region.delete(d); + + d = new Delete(T1, ts); + d.deleteColumn(c0, c0, ts+2); + region.delete(d); + + // 1 family marker, 1 column marker, 2 version markers + assertEquals(4, countDeleteMarkers(region)); + + region.flushcache(); + assertEquals(4, countDeleteMarkers(region)); + region.compactStores(false); + assertEquals(4, countDeleteMarkers(region)); + + // another put will push out the earlier put... + p = new Put(T1, ts+3); + p.add(c0, c0, T1); + region.put(p); + + region.flushcache(); + // no markers are collected, since there is an affected put + region.compactStores(true); + assertEquals(4, countDeleteMarkers(region)); + + // all markers remain, since we have the older row + // and we haven't pushed the inlined markers past MAX_VERSIONS + region.compactStores(true); + assertEquals(4, countDeleteMarkers(region)); + + // another put will push out the earlier put... + p = new Put(T1, ts+4); + p.add(c0, c0, T1); + region.put(p); + + // this pushed out the column and version marker + // but the family markers remains. THIS IS A PROBLEM! + region.compactStores(true); + assertEquals(1, countDeleteMarkers(region)); + + // no amount of compacting is getting this of this one + // KEEP_DELETED_CELLS=>TTL is an option to avoid this. + region.compactStores(true); + assertEquals(1, countDeleteMarkers(region)); + + HRegion.closeHRegion(region); + } + + /** * Verify correct range demarcation */ @Test public void testRanges() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, true); + HConstants.FOREVER, KeepDeletedCells.TRUE); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTimeMillis(); @@ -584,7 +663,7 @@ public class TestKeepDeletes { @Test public void testDeleteMarkerVersioning() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 1, - HConstants.FOREVER, true); + HConstants.FOREVER, KeepDeletedCells.TRUE); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTimeMillis(); @@ -676,7 +755,7 @@ public class TestKeepDeletes { */ public void testWithMixedCFs() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 1, - HConstants.FOREVER, true); + HConstants.FOREVER, KeepDeletedCells.TRUE); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTimeMillis(); @@ -727,7 +806,8 @@ public class TestKeepDeletes { */ @Test public void testWithMinVersions() throws Exception { - HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, true); + HTableDescriptor htd = + hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.TRUE); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000; // 2s in the past @@ -799,6 +879,51 @@ public class TestKeepDeletes { HRegion.closeHRegion(region); } + /** + * Test keeping deleted rows together with min versions set + * @throws Exception + */ + @Test + public void testWithTTL() throws Exception { + HTableDescriptor htd = + hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.TTL); + HRegion region = hbu.createLocalHRegion(htd, null, null); + + long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000; // 2s in the past + + Put p = new Put(T1, ts); + p.add(c0, c0, T3); + region.put(p); + + // place an old row, to make the family marker expires anyway + p = new Put(T2, ts-10); + p.add(c0, c0, T1); + region.put(p); + + checkGet(region, T1, c0, c0, ts+1, T3); + // place a family delete marker + Delete d = new Delete(T1, ts+2); + region.delete(d); + + checkGet(region, T1, c0, c0, ts+1, T3); + + // 3 families, one column delete marker + assertEquals(3, countDeleteMarkers(region)); + + region.flushcache(); + // no delete marker removes by the flush + assertEquals(3, countDeleteMarkers(region)); + + // but the Put is gone + checkGet(region, T1, c0, c0, ts+1); + + region.compactStores(true); + // all delete marker gone + assertEquals(0, countDeleteMarkers(region)); + + HRegion.closeHRegion(region); + } + private void checkGet(HRegion region, byte[] row, byte[] fam, byte[] col, long time, byte[]... vals) throws IOException { Get g = new Get(row); http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java index 3794ab6..61c12bc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueTestUtil; import org.apache.hadoop.hbase.MediumTests; @@ -93,8 +94,9 @@ public class TestMemStore extends TestCase { List<KeyValueScanner> memstorescanners = this.memstore.getScanners(0); Scan scan = new Scan(); List<Cell> result = new ArrayList<Cell>(); - ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, false, - 0, this.memstore.comparator); + ScanInfo scanInfo = + new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE, 0, + this.memstore.comparator); ScanType scanType = ScanType.USER_SCAN; StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); int count = 0; @@ -511,7 +513,7 @@ public class TestMemStore extends TestCase { } //starting from each row, validate results should contain the starting row for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) { - ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, false, + ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE, 0, this.memstore.comparator); ScanType scanType = ScanType.USER_SCAN; InternalScanner scanner = new StoreScanner(new Scan( http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java index 73a712f..0992410 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -64,7 +65,8 @@ public class TestMinVersions { */ @Test public void testGetClosestBefore() throws Exception { - HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, false); + HTableDescriptor htd = + hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE); HRegion region = hbu.createLocalHRegion(htd, null, null); try { @@ -113,7 +115,8 @@ public class TestMinVersions { @Test public void testStoreMemStore() throws Exception { // keep 3 versions minimum - HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, false); + HTableDescriptor htd = + hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE); HRegion region = hbu.createLocalHRegion(htd, null, null); // 2s in the past long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000; @@ -167,7 +170,8 @@ public class TestMinVersions { */ @Test public void testDelete() throws Exception { - HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, false); + HTableDescriptor htd = + hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE); HRegion region = hbu.createLocalHRegion(htd, null, null); // 2s in the past @@ -225,7 +229,8 @@ public class TestMinVersions { */ @Test public void testMemStore() throws Exception { - HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, false); + HTableDescriptor htd = + hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); HRegion region = hbu.createLocalHRegion(htd, null, null); // 2s in the past @@ -300,7 +305,8 @@ public class TestMinVersions { @Test public void testBaseCase() throws Exception { // 1 version minimum, 1000 versions maximum, ttl = 1s - HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, false); + HTableDescriptor htd = + hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); HRegion region = hbu.createLocalHRegion(htd, null, null); try { @@ -391,7 +397,8 @@ public class TestMinVersions { */ @Test public void testFilters() throws Exception { - HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, false); + HTableDescriptor htd = + hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); HRegion region = hbu.createLocalHRegion(htd, null, null); final byte [] c1 = COLUMNS[1]; http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java index e8f5a5a..aafa710 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java @@ -29,6 +29,7 @@ import java.util.NavigableSet; import org.apache.hadoop.hbase.HBaseTestCase; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValue.Type; @@ -93,7 +94,7 @@ public class TestQueryMatcher extends HBaseTestCase { private void _testMatch_ExplicitColumns(Scan scan, List<MatchCode> expected) throws IOException { // 2,4,5 ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, - 0, 1, ttl, false, 0, rowComparator), get.getFamilyMap().get(fam2), + 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), get.getFamilyMap().get(fam2), EnvironmentEdgeManager.currentTimeMillis() - ttl); List<KeyValue> memstore = new ArrayList<KeyValue>(); @@ -175,7 +176,7 @@ public class TestQueryMatcher extends HBaseTestCase { expected.add(ScanQueryMatcher.MatchCode.DONE); ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, - 0, 1, ttl, false, 0, rowComparator), null, + 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), null, EnvironmentEdgeManager.currentTimeMillis() - ttl); List<KeyValue> memstore = new ArrayList<KeyValue>(); @@ -229,7 +230,7 @@ public class TestQueryMatcher extends HBaseTestCase { long now = EnvironmentEdgeManager.currentTimeMillis(); ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, - 0, 1, testTTL, false, 0, rowComparator), get.getFamilyMap().get(fam2), + 0, 1, testTTL, KeepDeletedCells.FALSE, 0, rowComparator), get.getFamilyMap().get(fam2), now - testTTL); KeyValue [] kvs = new KeyValue[] { @@ -283,7 +284,7 @@ public class TestQueryMatcher extends HBaseTestCase { long now = EnvironmentEdgeManager.currentTimeMillis(); ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, - 0, 1, testTTL, false, 0, rowComparator), null, + 0, 1, testTTL, KeepDeletedCells.FALSE, 0, rowComparator), null, now - testTTL); KeyValue [] kvs = new KeyValue[] { @@ -338,7 +339,7 @@ public class TestQueryMatcher extends HBaseTestCase { byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException { long now = EnvironmentEdgeManager.currentTimeMillis(); // Set time to purge deletes to negative value to avoid it ever happening. - ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, false, -1L, rowComparator); + ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator); NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2); ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE, http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java index fa09263..f6532c6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.client.Durability; @@ -251,7 +252,7 @@ public class TestReversibleScanners { ScanType scanType = ScanType.USER_SCAN; ScanInfo scanInfo = new ScanInfo(FAMILYNAME, 0, Integer.MAX_VALUE, - Long.MAX_VALUE, false, 0, KeyValue.COMPARATOR); + Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR); // Case 1.Test a full reversed scan Scan scan = new Scan(); http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java index 05ac4b4..bd15a13 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -37,6 +37,7 @@ import junit.framework.TestCase; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueTestUtil; import org.apache.hadoop.hbase.MediumTests; @@ -54,7 +55,7 @@ public class TestStoreScanner extends TestCase { private static final String CF_STR = "cf"; final byte [] CF = Bytes.toBytes(CF_STR); private ScanInfo scanInfo = new ScanInfo(CF, 0, Integer.MAX_VALUE, - Long.MAX_VALUE, false, 0, KeyValue.COMPARATOR); + Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR); private ScanType scanType = ScanType.USER_SCAN; public void setUp() throws Exception { @@ -422,7 +423,7 @@ public class TestStoreScanner extends TestCase { List<KeyValueScanner> scanners = scanFixture(kvs); Scan scan = new Scan(); scan.setMaxVersions(1); - ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, false, 0, + ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR); ScanType scanType = ScanType.USER_SCAN; StoreScanner scanner = @@ -493,7 +494,7 @@ public class TestStoreScanner extends TestCase { Scan scan = new Scan(); scan.setMaxVersions(1); // scanner with ttl equal to 500 - ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, false, 0, + ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR); ScanType scanType = ScanType.USER_SCAN; StoreScanner scanner = @@ -556,7 +557,7 @@ public class TestStoreScanner extends TestCase { ScanInfo scanInfo = new ScanInfo(Bytes.toBytes("cf"), 0 /* minVersions */, 2 /* maxVersions */, 500 /* ttl */, - false /* keepDeletedCells */, + KeepDeletedCells.FALSE /* keepDeletedCells */, 200, /* timeToPurgeDeletes */ KeyValue.COMPARATOR); StoreScanner scanner = http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5d0e23/hbase-shell/src/main/ruby/hbase/admin.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index b29536c..00a1a84 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -649,7 +649,7 @@ module Hbase family.setBlocksize(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE) family.setMaxVersions(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS) family.setMinVersions(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS) - family.setKeepDeletedCells(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS) + family.setKeepDeletedCells(org.apache.hadoop.hbase.KeepDeletedCells.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS).to_s.upcase)) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS) family.setCompressTags(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESS_TAGS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESS_TAGS) family.setPrefetchBlocksOnOpen(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::PREFETCH_BLOCKS_ON_OPEN))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::PREFETCH_BLOCKS_ON_OPEN) family.setValue(COMPRESSION_COMPACT, arg.delete(COMPRESSION_COMPACT)) if arg.include?(COMPRESSION_COMPACT)
