http://git-wip-us.apache.org/repos/asf/hbase/blob/4a4c0120/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileArchiverNotifierImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileArchiverNotifierImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileArchiverNotifierImpl.java new file mode 100644 index 0000000..e139e4f --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileArchiverNotifierImpl.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellScanner; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.SnapshotType; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.quotas.FileArchiverNotifierImpl.SnapshotWithSize; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotManifest; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; +import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; +import org.apache.hbase.thirdparty.com.google.common.collect.Maps; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile; + +/** + * Test class for {@link FileArchiverNotifierImpl}. + */ +@Category(MediumTests.class) +public class TestFileArchiverNotifierImpl { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestFileArchiverNotifierImpl.class); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final AtomicLong COUNTER = new AtomicLong(); + + @Rule + public TestName testName = new TestName(); + + private Connection conn; + private Admin admin; + private SpaceQuotaHelperForTests helper; + private FileSystem fs; + private Configuration conf; + + @BeforeClass + public static void setUp() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + SpaceQuotaHelperForTests.updateConfigForQuotas(conf); + // Clean up the compacted files faster than normal (15s instead of 2mins) + conf.setInt("hbase.hfile.compaction.discharger.interval", 15 * 1000); + // Prevent the SnapshotQuotaObserverChore from running + conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_DELAY_KEY, 60 * 60 * 1000); + conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_PERIOD_KEY, 60 * 60 * 1000); + TEST_UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setup() throws Exception { + conn = TEST_UTIL.getConnection(); + admin = TEST_UTIL.getAdmin(); + helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER); + helper.removeAllQuotas(conn); + fs = TEST_UTIL.getTestFileSystem(); + conf = TEST_UTIL.getConfiguration(); + } + + @Test + public void testSnapshotSizePersistence() throws IOException { + final Admin admin = TEST_UTIL.getAdmin(); + final TableName tn = TableName.valueOf(testName.getMethodName()); + if (admin.tableExists(tn)) { + admin.disableTable(tn); + admin.deleteTable(tn); + } + TableDescriptor desc = TableDescriptorBuilder.newBuilder(tn).addColumnFamily( + ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE)).build(); + admin.createTable(desc); + + FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn); + List<SnapshotWithSize> snapshotsWithSizes = new ArrayList<>(); + try (Table table = conn.getTable(tn)) { + // Writing no values will result in no records written. + verify(table, () -> { + notifier.persistSnapshotSizes(table, snapshotsWithSizes); + assertEquals(0, count(table)); + }); + + verify(table, () -> { + snapshotsWithSizes.add(new SnapshotWithSize("ss1", 1024L)); + snapshotsWithSizes.add(new SnapshotWithSize("ss2", 4096L)); + notifier.persistSnapshotSizes(table, snapshotsWithSizes); + assertEquals(2, count(table)); + assertEquals(1024L, extractSnapshotSize(table, tn, "ss1")); + assertEquals(4096L, extractSnapshotSize(table, tn, "ss2")); + }); + } + } + + @Test + public void testIncrementalFileArchiving() throws Exception { + final Admin admin = TEST_UTIL.getAdmin(); + final TableName tn = TableName.valueOf(testName.getMethodName()); + if (admin.tableExists(tn)) { + admin.disableTable(tn); + admin.deleteTable(tn); + } + final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME); + final TableName tn1 = helper.createTableWithRegions(1); + admin.setQuota(QuotaSettingsFactory.limitTableSpace( + tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS)); + + // Write some data and flush it + helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE); + admin.flush(tn1); + + // Create a snapshot on the table + final String snapshotName1 = tn1 + "snapshot1"; + admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH)); + + FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn); + long t1 = notifier.getLastFullCompute(); + long snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1)); + assertEquals("The size of the snapshots should be zero", 0, snapshotSize); + assertTrue("Last compute time was not less than current compute time", + t1 < notifier.getLastFullCompute()); + + // No recently archived files and the snapshot should have no size + assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1)); + + // Invoke the addArchivedFiles method with no files + notifier.addArchivedFiles(Collections.emptySet()); + + // The size should not have changed + assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1)); + + notifier.addArchivedFiles(ImmutableSet.of(entry("a", 1024L), entry("b", 1024L))); + + // The size should not have changed + assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1)); + + // Pull one file referenced by the snapshot out of the manifest + Set<String> referencedFiles = getFilesReferencedBySnapshot(snapshotName1); + assertTrue("Found snapshot referenced files: " + referencedFiles, referencedFiles.size() >= 1); + String referencedFile = Iterables.getFirst(referencedFiles, null); + assertNotNull(referencedFile); + + // Report that a file this snapshot referenced was moved to the archive. This is a sign + // that the snapshot should now "own" the size of this file + final long fakeFileSize = 2048L; + notifier.addArchivedFiles(ImmutableSet.of(entry(referencedFile, fakeFileSize))); + + // Verify that the snapshot owns this file. + assertEquals(fakeFileSize, extractSnapshotSize(quotaTable, tn, snapshotName1)); + + // In reality, we did not actually move the file, so a "full" computation should re-set the + // size of the snapshot back to 0. + long t2 = notifier.getLastFullCompute(); + snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1)); + assertEquals(0, snapshotSize); + assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1)); + // We should also have no recently archived files after a re-computation + assertTrue("Last compute time was not less than current compute time", + t2 < notifier.getLastFullCompute()); + } + + @Test + public void testParseOldNamespaceSnapshotSize() throws Exception { + final Admin admin = TEST_UTIL.getAdmin(); + final TableName fakeQuotaTableName = TableName.valueOf(testName.getMethodName()); + final TableName tn = TableName.valueOf(testName.getMethodName() + "1"); + if (admin.tableExists(fakeQuotaTableName)) { + admin.disableTable(fakeQuotaTableName); + admin.deleteTable(fakeQuotaTableName); + } + TableDescriptor desc = TableDescriptorBuilder.newBuilder(fakeQuotaTableName).addColumnFamily( + ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE)) + .addColumnFamily(ColumnFamilyDescriptorBuilder.of(QuotaUtil.QUOTA_FAMILY_INFO)).build(); + admin.createTable(desc); + + final String ns = ""; + try (Table fakeQuotaTable = conn.getTable(fakeQuotaTableName)) { + FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn); + // Verify no record is treated as zero + assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns)); + + // Set an explicit value of zero + fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 0L)); + assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns)); + + // Set a non-zero value + fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 1024L)); + assertEquals(1024L, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns)); + } + } + + private long count(Table t) throws IOException { + try (ResultScanner rs = t.getScanner(new Scan())) { + long sum = 0; + for (Result r : rs) { + while (r.advance()) { + sum++; + } + } + return sum; + } + } + + private long extractSnapshotSize( + Table quotaTable, TableName tn, String snapshot) throws IOException { + Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot); + Result r = quotaTable.get(g); + assertNotNull(r); + CellScanner cs = r.cellScanner(); + assertTrue(cs.advance()); + Cell c = cs.current(); + assertNotNull(c); + return QuotaTableUtil.extractSnapshotSize( + c.getValueArray(), c.getValueOffset(), c.getValueLength()); + } + + private void verify(Table t, IOThrowingRunnable test) throws IOException { + admin.disableTable(t.getName()); + admin.truncateTable(t.getName(), false); + test.run(); + } + + @FunctionalInterface + private interface IOThrowingRunnable { + void run() throws IOException; + } + + private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException { + HashSet<String> files = new HashSet<>(); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( + snapshotName, FSUtils.getRootDir(conf)); + SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo( + fs, snapshotDir); + SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd); + // For each region referenced by the snapshot + for (SnapshotRegionManifest rm : manifest.getRegionManifests()) { + // For each column family in this region + for (FamilyFiles ff : rm.getFamilyFilesList()) { + // And each store file in that family + for (StoreFile sf : ff.getStoreFilesList()) { + files.add(sf.getName()); + } + } + } + return files; + } + + private <K,V> Entry<K,V> entry(K k, V v) { + return Maps.immutableEntry(k, v); + } +}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a4c0120/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java index f90ed82..fdc7ad3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.quotas; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.util.Collections; +import java.util.List; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.conf.Configuration; @@ -31,10 +33,15 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ClientServiceCallable; import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.RpcRetryingCaller; import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory; +import org.apache.hadoop.hbase.client.SnapshotType; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.junit.AfterClass; import org.junit.Before; @@ -69,8 +76,15 @@ public class TestLowLatencySpaceQuotas { Configuration conf = TEST_UTIL.getConfiguration(); // The default 1s period for QuotaObserverChore is good. SpaceQuotaHelperForTests.updateConfigForQuotas(conf); - // Set the period to read region size from HDFS to be very long + // Set the period/delay to read region size from HDFS to be very long conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 1000 * 120); + conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 1000 * 120); + // Set the same long period/delay to compute snapshot sizes + conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_PERIOD_KEY, 1000 * 120); + conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_DELAY_KEY, 1000 * 120); + // Clean up the compacted files faster than normal (5s instead of 2mins) + conf.setInt("hbase.hfile.compaction.discharger.interval", 5 * 1000); + TEST_UTIL.startMiniCluster(1); } @@ -225,4 +239,69 @@ public class TestLowLatencySpaceQuotas { } }); } + + @Test + public void testSnapshotSizes() throws Exception { + TableName tn = helper.createTableWithRegions(1); + // Set a quota + QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( + tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); + admin.setQuota(settings); + + // Write some data and flush it to disk. + final long sizePerBatch = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; + helper.writeData(tn, sizePerBatch); + admin.flush(tn); + + final String snapshot1 = "snapshot1"; + admin.snapshot(snapshot1, tn, SnapshotType.SKIPFLUSH); + + // Compute the size of the file for the Region we'll send to archive + Region region = Iterables.getOnlyElement(TEST_UTIL.getHBaseCluster().getRegions(tn)); + List<? extends Store> stores = region.getStores(); + long summer = 0; + for (Store store : stores) { + summer += store.getStorefilesSize(); + } + final long storeFileSize = summer; + + // Wait for the table to show the usage + TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { + @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { + return snapshot.getUsage() == storeFileSize; + } + }); + + // Spoof a "full" computation of snapshot size. Normally the chore handles this, but we want + // to test in the absence of this chore. + FileArchiverNotifier notifier = TEST_UTIL.getHBaseCluster().getMaster() + .getSnapshotQuotaObserverChore().getNotifierForTable(tn); + notifier.computeAndStoreSnapshotSizes(Collections.singletonList(snapshot1)); + + // Force a major compaction to create a new file and push the old file to the archive + TEST_UTIL.compact(tn, true); + + // After moving the old file to archive/, the space of this table should double + // We have a new file created by the majc referenced by the table and the snapshot still + // referencing the old file. + TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { + @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { + return snapshot.getUsage() >= 2 * storeFileSize; + } + }); + + try (Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME)) { + Result r = quotaTable.get(QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot1)); + assertTrue("Expected a non-null, non-empty Result", r != null && !r.isEmpty()); + assertTrue(r.advance()); + assertEquals("The snapshot's size should be the same as the origin store file", + storeFileSize, QuotaTableUtil.parseSnapshotSize(r.current())); + + r = quotaTable.get(QuotaTableUtil.createGetNamespaceSnapshotSize(tn.getNamespaceAsString())); + assertTrue("Expected a non-null, non-empty Result", r != null && !r.isEmpty()); + assertTrue(r.advance()); + assertEquals("The snapshot's size should be the same as the origin store file", + storeFileSize, QuotaTableUtil.parseSnapshotSize(r.current())); + } + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/4a4c0120/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSnapshotQuotaObserverChore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSnapshotQuotaObserverChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSnapshotQuotaObserverChore.java index 818d6ff..097e646 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSnapshotQuotaObserverChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSnapshotQuotaObserverChore.java @@ -18,33 +18,35 @@ package org.apache.hadoop.hbase.quotas; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter.Predicate; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.SnapshotDescription; import org.apache.hadoop.hbase.client.SnapshotType; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.quotas.SnapshotQuotaObserverChore.SnapshotWithSize; import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.NoFilesToDischarge; import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate; import org.apache.hadoop.hbase.regionserver.HStore; @@ -61,7 +63,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap; -import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hbase.thirdparty.com.google.common.collect.Multimap; /** @@ -114,50 +116,6 @@ public class TestSnapshotQuotaObserverChore { } @Test - public void testSnapshotSizePersistence() throws IOException { - final Admin admin = TEST_UTIL.getAdmin(); - final TableName tn = TableName.valueOf("quota_snapshotSizePersistence"); - if (admin.tableExists(tn)) { - admin.disableTable(tn); - admin.deleteTable(tn); - } - HTableDescriptor desc = new HTableDescriptor(tn); - desc.addFamily(new HColumnDescriptor(QuotaTableUtil.QUOTA_FAMILY_USAGE)); - admin.createTable(desc); - - Multimap<TableName,SnapshotWithSize> snapshotsWithSizes = HashMultimap.create(); - try (Table table = conn.getTable(tn)) { - // Writing no values will result in no records written. - verify(table, () -> { - testChore.persistSnapshotSizes(table, snapshotsWithSizes); - assertEquals(0, count(table)); - }); - - verify(table, () -> { - TableName originatingTable = TableName.valueOf("t1"); - snapshotsWithSizes.put(originatingTable, new SnapshotWithSize("ss1", 1024L)); - snapshotsWithSizes.put(originatingTable, new SnapshotWithSize("ss2", 4096L)); - testChore.persistSnapshotSizes(table, snapshotsWithSizes); - assertEquals(2, count(table)); - assertEquals(1024L, extractSnapshotSize(table, originatingTable, "ss1")); - assertEquals(4096L, extractSnapshotSize(table, originatingTable, "ss2")); - }); - - snapshotsWithSizes.clear(); - verify(table, () -> { - snapshotsWithSizes.put(TableName.valueOf("t1"), new SnapshotWithSize("ss1", 1024L)); - snapshotsWithSizes.put(TableName.valueOf("t2"), new SnapshotWithSize("ss2", 4096L)); - snapshotsWithSizes.put(TableName.valueOf("t3"), new SnapshotWithSize("ss3", 8192L)); - testChore.persistSnapshotSizes(table, snapshotsWithSizes); - assertEquals(3, count(table)); - assertEquals(1024L, extractSnapshotSize(table, TableName.valueOf("t1"), "ss1")); - assertEquals(4096L, extractSnapshotSize(table, TableName.valueOf("t2"), "ss2")); - assertEquals(8192L, extractSnapshotSize(table, TableName.valueOf("t3"), "ss3")); - }); - } - } - - @Test public void testSnapshotsFromTables() throws Exception { TableName tn1 = helper.createTableWithRegions(1); TableName tn2 = helper.createTableWithRegions(1); @@ -268,13 +226,13 @@ public class TestSnapshotQuotaObserverChore { "Expected to see the single snapshot: " + snapshotsToCompute, 1, snapshotsToCompute.size()); // Get the size of our snapshot - Multimap<TableName,SnapshotWithSize> snapshotsWithSize = testChore.computeSnapshotSizes( + Map<String,Long> namespaceSnapshotSizes = testChore.computeSnapshotSizes( snapshotsToCompute); - assertEquals(1, snapshotsWithSize.size()); - SnapshotWithSize sws = Iterables.getOnlyElement(snapshotsWithSize.get(tn1)); - assertEquals(snapshotName, sws.getName()); + assertEquals(1, namespaceSnapshotSizes.size()); + Long size = namespaceSnapshotSizes.get(tn1.getNamespaceAsString()); + assertNotNull(size); // The snapshot should take up no space since the table refers to it completely - assertEquals(0, sws.getSize()); + assertEquals(0, size.longValue()); // Write some more data, flush it, and then major_compact the table helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE); @@ -302,71 +260,132 @@ public class TestSnapshotQuotaObserverChore { snapshotsToCompute = testChore.getSnapshotsToComputeSize(); assertEquals( "Expected to see the single snapshot: " + snapshotsToCompute, 1, snapshotsToCompute.size()); - snapshotsWithSize = testChore.computeSnapshotSizes( + namespaceSnapshotSizes = testChore.computeSnapshotSizes( snapshotsToCompute); - assertEquals(1, snapshotsWithSize.size()); - sws = Iterables.getOnlyElement(snapshotsWithSize.get(tn1)); - assertEquals(snapshotName, sws.getName()); + assertEquals(1, namespaceSnapshotSizes.size()); + size = namespaceSnapshotSizes.get(tn1.getNamespaceAsString()); + assertNotNull(size); // The snapshot should take up the size the table originally took up - assertEquals(snapshotSize, sws.getSize()); + assertEquals(snapshotSize, size.longValue()); } @Test public void testPersistingSnapshotsForNamespaces() throws Exception { - Multimap<TableName,SnapshotWithSize> snapshotsWithSizes = HashMultimap.create(); TableName tn1 = TableName.valueOf("ns1:tn1"); TableName tn2 = TableName.valueOf("ns1:tn2"); TableName tn3 = TableName.valueOf("ns2:tn1"); TableName tn4 = TableName.valueOf("ns2:tn2"); TableName tn5 = TableName.valueOf("tn1"); - - snapshotsWithSizes.put(tn1, new SnapshotWithSize("", 1024L)); - snapshotsWithSizes.put(tn2, new SnapshotWithSize("", 1024L)); - snapshotsWithSizes.put(tn3, new SnapshotWithSize("", 512L)); - snapshotsWithSizes.put(tn4, new SnapshotWithSize("", 1024L)); - snapshotsWithSizes.put(tn5, new SnapshotWithSize("", 3072L)); - - Map<String,Long> nsSizes = testChore.groupSnapshotSizesByNamespace(snapshotsWithSizes); - assertEquals(3, nsSizes.size()); - assertEquals(2048L, (long) nsSizes.get("ns1")); - assertEquals(1536L, (long) nsSizes.get("ns2")); - assertEquals(3072L, (long) nsSizes.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR)); + // Shim in a custom factory to avoid computing snapshot sizes. + FileArchiverNotifierFactory test = new FileArchiverNotifierFactory() { + Map<TableName,Long> tableToSize = ImmutableMap.of( + tn1, 1024L, tn2, 1024L, tn3, 512L, tn4, 1024L, tn5, 3072L); + @Override + public FileArchiverNotifier get( + Connection conn, Configuration conf, FileSystem fs, TableName tn) { + return new FileArchiverNotifier() { + @Override public void addArchivedFiles(Set<Entry<String,Long>> fileSizes) + throws IOException {} + + @Override + public long computeAndStoreSnapshotSizes(Collection<String> currentSnapshots) + throws IOException { + return tableToSize.get(tn); + } + }; + } + }; + try { + FileArchiverNotifierFactoryImpl.setInstance(test); + + Multimap<TableName,String> snapshotsToCompute = HashMultimap.create(); + snapshotsToCompute.put(tn1, ""); + snapshotsToCompute.put(tn2, ""); + snapshotsToCompute.put(tn3, ""); + snapshotsToCompute.put(tn4, ""); + snapshotsToCompute.put(tn5, ""); + Map<String,Long> nsSizes = testChore.computeSnapshotSizes(snapshotsToCompute); + assertEquals(3, nsSizes.size()); + assertEquals(2048L, (long) nsSizes.get("ns1")); + assertEquals(1536L, (long) nsSizes.get("ns2")); + assertEquals(3072L, (long) nsSizes.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR)); + } finally { + FileArchiverNotifierFactoryImpl.reset(); + } } - private long count(Table t) throws IOException { - try (ResultScanner rs = t.getScanner(new Scan())) { - long sum = 0; - for (Result r : rs) { - while (r.advance()) { - sum++; + @Test + public void testBucketingFilesToSnapshots() throws Exception { + // Create a table and set a quota + TableName tn1 = helper.createTableWithRegions(1); + admin.setQuota(QuotaSettingsFactory.limitTableSpace( + tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS)); + + // Write some data and flush it + helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE); + admin.flush(tn1); + + final AtomicReference<Long> lastSeenSize = new AtomicReference<>(); + // Wait for the Master chore to run to see the usage (with a fudge factor) + TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) { + @Override + boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { + lastSeenSize.set(snapshot.getUsage()); + return snapshot.getUsage() > 230L * SpaceQuotaHelperForTests.ONE_KILOBYTE; + } + }); + + // Create a snapshot on the table + final String snapshotName1 = tn1 + "snapshot1"; + admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH)); + // Major compact the table to force a rewrite + TEST_UTIL.compact(tn1, true); + + // Make sure that the snapshot owns the size + final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME); + TEST_UTIL.waitFor(30_000, new Predicate<Exception>() { + @Override + public boolean evaluate() throws Exception { + Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName1); + Result r = quotaTable.get(g); + if (r == null || r.isEmpty()) { + return false; } + r.advance(); + Cell c = r.current(); + return lastSeenSize.get() == QuotaTableUtil.parseSnapshotSize(c); } - return sum; - } - } + }); - private long extractSnapshotSize( - Table quotaTable, TableName tn, String snapshot) throws IOException { - Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot); - Result r = quotaTable.get(g); - assertNotNull(r); - CellScanner cs = r.cellScanner(); - cs.advance(); - Cell c = cs.current(); - assertNotNull(c); - return QuotaTableUtil.extractSnapshotSize( - c.getValueArray(), c.getValueOffset(), c.getValueLength()); - } + // Create another snapshot on the table + final String snapshotName2 = tn1 + "snapshot2"; + admin.snapshot(new SnapshotDescription(snapshotName2, tn1, SnapshotType.SKIPFLUSH)); + // Major compact the table to force a rewrite + TEST_UTIL.compact(tn1, true); - private void verify(Table t, IOThrowingRunnable test) throws IOException { - admin.disableTable(t.getName()); - admin.truncateTable(t.getName(), false); - test.run(); - } + // Make sure that the snapshot owns the size + TEST_UTIL.waitFor(30_000, new Predicate<Exception>() { + @Override + public boolean evaluate() throws Exception { + Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName2); + Result r = quotaTable.get(g); + if (r == null || r.isEmpty()) { + return false; + } + r.advance(); + Cell c = r.current(); + return lastSeenSize.get() == QuotaTableUtil.parseSnapshotSize(c); + } + }); - @FunctionalInterface - private interface IOThrowingRunnable { - void run() throws IOException; + Get g = QuotaTableUtil.createGetNamespaceSnapshotSize(tn1.getNamespaceAsString()); + Result r = quotaTable.get(g); + assertNotNull(r); + assertFalse(r.isEmpty()); + r.advance(); + long size = QuotaTableUtil.parseSnapshotSize(r.current()); + // Two snapshots of equal size. + assertEquals(lastSeenSize.get() * 2, size); } /**
