HBASE-16904 ADDENDUM cleanup of snapshot related changes for fs redo. * missing header on SnapshotRestoreMetaChanges * neuter parts of ScanPerformanceEvaluation that need to be updated * fixup test class packages on Test*RestoreSnapshotHelper * fixup test class packages on TestMob*ExportSnapshot * fixup new javadoc warnings * fixup new findbugs warning * favor commenting out and placeholders that result in test failures over failures for test-compile
Signed-off-by: Umesh Agashe <uaga...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/81522345 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/81522345 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/81522345 Branch: refs/heads/hbase-14439 Commit: 815223453954d8a36f522f42b89ebf800c1964e5 Parents: 159a67c Author: Sean Busbey <bus...@apache.org> Authored: Mon Nov 28 15:41:59 2016 -0600 Committer: Sean Busbey <bus...@apache.org> Committed: Wed Nov 30 00:54:57 2016 -0600 ---------------------------------------------------------------------- .../apache/hadoop/hbase/fs/MasterStorage.java | 3 - .../procedure/CloneSnapshotProcedure.java | 1 - .../snapshot/SnapshotRestoreMetaChanges.java | 18 +++ .../hadoop/hbase/ScanPerformanceEvaluation.java | 14 ++- .../hbase/client/TestTableSnapshotScanner.java | 11 +- .../legacy/cleaner/TestSnapshotFromMaster.java | 9 +- .../snapshot/TestExportSnapshotNoCluster.java | 115 +++++++++++++++++++ .../legacy/snapshot/TestMobExportSnapshot.java | 66 +++++++++++ .../snapshot/TestMobRestoreSnapshotHelper.java | 46 ++++++++ .../snapshot/TestMobSecureExportSnapshot.java | 53 +++++++++ .../snapshot/TestRestoreSnapshotHelper.java | 8 +- .../snapshot/TestSecureExportSnapshot.java | 58 ++++++++++ .../snapshot/TestSnapshotHFileCleaner.java | 9 +- .../TestMultiTableSnapshotInputFormatImpl.java | 12 +- .../master/snapshot/TestSnapshotManager.java | 5 +- .../hbase/snapshot/SnapshotTestingUtils.java | 74 ++++++------ .../snapshot/TestExportSnapshotNoCluster.java | 115 ------------------- .../hbase/snapshot/TestMobExportSnapshot.java | 65 ----------- .../snapshot/TestMobRestoreSnapshotHelper.java | 46 -------- .../snapshot/TestMobSecureExportSnapshot.java | 53 --------- .../snapshot/TestSecureExportSnapshot.java | 58 ---------- .../snapshot/TestSnapshotDescriptionUtils.java | 3 +- 22 files changed, 438 insertions(+), 404 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java index a62cbb7..9f16018 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java @@ -370,7 +370,6 @@ public abstract class MasterStorage<IDENTIFIER extends StorageIdentifier> { * Returns all {@link HRegionInfo} for a snapshot * * @param snapshot - * @return * @throws IOException */ public Map<String, HRegionInfo> getSnapshotRegions(final SnapshotDescription snapshot) @@ -482,7 +481,6 @@ public abstract class MasterStorage<IDENTIFIER extends StorageIdentifier> { * @param snapshot * @param src Source {@link StorageContext} * @param dest Destination {@link StorageContext} - * @return * @throws IOException */ public abstract boolean changeSnapshotContext(SnapshotDescription snapshot, StorageContext src, @@ -518,7 +516,6 @@ public abstract class MasterStorage<IDENTIFIER extends StorageIdentifier> { * @param destHtd * @param monitor * @param status - * @return * @throws IOException */ public SnapshotRestoreMetaChanges restoreSnapshot(final SnapshotDescription snapshot, http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java index cbdc02f..570fb72 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java @@ -345,7 +345,6 @@ public class CloneSnapshotProcedure final TableName tableName, final List<HRegionInfo> newRegions) throws IOException { - final Configuration conf = env.getMasterConfiguration(); final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher(); getMonitorStatus().setStatus("Clone snapshot - creating regions for table: " + tableName); http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRestoreMetaChanges.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRestoreMetaChanges.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRestoreMetaChanges.java index 22899d5..b87fad6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRestoreMetaChanges.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRestoreMetaChanges.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hbase.snapshot; import org.apache.commons.logging.Log; http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/ScanPerformanceEvaluation.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ScanPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ScanPerformanceEvaluation.java index 24e9590..2fa7080 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ScanPerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ScanPerformanceEvaluation.java @@ -210,14 +210,15 @@ public class ScanPerformanceEvaluation extends AbstractHBaseTool { Scan scan = getScan(); scanOpenTimer.start(); - TableSnapshotScanner scanner = new TableSnapshotScanner(conf, restoreDir, snapshotName, scan); +// TableSnapshotScanner scanner = new TableSnapshotScanner(conf, restoreDir, snapshotName, scan); scanOpenTimer.stop(); long numRows = 0; long numCells = 0; scanTimer.start(); while (true) { - Result result = scanner.next(); +// Result result = scanner.next(); + Result result = null; if (result == null) { break; } @@ -226,10 +227,11 @@ public class ScanPerformanceEvaluation extends AbstractHBaseTool { numCells += result.rawCells().length; } scanTimer.stop(); - scanner.close(); +// scanner.close(); - ScanMetrics metrics = scanner.getScanMetrics(); - long totalBytes = metrics.countOfBytesInResults.get(); +// ScanMetrics metrics = scanner.getScanMetrics(); +// long totalBytes = metrics.countOfBytesInResults.get(); + long totalBytes = 0; double throughput = (double)totalBytes / scanTimer.elapsedTime(TimeUnit.SECONDS); double throughputRows = (double)numRows / scanTimer.elapsedTime(TimeUnit.SECONDS); double throughputCells = (double)numCells / scanTimer.elapsedTime(TimeUnit.SECONDS); @@ -239,7 +241,7 @@ public class ScanPerformanceEvaluation extends AbstractHBaseTool { System.out.println("total time to open scanner: " + scanOpenTimer.elapsedMillis() + " ms"); System.out.println("total time to scan: " + scanTimer.elapsedMillis() + " ms"); - System.out.println("Scan metrics:\n" + metrics.getMetricsMap()); +// System.out.println("Scan metrics:\n" + metrics.getMetricsMap()); System.out.println("total bytes: " + totalBytes + " bytes (" + StringUtils.humanReadableInt(totalBytes) + ")"); http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java index c0c7624..716d940 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java @@ -140,11 +140,12 @@ public class TestTableSnapshotScanner { Path restoreDir = util.getDataTestDirOnTestFS(snapshotName); Scan scan = new Scan(bbb, yyy); // limit the scan - TableSnapshotScanner scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir, - snapshotName, scan); - - verifyScanner(scanner, bbb, yyy); - scanner.close(); +// TableSnapshotScanner scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir, +// snapshotName, scan); +// +// verifyScanner(scanner, bbb, yyy); +// scanner.close(); + Assert.fail("test commented out"); } finally { if (!shutdownCluster) { util.getHBaseAdmin().deleteSnapshot(snapshotName); http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestSnapshotFromMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestSnapshotFromMaster.java index 8b8592b..17fb6bf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestSnapshotFromMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestSnapshotFromMaster.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Set; @@ -332,9 +333,11 @@ public class TestSnapshotFromMaster { FSUtils.logFileSystemState(fs, rootDir, LOG); // get the snapshot files for the table - Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); - Set<String> snapshotHFiles = SnapshotReferenceUtil.getHFileNames( - UTIL.getConfiguration(), fs, snapshotTable); +// Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); +// Set<String> snapshotHFiles = SnapshotReferenceUtil.getHFileNames( +// UTIL.getConfiguration(), fs, snapshotTable); + Path snapshotTable = null; + Set<String> snapshotHFiles = Collections.<String>emptySet(); // check that the files in the archive contain the ones that we need for the snapshot LOG.debug("Have snapshot hfiles:"); for (String fileName : snapshotHFiles) { http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotNoCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotNoCluster.java new file mode 100644 index 0000000..ee3c79d --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotNoCluster.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CategoryBasedTimeout; +import org.apache.hadoop.hbase.HBaseCommonTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestRule; + +/** + * Test Export Snapshot Tool + */ +@Category({MapReduceTests.class, MediumTests.class}) +public class TestExportSnapshotNoCluster { + @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). + withTimeout(this.getClass()).withLookingForStuckThread(true).build(); + private static final Log LOG = LogFactory.getLog(TestExportSnapshotNoCluster.class); + + protected final static HBaseCommonTestingUtility TEST_UTIL = new HBaseCommonTestingUtility(); + + private static FileSystem fs; + private static Path testDir; + + public static void setUpBaseConf(Configuration conf) { + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + conf.setInt("hbase.regionserver.msginterval", 100); + conf.setInt("hbase.client.pause", 250); + conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); + conf.setBoolean("hbase.master.enabletable.roundrobin", true); + conf.setInt("mapreduce.map.maxattempts", 10); + conf.set(HConstants.HBASE_DIR, testDir.toString()); + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + testDir = TEST_UTIL.getDataTestDir(); + fs = testDir.getFileSystem(TEST_UTIL.getConfiguration()); + + setUpBaseConf(TEST_UTIL.getConfiguration()); + } + + /** + * Mock a snapshot with files in the archive dir, + * two regions, and one reference file. + */ + @Test + public void testSnapshotWithRefsExportFileSystemState() throws Exception { + SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); + SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("tableWithRefsV1", + "tableWithRefsV1"); + testSnapshotWithRefsExportFileSystemState(builder); + + snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); + builder = snapshotMock.createSnapshotV2("tableWithRefsV2", "tableWithRefsV2"); + testSnapshotWithRefsExportFileSystemState(builder); + } + + /** + * Generates a couple of regions for the specified SnapshotMock, + * and then it will run the export and verification. + */ + private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder) + throws Exception { + Path[] r1Files = builder.addRegion(); + Path[] r2Files = builder.addRegion(); + builder.commit(); + int snapshotFilesCount = r1Files.length + r2Files.length; + + byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName()); + TableName tableName = builder.getTableDescriptor().getTableName(); + TestExportSnapshot.testExportFileSystemState(TEST_UTIL.getConfiguration(), + tableName, snapshotName, snapshotName, snapshotFilesCount, + testDir, getDestinationDir(), false, null); + } + + private Path getDestinationDir() { + Path path = new Path(new Path(testDir, "export-test"), "export-" + System.currentTimeMillis()); + LOG.info("HDFS export destination path: " + path); + return path; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobExportSnapshot.java new file mode 100644 index 0000000..7248827 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobExportSnapshot.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs.legacy.snapshot; + + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.mob.MobConstants; +import org.apache.hadoop.hbase.mob.MobUtils; +import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +/** + * Test Export Snapshot Tool + */ +@Category({VerySlowRegionServerTests.class, LargeTests.class}) +public class TestMobExportSnapshot extends TestExportSnapshot { + private final Log LOG = LogFactory.getLog(getClass()); + + public static void setUpBaseConf(Configuration conf) { + TestExportSnapshot.setUpBaseConf(conf); + conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + setUpBaseConf(TEST_UTIL.getConfiguration()); + TEST_UTIL.startMiniCluster(3); + } + + @Override + protected void createTable() throws Exception { + MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY); + } + + @Override + protected RegionPredicate getBypassRegionPredicate() { + return new RegionPredicate() { + @Override + public boolean evaluate(final HRegionInfo regionInfo) { + return MobUtils.isMobRegionInfo(regionInfo); + } + }; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobRestoreSnapshotHelper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobRestoreSnapshotHelper.java new file mode 100644 index 0000000..0c9884c --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobRestoreSnapshotHelper.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.mob.MobConstants; +import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils.SnapshotMock; +import org.junit.experimental.categories.Category; + +/** + * Test the restore/clone operation from a file-system point of view. + */ +@Category(SmallTests.class) +public class TestMobRestoreSnapshotHelper extends TestRestoreSnapshotHelper { + final Log LOG = LogFactory.getLog(getClass()); + + @Override + protected void setupConf(Configuration conf) { + conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); + } + + @Override + protected SnapshotMock createSnapshotMock() throws IOException { + return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobSecureExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobSecureExportSnapshot.java new file mode 100644 index 0000000..4144797 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestMobSecureExportSnapshot.java @@ -0,0 +1,53 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; +import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.security.access.AccessControlLists; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; + +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +/** + * Reruns TestMobExportSnapshot using MobExportSnapshot in secure mode. + */ +@Category({VerySlowRegionServerTests.class, LargeTests.class}) +public class TestMobSecureExportSnapshot extends TestMobExportSnapshot { + @BeforeClass + public static void setUpBeforeClass() throws Exception { + setUpBaseConf(TEST_UTIL.getConfiguration()); + + // set the always on security provider + UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(), + HadoopSecurityEnabledUserProviderForTesting.class); + + // setup configuration + SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); + + TEST_UTIL.startMiniCluster(3); + + // Wait for the ACL table to become available + TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestRestoreSnapshotHelper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestRestoreSnapshotHelper.java index 0154963..2cd7aba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestRestoreSnapshotHelper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestRestoreSnapshotHelper.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hbase.snapshot; +package org.apache.hadoop.hbase.fs.legacy.snapshot; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; @@ -170,8 +171,9 @@ public class TestRestoreSnapshotHelper { MonitoredTask status = Mockito.mock(MonitoredTask.class); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd); - return new RestoreSnapshotHelper(conf, manifest, - htdClone, monitor, status); +// return new RestoreSnapshotHelper(conf, manifest, +// htdClone, monitor, status); + return null; } private Path getReferredToFile(final String referenceName) { http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSecureExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSecureExportSnapshot.java new file mode 100644 index 0000000..f2eb3d5 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSecureExportSnapshot.java @@ -0,0 +1,58 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import org.apache.hadoop.hbase.CategoryBasedTimeout; +import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.security.access.AccessControlLists; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; + +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestRule; + +/** + * Reruns TestExportSnapshot using ExportSnapshot in secure mode. + */ +@Category({VerySlowRegionServerTests.class, LargeTests.class}) +public class TestSecureExportSnapshot extends TestExportSnapshot { + @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). + withTimeout(this.getClass()).withLookingForStuckThread(true).build(); + @BeforeClass + public static void setUpBeforeClass() throws Exception { + setUpBaseConf(TEST_UTIL.getConfiguration()); + + // set the always on security provider + UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(), + HadoopSecurityEnabledUserProviderForTesting.class); + + // setup configuration + SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); + + TEST_UTIL.startMiniCluster(3); + + // Wait for the ACL table to become available + TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotHFileCleaner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotHFileCleaner.java index 98cd136..5cd964b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotHFileCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotHFileCleaner.java @@ -90,7 +90,8 @@ public class TestSnapshotHFileCleaner { String snapshotName = "snapshot"; byte[] snapshot = Bytes.toBytes(snapshotName); TableName tableName = TableName.valueOf("table"); - Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); +// Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + Path snapshotDir = null; HRegionInfo mockRegion = new HRegionInfo(tableName); Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName()); Path familyDir = new Path(regionSnapshotDir, "family"); @@ -112,7 +113,7 @@ public class TestSnapshotHFileCleaner { class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { Collection<String> files = new HashSet<String>(); - files.addAll(SnapshotReferenceUtil.getHFileNames(TEST_UTIL.getConfiguration(), fs, snapshotDir)); +// files.addAll(SnapshotReferenceUtil.getHFileNames(TEST_UTIL.getConfiguration(), fs, snapshotDir)); return files; } } @@ -138,7 +139,7 @@ public class TestSnapshotHFileCleaner { } catch (CorruptedSnapshotException cse) { LOG.info("Expected exception " + cse); } finally { - fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); +// fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); } } @@ -165,7 +166,7 @@ public class TestSnapshotHFileCleaner { } catch (CorruptedSnapshotException cse) { LOG.info("Expected exception " + cse); } finally { - fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); +// fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormatImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormatImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormatImpl.java index b4b8056..1581d20 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormatImpl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormatImpl.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -67,9 +68,9 @@ public class TestMultiTableSnapshotInputFormatImpl { // feels weird to introduce a RestoreSnapshotHelperFactory and inject that, which would // probably be the more "pure" // way of doing things. This is the lesser of two evils, perhaps? - doNothing().when(this.subject). - restoreSnapshot(any(Configuration.class), any(String.class), any(Path.class), - any(Path.class), any(FileSystem.class)); +// doNothing().when(this.subject). +// restoreSnapshot(any(Configuration.class), any(String.class), any(Path.class), +// any(Path.class), any(FileSystem.class)); this.conf = new Configuration(); this.rootDir = new Path("file:///test-root-dir"); @@ -178,8 +179,9 @@ public class TestMultiTableSnapshotInputFormatImpl { Map<String, Path> snapshotDirs = subject.getSnapshotDirs(conf); for (Map.Entry<String, Path> entry : snapshotDirs.entrySet()) { - verify(this.subject).restoreSnapshot(eq(this.conf), eq(entry.getKey()), eq(this.rootDir), - eq(entry.getValue()), any(FileSystem.class)); + Assert.fail("restore snapshot."); +// verify(this.subject).restoreSnapshot(eq(this.conf), eq(entry.getKey()), eq(this.rootDir), +// eq(entry.getValue()), any(FileSystem.class)); } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java index e085347..3e0aa43 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java @@ -135,8 +135,9 @@ public class TestSnapshotManager { // Create a "test snapshot" Path rootDir = UTIL.getDataTestDir(); - Path testSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( - "testSnapshotSupportConfiguration", rootDir); +// Path testSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( +// "testSnapshotSupportConfiguration", rootDir); + Path testSnapshotDir = null; fs.mkdirs(testSnapshotDir); try { // force snapshot feature to be disabled, but snapshots are present http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index c9628f6..c7f0580 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.fs.MasterStorage; +import org.apache.hadoop.hbase.fs.legacy.LegacyLayout; import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor; import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotManifest; import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotManifestV1; @@ -206,11 +207,13 @@ public final class SnapshotTestingUtils { final Configuration conf = admin.getConfiguration(); // check snapshot dir - Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( - snapshotDescriptor, rootDir); +// Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( +// snapshotDescriptor, rootDir); + Path snapshotDir = null; assertTrue(fs.exists(snapshotDir)); - HBaseProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); +// HBaseProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); + HBaseProtos.SnapshotDescription desc = null; // Extract regions and families with store files final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); @@ -218,14 +221,15 @@ public final class SnapshotTestingUtils { SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc); Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap(); for (SnapshotRegionManifest regionManifest: regionManifests.values()) { - SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest, - new SnapshotReferenceUtil.StoreFileVisitor() { - @Override - public void storeFile(final HRegionInfo regionInfo, final String family, - final SnapshotRegionManifest.StoreFile storeFile) throws IOException { - snapshotFamilies.add(Bytes.toBytes(family)); - } - }); + Assert.fail("visiting snapshot region store files."); +// SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest, +// new SnapshotReferenceUtil.StoreFileVisitor() { +// @Override +// public void storeFile(final HRegionInfo regionInfo, final String family, +// final SnapshotRegionManifest.StoreFile storeFile) throws IOException { +// snapshotFamilies.add(Bytes.toBytes(family)); +// } +// }); } // Verify that there are store files in the specified families @@ -429,26 +433,28 @@ public final class SnapshotTestingUtils { // Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, // mfs.getRootDir()); Path snapshotDir = null; - HBaseProtos.SnapshotDescription snapshotDesc = - SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); +// HBaseProtos.SnapshotDescription snapshotDesc = +// SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); + HBaseProtos.SnapshotDescription snapshotDesc = null; final TableName table = TableName.valueOf(snapshotDesc.getTable()); final ArrayList corruptedFiles = new ArrayList(); final Configuration conf = util.getConfiguration(); - SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, - new SnapshotReferenceUtil.StoreFileVisitor() { - @Override - public void storeFile(final HRegionInfo regionInfo, final String family, - final SnapshotRegionManifest.StoreFile storeFile) throws IOException { - String region = regionInfo.getEncodedName(); - String hfile = storeFile.getName(); - HFileLink link = HFileLink.build(conf, table, region, family, hfile); - if (corruptedFiles.size() % 2 == 0) { - fs.delete(link.getAvailablePath(fs), true); - corruptedFiles.add(hfile); - } - } - }); +// SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, +// new SnapshotReferenceUtil.StoreFileVisitor() { +// @Override +// public void storeFile(final HRegionInfo regionInfo, final String family, +// final SnapshotRegionManifest.StoreFile storeFile) throws IOException { +// String region = regionInfo.getEncodedName(); +// String hfile = storeFile.getName(); +// HFileLink link = HFileLink.build(conf, table, region, family, hfile); +// if (corruptedFiles.size() % 2 == 0) { +// fs.delete(link.getAvailablePath(fs), true); +// corruptedFiles.add(hfile); +// } +// } +// }); + Assert.fail("visiting snapshot store files."); assertTrue(corruptedFiles.size() > 0); return corruptedFiles; @@ -497,7 +503,7 @@ public final class SnapshotTestingUtils { this.htd = htd; this.desc = desc; this.tableRegions = tableRegions; - this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); +// this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); LegacyTableDescriptor.createTableDescriptor( fs, snapshotDir, htd, false); } @@ -589,9 +595,9 @@ public final class SnapshotTestingUtils { FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir); for (FileStatus fileStatus : manifestFiles) { String fileName = fileStatus.getPath().getName(); - if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE) + if (fileName.endsWith(LegacyLayout.SNAPSHOTINFO_FILE) || fileName.endsWith(".tabledesc") - || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) { + || fileName.endsWith(LegacyLayout.SNAPSHOT_TMP_DIR_NAME)) { fs.delete(fileStatus.getPath(), true); } } @@ -622,8 +628,8 @@ public final class SnapshotTestingUtils { SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); manifest.addTableDescriptor(htd); manifest.consolidate(); - SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs); - snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir); +// SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs); +// snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir); return snapshotDir; } @@ -677,8 +683,8 @@ public final class SnapshotTestingUtils { .setVersion(version) .build(); - Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); - SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs); +// Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); +// SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs); return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions); } http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java deleted file mode 100644 index e2d7c11..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.snapshot; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CategoryBasedTimeout; -import org.apache.hadoop.hbase.HBaseCommonTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; - -/** - * Test Export Snapshot Tool - */ -@Category({MapReduceTests.class, MediumTests.class}) -public class TestExportSnapshotNoCluster { - @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). - withTimeout(this.getClass()).withLookingForStuckThread(true).build(); - private static final Log LOG = LogFactory.getLog(TestExportSnapshotNoCluster.class); - - protected final static HBaseCommonTestingUtility TEST_UTIL = new HBaseCommonTestingUtility(); - - private static FileSystem fs; - private static Path testDir; - - public static void setUpBaseConf(Configuration conf) { - conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); - conf.setInt("hbase.regionserver.msginterval", 100); - conf.setInt("hbase.client.pause", 250); - conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); - conf.setBoolean("hbase.master.enabletable.roundrobin", true); - conf.setInt("mapreduce.map.maxattempts", 10); - conf.set(HConstants.HBASE_DIR, testDir.toString()); - } - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - testDir = TEST_UTIL.getDataTestDir(); - fs = testDir.getFileSystem(TEST_UTIL.getConfiguration()); - - setUpBaseConf(TEST_UTIL.getConfiguration()); - } - - /** - * Mock a snapshot with files in the archive dir, - * two regions, and one reference file. - */ - @Test - public void testSnapshotWithRefsExportFileSystemState() throws Exception { - SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); - SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("tableWithRefsV1", - "tableWithRefsV1"); - testSnapshotWithRefsExportFileSystemState(builder); - - snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); - builder = snapshotMock.createSnapshotV2("tableWithRefsV2", "tableWithRefsV2"); - testSnapshotWithRefsExportFileSystemState(builder); - } - - /** - * Generates a couple of regions for the specified SnapshotMock, - * and then it will run the export and verification. - */ - private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder) - throws Exception { - Path[] r1Files = builder.addRegion(); - Path[] r2Files = builder.addRegion(); - builder.commit(); - int snapshotFilesCount = r1Files.length + r2Files.length; - - byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName()); - TableName tableName = builder.getTableDescriptor().getTableName(); - TestExportSnapshot.testExportFileSystemState(TEST_UTIL.getConfiguration(), - tableName, snapshotName, snapshotName, snapshotFilesCount, - testDir, getDestinationDir(), false, null); - } - - private Path getDestinationDir() { - Path path = new Path(new Path(testDir, "export-test"), "export-" + System.currentTimeMillis()); - LOG.info("HDFS export destination path: " + path); - return path; - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java deleted file mode 100644 index c375b0a..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.snapshot; - - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; -import org.junit.BeforeClass; -import org.junit.experimental.categories.Category; - -/** - * Test Export Snapshot Tool - */ -@Category({VerySlowRegionServerTests.class, LargeTests.class}) -public class TestMobExportSnapshot extends TestExportSnapshot { - private final Log LOG = LogFactory.getLog(getClass()); - - public static void setUpBaseConf(Configuration conf) { - TestExportSnapshot.setUpBaseConf(conf); - conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); - } - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - TEST_UTIL.startMiniCluster(3); - } - - @Override - protected void createTable() throws Exception { - MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY); - } - - @Override - protected RegionPredicate getBypassRegionPredicate() { - return new RegionPredicate() { - @Override - public boolean evaluate(final HRegionInfo regionInfo) { - return MobUtils.isMobRegionInfo(regionInfo); - } - }; - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java deleted file mode 100644 index 47bcb91..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils.SnapshotMock; -import org.junit.experimental.categories.Category; - -/** - * Test the restore/clone operation from a file-system point of view. - */ -@Category(SmallTests.class) -public class TestMobRestoreSnapshotHelper extends TestRestoreSnapshotHelper { - final Log LOG = LogFactory.getLog(getClass()); - - @Override - protected void setupConf(Configuration conf) { - conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); - } - - @Override - protected SnapshotMock createSnapshotMock() throws IOException { - return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java deleted file mode 100644 index 8154995..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; -import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.AccessControlLists; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; - -import org.junit.BeforeClass; -import org.junit.experimental.categories.Category; - -/** - * Reruns TestMobExportSnapshot using MobExportSnapshot in secure mode. - */ -@Category({VerySlowRegionServerTests.class, LargeTests.class}) -public class TestMobSecureExportSnapshot extends TestMobExportSnapshot { - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - - // set the always on security provider - UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(), - HadoopSecurityEnabledUserProviderForTesting.class); - - // setup configuration - SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); - - TEST_UTIL.startMiniCluster(3); - - // Wait for the ACL table to become available - TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java deleted file mode 100644 index f335e4e..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import org.apache.hadoop.hbase.CategoryBasedTimeout; -import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.AccessControlLists; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; - -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; - -/** - * Reruns TestExportSnapshot using ExportSnapshot in secure mode. - */ -@Category({VerySlowRegionServerTests.class, LargeTests.class}) -public class TestSecureExportSnapshot extends TestExportSnapshot { - @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). - withTimeout(this.getClass()).withLookingForStuckThread(true).build(); - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - - // set the always on security provider - UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(), - HadoopSecurityEnabledUserProviderForTesting.class); - - // setup configuration - SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); - - TEST_UTIL.startMiniCluster(3); - - // Wait for the ACL table to become available - TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java index f55bb2d..317e74c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java @@ -94,8 +94,9 @@ public class TestSnapshotDescriptionUtils { + " but shouldn't. Test file leak?", fs.exists(workingDir)); SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build(); try { - SnapshotDescriptionUtils.completeSnapshot(snapshot, root, workingDir, fs); +// SnapshotDescriptionUtils.completeSnapshot(snapshot, root, workingDir, fs); fail("Shouldn't successfully complete move of a non-existent directory."); + throw new IOException("dead code placeholder."); } catch (IOException e) { LOG.info("Correctly failed to move non-existant directory: " + e.getMessage()); }
