Repository: hbase Updated Branches: refs/heads/branch-1 066f3f754 -> 569fe82ac refs/heads/master 094d65e6f -> 4b91a6425
HBASE-14714 some cleanup to snapshot code Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4b91a642 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4b91a642 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4b91a642 Branch: refs/heads/master Commit: 4b91a6425eadc3578ec291496bbf7fde9742f96a Parents: 094d65e Author: Matteo Bertozzi <matteo.berto...@cloudera.com> Authored: Thu Oct 29 13:48:59 2015 -0700 Committer: Matteo Bertozzi <matteo.berto...@cloudera.com> Committed: Thu Oct 29 13:48:59 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hbase/HTableDescriptor.java | 15 +++- .../hbase/client/TableSnapshotScanner.java | 39 +++------ .../hadoop/hbase/master/MasterFileSystem.java | 4 + .../hbase/master/snapshot/SnapshotManager.java | 5 +- .../hbase/snapshot/RestoreSnapshotHelper.java | 66 ++++++-------- .../hadoop/hbase/snapshot/SnapshotManifest.java | 90 +++++++++----------- .../master/cleaner/TestSnapshotFromMaster.java | 14 +-- .../hadoop/hbase/mob/TestCachedMobFile.java | 2 - .../apache/hadoop/hbase/mob/TestMobFile.java | 1 - .../hbase/snapshot/MobSnapshotTestingUtils.java | 19 ----- .../hbase/snapshot/SnapshotTestingUtils.java | 30 +++---- .../snapshot/TestFlushSnapshotFromClient.java | 44 +++------- .../TestRestoreFlushSnapshotFromClient.java | 5 +- .../snapshot/TestRestoreSnapshotHelper.java | 11 +-- 14 files changed, 133 insertions(+), 212 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index f3d7312..1bd4e07 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -355,8 +355,21 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> { * @param desc The descriptor. */ public HTableDescriptor(final HTableDescriptor desc) { + this(desc.name, desc); + } + + /** + * Construct a table descriptor by cloning the descriptor passed as a parameter + * but using a different table name. + * <p> + * Makes a deep copy of the supplied descriptor. + * Can make a modifiable descriptor from an UnmodifyableHTableDescriptor. + * @param name Table name. + * @param desc The descriptor. + */ + public HTableDescriptor(final TableName name, final HTableDescriptor desc) { super(); - setName(desc.name); + setName(name); setMetaFlags(this.name); for (HColumnDescriptor c: desc.families.values()) { this.families.put(c.getName(), new HColumnDescriptor(c)); http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java index f817e70..4601ae4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java @@ -34,11 +34,7 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; -import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; -import org.apache.hadoop.hbase.snapshot.SnapshotManifest; import org.apache.hadoop.hbase.util.FSUtils; /** @@ -49,8 +45,8 @@ import org.apache.hadoop.hbase.util.FSUtils; * <p> * This also allows one to run the scan from an * online or offline hbase cluster. The snapshot files can be exported by using the - * {@link org.apache.hadoop.hbase.snapshot.ExportSnapshot} tool, - * to a pure-hdfs cluster, and this scanner can be used to + * {@link org.apache.hadoop.hbase.snapshot.ExportSnapshot} tool, + * to a pure-hdfs cluster, and this scanner can be used to * run the scan directly over the snapshot files. The snapshot should not be deleted while there * are open scanners reading from snapshot files. * @@ -125,23 +121,14 @@ public class TableSnapshotScanner extends AbstractClientScanner { } private void init() throws IOException { - Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); - SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); - SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); - - // load table descriptor - htd = manifest.getTableDescriptor(); - - List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); - if (regionManifests == null) { - throw new IllegalArgumentException("Snapshot seems empty"); - } - - regions = new ArrayList<HRegionInfo>(regionManifests.size()); - for (SnapshotRegionManifest regionManifest : regionManifests) { - // load region descriptor - HRegionInfo hri = HRegionInfo.convert(regionManifest.getRegionInfo()); - + final RestoreSnapshotHelper.RestoreMetaChanges meta = + RestoreSnapshotHelper.copySnapshotForScanner( + conf, fs, rootDir, restoreDir, snapshotName); + final List<HRegionInfo> restoredRegions = meta.getRegionsToAdd(); + + htd = meta.getTableDescriptor(); + regions = new ArrayList<HRegionInfo>(restoredRegions.size()); + for (HRegionInfo hri: restoredRegions) { if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), hri.getStartKey(), hri.getEndKey())) { regions.add(hri); @@ -150,11 +137,7 @@ public class TableSnapshotScanner extends AbstractClientScanner { // sort for regions according to startKey. Collections.sort(regions); - initScanMetrics(scan); - - RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, - rootDir, restoreDir, snapshotName); } @Override @@ -184,7 +167,7 @@ public class TableSnapshotScanner extends AbstractClientScanner { if (result == null) { currentRegionScanner.close(); currentRegionScanner = null; - } + } } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java index 9ec49c2..43ae2f8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java @@ -612,4 +612,8 @@ public class MasterFileSystem { public RecoveryMode getLogRecoveryMode() { return this.splitLogManager.getRecoveryMode(); } + + public void logFileSystemState(Log log) throws IOException { + FSUtils.logFileSystemState(fs, rootdir, log); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java index d367d6e..d430493 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java @@ -70,7 +70,6 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; -import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException; @@ -741,7 +740,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc); } } else { - HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc, tableName); + HTableDescriptor htd = new HTableDescriptor(tableName, snapshotTableDesc); if (cpHost != null) { cpHost.preCloneSnapshot(reqSnapshot, htd); } @@ -761,7 +760,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable } } } - + private void checkAndUpdateNamespaceQuota(SnapshotManifest manifest, TableName tableName) throws IOException { if (this.master.getMasterQuotaManager().isQuotaEnabled()) { http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index 81e653d..0f14f70 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -23,6 +23,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -40,7 +41,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.HFileArchiver; @@ -183,7 +183,7 @@ public class RestoreSnapshotHelper { return null; } - RestoreMetaChanges metaChanges = new RestoreMetaChanges(parentsMap); + RestoreMetaChanges metaChanges = new RestoreMetaChanges(tableDesc, parentsMap); // Take a copy of the manifest.keySet() since we are going to modify // this instance, by removing the regions already present in the restore dir. @@ -259,13 +259,19 @@ public class RestoreSnapshotHelper { */ public static class RestoreMetaChanges { private final Map<String, Pair<String, String> > parentsMap; + private final HTableDescriptor htd; private List<HRegionInfo> regionsToRestore = null; private List<HRegionInfo> regionsToRemove = null; private List<HRegionInfo> regionsToAdd = null; - RestoreMetaChanges(final Map<String, Pair<String, String> > parentsMap) { + RestoreMetaChanges(HTableDescriptor htd, Map<String, Pair<String, String> > parentsMap) { this.parentsMap = parentsMap; + this.htd = htd; + } + + public HTableDescriptor getTableDescriptor() { + return htd; } /** @@ -527,13 +533,12 @@ public class RestoreSnapshotHelper { * @return The set of files in the specified family directory. */ private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException { - Set<String> familyFiles = new HashSet<String>(); - FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir); - if (hfiles == null) return familyFiles; + if (hfiles == null) return Collections.emptySet(); - for (FileStatus hfileRef: hfiles) { - String hfileName = hfileRef.getPath().getName(); + Set<String> familyFiles = new HashSet<String>(hfiles.length); + for (int i = 0; i < hfiles.length; ++i) { + String hfileName = hfiles[i].getPath().getName(); familyFiles.add(hfileName); } @@ -685,7 +690,7 @@ public class RestoreSnapshotHelper { Path refPath = StoreFileInfo.getReferredToFile(new Path(new Path(new Path(new Path(snapshotTable .getNamespaceAsString(), snapshotTable.getQualifierAsString()), regionInfo - .getEncodedName()), familyDir.getName()), hfileName)); + .getEncodedName()), familyDir.getName()), hfileName)); String snapshotRegionName = refPath.getParent().getParent().getName(); String fileName = refPath.getName(); @@ -744,7 +749,11 @@ public class RestoreSnapshotHelper { * @return the new HRegion instance */ public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) { - HRegionInfo regionInfo = new HRegionInfo(tableDesc.getTableName(), + return cloneRegionInfo(tableDesc.getTableName(), snapshotRegionInfo); + } + + public static HRegionInfo cloneRegionInfo(TableName tableName, HRegionInfo snapshotRegionInfo) { + HRegionInfo regionInfo = new HRegionInfo(tableName, snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(), snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId()); regionInfo.setOffline(snapshotRegionInfo.isOffline()); @@ -759,9 +768,9 @@ public class RestoreSnapshotHelper { FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs)); if (regionDirs == null) return null; - List<HRegionInfo> regions = new LinkedList<HRegionInfo>(); - for (FileStatus regionDir: regionDirs) { - HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath()); + List<HRegionInfo> regions = new ArrayList<HRegionInfo>(regionDirs.length); + for (int i = 0; i < regionDirs.length; ++i) { + HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath()); regions.add(hri); } LOG.debug("found " + regions.size() + " regions for table=" + @@ -770,30 +779,6 @@ public class RestoreSnapshotHelper { } /** - * Create a new table descriptor cloning the snapshot table schema. - * - * @param snapshotTableDescriptor - * @param tableName - * @return cloned table descriptor - * @throws IOException - */ - public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor, - final TableName tableName) throws IOException { - HTableDescriptor htd = new HTableDescriptor(tableName); - for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) { - htd.addFamily(hcd); - } - for (Map.Entry<Bytes, Bytes> e: - snapshotTableDescriptor.getValues().entrySet()) { - htd.setValue(e.getKey(), e.getValue()); - } - for (Map.Entry<String, String> e: snapshotTableDescriptor.getConfiguration().entrySet()) { - htd.setConfiguration(e.getKey(), e.getValue()); - } - return htd; - } - - /** * Copy the snapshot files for a snapshot scanner, discards meta changes. * @param conf * @param fs @@ -802,8 +787,8 @@ public class RestoreSnapshotHelper { * @param snapshotName * @throws IOException */ - public static void copySnapshotForScanner(Configuration conf, FileSystem fs, Path rootDir, - Path restoreDir, String snapshotName) throws IOException { + public static RestoreMetaChanges copySnapshotForScanner(Configuration conf, FileSystem fs, + Path rootDir, Path restoreDir, String snapshotName) throws IOException { // ensure that restore dir is not under root dir if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) { throw new IllegalArgumentException("Filesystems for restore directory and HBase root " + @@ -826,11 +811,12 @@ public class RestoreSnapshotHelper { // in the base hbase root dir. RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, manifest, manifest.getTableDescriptor(), restoreDir, monitor, status, false); - helper.restoreHdfsRegions(); // TODO: parallelize. + RestoreMetaChanges metaChanges = helper.restoreHdfsRegions(); // TODO: parallelize. if (LOG.isDebugEnabled()) { LOG.debug("Restored table dir:" + restoreDir); FSUtils.logFileSystemState(fs, restoreDir, LOG); } + return metaChanges; } } http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index ca004db..d3b9812 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -176,33 +176,16 @@ public final class SnapshotManifest { monitor.rethrowException(); Path storePath = MobUtils.getMobFamilyPath(mobRegionPath, hcd.getNameAsString()); - if (!fs.exists(storePath)) { - continue; - } - FileStatus[] stats = fs.listStatus(storePath); - if (stats == null) { + List<StoreFileInfo> storeFiles = getStoreFiles(storePath); + if (storeFiles == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("No mob files under family: " + hcd.getNameAsString()); + } continue; } - List<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(); - for (FileStatus stat : stats) { - storeFiles.add(new StoreFileInfo(conf, fs, stat)); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Adding snapshot references for " + storeFiles + " mob files"); - } - // 2.2. iterate through all the mob files and create "references". - for (int i = 0, sz = storeFiles.size(); i < sz; i++) { - StoreFileInfo storeFile = storeFiles.get(i); - monitor.rethrowException(); + addReferenceFiles(visitor, regionData, familyData, storeFiles, true); - // create "reference" to this store file. - if (LOG.isDebugEnabled()) { - LOG.debug("Adding reference for mob file (" + (i + 1) + "/" + sz + "): " - + storeFile.getPath()); - } - visitor.storeFile(regionData, familyData, storeFile); - } visitor.familyClose(regionData, familyData); } visitor.regionClose(regionData); @@ -286,20 +269,11 @@ public final class SnapshotManifest { if (isMobRegion) { Path regionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable()); Path storePath = MobUtils.getMobFamilyPath(regionPath, familyName); - if (!fs.exists(storePath)) { - continue; - } - FileStatus[] stats = fs.listStatus(storePath); - if (stats == null) { - continue; - } - storeFiles = new ArrayList<StoreFileInfo>(); - for (FileStatus stat : stats) { - storeFiles.add(new StoreFileInfo(conf, fs, stat)); - } + storeFiles = getStoreFiles(storePath); } else { storeFiles = regionFs.getStoreFiles(familyName); } + if (storeFiles == null) { if (LOG.isDebugEnabled()) { LOG.debug("No files under family: " + familyName); @@ -308,21 +282,9 @@ public final class SnapshotManifest { } // 2.1. build the snapshot reference for the store - if (LOG.isDebugEnabled()) { - LOG.debug("Adding snapshot references for " + storeFiles + " hfiles"); - } - - // 2.2. iterate through all the store's files and create "references". - int i = 0; - int sz = storeFiles.size(); - for (StoreFileInfo storeFile: storeFiles) { - monitor.rethrowException(); + // iterate through all the store's files and create "references". + addReferenceFiles(visitor, regionData, familyData, storeFiles, false); - // create "reference" to this store file. - LOG.debug("Adding reference for file (" + (++i) + "/" + sz + "): " - + storeFile.getPath()); - visitor.storeFile(regionData, familyData, storeFile); - } visitor.familyClose(regionData, familyData); } } @@ -335,6 +297,38 @@ public final class SnapshotManifest { } } + private List<StoreFileInfo> getStoreFiles(Path storeDir) throws IOException { + FileStatus[] stats = FSUtils.listStatus(fs, storeDir); + if (stats == null) return null; + + ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(stats.length); + for (int i = 0; i < stats.length; ++i) { + storeFiles.add(new StoreFileInfo(conf, fs, stats[i])); + } + return storeFiles; + } + + private void addReferenceFiles(RegionVisitor visitor, Object regionData, Object familyData, + Collection<StoreFileInfo> storeFiles, boolean isMob) throws IOException { + final String fileType = isMob ? "mob file" : "hfile"; + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Adding snapshot references for %s %ss", storeFiles, fileType)); + } + + int i = 0; + int sz = storeFiles.size(); + for (StoreFileInfo storeFile: storeFiles) { + monitor.rethrowException(); + + LOG.debug(String.format("Adding reference for %s (%d/%d): %s", + fileType, ++i, sz, storeFile.getPath())); + + // create "reference" to this store file. + visitor.storeFile(regionData, familyData, storeFile); + } + } + /** * Load the information in the SnapshotManifest. Called by SnapshotManifest.open() * http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java index 687b6ae..20b0642 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java @@ -22,9 +22,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Set; @@ -380,17 +378,7 @@ public class TestSnapshotFromMaster { private final Collection<String> getArchivedHFiles(Path archiveDir, Path rootDir, FileSystem fs, TableName tableName) throws IOException { Path tableArchive = FSUtils.getTableDir(archiveDir, tableName); - Path[] archivedHFiles = SnapshotTestingUtils.listHFiles(fs, tableArchive); - List<String> files = new ArrayList<String>(archivedHFiles.length); - LOG.debug("Have archived hfiles: " + tableArchive); - for (Path file : archivedHFiles) { - LOG.debug(file); - files.add(file.getName()); - } - // sort the archived files - - Collections.sort(files); - return files; + return SnapshotTestingUtils.listHFileNames(fs, tableArchive); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java index 4bb525d..7eb63d1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java @@ -26,7 +26,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; @@ -36,7 +35,6 @@ import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java index e2dced9..7523ca5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; import org.junit.Test; import org.junit.experimental.categories.Category; http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java index a278244..51b44ce 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java @@ -88,25 +88,6 @@ public class MobSnapshotTestingUtils { /** * Return the number of rows in the given table. */ - public static int countMobRows(final Table table) throws IOException { - Scan scan = new Scan(); - ResultScanner results = table.getScanner(scan); - int count = 0; - for (Result res : results) { - count++; - List<Cell> cells = res.listCells(); - for (Cell cell : cells) { - // Verify the value - Assert.assertTrue(CellUtil.cloneValue(cell).length > 0); - } - } - results.close(); - return count; - } - - /** - * Return the number of rows in the given table. - */ public static int countMobRows(final Table table, final byte[]... families) throws IOException { Scan scan = new Scan(); http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index 39c8f21..96e3990 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -22,7 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; -import java.util.Arrays; +import java.util.Collections; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -149,6 +149,14 @@ public class SnapshotTestingUtils { tableName); } + public static void confirmSnapshotValid(HBaseTestingUtility testUtil, + SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family) + throws IOException { + MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem(); + confirmSnapshotValid(snapshotDescriptor, tableName, family, + mfs.getRootDir(), testUtil.getHBaseAdmin(), mfs.getFileSystem()); + } + /** * Confirm that the snapshot contains references to all the files that should * be in the snapshot. @@ -335,28 +343,18 @@ public class SnapshotTestingUtils { * @return array of the current HFiles in the table (could be a zero-length array) * @throws IOException on unexecpted error reading the FS */ - public static Path[] listHFiles(final FileSystem fs, final Path tableDir) + public static ArrayList<String> listHFileNames(final FileSystem fs, final Path tableDir) throws IOException { - final ArrayList<Path> hfiles = new ArrayList<Path>(); + final ArrayList<String> hfiles = new ArrayList<String>(); FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() { @Override public void storeFile(final String region, final String family, final String hfileName) throws IOException { - hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName)))); + hfiles.add(hfileName); } }); - return hfiles.toArray(new Path[hfiles.size()]); - } - - public static String[] listHFileNames(final FileSystem fs, final Path tableDir) - throws IOException { - Path[] files = listHFiles(fs, tableDir); - String[] names = new String[files.length]; - for (int i = 0; i < files.length; ++i) { - names[i] = files[i].getName(); - } - Arrays.sort(names); - return names; + Collections.sort(hfiles); + return hfiles; } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java index b30bcc9..dbf2f0d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -48,7 +48,6 @@ import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -138,8 +137,7 @@ public class TestFlushSnapshotFromClient { SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); LOG.debug("FS state before snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); // take a snapshot of the enabled table String snapshotString = "offlineTableSnapshot"; @@ -152,14 +150,10 @@ public class TestFlushSnapshotFromClient { snapshot, TABLE_NAME); // make sure its a valid snapshot - FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); - Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); LOG.debug("FS state after snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); - SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, - admin, fs); + SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); } /** @@ -177,8 +171,7 @@ public class TestFlushSnapshotFromClient { UTIL.loadTable(table, TEST_FAM); LOG.debug("FS state before snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); // take a snapshot of the enabled table String snapshotString = "skipFlushTableSnapshot"; @@ -191,14 +184,10 @@ public class TestFlushSnapshotFromClient { snapshot, TABLE_NAME); // make sure its a valid snapshot - FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); - Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); LOG.debug("FS state after snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); - SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, - admin, fs); + SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); admin.deleteSnapshot(snapshot); snapshots = admin.listSnapshots(); @@ -220,8 +209,7 @@ public class TestFlushSnapshotFromClient { SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); LOG.debug("FS state before snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); // take a snapshot of the enabled table String snapshotString = "offlineTableSnapshot"; @@ -239,14 +227,10 @@ public class TestFlushSnapshotFromClient { snapshot, TABLE_NAME); // make sure its a valid snapshot - FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); - Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); LOG.debug("FS state after snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); - SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, - admin, fs); + SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); } @Test (timeout=300000) @@ -293,8 +277,8 @@ public class TestFlushSnapshotFromClient { HMaster master = UTIL.getMiniHBaseCluster().getMaster(); SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200); LOG.info(" === Async Snapshot Completed ==="); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); + // make sure we get the snapshot SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot); } @@ -494,7 +478,7 @@ public class TestFlushSnapshotFromClient { } // dump for debugging - logFSTree(FSUtils.getRootDir(UTIL.getConfiguration())); + UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); List<SnapshotDescription> taken = admin.listSnapshots(); int takenSize = taken.size(); @@ -518,10 +502,6 @@ public class TestFlushSnapshotFromClient { UTIL.deleteTable(TABLE2_NAME); } - private void logFSTree(Path root) throws IOException { - FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG); - } - private void waitRegionsAfterMerge(final long numRegionsAfterMerge) throws IOException, InterruptedException { Admin admin = UTIL.getHBaseAdmin(); http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java index 99ece6c..877ee21 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java @@ -27,14 +27,12 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -208,8 +206,7 @@ public class TestRestoreFlushSnapshotFromClient { // Helpers // ========================================================================== private void logFSTree() throws IOException { - MasterFileSystem mfs = UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); - FSUtils.logFileSystemState(mfs.getFileSystem(), mfs.getRootDir(), LOG); + UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); } protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName, http://git-wip-us.apache.org/repos/asf/hbase/blob/4b91a642/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java index 0b28cb2..a251c1c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -124,12 +125,12 @@ public class TestRestoreSnapshotHelper { private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd, final HTableDescriptor htdClone) throws IOException { - String[] files = SnapshotTestingUtils.listHFileNames(fs, + List<String> files = SnapshotTestingUtils.listHFileNames(fs, FSUtils.getTableDir(rootDir, htdClone.getTableName())); - assertEquals(12, files.length); - for (int i = 0; i < files.length; i += 2) { - String linkFile = files[i]; - String refFile = files[i+1]; + assertEquals(12, files.size()); + for (int i = 0; i < files.size(); i += 2) { + String linkFile = files.get(i); + String refFile = files.get(i+1); assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile)); assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile)); assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile));
