Repository: hbase Updated Branches: refs/heads/hbase-11339 33fc1918d -> 30bc7d559
HBASE-12646 SnapshotInfo tool does not find mob data in snapshots Adds MobPath to HFileLink Resolver Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/30bc7d55 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/30bc7d55 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/30bc7d55 Branch: refs/heads/hbase-11339 Commit: 30bc7d5598bc7bce4a4482026a7810e9ec3c04f7 Parents: 33fc191 Author: Jonathan M Hsieh <[email protected]> Authored: Sat Dec 6 08:47:14 2014 -0800 Committer: Jonathan M Hsieh <[email protected]> Committed: Wed Dec 10 11:46:02 2014 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/io/HFileLink.java | 29 +++++++++-- .../hadoop/hbase/snapshot/SnapshotInfo.java | 52 ++++++++++++++------ 2 files changed, 60 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/30bc7d55/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java index 5b2b874..6e97a76 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.FSUtils; @@ -89,6 +90,7 @@ public class HFileLink extends FileLink { private final Path archivePath; private final Path originPath; + private final Path mobPath; private final Path tempPath; /** @@ -103,14 +105,26 @@ public class HFileLink extends FileLink { /** * @param rootDir Path to the root directory where hbase files are stored * @param archiveDir Path to the hbase archive directory + * @param mobDir path to the hbase mob directory + * @param path The path of the HFile Link. + */ + public HFileLink(final Path rootDir, final Path archiveDir, final Path mobDir, final Path path) { + Path hfilePath = getRelativeTablePath(path); + this.tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath); + this.originPath = new Path(rootDir, hfilePath); + this.mobPath = new Path(mobDir, hfilePath); + this.archivePath = new Path(archiveDir, hfilePath); + setLocations(originPath, mobPath, tempPath, archivePath); + } + + + /** + * @param rootDir Path to the root directory where hbase files are stored + * @param archiveDir Path to the hbase archive directory * @param path The path of the HFile Link. */ public HFileLink(final Path rootDir, final Path archiveDir, final Path path) { - Path hfilePath = getRelativeTablePath(path); - this.tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath); - this.originPath = new Path(rootDir, hfilePath); - this.archivePath = new Path(archiveDir, hfilePath); - setLocations(originPath, tempPath, archivePath); + this(rootDir, archiveDir, new Path(rootDir, MobConstants.MOB_DIR_NAME), path); } /** @@ -159,6 +173,11 @@ public class HFileLink extends FileLink { } /** + * @return the path of the mob hfiles. + */ + public Path getMobPath() { return this.mobPath; } + + /** * @param path Path to check. * @return True if the path is a HFileLink. */ http://git-wip-us.apache.org/repos/asf/hbase/blob/30bc7d55/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java index de25394..ae7afcc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java @@ -115,14 +115,16 @@ public final class SnapshotInfo extends Configured implements Tool { } } - private AtomicInteger hfileArchiveCount = new AtomicInteger(); + private AtomicInteger hfilesArchiveCount = new AtomicInteger(); private AtomicInteger hfilesCorrupted = new AtomicInteger(); private AtomicInteger hfilesMissing = new AtomicInteger(); private AtomicInteger hfilesCount = new AtomicInteger(); + private AtomicInteger hfilesMobCount = new AtomicInteger(); private AtomicInteger logsMissing = new AtomicInteger(); private AtomicInteger logsCount = new AtomicInteger(); - private AtomicLong hfileArchiveSize = new AtomicLong(); - private AtomicLong hfileSize = new AtomicLong(); + private AtomicLong hfilesArchiveSize = new AtomicLong(); + private AtomicLong hfilesSize = new AtomicLong(); + private AtomicLong hfilesMobSize = new AtomicLong(); private AtomicLong logSize = new AtomicLong(); private final SnapshotDescription snapshot; @@ -152,14 +154,17 @@ public final class SnapshotInfo extends Configured implements Tool { /** @return the number of available store files */ public int getStoreFilesCount() { - return hfilesCount.get() + hfileArchiveCount.get(); + return hfilesCount.get() + hfilesArchiveCount.get() + hfilesMobCount.get(); } /** @return the number of available store files in the archive */ public int getArchivedStoreFilesCount() { - return hfileArchiveCount.get(); + return hfilesArchiveCount.get(); } + /** @return the number of available store files in the mob dir */ + public int getMobStoreFilesCount() { return hfilesMobCount.get(); } + /** @return the number of available log files */ public int getLogsCount() { return logsCount.get(); @@ -182,22 +187,30 @@ public final class SnapshotInfo extends Configured implements Tool { /** @return the total size of the store files referenced by the snapshot */ public long getStoreFilesSize() { - return hfileSize.get() + hfileArchiveSize.get(); + return hfilesSize.get() + hfilesArchiveSize.get() + hfilesMobSize.get(); } /** @return the total size of the store files shared */ public long getSharedStoreFilesSize() { - return hfileSize.get(); + return hfilesSize.get(); } /** @return the total size of the store files in the archive */ public long getArchivedStoreFileSize() { - return hfileArchiveSize.get(); + return hfilesArchiveSize.get(); } + /** @return the total size of the store files in the mob store*/ + public long getMobStoreFilesSize() { return hfilesMobSize.get(); } + /** @return the percentage of the shared store files */ public float getSharedStoreFilePercentage() { - return ((float)hfileSize.get() / (hfileSize.get() + hfileArchiveSize.get())) * 100; + return ((float) hfilesSize.get() / (getStoreFilesSize())) * 100; + } + + /** @return the percentage of the mob store files */ + public float getMobStoreFilePercentage() { + return ((float) hfilesMobSize.get() / (getStoreFilesSize())) * 100; } /** @return the total log size */ @@ -209,7 +222,7 @@ public final class SnapshotInfo extends Configured implements Tool { * Add the specified store file to the stats * @param region region encoded Name * @param family family name - * @param hfile store file name + * @param storeFile store file name * @return the store file information */ FileInfo addStoreFile(final HRegionInfo region, final String family, @@ -222,11 +235,15 @@ public final class SnapshotInfo extends Configured implements Tool { try { if ((inArchive = fs.exists(link.getArchivePath()))) { size = fs.getFileStatus(link.getArchivePath()).getLen(); - hfileArchiveSize.addAndGet(size); - hfileArchiveCount.incrementAndGet(); + hfilesArchiveSize.addAndGet(size); + hfilesArchiveCount.incrementAndGet(); + } else if (inArchive = fs.exists(link.getMobPath())) { + size = fs.getFileStatus(link.getMobPath()).getLen(); + hfilesMobSize.addAndGet(size); + hfilesMobCount.incrementAndGet(); } else { size = link.getFileStatus(fs).getLen(); - hfileSize.addAndGet(size); + hfilesSize.addAndGet(size); hfilesCount.incrementAndGet(); } isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size); @@ -442,11 +459,14 @@ public final class SnapshotInfo extends Configured implements Tool { } if (showStats) { - System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n", - stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(), + System.out.printf("%d HFiles (%d in archive, %d in mob storage), total size %s " + + "(%.2f%% %s shared with the source table, %.2f%% %s in mob dir)%n", + stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(), stats.getMobStoreFilesCount(), fileSizeToString(stats.getStoreFilesSize()), stats.getSharedStoreFilePercentage(), - fileSizeToString(stats.getSharedStoreFilesSize()) + fileSizeToString(stats.getSharedStoreFilesSize()), + stats.getMobStoreFilePercentage(), + fileSizeToString(stats.getMobStoreFilesSize()) ); System.out.printf("%d Logs, total size %s%n", stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
