Repository: hbase Updated Branches: refs/heads/branch-2 49501cd5b -> 64a32e673
HBASE-21171 [amv2] Tool to parse a directory of MasterProcWALs standalone Signed-off-by: Mike Drob <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/64a32e67 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/64a32e67 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/64a32e67 Branch: refs/heads/branch-2 Commit: 64a32e67337178342560f2960e6c4972bbe9f31c Parents: 49501cd Author: Michael Stack <[email protected]> Authored: Fri Sep 7 17:07:11 2018 -0700 Committer: Michael Stack <[email protected]> Committed: Sun Sep 9 09:28:08 2018 -0700 ---------------------------------------------------------------------- .../store/wal/ProcedureWALPrettyPrinter.java | 1 + .../procedure2/store/wal/WALProcedureStore.java | 38 ++++++++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/64a32e67/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java ---------------------------------------------------------------------- diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java index c692365..a7712b1 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java @@ -47,6 +47,7 @@ import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser; /** * ProcedureWALPrettyPrinter prints the contents of a given ProcedureWAL file + * @see WALProcedureStore#main(String[]) if you want to check parse of a directory of WALs. */ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) @InterfaceStability.Evolving http://git-wip-us.apache.org/repos/asf/hbase/blob/64a32e67/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java ---------------------------------------------------------------------- diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java index 975452a..70c71f5 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java @@ -43,9 +43,11 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreBase; import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker; import org.apache.hadoop.hbase.procedure2.util.ByteSlot; @@ -64,6 +66,8 @@ import org.apache.hbase.thirdparty.org.apache.commons.collections4.queue.Circula /** * WAL implementation of the ProcedureStore. + * @see ProcedureWALPrettyPrinter for printing content of a single WAL. + * @see #main(String[]) to parse a directory of MasterWALProcs. */ @InterfaceAudience.Private @InterfaceStability.Evolving @@ -424,7 +428,6 @@ public class WALProcedureStore extends ProcedureStoreBase { it.next(); // Skip the current log ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() { - long count = 0; @Override public void setMaxProcId(long maxProcId) { @@ -433,11 +436,6 @@ public class WALProcedureStore extends ProcedureStoreBase { @Override public void load(ProcedureIterator procIter) throws IOException { - if ((++count % 1000) == 0) { - // Log every 1000 procedures otherwise it looks like Master is dead if loads of WALs - // and procedures to load. - LOG.debug("Loaded {} procedures", this.count); - } loader.load(procIter); } @@ -1324,4 +1322,32 @@ public class WALProcedureStore extends ProcedureStoreBase { log.close(); return log; } + + /** + * Parses a directory of WALs building up ProcedureState. + * For testing parse and profiling. + * @param args Include pointer to directory of WAL files for a store instance to parse & load. + */ + public static void main(String [] args) throws IOException { + Configuration conf = HBaseConfiguration.create(); + if (args == null || args.length != 1) { + System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR."); + System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR"); + System.exit(-1); + } + WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null, + new WALProcedureStore.LeaseRecovery() { + @Override + public void recoverFileLease(FileSystem fs, Path path) throws IOException { + // no-op + } + }); + try { + store.start(16); + ProcedureExecutor pe = new ProcedureExecutor(conf, new Object()/*Pass anything*/, store); + pe.init(1, true); + } finally { + store.stop(true); + } + } }
