This is an automated email from the ASF dual-hosted git repository.
nihaljain pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-operator-tools.git
The following commit(s) were added to refs/heads/master by this push:
new ed47187 HBASE-27724 addFsRegionsMissingInMeta command should support
dumping … (#131)
ed47187 is described below
commit ed47187291878f106613cb644ad9679fef7fc9a7
Author: Nihal Jain <nihaljain...@gmail.com>
AuthorDate: Sat Aug 19 22:53:59 2023 +0530
HBASE-27724 addFsRegionsMissingInMeta command should support dumping …
(#131)
* HBASE-27724 addFsRegionsMissingInMeta command should support dumping
region list into file(s) which can be passed as input(s) to assigns command
* Sync doc
---
hbase-hbck2/README.md | 26 ++++--
.../src/main/java/org/apache/hbase/HBCK2.java | 99 +++++++++++++++++++---
2 files changed, 107 insertions(+), 18 deletions(-)
diff --git a/hbase-hbck2/README.md b/hbase-hbck2/README.md
index 891d851..0c01eeb 100644
--- a/hbase-hbck2/README.md
+++ b/hbase-hbck2/README.md
@@ -100,9 +100,12 @@ Options:
-z,--zookeeper.znode.parent <arg> parent znode of hbase
ensemble
Command:
- addFsRegionsMissingInMeta <NAMESPACE|NAMESPACE:TABLENAME>...|-i
<INPUT_FILE>...
+ addFsRegionsMissingInMeta [OPTIONS]
+[<NAMESPACE|NAMESPACE:TABLENAME>...|-i <INPUTFILES>...]
Options:
- -i,--inputFiles take one or more input files of namespace of table names
+ -i,--inputFiles take one or more files of namespace or table names
+ -o,--outputFile name/prefix of the file(s) to dump region names
+ -n,--numLines number of lines to be written to each output file
To be used when regions missing from hbase:meta but directories
are present still in HDFS. Can happen if user has run _hbck1_
'OfflineMetaRepair' against an hbase-2.x cluster. Needs hbase:meta
@@ -121,12 +124,23 @@ Command:
namespace 'n2':
$ HBCK2 addFsRegionsMissingInMeta default:tbl_1 n1:tbl_2 n2
Returns HBCK2 an 'assigns' command with all re-inserted regions.
- If -i or --inputFiles is specified, pass one or more input file names.
- Each file contains <NAMESPACE|NAMESPACE:TABLENAME>, one per line. For
example:
- For example:
- $ HBCK2 addFsRegionsMissingInMeta -i fileName1 fileName2
SEE ALSO: reportMissingRegionsInMeta
SEE ALSO: fixMeta
+ If -i or --inputFiles is specified, pass one or more input file names.
+ Each file contains <NAMESPACE|NAMESPACE:TABLENAME>, one per line.
+ For example:
+ $ HBCK2 addFsRegionsMissingInMeta -i fileName1 fileName2
+ If -o or --outputFile is specified, the output file(s) can be passed as
+ input to assigns command via -i or -inputFiles option.
+ If -n or --numLines is specified, and say it is set to 100, this will
+ create files with prefix as value passed by -o or --outputFile option.
+ Each file will have 100 region names (max.), one per line.
+ For example:
+ $ HBCK2 addFsRegionsMissingInMeta -o outputFilePrefix -n 100
+ -i fileName1 fileName2
+ But if -n is not specified, but -o is specified, it will dump all
+ region names in a single file, one per line.
+ NOTE: -n option is applicable only if -o option is specified.
assigns [OPTIONS] <ENCODED_REGIONNAME/INPUTFILES_FOR_REGIONNAMES>...
Options:
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
index 5918583..0d060c7 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
@@ -17,6 +17,7 @@
*/
package org.apache.hbase;
+import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -37,6 +38,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.stream.Collectors;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.LineIterator;
import org.apache.hadoop.conf.Configuration;
@@ -67,6 +69,7 @@ import org.apache.logging.log4j.core.config.Configurator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser;
import org.apache.hbase.thirdparty.org.apache.commons.cli.DefaultParser;
@@ -384,6 +387,68 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
}
}
+ private Pair<List<String>, List<Exception>>
+ addMissingRegionsInMetaForTablesWrapper(String... args) throws IOException
{
+ // Init
+ Options options = new Options();
+ Option outputFile =
Option.builder("o").longOpt("outputFile").hasArg().build();
+ options.addOption(outputFile);
+ Option numberOfLinesPerFile =
Option.builder("n").longOpt("numLines").hasArg().build();
+ options.addOption(numberOfLinesPerFile);
+ Option inputFile = Option.builder("i").longOpt("inputFiles").build();
+ options.addOption(inputFile);
+
+ // 1st element is region name list and 2nd element is error list
+ final Pair<List<String>, List<Exception>> result =
+ Pair.newPair(new ArrayList<>(), new ArrayList<>());
+ final List<String> regionsList = result.getFirst();
+ final List<Exception> errorList = result.getSecond();
+
+ // Parse command-line
+ CommandLine commandLine = getCommandLine(args, options);
+ if (commandLine == null) {
+ return result;
+ }
+
+ boolean outputFileFlag = commandLine.hasOption(outputFile.getOpt());
+ boolean inputFileFlag = commandLine.hasOption(inputFile.getOpt());
+ boolean numberOfLinesPerFileFlag =
commandLine.hasOption(numberOfLinesPerFile.getOpt());
+
+ final List<String> namespacesTables =
+ getFromArgsOrFiles(commandLine.getArgList(), inputFileFlag);
+ final List<Future<List<String>>> addedRegions =
+ addMissingRegionsInMetaForTables(namespacesTables.toArray(new String[]
{}));
+
+ for (Future<List<String>> f : addedRegions) {
+ try {
+ regionsList.addAll(f.get());
+ } catch (InterruptedException | ExecutionException e) {
+ errorList.add(e);
+ }
+ }
+
+ if (outputFileFlag) {
+ String fileNameOrPrefix =
commandLine.getOptionValue(outputFile.getOpt());
+ if (numberOfLinesPerFileFlag) {
+ int numberOfRegionsPerFile =
+
Integer.parseInt(commandLine.getOptionValue(numberOfLinesPerFile.getOpt()));
+ final List<List<String>> partition = Lists.partition(regionsList,
numberOfRegionsPerFile);
+ for (int i = 0; i < partition.size(); i++) {
+ // Dump to file
+ File file = new File(fileNameOrPrefix + "." + i);
+ System.out.println("Dumping region names to file: " +
file.getAbsolutePath());
+ FileUtils.writeLines(file, partition.get(i));
+ }
+ } else {
+ File file = new File(fileNameOrPrefix);
+ System.out.println("Dumping region names to file: " +
file.getAbsolutePath());
+ FileUtils.writeLines(file, regionsList);
+ }
+ }
+
+ return result;
+ }
+
List<Long> assigns(Hbck hbck, String[] args) throws IOException {
Options options = new Options();
Option override = Option.builder("o").longOpt("override").build();
@@ -547,10 +612,12 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
}
private static void usageAddFsRegionsMissingInMeta(PrintWriter writer) {
- writer.println(" " + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + "
[<NAMESPACE|"
+ writer.println(" " + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + " [OPTIONS]
[<NAMESPACE|"
+ "NAMESPACE:TABLENAME>...|-i <INPUTFILES>...]");
writer.println(" Options:");
writer.println(" -i,--inputFiles take one or more files of namespace
or table names");
+ writer.println(" -o,--outputFile name/prefix of the file(s) to dump
region names");
+ writer.println(" -n,--numLines number of lines to be written to each
output file");
writer.println(" To be used when regions missing from hbase:meta but
directories");
writer.println(" are present still in HDFS. Can happen if user has run
_hbck1_");
writer.println(" 'OfflineMetaRepair' against an hbase-2.x cluster. Needs
hbase:meta");
@@ -577,6 +644,18 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
writer.println(" For example:");
writer.println(
" $ HBCK2 " + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + " -i
fileName1 fileName2");
+ writer.println(" If -o or --outputFile is specified, the output file(s)
can be passed as");
+ writer.println(" input to assigns command via -i or -inputFiles
option.");
+ writer.println(" If -n or --numLines is specified, and say it is set to
100, this will");
+ writer.println(" create files with prefix as value passed by -o or
--outputFile option.");
+ writer.println(" Each file will have 100 region names (max.), one per
line.");
+ writer.println(" For example:");
+ writer.println(
+ " $ HBCK2 " + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + " -o
outputFilePrefix -n 100");
+ writer.println(" -i fileName1 fileName2");
+ writer.println(" But if -n is not specified, but -o is specified, it
will dump all");
+ writer.println(" region names in a single file, one per line.");
+ writer.println(" NOTE: -n option is applicable only if -o option is
specified.");
}
private static void usageAssigns(PrintWriter writer) {
@@ -1116,18 +1195,14 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
showErrorMessage(command + " takes one or more table names.");
return EXIT_FAILURE;
}
- List<Future<List<String>>> addedRegions =
- addMissingRegionsInMetaForTables(purgeFirst(commands));
- List<String> regionNames = new ArrayList<>();
- List<Exception> errors = new ArrayList<>();
- for (Future<List<String>> f : addedRegions) {
- try {
- regionNames.addAll(f.get());
- } catch (InterruptedException | ExecutionException e) {
- errors.add(e);
- }
+
+ try {
+ Pair<List<String>, List<Exception>> result =
+ addMissingRegionsInMetaForTablesWrapper(purgeFirst(commands));
+ System.out.println(formatReAddedRegionsMessage(result.getFirst(),
result.getSecond()));
+ } catch (Exception e) {
+ return EXIT_FAILURE;
}
- System.out.println(formatReAddedRegionsMessage(regionNames, errors));
break;
case REPORT_MISSING_REGIONS_IN_META:
