Repository: hbase Updated Branches: refs/heads/branch-1 b1c60940d -> 6b233c433
HBASE-8386 deprecate TableMapReduce.addDependencyJars(Configuration, class<?> ...) Signed-off-by: Matteo Bertozzi <[email protected]> Conflicts: hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6b233c43 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6b233c43 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6b233c43 Branch: refs/heads/branch-1 Commit: 6b233c4332a084d87898ece0783912326e1e13f7 Parents: 3d4b533 Author: Sean Busbey <[email protected]> Authored: Tue Jul 19 07:21:26 2016 -0500 Committer: Sean Busbey <[email protected]> Committed: Fri Aug 5 14:08:50 2016 -0500 ---------------------------------------------------------------------- .../test/IntegrationTestBigLinkedList.java | 6 +++-- .../test/IntegrationTestLoadAndVerify.java | 4 +-- ...tionTestWithCellVisibilityLoadAndVerify.java | 2 +- .../hadoop/hbase/mapred/TableMapReduceUtil.java | 2 +- .../apache/hadoop/hbase/mapreduce/Import.java | 6 ++--- .../hadoop/hbase/mapreduce/ImportTsv.java | 2 +- .../hbase/mapreduce/TableMapReduceUtil.java | 28 ++++++++++++++++++-- .../hadoop/hbase/mapreduce/WALPlayer.java | 4 +-- .../hadoop/hbase/PerformanceEvaluation.java | 2 +- .../mapred/TestTableSnapshotInputFormat.java | 2 +- .../mapreduce/TestTableSnapshotInputFormat.java | 2 +- src/main/asciidoc/_chapters/mapreduce.adoc | 2 +- 12 files changed, 44 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index 1f57e07..2d828f4 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -780,7 +780,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { job.getConfiguration().setBoolean("mapreduce.map.speculative", false); TableMapReduceUtil.addDependencyJars(job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), + AbstractHBaseTool.class); TableMapReduceUtil.initCredentials(job); boolean success = jobCompletion(job); @@ -1287,7 +1288,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan, VerifyMapper.class, BytesWritable.class, BytesWritable.class, job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), + AbstractHBaseTool.class); job.getConfiguration().setBoolean("mapreduce.map.speculative", false); http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java index 9ecb9ca..30d5a0d 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java @@ -348,7 +348,7 @@ public void cleanUpCluster() throws Exception { TableMapReduceUtil.addDependencyJars(job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class); TableMapReduceUtil.initCredentials(job); assertTrue(job.waitForCompletion(true)); return job; @@ -372,7 +372,7 @@ public void cleanUpCluster() throws Exception { TableMapReduceUtil.initTableMapperJob( htd.getTableName().getNameAsString(), scan, VerifyMapper.class, BytesWritable.class, BytesWritable.class, job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class); int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING); TableMapReduceUtil.setScannerCaching(job, scannerCaching); http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java index 4285309..906bf33 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java @@ -330,7 +330,7 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT scan.setAuthorizations(new Authorizations(auths)); TableMapReduceUtil.initTableMapperJob(htd.getTableName().getNameAsString(), scan, VerifyMapper.class, NullWritable.class, NullWritable.class, job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class); int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING); TableMapReduceUtil.setScannerCaching(job, scannerCaching); job.setNumReduceTasks(0); http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java index 2a040d9..d5f225f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java @@ -368,7 +368,7 @@ public class TableMapReduceUtil { */ public static void addDependencyJars(JobConf job) throws IOException { org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addHBaseDependencyJars(job); - org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars( + org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses( job, // when making changes here, consider also mapreduce.TableMapReduceUtil // pull job classes http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java index 0a5d937..d315b81 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java @@ -603,7 +603,7 @@ public class Import { try { Class<? extends Filter> filter = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class); if (filter != null) { - TableMapReduceUtil.addDependencyJars(conf, filter); + TableMapReduceUtil.addDependencyJarsForClasses(conf, filter); } } catch (Exception e) { throw new IOException(e); @@ -630,7 +630,7 @@ public class Import { fs.deleteOnExit(partitionsPath); job.setPartitionerClass(KeyValueWritableComparablePartitioner.class); job.setNumReduceTasks(regionLocator.getStartKeys().length); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), com.google.common.base.Preconditions.class); } } else if (hfileOutPath != null) { @@ -644,7 +644,7 @@ public class Import { job.setMapOutputKeyClass(ImmutableBytesWritable.class); job.setMapOutputValueClass(KeyValue.class); HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), com.google.common.base.Preconditions.class); } } else { http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java index 6ce1a28..323e1e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java @@ -589,7 +589,7 @@ public class ImportTsv extends Configured implements Tool { KeyValueSerialization.class.getName()); } TableMapReduceUtil.addDependencyJars(job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), com.google.common.base.Function.class /* Guava used by TsvParser */); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index 458464f..a939ef3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -778,7 +778,7 @@ public class TableMapReduceUtil { " Continuing without it."); } - addDependencyJars(conf, + addDependencyJarsForClasses(conf, // explicitly pull a class from each module org.apache.hadoop.hbase.HConstants.class, // hbase-common org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol @@ -826,7 +826,7 @@ public class TableMapReduceUtil { public static void addDependencyJars(Job job) throws IOException { addHBaseDependencyJars(job.getConfiguration()); try { - addDependencyJars(job.getConfiguration(), + addDependencyJarsForClasses(job.getConfiguration(), // when making changes here, consider also mapred.TableMapReduceUtil // pull job classes job.getMapOutputKeyClass(), @@ -846,9 +846,33 @@ public class TableMapReduceUtil { * Add the jars containing the given classes to the job's configuration * such that JobClient will ship them to the cluster and add them to * the DistributedCache. + * @deprecated rely on {@link #addDependencyJars(Job)} instead. */ + @Deprecated public static void addDependencyJars(Configuration conf, Class<?>... classes) throws IOException { + LOG.warn("The addDependencyJars(Configuration, Class<?>...) method has been deprecated since it" + + " is easy to use incorrectly. Most users should rely on addDependencyJars(Job) " + + "instead. See HBASE-8386 for more details."); + addDependencyJarsForClasses(conf, classes); + } + + /** + * Add the jars containing the given classes to the job's configuration + * such that JobClient will ship them to the cluster and add them to + * the DistributedCache. + * + * N.B. that this method at most adds one jar per class given. If there is more than one + * jar available containing a class with the same name as a given class, we don't define + * which of those jars might be chosen. + * + * @param conf The Hadoop Configuration to modify + * @param classes will add just those dependencies needed to find the given classes + * @throws IOException if an underlying library call fails. + */ + @InterfaceAudience.Private + public static void addDependencyJarsForClasses(Configuration conf, + Class<?>... classes) throws IOException { FileSystem localFs = FileSystem.getLocal(conf); Set<String> jars = new HashSet<String>(); http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index 7ffd222..377b6ea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -284,7 +284,7 @@ public class WALPlayer extends Configured implements Tool { RegionLocator regionLocator = conn.getRegionLocator(tableName)) { HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator); } - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), com.google.common.base.Preconditions.class); } else { // output to live cluster @@ -297,7 +297,7 @@ public class WALPlayer extends Configured implements Tool { } String codecCls = WALCellCodec.getWALCellCodecClass(conf); try { - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), Class.forName(codecCls)); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), Class.forName(codecCls)); } catch (Exception e) { throw new IOException("Cannot determine wal codec class " + codecCls, e); } http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index 40ddc39..16ea209 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -498,7 +498,7 @@ public class PerformanceEvaluation extends Configured implements Tool { TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs")); TableMapReduceUtil.addDependencyJars(job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), Histogram.class, // yammer metrics ObjectMapper.class); // jackson-mapper-asl http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java index c3771c5..60f19a2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java @@ -242,7 +242,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa JobConf jobConf = new JobConf(util.getConfiguration()); jobConf.setJarByClass(util.getClass()); - org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf, + org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses(jobConf, TestTableSnapshotInputFormat.class); TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS, http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java index 1b80590..349f96e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java @@ -275,7 +275,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa Scan scan = new Scan(startRow, endRow); // limit the scan job.setJarByClass(util.getClass()); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), TestTableSnapshotInputFormat.class); TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, http://git-wip-us.apache.org/repos/asf/hbase/blob/6b233c43/src/main/asciidoc/_chapters/mapreduce.adoc ---------------------------------------------------------------------- diff --git a/src/main/asciidoc/_chapters/mapreduce.adoc b/src/main/asciidoc/_chapters/mapreduce.adoc index 2a42af2..0792c04 100644 --- a/src/main/asciidoc/_chapters/mapreduce.adoc +++ b/src/main/asciidoc/_chapters/mapreduce.adoc @@ -71,7 +71,7 @@ This example assumes you use a BASH-compatible shell. $ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/lib/hbase-server-VERSION.jar rowcounter usertable ---- -When the command runs, internally, the HBase JAR finds the dependencies it needs for ZooKeeper, Guava, and its other dependencies on the passed `HADOOP_CLASSPATH` and adds the JARs to the MapReduce job configuration. +When the command runs, internally, the HBase JAR finds the dependencies it needs and adds them to the MapReduce job configuration. See the source at `TableMapReduceUtil#addDependencyJars(org.apache.hadoop.mapreduce.Job)` for how this is done. The command `hbase mapredcp` can also help you dump the CLASSPATH entries required by MapReduce, which are the same jars `TableMapReduceUtil#addDependencyJars` would add.
