[11/50] [abbrv] hadoop git commit: MAPREDUCE-6248. Exposed the internal MapReduce job's information as a public API in DistCp. Contributed by Jing Zhao.
MAPREDUCE-6248. Exposed the internal MapReduce job's information as a public
API in DistCp. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bab6209c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bab6209c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bab6209c
Branch: refs/heads/HDFS-7285
Commit: bab6209c170d1127680f8d0e975e2e54e9c63ccc
Parents: ff1b358
Author: Vinod Kumar Vavilapalli vino...@apache.org
Authored: Tue Mar 3 16:28:22 2015 -0800
Committer: Jing Zhao ji...@apache.org
Committed: Mon Mar 9 13:11:23 2015 -0700
--
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../java/org/apache/hadoop/tools/DistCp.java| 47 +++-
2 files changed, 39 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bab6209c/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 7a2eff3..b2ae9d9 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -320,6 +320,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-5612. Add javadoc for TaskCompletionEvent.Status.
(Chris Palmer via aajisaka)
+MAPREDUCE-6248. Exposed the internal MapReduce job's information as a
public
+API in DistCp. (Jing Zhao via vinodkv)
+
OPTIMIZATIONS
MAPREDUCE-6169. MergeQueue should release reference to the current item
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bab6209c/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
--
diff --git
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index 28535a7..b80aeb8 100644
---
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.tools;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
@@ -51,12 +53,14 @@ import com.google.common.annotations.VisibleForTesting;
* launch the copy-job. DistCp may alternatively be sub-classed to fine-tune
* behaviour.
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class DistCp extends Configured implements Tool {
/**
- * Priority of the ResourceManager shutdown hook.
+ * Priority of the shutdown hook.
*/
- public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ static final int SHUTDOWN_HOOK_PRIORITY = 30;
private static final Log LOG = LogFactory.getLog(DistCp.class);
@@ -66,7 +70,7 @@ public class DistCp extends Configured implements Tool {
private static final String PREFIX = _distcp;
private static final String WIP_PREFIX = ._WIP_;
private static final String DISTCP_DEFAULT_XML = distcp-default.xml;
- public static final Random rand = new Random();
+ static final Random rand = new Random();
private boolean submitted;
private FileSystem jobFS;
@@ -90,7 +94,7 @@ public class DistCp extends Configured implements Tool {
* To be used with the ToolRunner. Not for public consumption.
*/
@VisibleForTesting
- public DistCp() {}
+ DistCp() {}
/**
* Implementation of Tool::run(). Orchestrates the copy of source file(s)
@@ -100,6 +104,7 @@ public class DistCp extends Configured implements Tool {
* @param argv List of arguments passed to DistCp, from the ToolRunner.
* @return On success, it returns 0. Else, -1.
*/
+ @Override
public int run(String[] argv) {
if (argv.length 1) {
OptionsParser.usage();
@@ -145,9 +150,21 @@ public class DistCp extends Configured implements Tool {
* @throws Exception
*/
public Job execute() throws Exception {
+Job job = createAndSubmitJob();
+
+if (inputOptions.shouldBlock()) {
+ waitForJobCompletion(job);
+}
+return job;
+ }
+
+ /**
+ * Create and submit the mapreduce job.
+ * @return The mapreduce job object that has been submitted
+ */
+ public Job createAndSubmitJob() throws Exception {
assert inputOptions != null;
assert getConf() != null;
-
Job job = null;
try {
synchronized(this) {
@@ -169,16 +186,24 @@ public class DistCp extends Configured implements Tool {
String jobID =
[04/27] hadoop git commit: MAPREDUCE-6248. Exposed the internal MapReduce job's information as a public API in DistCp. Contributed by Jing Zhao.
MAPREDUCE-6248. Exposed the internal MapReduce job's information as a public
API in DistCp. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5af693fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5af693fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5af693fd
Branch: refs/heads/YARN-2928
Commit: 5af693fde26755b6f175bd65f93cf4a80de0d1e0
Parents: b2f1ec3
Author: Vinod Kumar Vavilapalli vino...@apache.org
Authored: Tue Mar 3 16:28:22 2015 -0800
Committer: Vinod Kumar Vavilapalli vino...@apache.org
Committed: Tue Mar 3 16:28:41 2015 -0800
--
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../java/org/apache/hadoop/tools/DistCp.java| 47 +++-
2 files changed, 39 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5af693fd/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 7a2eff3..b2ae9d9 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -320,6 +320,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-5612. Add javadoc for TaskCompletionEvent.Status.
(Chris Palmer via aajisaka)
+MAPREDUCE-6248. Exposed the internal MapReduce job's information as a
public
+API in DistCp. (Jing Zhao via vinodkv)
+
OPTIMIZATIONS
MAPREDUCE-6169. MergeQueue should release reference to the current item
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5af693fd/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
--
diff --git
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index 28535a7..b80aeb8 100644
---
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.tools;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
@@ -51,12 +53,14 @@ import com.google.common.annotations.VisibleForTesting;
* launch the copy-job. DistCp may alternatively be sub-classed to fine-tune
* behaviour.
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class DistCp extends Configured implements Tool {
/**
- * Priority of the ResourceManager shutdown hook.
+ * Priority of the shutdown hook.
*/
- public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ static final int SHUTDOWN_HOOK_PRIORITY = 30;
private static final Log LOG = LogFactory.getLog(DistCp.class);
@@ -66,7 +70,7 @@ public class DistCp extends Configured implements Tool {
private static final String PREFIX = _distcp;
private static final String WIP_PREFIX = ._WIP_;
private static final String DISTCP_DEFAULT_XML = distcp-default.xml;
- public static final Random rand = new Random();
+ static final Random rand = new Random();
private boolean submitted;
private FileSystem jobFS;
@@ -90,7 +94,7 @@ public class DistCp extends Configured implements Tool {
* To be used with the ToolRunner. Not for public consumption.
*/
@VisibleForTesting
- public DistCp() {}
+ DistCp() {}
/**
* Implementation of Tool::run(). Orchestrates the copy of source file(s)
@@ -100,6 +104,7 @@ public class DistCp extends Configured implements Tool {
* @param argv List of arguments passed to DistCp, from the ToolRunner.
* @return On success, it returns 0. Else, -1.
*/
+ @Override
public int run(String[] argv) {
if (argv.length 1) {
OptionsParser.usage();
@@ -145,9 +150,21 @@ public class DistCp extends Configured implements Tool {
* @throws Exception
*/
public Job execute() throws Exception {
+Job job = createAndSubmitJob();
+
+if (inputOptions.shouldBlock()) {
+ waitForJobCompletion(job);
+}
+return job;
+ }
+
+ /**
+ * Create and submit the mapreduce job.
+ * @return The mapreduce job object that has been submitted
+ */
+ public Job createAndSubmitJob() throws Exception {
assert inputOptions != null;
assert getConf() != null;
-
Job job = null;
try {
synchronized(this) {
@@ -169,16 +186,24 @@ public class DistCp extends Configured implements Tool {
String jobID =
hadoop git commit: MAPREDUCE-6248. Exposed the internal MapReduce job's information as a public API in DistCp. Contributed by Jing Zhao.
Repository: hadoop
Updated Branches:
refs/heads/trunk b2f1ec312 - 5af693fde
MAPREDUCE-6248. Exposed the internal MapReduce job's information as a public
API in DistCp. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5af693fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5af693fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5af693fd
Branch: refs/heads/trunk
Commit: 5af693fde26755b6f175bd65f93cf4a80de0d1e0
Parents: b2f1ec3
Author: Vinod Kumar Vavilapalli vino...@apache.org
Authored: Tue Mar 3 16:28:22 2015 -0800
Committer: Vinod Kumar Vavilapalli vino...@apache.org
Committed: Tue Mar 3 16:28:41 2015 -0800
--
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../java/org/apache/hadoop/tools/DistCp.java| 47 +++-
2 files changed, 39 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5af693fd/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 7a2eff3..b2ae9d9 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -320,6 +320,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-5612. Add javadoc for TaskCompletionEvent.Status.
(Chris Palmer via aajisaka)
+MAPREDUCE-6248. Exposed the internal MapReduce job's information as a
public
+API in DistCp. (Jing Zhao via vinodkv)
+
OPTIMIZATIONS
MAPREDUCE-6169. MergeQueue should release reference to the current item
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5af693fd/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
--
diff --git
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index 28535a7..b80aeb8 100644
---
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.tools;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
@@ -51,12 +53,14 @@ import com.google.common.annotations.VisibleForTesting;
* launch the copy-job. DistCp may alternatively be sub-classed to fine-tune
* behaviour.
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class DistCp extends Configured implements Tool {
/**
- * Priority of the ResourceManager shutdown hook.
+ * Priority of the shutdown hook.
*/
- public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ static final int SHUTDOWN_HOOK_PRIORITY = 30;
private static final Log LOG = LogFactory.getLog(DistCp.class);
@@ -66,7 +70,7 @@ public class DistCp extends Configured implements Tool {
private static final String PREFIX = _distcp;
private static final String WIP_PREFIX = ._WIP_;
private static final String DISTCP_DEFAULT_XML = distcp-default.xml;
- public static final Random rand = new Random();
+ static final Random rand = new Random();
private boolean submitted;
private FileSystem jobFS;
@@ -90,7 +94,7 @@ public class DistCp extends Configured implements Tool {
* To be used with the ToolRunner. Not for public consumption.
*/
@VisibleForTesting
- public DistCp() {}
+ DistCp() {}
/**
* Implementation of Tool::run(). Orchestrates the copy of source file(s)
@@ -100,6 +104,7 @@ public class DistCp extends Configured implements Tool {
* @param argv List of arguments passed to DistCp, from the ToolRunner.
* @return On success, it returns 0. Else, -1.
*/
+ @Override
public int run(String[] argv) {
if (argv.length 1) {
OptionsParser.usage();
@@ -145,9 +150,21 @@ public class DistCp extends Configured implements Tool {
* @throws Exception
*/
public Job execute() throws Exception {
+Job job = createAndSubmitJob();
+
+if (inputOptions.shouldBlock()) {
+ waitForJobCompletion(job);
+}
+return job;
+ }
+
+ /**
+ * Create and submit the mapreduce job.
+ * @return The mapreduce job object that has been submitted
+ */
+ public Job createAndSubmitJob() throws Exception {
assert inputOptions != null;
assert getConf() != null;
-
Job job = null;
try {
synchronized(this) {
@@ -169,16 +186,24 @@ public class
hadoop git commit: MAPREDUCE-6248. Exposed the internal MapReduce job's information as a public API in DistCp. Contributed by Jing Zhao.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 888a44563 - 6604613bc
MAPREDUCE-6248. Exposed the internal MapReduce job's information as a public
API in DistCp. Contributed by Jing Zhao.
(cherry picked from commit 5af693fde26755b6f175bd65f93cf4a80de0d1e0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6604613b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6604613b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6604613b
Branch: refs/heads/branch-2
Commit: 6604613bc5778564b66a8bfda542c0086cada17f
Parents: 888a445
Author: Vinod Kumar Vavilapalli vino...@apache.org
Authored: Tue Mar 3 16:28:22 2015 -0800
Committer: Vinod Kumar Vavilapalli vino...@apache.org
Committed: Tue Mar 3 16:29:36 2015 -0800
--
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../java/org/apache/hadoop/tools/DistCp.java| 47 +++-
2 files changed, 39 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6604613b/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index ad74745..ed3022a 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -72,6 +72,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-5612. Add javadoc for TaskCompletionEvent.Status.
(Chris Palmer via aajisaka)
+MAPREDUCE-6248. Exposed the internal MapReduce job's information as a
public
+API in DistCp. (Jing Zhao via vinodkv)
+
OPTIMIZATIONS
MAPREDUCE-6169. MergeQueue should release reference to the current item
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6604613b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
--
diff --git
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index d202f0a..08c88bf 100644
---
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.tools;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
@@ -51,12 +53,14 @@ import com.google.common.annotations.VisibleForTesting;
* launch the copy-job. DistCp may alternatively be sub-classed to fine-tune
* behaviour.
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class DistCp extends Configured implements Tool {
/**
- * Priority of the ResourceManager shutdown hook.
+ * Priority of the shutdown hook.
*/
- public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ static final int SHUTDOWN_HOOK_PRIORITY = 30;
private static final Log LOG = LogFactory.getLog(DistCp.class);
@@ -66,7 +70,7 @@ public class DistCp extends Configured implements Tool {
private static final String PREFIX = _distcp;
private static final String WIP_PREFIX = ._WIP_;
private static final String DISTCP_DEFAULT_XML = distcp-default.xml;
- public static final Random rand = new Random();
+ static final Random rand = new Random();
private boolean submitted;
private FileSystem jobFS;
@@ -90,7 +94,7 @@ public class DistCp extends Configured implements Tool {
* To be used with the ToolRunner. Not for public consumption.
*/
@VisibleForTesting
- public DistCp() {}
+ DistCp() {}
/**
* Implementation of Tool::run(). Orchestrates the copy of source file(s)
@@ -100,6 +104,7 @@ public class DistCp extends Configured implements Tool {
* @param argv List of arguments passed to DistCp, from the ToolRunner.
* @return On success, it returns 0. Else, -1.
*/
+ @Override
public int run(String[] argv) {
if (argv.length 1) {
OptionsParser.usage();
@@ -145,9 +150,21 @@ public class DistCp extends Configured implements Tool {
* @throws Exception, on failure.
*/
public Job execute() throws Exception {
+Job job = createAndSubmitJob();
+
+if (inputOptions.shouldBlock()) {
+ waitForJobCompletion(job);
+}
+return job;
+ }
+
+ /**
+ * Create and submit the mapreduce job.
+ * @return The mapreduce job object that has been submitted
+ */
+ public Job createAndSubmitJob() throws Exception {
assert inputOptions != null;
assert getConf() != null;
-
