MAPREDUCE-7059. Downward Compatibility issue: MR job fails because of unknown setErasureCodingPolicy method from 3.x client to HDFS 2.x cluster. Contributed by Jiandan Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4c90cec Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4c90cec Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4c90cec Branch: refs/heads/YARN-7055 Commit: c4c90cec4372cfceeb676c84522badcb454f87f9 Parents: e15126b Author: Weiwei Yang <w...@apache.org> Authored: Thu Mar 1 10:18:53 2018 +0800 Committer: Rohith Sharma K S <rohithsharm...@apache.org> Committed: Fri Mar 2 11:08:28 2018 +0530 ---------------------------------------------------------------------- .../hadoop/mapreduce/JobResourceUploader.java | 29 +++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4c90cec/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java index 03b29bd..1dbecb6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java @@ -37,6 +37,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.ipc.RpcNoSuchMethodException; import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -171,7 +173,7 @@ class JobResourceUploader { if (!conf.getBoolean(MRJobConfig.MR_AM_STAGING_DIR_ERASURECODING_ENABLED, MRJobConfig.DEFAULT_MR_AM_STAGING_ERASURECODING_ENABLED)) { - disableErasureCodingForPath(jtFs, submitJobDir); + disableErasureCodingForPath(submitJobDir); } // Get the resources that have been added via command line arguments in the @@ -874,13 +876,26 @@ class JobResourceUploader { return finalPath; } - private void disableErasureCodingForPath(FileSystem fs, Path path) + private void disableErasureCodingForPath(Path path) throws IOException { - if (jtFs instanceof DistributedFileSystem) { - LOG.info("Disabling Erasure Coding for path: " + path); - DistributedFileSystem dfs = (DistributedFileSystem) jtFs; - dfs.setErasureCodingPolicy(path, - SystemErasureCodingPolicies.getReplicationPolicy().getName()); + try { + if (jtFs instanceof DistributedFileSystem) { + LOG.info("Disabling Erasure Coding for path: " + path); + DistributedFileSystem dfs = (DistributedFileSystem) jtFs; + dfs.setErasureCodingPolicy(path, + SystemErasureCodingPolicies.getReplicationPolicy().getName()); + } + } catch (RemoteException e) { + if (!RpcNoSuchMethodException.class.getName().equals(e.getClassName())) { + throw e; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug( + "Ignore disabling erasure coding for path {} because method " + + "disableErasureCodingForPath doesn't exist, probably " + + "talking to a lower version HDFS.", path.toString(), e); + } + } } } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
