Repository: incubator-reef Updated Branches: refs/heads/master 871071203 -> f47abcf59
[REEF-204] Fixed failure to react to misconfigured Hadoop 2.6 clusters. YARN 2.6 added `YarnConfiguration.YARN_APPLICATION_CLASSPATH` and `DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH`. We assumed that their presence also indicates that they are configured correctly. However, HDInsight 3.2 uses Hadoop 2.6 but does NOT configure `YarnConfiguration.YARN_APPLICATION_CLASSPATH` correctly. This change checks whether this option is empty. If it it is, we now revert to the legacy behavior of Hadoop 2.4 and earlier: The classpath is constructed from environment variables (%HADOOP_HOME% and such). JIRA: [REEF-204](https://issues.apache.org/jira/browse/REEF-204) Pull Request: This closes #112 Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/f47abcf5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/f47abcf5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/f47abcf5 Branch: refs/heads/master Commit: f47abcf59a748d04f239cbbaea448a794504ae11 Parents: 8710712 Author: Beysim Sezgin <[email protected]> Authored: Fri Mar 13 16:49:40 2015 -0700 Committer: Markus Weimer <[email protected]> Committed: Fri Mar 13 17:28:58 2015 -0700 ---------------------------------------------------------------------- ...safeHDInsightRuntimeConfigurationStatic.java | 3 ++ .../reef/runtime/yarn/ClassPathBuilder.java | 16 +++++++- .../runtime/yarn/YarnClasspathProvider.java | 39 +++++++++++++++----- 3 files changed, 46 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/f47abcf5/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java index 37a46bf..cf7686d 100644 --- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java +++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java @@ -24,8 +24,10 @@ import org.apache.reef.client.RunningJob; import org.apache.reef.runtime.common.client.REEFImplementation; import org.apache.reef.runtime.common.client.RunningJobImpl; import org.apache.reef.runtime.common.client.api.JobSubmissionHandler; +import org.apache.reef.runtime.common.files.RuntimeClasspathProvider; import org.apache.reef.runtime.common.launch.REEFMessageCodec; import org.apache.reef.runtime.hdinsight.client.sslhacks.UnsafeClientConstructor; +import org.apache.reef.runtime.yarn.YarnClasspathProvider; import org.apache.reef.tang.formats.ConfigurationModule; import org.apache.reef.tang.formats.ConfigurationModuleBuilder; import org.apache.reef.util.logging.LoggingSetup; @@ -41,6 +43,7 @@ public final class UnsafeHDInsightRuntimeConfigurationStatic extends Configurati public static final ConfigurationModule CONF = new UnsafeHDInsightRuntimeConfigurationStatic() .bindImplementation(REEF.class, REEFImplementation.class) + .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class) .bindImplementation(RunningJob.class, RunningJobImpl.class) .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class) .bindImplementation(JobSubmissionHandler.class, HDInsightJobSubmissionHandler.class) http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/f47abcf5/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java index fa7780b..5a96b87 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java @@ -45,8 +45,8 @@ final class ClassPathBuilder { */ private static boolean couldBeYarnConfigurationPath(final String path) { return path.contains("conf") || - path.contains("etc") || - path.contains(HadoopEnvironment.HADOOP_CONF_DIR); + path.contains("etc") || + path.contains(HadoopEnvironment.HADOOP_CONF_DIR); } /** @@ -94,6 +94,18 @@ final class ClassPathBuilder { } /** + * Adds all the given entries to the classpath suffix. + * + * @param entries + */ + void addAllToSuffix(final String... entries) { + for (final String classPathEntry : entries) { + this.addToPrefix(classPathEntry); + } + } + + + /** * @return the suffix in an immutable list. */ List<String> getSuffixAsImmutableList() { http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/f47abcf5/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java index c5a83a3..0a732ad 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java @@ -23,6 +23,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.reef.runtime.common.files.RuntimeClasspathProvider; import org.apache.reef.util.OSUtils; +import org.apache.reef.util.Optional; import javax.inject.Inject; import java.util.List; @@ -47,7 +48,6 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider { // Used when we can't get a classpath from YARN private static final String[] LEGACY_CLASSPATH_LIST = new String[]{ - HADOOP_CONF_DIR, HADOOP_HOME + "/*", HADOOP_HOME + "/lib/*", HADOOP_COMMON_HOME + "/*", @@ -76,19 +76,21 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider { boolean needsLegacyClasspath = false; // will be set to true below whenever we encounter issues with the YARN Configuration final ClassPathBuilder builder = new ClassPathBuilder(); + try { // Add the classpath actually configured on this cluster - final String[] yarnClassPath = yarnConfiguration.getTrimmedStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH); - if (null == yarnClassPath || yarnClassPath.length == 0) { - needsLegacyClasspath = true; - } else { - builder.addAll(yarnClassPath); - } + final Optional<String[]> yarnClassPath = getTrimmedStrings(yarnConfiguration, YarnConfiguration.YARN_APPLICATION_CLASSPATH); final String[] yarnDefaultClassPath = YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH; - if (null == yarnDefaultClassPath || yarnDefaultClassPath.length == 0) { - LOG.log(Level.SEVERE, "YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH is empty. This indicates a broken cluster configuration"); + + if (!yarnClassPath.isPresent()) { + LOG.log(Level.SEVERE, "YarnConfiguration.YARN_APPLICATION_CLASSPATH is empty. This indicates a broken cluster configuration."); needsLegacyClasspath = true; } else { + if (LOG.isLoggable(CLASSPATH_LOG_LEVEL)) { + LOG.log(CLASSPATH_LOG_LEVEL, "YarnConfiguration.YARN_APPLICATION_CLASSPATH is [" + StringUtils.join(yarnClassPath.get(), '|') + "]"); + LOG.log(CLASSPATH_LOG_LEVEL, "YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH is [" + StringUtils.join(yarnDefaultClassPath, '|') + "]"); + } + builder.addAll(yarnClassPath.get()); builder.addAll(yarnDefaultClassPath); } } catch (final NoSuchFieldError e) { @@ -99,7 +101,8 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider { } if (needsLegacyClasspath) { - builder.addAll(LEGACY_CLASSPATH_LIST); + builder.addToPrefix(HADOOP_CONF_DIR); + builder.addAllToSuffix(LEGACY_CLASSPATH_LIST); } this.classPathPrefix = builder.getPrefixAsImmutableList(); @@ -107,6 +110,22 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider { this.logClasspath(); } + /** + * Fetches the string[] under the given key, if it exists and contains entries (.length >0) + * + * @param configuration + * @param key + * @return + */ + private static Optional<String[]> getTrimmedStrings(final YarnConfiguration configuration, final String key) { + final String[] result = configuration.getTrimmedStrings(key); + if (null == result || result.length == 0) { + return Optional.empty(); + } else { + return Optional.of(result); + } + } + @Override public List<String> getDriverClasspathPrefix() { return this.classPathPrefix;
