WIP on fixes.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8e5dcd2f Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8e5dcd2f Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8e5dcd2f Branch: refs/heads/ignite-3909 Commit: 8e5dcd2f0f4d9da3585e5aa3c2ef4506db23d362 Parents: 2474e2b Author: vozerov-gridgain <voze...@gridgain.com> Authored: Thu Sep 15 18:27:11 2016 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Thu Sep 15 18:27:11 2016 +0300 ---------------------------------------------------------------------- .../processors/hadoop/HadoopClassLoader.java | 161 ++++++++++--------- .../internal/processors/hadoop/HadoopUtils.java | 3 + .../processors/hadoop/v2/HadoopV2Job.java | 2 +- .../hadoop/HadoopClassLoaderTest.java | 4 +- 4 files changed, 94 insertions(+), 76 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java index 2e0e271..a921f03 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java @@ -61,13 +61,11 @@ import org.objectweb.asm.commons.RemappingClassAdapter; * unavailable for parent. */ public class HadoopClassLoader extends URLClassLoader implements ClassCache { - static { - // We are very parallel capable. - registerAsParallelCapable(); - } + /** Name of the Hadoop shutdown hook manager class. */ + public static final String SHUTDOWN_HOOK_MGR_CLS_NAME = "org.apache.hadoop.util.ShutdownHookManager"; /** Name of the Hadoop daemon class. */ - public static final String HADOOP_DAEMON_CLASS_NAME = "org.apache.hadoop.util.Daemon"; + public static final String HADOOP_DAEMON_CLS_NAME = "org.apache.hadoop.util.Daemon"; /** Name of libhadoop library. */ private static final String LIBHADOOP = "hadoop."; @@ -97,6 +95,11 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { /** Native library names. */ private final String[] libNames; + static { + // We are very parallel capable. + registerAsParallelCapable(); + } + /** * Gets name for Job class loader. The name is specific for local node id. * @param locNodeId The local node id. @@ -192,57 +195,19 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { } } - /** - * Need to parse only Ignite Hadoop and IGFS classes. - * - * @param cls Class name. - * @return {@code true} if we need to check this class. - */ - private static boolean isHadoopIgfs(String cls) { - String ignitePkgPrefix = "org.apache.ignite"; - - int len = ignitePkgPrefix.length(); - - return cls.startsWith(ignitePkgPrefix) && ( - cls.indexOf("igfs.", len) != -1 || - cls.indexOf(".fs.", len) != -1 || - cls.indexOf("hadoop.", len) != -1); - } - - /** - * @param cls Class name. - * @return {@code true} If this is Hadoop class. - */ - private static boolean isHadoop(String cls) { - return cls.startsWith("org.apache.hadoop."); - } - /** {@inheritDoc} */ @Override protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException { try { - if (isHadoop(name)) { // Always load Hadoop classes explicitly, since Hadoop can be available in App classpath. - if (name.endsWith(".util.ShutdownHookManager")) // Dirty hack to get rid of Hadoop shutdown hooks. - return loadFromBytes(name, HadoopShutdownHookManager.class.getName()); - else if (name.equals(HADOOP_DAEMON_CLASS_NAME)) - // We replace this in order to be able to forcibly stop some daemon threads - // that otherwise never stop (e.g. PeerCache runnables): - return loadFromBytes(name, HadoopDaemon.class.getName()); - + if (name.equals(SHUTDOWN_HOOK_MGR_CLS_NAME)) + // Dirty hack to get rid of Hadoop shutdown hooks. + return loadFromBytes(name, HadoopShutdownHookManager.class.getName()); + else if (name.equals(HADOOP_DAEMON_CLS_NAME)) + // We replace this in order to be able to forcibly stop some daemon threads + // that otherwise never stop (e.g. PeerCache runnables): + return loadFromBytes(name, HadoopDaemon.class.getName()); + + if (hasExternalDependencies(name)) return loadClassExplicitly(name, resolve); - } - - if (isHadoopIgfs(name)) { // For Ignite Hadoop and IGFS classes we have to check if they depend on Hadoop. - Boolean hasDeps = cache.get(name); - - if (hasDeps == null) { - hasDeps = hasExternalDependencies(name); - - cache.put(name, hasDeps); - } - - if (hasDeps) - return loadClassExplicitly(name, resolve); - } return super.loadClass(name, resolve); } @@ -362,14 +327,39 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { * @return {@code True} if class has external dependencies. */ boolean hasExternalDependencies(String clsName) { - CollectingContext ctx = new CollectingContext(); + Boolean res = cache.get(clsName); + + if (res == null) { + CollectingContext ctx = new CollectingContext(); + + ctx.annVisitor = new CollectingAnnotationVisitor(ctx); + ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor); + ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor); + ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, ctx.mthdVisitor, ctx.fldVisitor); + + return hasExternalDependencies(clsName, ctx); + } + else + return res; + } + + /** + * Check whether class has external dependencies on Hadoop. + * + * @param clsName Class name. + * @param ctx Context. + * @return {@code True} if class has external dependencies. + */ + private boolean hasExternalDependencies(String clsName, CollectingContext ctx) { + Boolean res = cache.get(clsName); - ctx.annVisitor = new CollectingAnnotationVisitor(ctx); - ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor); - ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor); - ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, ctx.mthdVisitor, ctx.fldVisitor); + if (res == null) { + res = hasExternalDependencies0(clsName, ctx); - return hasExternalDependencies(clsName, ctx); + cache.put(clsName, res); + } + + return res; } /** @@ -379,9 +369,13 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { * @param ctx Context. * @return {@code true} If the class has external dependencies. */ - boolean hasExternalDependencies(String clsName, CollectingContext ctx) { - if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea sucks, so filtering explicitly as external. - return true; + private boolean hasExternalDependencies0(String clsName, CollectingContext ctx) { + Boolean res = hasDependencyPredefined(clsName); + + if (res != null) + return res; + + System.out.println(">>> EXT: " + clsName); // Try to get from parent to check if the type accessible. InputStream in = loadClassBytes(getParent(), clsName); @@ -389,16 +383,16 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { if (in == null) // The class is external itself, it must be loaded from this class loader. return true; - if (!isHadoopIgfs(clsName)) // Other classes should not have external dependencies. - return false; - final ClassReader rdr; try { rdr = new ClassReader(in); } - catch (IOException e) { - throw new RuntimeException("Failed to read class: " + clsName, e); + catch (Exception e) { + System.out.println(">>> ERR: " + clsName); + + return false; + //throw new RuntimeException("Failed to read class: " + clsName, e); } ctx.visited.add(clsName); @@ -416,15 +410,39 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { String parentCls = clsName.substring(0, idx); + // TODO: Looks suspicious. if (ctx.visited.contains(parentCls)) return false; - Boolean res = cache.get(parentCls); + return hasExternalDependencies(parentCls, ctx); + } + + /** + * Whether we know in advance whether class has dependency or not. + * + * @param cls Class. + * @return Result. + */ + @Nullable private static Boolean hasDependencyPredefined(String cls) { + // 1. Java systm classes never has dependencies. + if (cls.startsWith("java.") || cls.startsWith("javax.") || cls.startsWith("sun.") || cls.startsWith("com.sun.")) + return false; + + // 2. Some other well-known packages. + if (cls.startsWith("org.jsr166.") || cls.startsWith("org.w3c.") || cls.startsWith("org.xml.sax.") || cls.startsWith("org.slf4j.") || cls.startsWith("com.google.common.")) + return false; - if (res == null) - res = hasExternalDependencies(parentCls, ctx); + // 3. Special handling for "org.apache" + if (cls.startsWith("org.apache.")) { + if (cls.startsWith("org.apache.ignite")) + return cls.contains(".hadoop.") || cls.contains(".igfs.") || cls.contains(".fs."); - return res; + if (cls.startsWith("org.apache.hadoop")) + return true; + } + + // No more guesses, will parse the class. + return null; } /** @@ -572,9 +590,6 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { assert depCls.charAt(0) != 'L' : depCls; assert validateClassName(depCls) : depCls; - if (depCls.startsWith("java.") || depCls.startsWith("javax.")) // Filter out platform classes. - return; - if (visited.contains(depCls)) return; http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java index 65d9810..6d3866b 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java @@ -277,6 +277,9 @@ public class HadoopUtils { for (Map.Entry<String, String> entry : jobConf) props.put(entry.getKey(), entry.getValue()); + props.put("fs.defaultFS", "dummy:///test/"); + props.put("fs.dummy.impl", "com.mapr.DummyFileSystem"); + return new HadoopDefaultJobInfo(jobConf.getJobName(), jobConf.getUser(), hasCombiner, numReduces, props); } http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java index a0f30eb..1264e48 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java @@ -382,7 +382,7 @@ public class HadoopV2Job implements HadoopJob { * @throws Exception On error. */ private void stopHadoopFsDaemons(ClassLoader ldr) throws Exception { - Class<?> daemonCls = ldr.loadClass(HadoopClassLoader.HADOOP_DAEMON_CLASS_NAME); + Class<?> daemonCls = ldr.loadClass(HadoopClassLoader.HADOOP_DAEMON_CLS_NAME); Method m = daemonCls.getMethod("dequeueAndStopAll"); http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java index 02d98d0..2fd7777 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java @@ -94,7 +94,7 @@ public class HadoopClassLoaderTest extends TestCase { CircularWithoutHadoop.class, }; - for (Class c: positiveClasses) + for (Class c : positiveClasses) assertTrue(c.getName(), ldr.hasExternalDependencies(c.getName())); // Negative cases: @@ -104,7 +104,7 @@ public class HadoopClassLoaderTest extends TestCase { Without.class, }; - for (Class c: negativeClasses) + for (Class c : negativeClasses) assertFalse(c.getName(), ldr.hasExternalDependencies(c.getName())); } } \ No newline at end of file
