WIP on fixes.

Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8e5dcd2f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8e5dcd2f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8e5dcd2f

Branch: refs/heads/ignite-3909
Commit: 8e5dcd2f0f4d9da3585e5aa3c2ef4506db23d362
Parents: 2474e2b
Author: vozerov-gridgain <voze...@gridgain.com>
Authored: Thu Sep 15 18:27:11 2016 +0300
Committer: vozerov-gridgain <voze...@gridgain.com>
Committed: Thu Sep 15 18:27:11 2016 +0300

----------------------------------------------------------------------
 .../processors/hadoop/HadoopClassLoader.java    | 161 ++++++++++---------
 .../internal/processors/hadoop/HadoopUtils.java |   3 +
 .../processors/hadoop/v2/HadoopV2Job.java       |   2 +-
 .../hadoop/HadoopClassLoaderTest.java           |   4 +-
 4 files changed, 94 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 2e0e271..a921f03 100644
--- 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -61,13 +61,11 @@ import org.objectweb.asm.commons.RemappingClassAdapter;
  * unavailable for parent.
  */
 public class HadoopClassLoader extends URLClassLoader implements ClassCache {
-    static {
-        // We are very parallel capable.
-        registerAsParallelCapable();
-    }
+    /** Name of the Hadoop shutdown hook manager class. */
+    public static final String SHUTDOWN_HOOK_MGR_CLS_NAME = 
"org.apache.hadoop.util.ShutdownHookManager";
 
     /** Name of the Hadoop daemon class. */
-    public static final String HADOOP_DAEMON_CLASS_NAME = 
"org.apache.hadoop.util.Daemon";
+    public static final String HADOOP_DAEMON_CLS_NAME = 
"org.apache.hadoop.util.Daemon";
 
     /** Name of libhadoop library. */
     private static final String LIBHADOOP = "hadoop.";
@@ -97,6 +95,11 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
     /** Native library names. */
     private final String[] libNames;
 
+    static {
+        // We are very parallel capable.
+        registerAsParallelCapable();
+    }
+
     /**
      * Gets name for Job class loader. The name is specific for local node id.
      * @param locNodeId The local node id.
@@ -192,57 +195,19 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
         }
     }
 
-    /**
-     * Need to parse only Ignite Hadoop and IGFS classes.
-     *
-     * @param cls Class name.
-     * @return {@code true} if we need to check this class.
-     */
-    private static boolean isHadoopIgfs(String cls) {
-        String ignitePkgPrefix = "org.apache.ignite";
-
-        int len = ignitePkgPrefix.length();
-
-        return cls.startsWith(ignitePkgPrefix) && (
-            cls.indexOf("igfs.", len) != -1 ||
-            cls.indexOf(".fs.", len) != -1 ||
-            cls.indexOf("hadoop.", len) != -1);
-    }
-
-    /**
-     * @param cls Class name.
-     * @return {@code true} If this is Hadoop class.
-     */
-    private static boolean isHadoop(String cls) {
-        return cls.startsWith("org.apache.hadoop.");
-    }
-
     /** {@inheritDoc} */
     @Override protected Class<?> loadClass(String name, boolean resolve) 
throws ClassNotFoundException {
         try {
-            if (isHadoop(name)) { // Always load Hadoop classes explicitly, 
since Hadoop can be available in App classpath.
-                if (name.endsWith(".util.ShutdownHookManager"))  // Dirty hack 
to get rid of Hadoop shutdown hooks.
-                    return loadFromBytes(name, 
HadoopShutdownHookManager.class.getName());
-                else if (name.equals(HADOOP_DAEMON_CLASS_NAME))
-                    // We replace this in order to be able to forcibly stop 
some daemon threads
-                    // that otherwise never stop (e.g. PeerCache runnables):
-                    return loadFromBytes(name, HadoopDaemon.class.getName());
-
+            if (name.equals(SHUTDOWN_HOOK_MGR_CLS_NAME))
+                // Dirty hack to get rid of Hadoop shutdown hooks.
+                return loadFromBytes(name, 
HadoopShutdownHookManager.class.getName());
+            else if (name.equals(HADOOP_DAEMON_CLS_NAME))
+                // We replace this in order to be able to forcibly stop some 
daemon threads
+                // that otherwise never stop (e.g. PeerCache runnables):
+                return loadFromBytes(name, HadoopDaemon.class.getName());
+
+            if (hasExternalDependencies(name))
                 return loadClassExplicitly(name, resolve);
-            }
-
-            if (isHadoopIgfs(name)) { // For Ignite Hadoop and IGFS classes we 
have to check if they depend on Hadoop.
-                Boolean hasDeps = cache.get(name);
-
-                if (hasDeps == null) {
-                    hasDeps = hasExternalDependencies(name);
-
-                    cache.put(name, hasDeps);
-                }
-
-                if (hasDeps)
-                    return loadClassExplicitly(name, resolve);
-            }
 
             return super.loadClass(name, resolve);
         }
@@ -362,14 +327,39 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
      * @return {@code True} if class has external dependencies.
      */
     boolean hasExternalDependencies(String clsName) {
-        CollectingContext ctx = new CollectingContext();
+        Boolean res = cache.get(clsName);
+
+        if (res == null) {
+            CollectingContext ctx = new CollectingContext();
+
+            ctx.annVisitor = new CollectingAnnotationVisitor(ctx);
+            ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor);
+            ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor);
+            ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, 
ctx.mthdVisitor, ctx.fldVisitor);
+
+            return hasExternalDependencies(clsName, ctx);
+        }
+        else
+            return res;
+    }
+
+    /**
+     * Check whether class has external dependencies on Hadoop.
+     *
+     * @param clsName Class name.
+     * @param ctx Context.
+     * @return {@code True} if class has external dependencies.
+     */
+    private boolean hasExternalDependencies(String clsName, CollectingContext 
ctx) {
+        Boolean res = cache.get(clsName);
 
-        ctx.annVisitor = new CollectingAnnotationVisitor(ctx);
-        ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor);
-        ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor);
-        ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, 
ctx.mthdVisitor, ctx.fldVisitor);
+        if (res == null) {
+            res = hasExternalDependencies0(clsName, ctx);
 
-        return hasExternalDependencies(clsName, ctx);
+            cache.put(clsName, res);
+        }
+
+        return res;
     }
 
     /**
@@ -379,9 +369,13 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
      * @param ctx Context.
      * @return {@code true} If the class has external dependencies.
      */
-    boolean hasExternalDependencies(String clsName, CollectingContext ctx) {
-        if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea 
sucks, so filtering explicitly as external.
-            return true;
+    private boolean hasExternalDependencies0(String clsName, CollectingContext 
ctx) {
+        Boolean res = hasDependencyPredefined(clsName);
+
+        if (res != null)
+            return res;
+
+        System.out.println(">>> EXT: " + clsName);
 
         // Try to get from parent to check if the type accessible.
         InputStream in = loadClassBytes(getParent(), clsName);
@@ -389,16 +383,16 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
         if (in == null) // The class is external itself, it must be loaded 
from this class loader.
             return true;
 
-        if (!isHadoopIgfs(clsName)) // Other classes should not have external 
dependencies.
-            return false;
-
         final ClassReader rdr;
 
         try {
             rdr = new ClassReader(in);
         }
-        catch (IOException e) {
-            throw new RuntimeException("Failed to read class: " + clsName, e);
+        catch (Exception e) {
+            System.out.println(">>> ERR: " + clsName);
+
+            return false;
+            //throw new RuntimeException("Failed to read class: " + clsName, 
e);
         }
 
         ctx.visited.add(clsName);
@@ -416,15 +410,39 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
 
         String parentCls = clsName.substring(0, idx);
 
+        // TODO: Looks suspicious.
         if (ctx.visited.contains(parentCls))
             return false;
 
-        Boolean res = cache.get(parentCls);
+        return hasExternalDependencies(parentCls, ctx);
+    }
+
+    /**
+     * Whether we know in advance whether class has dependency or not.
+     *
+     * @param cls Class.
+     * @return Result.
+     */
+    @Nullable private static Boolean hasDependencyPredefined(String cls) {
+        // 1. Java systm classes never has dependencies.
+        if (cls.startsWith("java.") || cls.startsWith("javax.") || 
cls.startsWith("sun.") || cls.startsWith("com.sun."))
+            return false;
+
+        // 2. Some other well-known packages.
+        if (cls.startsWith("org.jsr166.") ||  cls.startsWith("org.w3c.") || 
cls.startsWith("org.xml.sax.") || cls.startsWith("org.slf4j.") || 
cls.startsWith("com.google.common."))
+            return false;
 
-        if (res == null)
-            res = hasExternalDependencies(parentCls, ctx);
+        // 3. Special handling for "org.apache"
+        if (cls.startsWith("org.apache.")) {
+            if (cls.startsWith("org.apache.ignite"))
+                return cls.contains(".hadoop.") || cls.contains(".igfs.") || 
cls.contains(".fs.");
 
-        return res;
+            if (cls.startsWith("org.apache.hadoop"))
+                return true;
+        }
+
+        // No more guesses, will parse the class.
+        return null;
     }
 
     /**
@@ -572,9 +590,6 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
             assert depCls.charAt(0) != 'L' : depCls;
             assert validateClassName(depCls) : depCls;
 
-            if (depCls.startsWith("java.") || depCls.startsWith("javax.")) // 
Filter out platform classes.
-                return;
-
             if (visited.contains(depCls))
                 return;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
index 65d9810..6d3866b 100644
--- 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
+++ 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
@@ -277,6 +277,9 @@ public class HadoopUtils {
         for (Map.Entry<String, String> entry : jobConf)
             props.put(entry.getKey(), entry.getValue());
 
+        props.put("fs.defaultFS", "dummy:///test/");
+        props.put("fs.dummy.impl", "com.mapr.DummyFileSystem");
+
         return new HadoopDefaultJobInfo(jobConf.getJobName(), 
jobConf.getUser(), hasCombiner, numReduces, props);
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
index a0f30eb..1264e48 100644
--- 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
+++ 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
@@ -382,7 +382,7 @@ public class HadoopV2Job implements HadoopJob {
      * @throws Exception On error.
      */
     private void stopHadoopFsDaemons(ClassLoader ldr) throws Exception {
-        Class<?> daemonCls = 
ldr.loadClass(HadoopClassLoader.HADOOP_DAEMON_CLASS_NAME);
+        Class<?> daemonCls = 
ldr.loadClass(HadoopClassLoader.HADOOP_DAEMON_CLS_NAME);
 
         Method m = daemonCls.getMethod("dequeueAndStopAll");
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
 
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
index 02d98d0..2fd7777 100644
--- 
a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
+++ 
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
@@ -94,7 +94,7 @@ public class HadoopClassLoaderTest extends TestCase {
             CircularWithoutHadoop.class,
         };
 
-        for (Class c: positiveClasses)
+        for (Class c : positiveClasses)
             assertTrue(c.getName(), ldr.hasExternalDependencies(c.getName()));
 
         // Negative cases:
@@ -104,7 +104,7 @@ public class HadoopClassLoaderTest extends TestCase {
             Without.class,
         };
 
-        for (Class c: negativeClasses)
+        for (Class c : negativeClasses)
             assertFalse(c.getName(), ldr.hasExternalDependencies(c.getName()));
     }
 }
\ No newline at end of file

Reply via email to