Repository: ignite Updated Branches: refs/heads/ignite-3909 5fa1e6068 -> a6ed71b2b
Finalization. Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a68826dd Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a68826dd Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a68826dd Branch: refs/heads/ignite-3909 Commit: a68826dd9fb0383e61f17bb72e26c90c3856bbca Parents: 5fa1e60 Author: vozerov-gridgain <voze...@gridgain.com> Authored: Thu Sep 15 19:14:02 2016 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Thu Sep 15 19:14:02 2016 +0300 ---------------------------------------------------------------------- .../src/main/java/com/mapr/DummyFileSystem.java | 99 ++++++++++++++++++++ .../processors/hadoop/HadoopClassLoader.java | 71 ++++++++++---- 2 files changed, 150 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/a68826dd/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java b/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java new file mode 100644 index 0000000..63312c8 --- /dev/null +++ b/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java @@ -0,0 +1,99 @@ +package com.mapr; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Progressable; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URI; + +/** + * Created by vozerov on 9/15/2016. + */ +public class DummyFileSystem extends FileSystem { + + public DummyFileSystem() throws IOException { + try { + Method m = UserGroupInformation.class.getDeclaredMethod("reset"); + + m.setAccessible(true); + + m.invoke(null); + + Field f = Groups.class.getDeclaredField("GROUPS"); + + f.setAccessible(true); + + f.set(null, null); + } + catch (Exception ignore) { + throw new RuntimeException(ignore); + } + + UserGroupInformation.getLoginUser(); + } + + @Override + public URI getUri() { + return null; + } + + @Override + public FSDataInputStream open(Path f, int bufferSize) throws IOException { + return null; + } + + @Override + public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { + return null; + } + + @Override + public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException { + return null; + } + + @Override + public boolean rename(Path src, Path dst) throws IOException { + return false; + } + + @Override + public boolean delete(Path f, boolean recursive) throws IOException { + return false; + } + + @Override + public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException { + return new FileStatus[0]; + } + + @Override + public void setWorkingDirectory(Path new_dir) { + + } + + @Override + public Path getWorkingDirectory() { + return null; + } + + @Override + public boolean mkdirs(Path f, FsPermission permission) throws IOException { + return false; + } + + @Override + public FileStatus getFileStatus(Path f) throws IOException { + return null; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/a68826dd/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java index c0b671e..f457ffa 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java @@ -410,43 +410,74 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { String parentCls = clsName.substring(0, idx); - // TODO: Looks suspicious. - if (ctx.visited.contains(parentCls)) - return false; - return hasExternalDependencies(parentCls, ctx); } /** + * Check whether class name starts with particular prefix. + * + * @param cls Class name. + * @param skip Prefix to skip. + * @param prefix Prefix. + * @return {@code True} if starts with. + */ + private static boolean startsWith(String cls, String skip, String prefix) { + assert cls.startsWith(skip); + + return cls.startsWith(prefix, skip.length()); + } + + private static final String PKG_ORG = "org."; + + private static final String PKG_ORG_APACHE = "org.apache."; + + /** * Whether we know in advance whether class has dependency or not. * * @param cls Class. * @return Result. */ @Nullable private static Boolean hasDependencyPredefined(String cls) { - // 1. Java systm classes never has dependencies. - if (cls.startsWith("java.") || cls.startsWith("javax.") || cls.startsWith("sun.") || cls.startsWith("com.sun.")) - return false; - - // 2. Some other well-known packages. - if (cls.startsWith("org.jsr166.") || cls.startsWith("org.w3c.") || cls.startsWith("org.xml.sax.") || cls.startsWith("org.slf4j.") || cls.startsWith("com.google.common.")) - return false; + // Large "org" group. + if (cls.startsWith(PKG_ORG)) { + // Large "apache" group. + if (cls.startsWith(PKG_ORG_APACHE)) { + // Hadoop classes always have dependencies. + if (startsWith(cls, PKG_ORG_APACHE, "hadoop.")) + return true; + + // Filter out Ignite classes which definitely do not have dependencies. + if (startsWith(cls, PKG_ORG_APACHE, "ignite.")) { + if (!cls.contains(".hadoop.") && !cls.contains(".igfs.") && !cls.contains(".fs.")) + return false; + } - // 3. Special handling for "org.apache" - if (cls.startsWith("org.apache.")) { - if (cls.startsWith("org.apache.ignite")) { - if (!cls.contains(".hadoop.") && !cls.contains(".igfs.") && !cls.contains(".fs.")) + // Other well-known "org.apache" packages. + if (startsWith(cls, PKG_ORG_APACHE, "commons.") || + startsWith(cls, PKG_ORG_APACHE, "log4j.") || + startsWith(cls, PKG_ORG_APACHE, "xerces.")) return false; } - if (cls.startsWith("org.apache.hadoop")) - return true; - - if (cls.startsWith("org.apache.xerces") || cls.startsWith("org.apache.log4j") || cls.startsWith("org.apache.commons.logging") || cls.startsWith("org.apache.commons.lang") || cls.startsWith("org.apache.commons.collections") - || cls.startsWith("org.apache.commons.configuration")) + // Other well-known "org" packages. + if (startsWith(cls, PKG_ORG, "jsr166.") || + startsWith(cls, PKG_ORG, "w3c.") || + startsWith(cls, PKG_ORG, "slf4j.") || + startsWith(cls, PKG_ORG, "xml.sax.")) return false; } + // Filter out Java system packages. + if (cls.startsWith("java.") || + cls.startsWith("javax.") || + cls.startsWith("sun.") || + cls.startsWith("com.sun.")) + return false; + + // Other well-known packages. + if (cls.startsWith("com.google.common")) + return false; + // No more guesses, will parse the class. return null; }