[03/32] git commit: ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that were created before the VFS ClassLoader was created.
ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that were created before the VFS ClassLoader was created. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/db1b63c4 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/db1b63c4 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/db1b63c4 Branch: refs/heads/ACCUMULO-378 Commit: db1b63c4cb9f2f1bfdb45765b9c79f84105d601a Parents: 1193f4b Author: Dave Marion dlmar...@hotmail.com Authored: Wed May 21 21:11:26 2014 -0400 Committer: Dave Marion dlmar...@hotmail.com Committed: Wed May 21 21:11:26 2014 -0400 -- .../classloader/vfs/AccumuloVFSClassLoader.java | 19 +++ 1 file changed, 19 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/db1b63c4/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java -- diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java index fa20725..d4044a5 100644 --- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java +++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.List; import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileObject; import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider; import org.apache.commons.vfs2.CacheStrategy; import org.apache.commons.vfs2.FileObject; @@ -37,6 +38,8 @@ import org.apache.commons.vfs2.cache.SoftRefFilesCache; import org.apache.commons.vfs2.impl.DefaultFileSystemManager; import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory; import org.apache.commons.vfs2.impl.VFSClassLoader; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.Logger; /** @@ -213,6 +216,22 @@ public class AccumuloVFSClassLoader { // Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, vfs, parent)); loader = localLoader; + + //An HDFS FileSystem and Configuration object were created for each unique HDFS namespace in the call to resolve above. + //The HDFS Client did us a favor and cached these objects so that the next time someone calls FileSystem.get(uri), they + //get the cached object. However, these objects were created not with the system VFS classloader, but the classloader above + //it. We need to override the classloader on the Configuration objects. Ran into an issue were log recovery was being attempted + //and SequenceFile$Reader was trying to instantiate the key class via WritableName.getClass(String, Configuration) + for (FileObject fo : vfsCP) { +if (fo instanceof HdfsFileObject) { + String uri = fo.getName().getRootURI(); + Configuration c = new Configuration(true); + c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri); + FileSystem fs = FileSystem.get(c); + fs.getConf().setClassLoader(loader.getClassLoader()); +} + } + } } }
git commit: ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that were created before the VFS ClassLoader was created.
Repository: accumulo Updated Branches: refs/heads/1.6.1-SNAPSHOT 1193f4b84 - db1b63c4c ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that were created before the VFS ClassLoader was created. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/db1b63c4 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/db1b63c4 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/db1b63c4 Branch: refs/heads/1.6.1-SNAPSHOT Commit: db1b63c4cb9f2f1bfdb45765b9c79f84105d601a Parents: 1193f4b Author: Dave Marion dlmar...@hotmail.com Authored: Wed May 21 21:11:26 2014 -0400 Committer: Dave Marion dlmar...@hotmail.com Committed: Wed May 21 21:11:26 2014 -0400 -- .../classloader/vfs/AccumuloVFSClassLoader.java | 19 +++ 1 file changed, 19 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/db1b63c4/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java -- diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java index fa20725..d4044a5 100644 --- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java +++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.List; import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileObject; import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider; import org.apache.commons.vfs2.CacheStrategy; import org.apache.commons.vfs2.FileObject; @@ -37,6 +38,8 @@ import org.apache.commons.vfs2.cache.SoftRefFilesCache; import org.apache.commons.vfs2.impl.DefaultFileSystemManager; import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory; import org.apache.commons.vfs2.impl.VFSClassLoader; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.Logger; /** @@ -213,6 +216,22 @@ public class AccumuloVFSClassLoader { // Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, vfs, parent)); loader = localLoader; + + //An HDFS FileSystem and Configuration object were created for each unique HDFS namespace in the call to resolve above. + //The HDFS Client did us a favor and cached these objects so that the next time someone calls FileSystem.get(uri), they + //get the cached object. However, these objects were created not with the system VFS classloader, but the classloader above + //it. We need to override the classloader on the Configuration objects. Ran into an issue were log recovery was being attempted + //and SequenceFile$Reader was trying to instantiate the key class via WritableName.getClass(String, Configuration) + for (FileObject fo : vfsCP) { +if (fo instanceof HdfsFileObject) { + String uri = fo.getName().getRootURI(); + Configuration c = new Configuration(true); + c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri); + FileSystem fs = FileSystem.get(c); + fs.getConf().setClassLoader(loader.getClassLoader()); +} + } + } } }
[1/2] git commit: ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that were created before the VFS ClassLoader was created.
Repository: accumulo Updated Branches: refs/heads/master f985d3c3b - 58e7665ae ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that were created before the VFS ClassLoader was created. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/db1b63c4 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/db1b63c4 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/db1b63c4 Branch: refs/heads/master Commit: db1b63c4cb9f2f1bfdb45765b9c79f84105d601a Parents: 1193f4b Author: Dave Marion dlmar...@hotmail.com Authored: Wed May 21 21:11:26 2014 -0400 Committer: Dave Marion dlmar...@hotmail.com Committed: Wed May 21 21:11:26 2014 -0400 -- .../classloader/vfs/AccumuloVFSClassLoader.java | 19 +++ 1 file changed, 19 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/db1b63c4/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java -- diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java index fa20725..d4044a5 100644 --- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java +++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.List; import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileObject; import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider; import org.apache.commons.vfs2.CacheStrategy; import org.apache.commons.vfs2.FileObject; @@ -37,6 +38,8 @@ import org.apache.commons.vfs2.cache.SoftRefFilesCache; import org.apache.commons.vfs2.impl.DefaultFileSystemManager; import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory; import org.apache.commons.vfs2.impl.VFSClassLoader; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.Logger; /** @@ -213,6 +216,22 @@ public class AccumuloVFSClassLoader { // Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, vfs, parent)); loader = localLoader; + + //An HDFS FileSystem and Configuration object were created for each unique HDFS namespace in the call to resolve above. + //The HDFS Client did us a favor and cached these objects so that the next time someone calls FileSystem.get(uri), they + //get the cached object. However, these objects were created not with the system VFS classloader, but the classloader above + //it. We need to override the classloader on the Configuration objects. Ran into an issue were log recovery was being attempted + //and SequenceFile$Reader was trying to instantiate the key class via WritableName.getClass(String, Configuration) + for (FileObject fo : vfsCP) { +if (fo instanceof HdfsFileObject) { + String uri = fo.getName().getRootURI(); + Configuration c = new Configuration(true); + c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri); + FileSystem fs = FileSystem.get(c); + fs.getConf().setClassLoader(loader.getClassLoader()); +} + } + } } }