Repository: storm Updated Branches: refs/heads/master 90ca7fa0c -> bfa26e4e5
STORM-2671: remove the storm rename hack Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fe961e6b Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fe961e6b Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fe961e6b Branch: refs/heads/master Commit: fe961e6bf6c810b8cb4f18bd274e49ab889cce6a Parents: 6ae02e4 Author: Robert (Bobby) Evans <[email protected]> Authored: Wed Aug 2 10:01:15 2017 -0500 Committer: Robert (Bobby) Evans <[email protected]> Committed: Wed Aug 2 10:37:13 2017 -0500 ---------------------------------------------------------------------- bin/storm.py | 45 +-- pom.xml | 1 - .../daemon/ClientJarTransformerRunner.java | 49 --- .../org/apache/storm/daemon/JarTransformer.java | 31 -- storm-rename-hack/pom.xml | 131 ------- .../org/apache/storm/hack/DefaultShader.java | 391 ------------------- .../main/java/org/apache/storm/hack/IOUtil.java | 41 -- .../org/apache/storm/hack/ShadeRequest.java | 69 ---- .../apache/storm/hack/StormShadeRequest.java | 41 -- .../storm/hack/StormShadeTransformer.java | 32 -- .../apache/storm/hack/relocation/Relocator.java | 40 -- .../storm/hack/relocation/SimpleRelocator.java | 97 ----- .../storm/hack/resource/ClojureTransformer.java | 71 ---- .../hack/resource/ResourceTransformer.java | 46 --- .../java/org/apache/storm/DaemonConfig.java | 10 - 15 files changed, 12 insertions(+), 1083 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/bin/storm.py ---------------------------------------------------------------------- diff --git a/bin/storm.py b/bin/storm.py index 5de49f9..a97f9b3 100755 --- a/bin/storm.py +++ b/bin/storm.py @@ -303,39 +303,18 @@ def run_client_jar(jarfile, klass, args, daemon=False, client=True, extrajvmopts local_jars = DEP_JARS_OPTS artifact_to_file_jars = resolve_dependencies(DEP_ARTIFACTS_OPTS, DEP_ARTIFACTS_REPOSITORIES_OPTS, DEP_PROXY_URL, DEP_PROXY_USERNAME, DEP_PROXY_PASSWORD) - transform_class = confvalue("client.jartransformer.class", [CLUSTER_CONF_DIR]) - if (transform_class != None and transform_class != "null"): - tmpjar = os.path.join(tempfile.gettempdir(), uuid.uuid1().hex+".jar") - exec_storm_class("org.apache.storm.daemon.ClientJarTransformerRunner", args=[transform_class, jarfile, tmpjar], fork=True, daemon=False) - extra_jars = [tmpjar, USER_CONF_DIR, STORM_BIN_DIR] - extra_jars.extend(local_jars) - extra_jars.extend(artifact_to_file_jars.values()) - topology_runner_exit_code = exec_storm_class( - klass, - jvmtype="-client", - extrajars=extra_jars, - args=args, - daemon=daemon, - client=client, - fork=True, - jvmopts=JAR_JVM_OPTS + extrajvmopts + ["-Dstorm.jar=" + tmpjar] + - ["-Dstorm.dependency.jars=" + ",".join(local_jars)] + - ["-Dstorm.dependency.artifacts=" + json.dumps(artifact_to_file_jars)]) - os.remove(tmpjar) - sys.exit(topology_runner_exit_code) - else: - extra_jars=[jarfile, USER_CONF_DIR, STORM_BIN_DIR] - extra_jars.extend(local_jars) - extra_jars.extend(artifact_to_file_jars.values()) - exec_storm_class( - klass, - jvmtype="-client", - extrajars=extra_jars, - args=args, - daemon=False, - jvmopts=JAR_JVM_OPTS + extrajvmopts + ["-Dstorm.jar=" + jarfile] + - ["-Dstorm.dependency.jars=" + ",".join(local_jars)] + - ["-Dstorm.dependency.artifacts=" + json.dumps(artifact_to_file_jars)]) + extra_jars=[jarfile, USER_CONF_DIR, STORM_BIN_DIR] + extra_jars.extend(local_jars) + extra_jars.extend(artifact_to_file_jars.values()) + exec_storm_class( + klass, + jvmtype="-client", + extrajars=extra_jars, + args=args, + daemon=False, + jvmopts=JAR_JVM_OPTS + extrajvmopts + ["-Dstorm.jar=" + jarfile] + + ["-Dstorm.dependency.jars=" + ",".join(local_jars)] + + ["-Dstorm.dependency.artifacts=" + json.dumps(artifact_to_file_jars)]) def local(jarfile, klass, *args): """Syntax: [storm local topology-jar-path class ...] http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 9d890d9..319854b 100644 --- a/pom.xml +++ b/pom.xml @@ -329,7 +329,6 @@ <module>storm-server</module> <module>storm-core</module> <module>storm-webapp</module> - <module>storm-rename-hack</module> <module>storm-clojure</module> <module>storm-clojure-test</module> <module>storm-submit-tools</module> http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-core/src/jvm/org/apache/storm/daemon/ClientJarTransformerRunner.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/org/apache/storm/daemon/ClientJarTransformerRunner.java b/storm-core/src/jvm/org/apache/storm/daemon/ClientJarTransformerRunner.java deleted file mode 100644 index 3916468..0000000 --- a/storm-core/src/jvm/org/apache/storm/daemon/ClientJarTransformerRunner.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.daemon; - -import org.apache.storm.utils.ReflectionUtils; - -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.InputStream; - -/** - * Main executable to load and run a jar transformer - */ -public class ClientJarTransformerRunner { - public static void main(String [] args) throws IOException { - JarTransformer transformer = newJarTransformerInstance(args[0]); - InputStream in = new FileInputStream(args[1]); - OutputStream out = new FileOutputStream(args[2]); - transformer.transform(in, out); - in.close(); - out.close(); - } - - private static JarTransformer newJarTransformerInstance(String klass) { - JarTransformer ret = null; - if (klass != null) { - ret = (JarTransformer) ReflectionUtils.newInstance(klass); - } - return ret; - } -} http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-core/src/jvm/org/apache/storm/daemon/JarTransformer.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/org/apache/storm/daemon/JarTransformer.java b/storm-core/src/jvm/org/apache/storm/daemon/JarTransformer.java deleted file mode 100644 index fc4e2af..0000000 --- a/storm-core/src/jvm/org/apache/storm/daemon/JarTransformer.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.daemon; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -/** - * A plugin that can be used to transform a jar file in nimbus before it - * is used by a topology. - */ -public interface JarTransformer { - public void transform(InputStream input, OutputStream output) throws IOException; -} http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-rename-hack/pom.xml ---------------------------------------------------------------------- diff --git a/storm-rename-hack/pom.xml b/storm-rename-hack/pom.xml deleted file mode 100644 index d602872..0000000 --- a/storm-rename-hack/pom.xml +++ /dev/null @@ -1,131 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <artifactId>storm</artifactId> - <groupId>org.apache.storm</groupId> - <version>2.0.0-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - - - <artifactId>storm-rename-hack</artifactId> - <packaging>jar</packaging> - <name>storm-rename-hack</name> - - <properties> - <mavenVersion>3.0</mavenVersion> - <asmVersion>5.0.2</asmVersion> - </properties> - - <dependencies> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-core</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm</artifactId> - <version>${asmVersion}</version> - </dependency> - <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm-commons</artifactId> - <version>${asmVersion}</version> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <keepDependenciesWithProvidedScope>false</keepDependenciesWithProvidedScope> - <createDependencyReducedPom>true</createDependencyReducedPom> - <artifactSet> - <includes> - <include>com.google.guava:guava</include> - <include>org.ow2.asm:asm</include> - <include>org.ow2.asm:asm-commons</include> - </includes> - </artifactSet> - <relocations> - <relocation> - <pattern>com.google</pattern> - <shadedPattern>org.apache.storm.hack.shade.com.google</shadedPattern> - </relocation> - <relocation> - <pattern>org.objectweb.asm</pattern> - <shadedPattern>org.apache.storm.hack.shade.org.objectweb.asm</shadedPattern> - </relocation> - </relocations> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.sf</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.dsa</exclude> - <exclude>META-INF/*.RSA</exclude> - <exclude>META-INF/*.rsa</exclude> - <exclude>META-INF/*.EC</exclude> - <exclude>META-INF/*.ec</exclude> - <exclude>META-INF/MSFTSIG.SF</exclude> - <exclude>META-INF/MSFTSIG.RSA</exclude> - </excludes> - </filter> - </filters> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <transformers> - <transformer - implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> - <transformer - implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - </transformer> - </transformers> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-checkstyle-plugin</artifactId> - <!--Note - the version would be inherited--> - <configuration> - <maxAllowedViolations>408</maxAllowedViolations> - </configuration> - </plugin> - </plugins> - </build> -</project> http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-rename-hack/src/main/java/org/apache/storm/hack/DefaultShader.java ---------------------------------------------------------------------- diff --git a/storm-rename-hack/src/main/java/org/apache/storm/hack/DefaultShader.java b/storm-rename-hack/src/main/java/org/apache/storm/hack/DefaultShader.java deleted file mode 100644 index f6867b7..0000000 --- a/storm-rename-hack/src/main/java/org/apache/storm/hack/DefaultShader.java +++ /dev/null @@ -1,391 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.storm.hack; - -import org.apache.storm.hack.relocation.Relocator; -import org.apache.storm.hack.resource.ResourceTransformer; -import org.objectweb.asm.ClassReader; -import org.objectweb.asm.ClassVisitor; -import org.objectweb.asm.ClassWriter; -import org.objectweb.asm.commons.Remapper; -import org.objectweb.asm.commons.RemappingClassAdapter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.jar.JarEntry; -import java.util.jar.JarInputStream; -import java.util.jar.JarOutputStream; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.zip.ZipException; - -/** - * This is based off of - * - * https://github.com/apache/maven-plugins.git - * - * maven-shade-plugin-2.4.1 - */ -public class DefaultShader { - private static final Logger LOG = LoggerFactory.getLogger(DefaultShader.class); - - public void shadeJarStream(ShadeRequest shadeRequest, InputStream in, final OutputStream fileOutputStream) - throws IOException { - Set<String> resources = new HashSet<>(); - - List<ResourceTransformer> transformers = - new ArrayList<>( shadeRequest.getResourceTransformers() ); - LOG.debug("Transformers {}", transformers); - - RelocatorRemapper remapper = new RelocatorRemapper( shadeRequest.getRelocators() ); - LOG.debug("Remapper {}", remapper); - - JarOutputStream jos = new JarOutputStream( new BufferedOutputStream( fileOutputStream ) ); - - JarInputStream jarFile = new JarInputStream(in); - - for ( JarEntry entry = jarFile.getNextJarEntry(); entry != null; entry = jarFile.getNextJarEntry()) - { - String name = entry.getName(); - LOG.debug("Processing " + name); - remapper.setClassName(name); - if ( "META-INF/INDEX.LIST".equals( name ) ) - { - LOG.debug("Skipping INDEX.LIST..."); - // we cannot allow the jar indexes to be copied over or the - // jar is useless. Ideally, we could create a new one - // later - continue; - } - - if ( !entry.isDirectory() ) - { - InputStream is = jarFile; - - String mappedName = remapper.map( name ); - LOG.debug(name + " -> " + mappedName); - - int idx = mappedName.lastIndexOf( '/' ); - if ( idx != -1 ) - { - // make sure dirs are created - String dir = mappedName.substring( 0, idx ); - if ( !resources.contains( dir ) ) - { - addDirectory( resources, jos, dir ); - } - } - - if ( name.endsWith( ".class" ) ) - { - addRemappedClass( remapper, jos, name, is ); - } - else if ( name.endsWith( ".java" ) ) - { - // Avoid duplicates - if ( resources.contains( mappedName ) ) - { - continue; - } - - addJavaSource( resources, jos, mappedName, is, shadeRequest.getRelocators() ); - } - else - { - if ( !resourceTransformed( transformers, mappedName, is, shadeRequest.getRelocators() ) ) - { - // Avoid duplicates that aren't accounted for by the resource transformers - if ( resources.contains( mappedName ) ) - { - continue; - } - - addResource( resources, jos, mappedName, is ); - } - } - } - } - - jarFile.close(); - - for ( ResourceTransformer transformer : transformers ) - { - if ( transformer.hasTransformedResource() ) - { - transformer.modifyOutputStream( jos ); - } - } - - jos.close(); - } - - private void addDirectory( Set<String> resources, JarOutputStream jos, String name ) - throws IOException { - if (name.lastIndexOf('/') > 0) { - String parent = name.substring(0, name.lastIndexOf('/')); - if (!resources.contains(parent)) { - addDirectory(resources, jos, parent); - } - } - - // directory entries must end in "/" - JarEntry entry = new JarEntry(name + "/"); - LOG.debug("Adding JAR directory " + entry); - jos.putNextEntry(entry); - - resources.add(name); - } - - private void addRemappedClass( RelocatorRemapper remapper, JarOutputStream jos, String name, - InputStream is ) - throws IOException - { - LOG.debug("Remapping class... "+name); - if ( !remapper.hasRelocators() ) - { - try - { - LOG.debug("Just copy class..."); - jos.putNextEntry( new JarEntry( name ) ); - IOUtil.copy( is, jos ); - } - catch ( ZipException e ) - { - LOG.info( "zip exception ", e); - } - - return; - } - - ClassReader cr = new ClassReader( is ); - - // We don't pass the ClassReader here. This forces the ClassWriter to rebuild the constant pool. - // Copying the original constant pool should be avoided because it would keep references - // to the original class names. This is not a problem at runtime (because these entries in the - // constant pool are never used), but confuses some tools such as Felix' maven-bundle-plugin - // that use the constant pool to determine the dependencies of a class. - ClassWriter cw = new ClassWriter( 0 ); - - final String pkg = name.substring( 0, name.lastIndexOf( '/' ) + 1 ); - ClassVisitor cv = new RemappingClassAdapter( cw, remapper ) - { - @Override - public void visitSource( final String source, final String debug ) - { - LOG.debug("visitSource "+source); - if ( source == null ) - { - super.visitSource( source, debug ); - } - else - { - final String fqSource = pkg + source; - final String mappedSource = remapper.map( fqSource ); - final String filename = mappedSource.substring( mappedSource.lastIndexOf( '/' ) + 1 ); - LOG.debug("Remapped to "+filename); - super.visitSource( filename, debug ); - } - } - }; - - try - { - cr.accept( cv, ClassReader.EXPAND_FRAMES ); - } - catch ( Throwable ise ) - { - throw new IOException( "Error in ASM processing class " + name, ise ); - } - - byte[] renamedClass = cw.toByteArray(); - - // Need to take the .class off for remapping evaluation - String mappedName = remapper.map( name.substring( 0, name.indexOf( '.' ) ) ); - LOG.debug("Remapped class name to "+mappedName); - - try - { - // Now we put it back on so the class file is written out with the right extension. - jos.putNextEntry( new JarEntry( mappedName + ".class" ) ); - jos.write(renamedClass); - } - catch ( ZipException e ) - { - LOG.info( "zip exception ", e); - } - } - - private boolean resourceTransformed( List<ResourceTransformer> resourceTransformers, String name, InputStream is, - List<Relocator> relocators ) - throws IOException - { - boolean resourceTransformed = false; - - for ( ResourceTransformer transformer : resourceTransformers ) - { - if ( transformer.canTransformResource( name ) ) - { - LOG.debug( "Transforming " + name + " using " + transformer.getClass().getName() ); - - transformer.processResource( name, is, relocators ); - - resourceTransformed = true; - - break; - } - } - return resourceTransformed; - } - - private void addJavaSource( Set<String> resources, JarOutputStream jos, String name, InputStream is, - List<Relocator> relocators ) - throws IOException - { - jos.putNextEntry( new JarEntry( name ) ); - - String sourceContent = IOUtil.toString( new InputStreamReader( is, "UTF-8" ) ); - - for ( Relocator relocator : relocators ) - { - sourceContent = relocator.applyToSourceContent( sourceContent ); - } - - OutputStreamWriter writer = new OutputStreamWriter( jos, "UTF-8" ); - writer.append(sourceContent); - writer.flush(); - - resources.add( name ); - } - - private void addResource( Set<String> resources, JarOutputStream jos, String name, InputStream is ) - throws IOException - { - jos.putNextEntry( new JarEntry( name ) ); - - IOUtil.copy( is, jos ); - - resources.add( name ); - } - - class RelocatorRemapper extends Remapper - { - - private final Pattern classPattern = Pattern.compile( "(\\[*)?L(.+);" ); - - private final List<Relocator> relocators; - - private final HashSet<String> warned = new HashSet<>(); - - private String className = "UNKNOWN"; - - public RelocatorRemapper( List<Relocator> relocators) - { - this.relocators = relocators; - } - - public boolean hasRelocators() - { - return !relocators.isEmpty(); - } - - public void setClassName(String className) { - this.className = className; - } - - @Override - public Object mapValue( Object object ) - { - if ( object instanceof String ) - { - String name = (String) object; - String value = name; - - String prefix = ""; - String suffix = ""; - - Matcher m = classPattern.matcher( name ); - if ( m.matches() ) - { - prefix = m.group( 1 ) + "L"; - suffix = ";"; - name = m.group( 2 ); - } - - for ( Relocator r : relocators ) - { - if ( r.canRelocateClass( name ) ) - { - value = prefix + r.relocateClass( name ) + suffix; - break; - } - else if ( r.canRelocatePath( name ) ) - { - value = prefix + r.relocatePath( name ) + suffix; - break; - } - } - - return value; - } - - return super.mapValue( object ); - } - - @Override - public String map( String name ) - { - String orig = name; - String value = name; - - String prefix = ""; - String suffix = ""; - - Matcher m = classPattern.matcher( name ); - if ( m.matches() ) - { - prefix = m.group( 1 ) + "L"; - suffix = ";"; - name = m.group( 2 ); - } - - for ( Relocator r : relocators ) - { - if ( r.canRelocatePath( name ) ) - { - value = prefix + r.relocatePath( name ) + suffix; - if (!warned.contains(orig)) { - LOG.warn("Relocating {} to {} in {} please modify your code to use the new namespace", orig, value, className); - warned.add(orig); - } - break; - } - } - - return value; - } - - } - -} http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-rename-hack/src/main/java/org/apache/storm/hack/IOUtil.java ---------------------------------------------------------------------- diff --git a/storm-rename-hack/src/main/java/org/apache/storm/hack/IOUtil.java b/storm-rename-hack/src/main/java/org/apache/storm/hack/IOUtil.java deleted file mode 100644 index 5fc304c..0000000 --- a/storm-rename-hack/src/main/java/org/apache/storm/hack/IOUtil.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.hack; - -import java.io.*; - -public class IOUtil { - public static void copy(InputStream in, OutputStream out) throws IOException { - byte [] buffer = new byte[4096]; - int read; - while ((read = in.read(buffer)) > 0) { - out.write(buffer, 0, read); - } - } - - public static String toString(Reader reader) throws IOException { - StringWriter ret = new StringWriter(); - char [] buffer = new char[4096]; - int read; - while ((read = reader.read(buffer)) > 0) { - ret.write(buffer, 0, read); - } - return ret.toString(); - } -} http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-rename-hack/src/main/java/org/apache/storm/hack/ShadeRequest.java ---------------------------------------------------------------------- diff --git a/storm-rename-hack/src/main/java/org/apache/storm/hack/ShadeRequest.java b/storm-rename-hack/src/main/java/org/apache/storm/hack/ShadeRequest.java deleted file mode 100644 index ef480c2..0000000 --- a/storm-rename-hack/src/main/java/org/apache/storm/hack/ShadeRequest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.storm.hack; - -import org.apache.storm.hack.relocation.Relocator; -import org.apache.storm.hack.resource.ResourceTransformer; - -import java.util.List; - -/** - * This is based off of - * - * https://github.com/apache/maven-plugins.git - * - * maven-shade-plugin-2.4.1 - */ -public class ShadeRequest -{ - private List<Relocator> relocators; - - private List<ResourceTransformer> resourceTransformers; - - public List<Relocator> getRelocators() - { - return relocators; - } - - /** - * The relocators. - * - * @param relocators - */ - public void setRelocators( List<Relocator> relocators ) - { - this.relocators = relocators; - } - - public List<ResourceTransformer> getResourceTransformers() - { - return resourceTransformers; - } - - /** - * The transformers. - * - * @param resourceTransformers - */ - public void setResourceTransformers( List<ResourceTransformer> resourceTransformers ) - { - this.resourceTransformers = resourceTransformers; - } -} http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeRequest.java ---------------------------------------------------------------------- diff --git a/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeRequest.java b/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeRequest.java deleted file mode 100644 index 03d60b7..0000000 --- a/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeRequest.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.hack; - -import org.apache.storm.hack.relocation.Relocator; -import org.apache.storm.hack.relocation.SimpleRelocator; -import org.apache.storm.hack.resource.ClojureTransformer; -import org.apache.storm.hack.resource.ResourceTransformer; - -import java.util.Arrays; - -public class StormShadeRequest { - public static ShadeRequest makeRequest() { - ShadeRequest request = new ShadeRequest(); - request.setRelocators(Arrays.asList( - (Relocator)new SimpleRelocator("backtype.storm", "org.apache.storm"), - (Relocator)new SimpleRelocator("storm.trident", "org.apache.storm.trident"), - (Relocator)new SimpleRelocator("org.apache.thrift7", "org.apache.storm.thrift") - )); - request.setResourceTransformers(Arrays.asList( - (ResourceTransformer)new ClojureTransformer() - )); - return request; - } -} http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeTransformer.java ---------------------------------------------------------------------- diff --git a/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeTransformer.java b/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeTransformer.java deleted file mode 100644 index 8afbcea..0000000 --- a/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeTransformer.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.hack; - -import org.apache.storm.daemon.JarTransformer; - -import java.io.*; - -public class StormShadeTransformer implements JarTransformer { - @Override - public void transform(InputStream input, OutputStream output) throws IOException { - DefaultShader shader = new DefaultShader(); - ShadeRequest request = StormShadeRequest.makeRequest(); - shader.shadeJarStream(request,input, output); - } -} http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-rename-hack/src/main/java/org/apache/storm/hack/relocation/Relocator.java ---------------------------------------------------------------------- diff --git a/storm-rename-hack/src/main/java/org/apache/storm/hack/relocation/Relocator.java b/storm-rename-hack/src/main/java/org/apache/storm/hack/relocation/Relocator.java deleted file mode 100644 index f8b9714..0000000 --- a/storm-rename-hack/src/main/java/org/apache/storm/hack/relocation/Relocator.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.storm.hack.relocation; - -/** - * This is based off of - * - * https://github.com/apache/maven-plugins.git - * - * maven-shade-plugin-2.4.1 - */ -public interface Relocator -{ - boolean canRelocatePath( String clazz ); - - String relocatePath( String clazz ); - - boolean canRelocateClass( String clazz ); - - String relocateClass( String clazz ); - - String applyToSourceContent( String sourceContent ); -} http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-rename-hack/src/main/java/org/apache/storm/hack/relocation/SimpleRelocator.java ---------------------------------------------------------------------- diff --git a/storm-rename-hack/src/main/java/org/apache/storm/hack/relocation/SimpleRelocator.java b/storm-rename-hack/src/main/java/org/apache/storm/hack/relocation/SimpleRelocator.java deleted file mode 100644 index 5c58680..0000000 --- a/storm-rename-hack/src/main/java/org/apache/storm/hack/relocation/SimpleRelocator.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.storm.hack.relocation; - -/** - * This is based off of - * - * https://github.com/apache/maven-plugins.git - * - * maven-shade-plugin-2.4.1 - */ -public class SimpleRelocator - implements Relocator -{ - - private final String pattern; - - private final String pathPattern; - - private final String shadedPattern; - - private final String shadedPathPattern; - - public SimpleRelocator( String patt, String shadedPattern) - { - if ( patt == null ) - { - this.pattern = ""; - this.pathPattern = ""; - } - else - { - this.pattern = patt.replace( '/', '.' ); - this.pathPattern = patt.replace( '.', '/' ); - } - - if ( shadedPattern != null ) - { - this.shadedPattern = shadedPattern.replace( '/', '.' ); - this.shadedPathPattern = shadedPattern.replace( '.', '/' ); - } - else - { - this.shadedPattern = "hidden." + this.pattern; - this.shadedPathPattern = "hidden/" + this.pathPattern; - } - } - - public boolean canRelocatePath( String path ) - { - if ( path.endsWith( ".class" ) ) - { - path = path.substring( 0, path.length() - 6 ); - } - - // Allow for annoying option of an extra / on the front of a path. See MSHADE-119; comes from - // getClass().getResource("/a/b/c.properties"). - return path.startsWith( pathPattern ) || path.startsWith ( "/" + pathPattern ); - } - - public boolean canRelocateClass( String clazz ) - { - return clazz.indexOf( '/' ) < 0 && canRelocatePath( clazz.replace( '.', '/' ) ); - } - - public String relocatePath( String path ) - { - return path.replaceFirst( pathPattern, shadedPathPattern ); - } - - public String relocateClass( String clazz ) - { - return clazz.replaceFirst( pattern, shadedPattern ); - } - - public String applyToSourceContent( String sourceContent ) - { - return sourceContent.replaceAll( "\\b" + pattern, shadedPattern ); - } -} http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-rename-hack/src/main/java/org/apache/storm/hack/resource/ClojureTransformer.java ---------------------------------------------------------------------- diff --git a/storm-rename-hack/src/main/java/org/apache/storm/hack/resource/ClojureTransformer.java b/storm-rename-hack/src/main/java/org/apache/storm/hack/resource/ClojureTransformer.java deleted file mode 100644 index 4b34e6a..0000000 --- a/storm-rename-hack/src/main/java/org/apache/storm/hack/resource/ClojureTransformer.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.hack.resource; - -import org.apache.storm.hack.relocation.Relocator; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; -import java.util.List; -import java.util.jar.JarOutputStream; -import java.util.jar.JarEntry; - -public class ClojureTransformer implements ResourceTransformer { - - private final HashMap<String, String> entries = new HashMap<>(); - - @Override - public boolean canTransformResource(String s) { - if(s.endsWith(".clj")){ - return true; - } - return false; - } - - @Override - public void processResource(String s, InputStream inputStream, List<Relocator> relocators) throws IOException { - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - int b; - while((b = inputStream.read()) != -1){ - out.write(b); - } - String data = out.toString(); - - for(Relocator rel : relocators){ - data = rel.applyToSourceContent(data); - } - this.entries.put(s, data); - } - - @Override - public boolean hasTransformedResource() { - return !entries.isEmpty(); - } - - @Override - public void modifyOutputStream(JarOutputStream jarOut) throws IOException { - for(String key : this.entries.keySet()){ - jarOut.putNextEntry(new JarEntry(key)); - jarOut.write(this.entries.get(key).getBytes()); - } - } -} http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-rename-hack/src/main/java/org/apache/storm/hack/resource/ResourceTransformer.java ---------------------------------------------------------------------- diff --git a/storm-rename-hack/src/main/java/org/apache/storm/hack/resource/ResourceTransformer.java b/storm-rename-hack/src/main/java/org/apache/storm/hack/resource/ResourceTransformer.java deleted file mode 100644 index b63e896..0000000 --- a/storm-rename-hack/src/main/java/org/apache/storm/hack/resource/ResourceTransformer.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.storm.hack.resource; - -import org.apache.storm.hack.relocation.Relocator; - -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.jar.JarOutputStream; - -/** - * This is based off of - * - * https://github.com/apache/maven-plugins.git - * - * maven-shade-plugin-2.4.1 - */ -public interface ResourceTransformer -{ - boolean canTransformResource( String resource ); - - void processResource( String resource, InputStream is, List<Relocator> relocators ) - throws IOException; - - boolean hasTransformedResource(); - - void modifyOutputStream(JarOutputStream jarOut) throws IOException; -} http://git-wip-us.apache.org/repos/asf/storm/blob/fe961e6b/storm-server/src/main/java/org/apache/storm/DaemonConfig.java ---------------------------------------------------------------------- diff --git a/storm-server/src/main/java/org/apache/storm/DaemonConfig.java b/storm-server/src/main/java/org/apache/storm/DaemonConfig.java index 6821d21..a1292e0 100644 --- a/storm-server/src/main/java/org/apache/storm/DaemonConfig.java +++ b/storm-server/src/main/java/org/apache/storm/DaemonConfig.java @@ -826,16 +826,6 @@ public class DaemonConfig implements Validated { public static final String NIMBUS_CODE_SYNC_FREQ_SECS = "nimbus.code.sync.freq.secs"; /** - * An implementation of @{link org.apache.storm.daemon.JarTransformer} that will can be used to transform a jar - * file before storm jar runs with it. Use with extreme caution. - * If you want to enable a transition between org.apache.storm and org.apache.storm to run older topologies - * you can set this to org.apache.storm.hack.StormShadeTransformer. But this is likely to be deprecated in - * future releases. - */ - @isString - public static final String CLIENT_JAR_TRANSFORMER = "client.jartransformer.class"; - - /** * The plugin to be used for resource isolation */ @isImplementationOfClass(implementsClass = ResourceIsolationInterface.class)
