This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a commit to branch ignite-13618-javapoet in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/ignite-13618-javapoet by this push: new 3c7ef24 WIP. Implement classloader for dynamic classes. Fix tests. 3c7ef24 is described below commit 3c7ef24763bf1df88c244736cbecd6e704ffbfd5 Author: Andrew Mashenkov <andrey.mashen...@gmail.com> AuthorDate: Wed Dec 16 01:19:19 2020 +0300 WIP. Implement classloader for dynamic classes. Fix tests. --- .../internal/schema/marshaller/MarshallerUtil.java | 63 ++++++++++++++++------ .../schema/marshaller/SerializerFactory.java | 4 +- ...izerGenerator.java => SerializerGenerator.java} | 20 ++----- .../benchmarks/SerializerBenchmarkTest.java | 2 + .../schema/marshaller/JavaSerializerTest.java | 60 ++++++++++----------- 5 files changed, 85 insertions(+), 64 deletions(-) diff --git a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/MarshallerUtil.java b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/MarshallerUtil.java index e04fc64..4f36129 100644 --- a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/MarshallerUtil.java +++ b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/MarshallerUtil.java @@ -144,21 +144,19 @@ public final class MarshallerUtil { public static ClassLoader compileCode(JavaFile javafile, Map<String, byte[]> classes) { final JavaCompiler cmp = ToolProvider.getSystemJavaCompiler(); - final Map<String, byte[]> classes0 = classes == null ? new HashMap<>() : classes; + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); - final MemoryClassLoader loader = new MemoryClassLoader(classes0, ClassLoader.getSystemClassLoader()); + Map<String, byte[]> classes = classLoader instanceof MemoryClassLoader ? + ((MemoryClassLoader)classLoader).classBytes : new ConcurrentHashMap<>(); - try (final MemoryJavaFileManager fileManager = new MemoryJavaFileManager(cmp.getStandardFileManager(null, null, null), classes0) { - @Override public ClassLoader getClassLoader(Location location) { - return loader; - } - }) { + try (final MemoryJavaFileManager fileManager = new MemoryJavaFileManager(cmp.getStandardFileManager(null, null, null), classes)) { DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<>(); JavaCompiler.CompilationTask task = cmp.getTask(null, fileManager, diagnostics, null, null, Collections.singletonList(javafile.toJavaFileObject())); if (task.call()) - return loader; + return classLoader instanceof MemoryClassLoader ? classLoader : + new MemoryClassLoader(classes, ClassLoader.getSystemClassLoader()); // TODO: write to log. for (Diagnostic diagnostic : diagnostics.getDiagnostics()) { @@ -187,8 +185,17 @@ public final class MarshallerUtil { this.classes = classes; } - Map<String, byte[]> getClasses() { - return classes; + private static class MemoryJavaFileObject extends SimpleJavaFileObject { + private final byte[] bytes; + + public MemoryJavaFileObject(String className, byte[] bytes) { + super(URI.create(className + Kind.CLASS.extension), Kind.CLASS); + this.bytes = bytes; + } + + @Override public InputStream openInputStream() { + return new ByteArrayInputStream(bytes); + } } /** @@ -236,6 +243,32 @@ public final class MarshallerUtil { return it; } + @Override public String inferBinaryName(Location location, JavaFileObject jfo) { + if (!(jfo instanceof MemoryJavaFileObject)) { + String result = super.inferBinaryName(location, jfo); + assert result != null; + return result; + } + + // A [Java]FileObject's "name" looks like this: "/orc/codehaus/commons/compiler/Foo.java". + // A [Java]FileObject's "binary name" looks like "java.lang.annotation.Retention". + + String bn = jfo.getName(); + if (bn.startsWith("/")) + bn = bn.substring(1); + + if (!bn.endsWith(jfo.getKind().extension)) { + throw new AssertionError( + "Name \"" + jfo.getName() + "\" does not match kind \"" + jfo.getKind() + "\"" + ); + } + bn = bn.substring(0, bn.length() - jfo.getKind().extension.length()); + + bn = bn.replace('/', '.'); + + return bn; + } + @Override public JavaFileObject getJavaFileForInput(Location location, String className, Kind kind) throws IOException { if (location == StandardLocation.CLASS_OUTPUT) { JavaFileObject javaFileObject = getJavaFileObjectByName(className); @@ -250,13 +283,9 @@ public final class MarshallerUtil { @Nullable private JavaFileObject getJavaFileObjectByName(String className) { final byte[] bytes = classes.get(className); - if (bytes != null) { - return new SimpleJavaFileObject(URI.create(className), JavaFileObject.Kind.CLASS) { - @Override public InputStream openInputStream() { - return new ByteArrayInputStream(bytes); - } - }; - } + if (bytes != null) + return new MemoryJavaFileObject(className, bytes); + return null; } diff --git a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/SerializerFactory.java b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/SerializerFactory.java index d9a8295..ce88b18 100644 --- a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/SerializerFactory.java +++ b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/SerializerFactory.java @@ -18,7 +18,7 @@ package org.apache.ignite.internal.schema.marshaller; import org.apache.ignite.internal.schema.SchemaDescriptor; -import org.apache.ignite.internal.schema.marshaller.generator.JaninoSerializerGenerator; +import org.apache.ignite.internal.schema.marshaller.generator.SerializerGenerator; import org.apache.ignite.internal.schema.marshaller.reflection.JavaSerializerFactory; /** @@ -30,7 +30,7 @@ public interface SerializerFactory { * @return Serializer factory back by code generator. */ public static SerializerFactory createJaninoSerializerFactory() { - return new JaninoSerializerGenerator(); + return new SerializerGenerator(); } /** diff --git a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/JaninoSerializerGenerator.java b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/SerializerGenerator.java similarity index 96% rename from modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/JaninoSerializerGenerator.java rename to modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/SerializerGenerator.java index 9de4011..42786bf 100644 --- a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/JaninoSerializerGenerator.java +++ b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/SerializerGenerator.java @@ -26,11 +26,8 @@ import com.squareup.javapoet.TypeName; import com.squareup.javapoet.TypeSpec; import java.lang.invoke.MethodHandles; import java.lang.reflect.InvocationTargetException; -import java.util.HashMap; -import java.util.Map; import javax.annotation.processing.Generated; import javax.lang.model.element.Modifier; -import javax.tools.JavaFileManager; import org.apache.ignite.internal.schema.ByteBufferTuple; import org.apache.ignite.internal.schema.Columns; import org.apache.ignite.internal.schema.SchemaDescriptor; @@ -44,19 +41,16 @@ import org.apache.ignite.internal.schema.marshaller.SerializerFactory; import org.apache.ignite.internal.util.ObjectFactory; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.jetbrains.annotations.TestOnly; /** * {@link Serializer} code generator backed with Janino. */ -public class JaninoSerializerGenerator implements SerializerFactory { +public class SerializerGenerator implements SerializerFactory { public static final String SERIALIZER_PACKAGE_NAME = "org.apache.ignite.internal.schema.marshaller"; public static final String SERIALIZER_CLASS_NAME_PREFIX = "JaninoSerializerForSchema_"; - private Map<String, byte[]> classes; - /** {@inheritDoc} */ @Override public Serializer create( SchemaDescriptor schema, @@ -71,7 +65,7 @@ public class JaninoSerializerGenerator implements SerializerFactory { //TODO: pass code to logger on trace level. System.out.println(javaFile.toString()); - final ClassLoader loader = MarshallerUtil.compileCode(javaFile, classes); + final ClassLoader loader = MarshallerUtil.compileCode(javaFile); try { final Class<?> aClass = loader.loadClass(javaFile.packageName + '.' + className); @@ -86,11 +80,6 @@ public class JaninoSerializerGenerator implements SerializerFactory { } } - @TestOnly - public void setClasses(Map<String, byte[]> classes) { - this.classes = classes; - } - /** * Generates serializer code. * @@ -103,7 +92,7 @@ public class JaninoSerializerGenerator implements SerializerFactory { try { // Build field accessor generators. final MarshallerExprGenerator keyMarsh = createObjectMarshaller(keyClass, "keyFactory", schema.keyColumns(), 0); - final MarshallerExprGenerator valMarsh = createObjectMarshaller(valClass, "valFactory", schema.valueColumns(), schema.valueColumns().length()); + final MarshallerExprGenerator valMarsh = createObjectMarshaller(valClass, "valFactory", schema.valueColumns(), schema.keyColumns().length()); final TypeSpec.Builder classBuilder = TypeSpec.classBuilder(className) .addSuperinterface(Serializer.class) @@ -171,7 +160,8 @@ public class JaninoSerializerGenerator implements SerializerFactory { classBuilder.addStaticBlock(staticInitBuilder.build()); } - private MethodSpec generateHelpersMetod(SchemaDescriptor schema, MarshallerExprGenerator keyMarsh, MarshallerExprGenerator valMarsh) { + private MethodSpec generateHelpersMetod(SchemaDescriptor schema, MarshallerExprGenerator keyMarsh, + MarshallerExprGenerator valMarsh) { final MethodSpec.Builder builder = MethodSpec .methodBuilder("createAssembler") .addModifiers(Modifier.PRIVATE) diff --git a/modules/commons/src/test/java/org/apache/ignite/internal/benchmarks/SerializerBenchmarkTest.java b/modules/commons/src/test/java/org/apache/ignite/internal/benchmarks/SerializerBenchmarkTest.java index e84e73b..f574dec 100644 --- a/modules/commons/src/test/java/org/apache/ignite/internal/benchmarks/SerializerBenchmarkTest.java +++ b/modules/commons/src/test/java/org/apache/ignite/internal/benchmarks/SerializerBenchmarkTest.java @@ -94,6 +94,8 @@ public class SerializerBenchmarkTest { */ @Setup public void init() throws Exception { + Thread.currentThread().setContextClassLoader(MarshallerUtil.dynamicClassLoader()); + long seed = System.currentTimeMillis(); rnd = new Random(seed); diff --git a/modules/commons/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java b/modules/commons/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java index a9589e9..d10fdf7 100644 --- a/modules/commons/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java +++ b/modules/commons/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java @@ -37,7 +37,7 @@ import org.apache.ignite.internal.schema.NativeType; import org.apache.ignite.internal.schema.NativeTypeSpec; import org.apache.ignite.internal.schema.SchemaDescriptor; import org.apache.ignite.internal.schema.TestUtils; -import org.apache.ignite.internal.schema.marshaller.generator.JaninoSerializerGenerator; +import org.apache.ignite.internal.schema.marshaller.generator.SerializerGenerator; import org.apache.ignite.internal.schema.marshaller.reflection.JavaSerializerFactory; import org.apache.ignite.internal.util.ObjectFactory; import org.junit.jupiter.api.BeforeEach; @@ -71,7 +71,7 @@ public class JavaSerializerTest { */ private static List<SerializerFactory> serializerFactoryProvider() { return Arrays.asList( - new JaninoSerializerGenerator(), + new SerializerGenerator(), new JavaSerializerFactory() ); } @@ -265,46 +265,49 @@ public class JavaSerializerTest { assertThrows(IllegalStateException.class, () -> factory.create(schema, key.getClass(), val.getClass()), "Class has no default constructor: class=org.apache.ignite.internal.schema.marshaller.JavaSerializerTest$WrongTestObject" - ); + ); } - /** * */ @ParameterizedTest @MethodSource("serializerFactoryProvider") public void testClassLoader(SerializerFactory factory) throws SerializationException { - Column[] keyCols = new Column[] { - new Column("key", LONG, false) - }; + final ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(MarshallerUtil.dynamicClassLoader()); - Column[] valCols = new Column[] { - new Column("col0", LONG, false), - new Column("col1", LONG, false), - new Column("col2", LONG, false), - }; + Column[] keyCols = new Column[] { + new Column("key", LONG, false) + }; - SchemaDescriptor schema = new SchemaDescriptor(1, new Columns(keyCols), new Columns(valCols)); + Column[] valCols = new Column[] { + new Column("col0", LONG, false), + new Column("col1", LONG, false), + new Column("col2", LONG, false), + }; - final Map<String, byte[]> classes = new HashMap<>(); - final Class<?> valClass = createGeneratedObjectClass(Long.class, classes); - final ObjectFactory<?> objFactory = new ObjectFactory<>(valClass); + SchemaDescriptor schema = new SchemaDescriptor(1, new Columns(keyCols), new Columns(valCols)); - final Long key = rnd.nextLong(); + final Class<?> valClass = createGeneratedObjectClass(Long.class); + final ObjectFactory<?> objFactory = new ObjectFactory<>(valClass); - if (factory instanceof JaninoSerializerGenerator) - ((JaninoSerializerGenerator)factory).setClasses(classes); + final Long key = rnd.nextLong(); - Serializer serializer = factory.create(schema, key.getClass(), valClass); + Serializer serializer = factory.create(schema, key.getClass(), valClass); - byte[] bytes = serializer.serialize(key, objFactory.create()); + byte[] bytes = serializer.serialize(key, objFactory.create()); - Object key1 = serializer.deserializeKey(bytes); - Object val1 = serializer.deserializeValue(bytes); + Object key1 = serializer.deserializeKey(bytes); + Object val1 = serializer.deserializeValue(bytes); - assertTrue(key.getClass().isInstance(key1)); - assertTrue(valClass.isInstance(val1)); + assertTrue(key.getClass().isInstance(key1)); + assertTrue(valClass.isInstance(val1)); + } + finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); + } } /** @@ -367,10 +370,9 @@ public class JavaSerializerTest { * Generate class for test objects. * * @param fieldType Field type. - * @param classes * @return Generated test object class. */ - private Class<?> createGeneratedObjectClass(Class<?> fieldType, Map<String, byte[]> classes) { + private Class<?> createGeneratedObjectClass(Class<?> fieldType) { final String packageName = getClass().getPackageName(); final String className = "GeneratedTestObject"; @@ -392,10 +394,8 @@ public class JavaSerializerTest { final JavaFile javaFile = JavaFile.builder(packageName, classBuilder.build()).build(); - final ClassLoader loader = MarshallerUtil.compileCode(javaFile, classes); - try { - return loader.loadClass(packageName + '.' + className); + return MarshallerUtil.compileCode(javaFile).loadClass(packageName + '.' + className); } catch (Exception ex) { throw new IllegalStateException("Failed to compile/instantiate generated Serializer.", ex);