This is an automated email from the ASF dual-hosted git repository. chaokunyang pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push: new a0a6d7b7 feat(java): scoped meta share mode for type forward/backward compaibility (#1660) a0a6d7b7 is described below commit a0a6d7b7f494f167ac46592dfc8b2a8c4706c5f2 Author: Shawn Yang <shawn.ck.y...@gmail.com> AuthorDate: Thu May 30 14:19:29 2024 +0800 feat(java): scoped meta share mode for type forward/backward compaibility (#1660) ## What does this PR do? This PR implements scoped meta share mode for type forward/backward compaibility ## Related issues #202 ## Does this PR introduce any user-facing change? <!-- If any user-facing interface changes, please [open an issue](https://github.com/apache/incubator-fury/issues/new/choose) describing the need to do so and update the document if necessary. --> - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark Perf increased from `1900102.586` to `2430410.064` ``` Before: Benchmark (bufferType) (objectType) (references) Mode Cnt Score Error Units fury_deserialize array MEDIA_CONTENT false thrpt 10 2734151.212 ± 253921.628 ops/s fury_deserialize_compatible array MEDIA_CONTENT false thrpt 10 1900102.586 ± 62176.872 ops/s furymetashared_deserialize_compatible array MEDIA_CONTENT false thrpt 10 3011439.327 ± 260518.752 ops/s After: Benchmark (bufferType) (objectType) (references) Mode Cnt Score Error Units fury_deserialize array MEDIA_CONTENT false thrpt 10 2661186.814 ± 279377.198 ops/s fury_deserialize_compatible array MEDIA_CONTENT false thrpt 10 2430410.064 ± 164165.865 ops/s furymetashared_deserialize_compatible array MEDIA_CONTENT false thrpt 10 3098083.064 ± 259391.053 ops/s ``` Size decreased from **732 to 577**: ``` Before 2024-05-30 01:00:49 INFO FuryState:157 [fury_deserialize_compatible-jmh-worker-1] - ======> Fury | MEDIA_CONTENT | false | array | 732 | After 2024-05-30 12:57:00 INFO FuryState:157 [fury_deserialize_compatible-jmh-worker-1] - ======> Fury | MEDIA_CONTENT | false | array | 577 | ``` The --- .../integration_tests/RecordSerializersTest.java | 8 +- java/benchmark/pom.xml | 6 ++ .../fury/benchmark/UserTypeDeserializeSuite.java | 8 +- .../fury/benchmark/UserTypeSerializeSuite.java | 6 +- .../org/apache/fury/benchmark/state/FuryState.java | 8 +- .../org/apache/fury/benchmark/state/KryoState.java | 4 + .../src/main/java/org/apache/fury/Fury.java | 20 ++-- .../fury/builder/MetaSharedCodecBuilder.java | 2 +- .../apache/fury/builder/ObjectCodecBuilder.java | 2 +- .../main/java/org/apache/fury/config/Config.java | 26 ++++-- .../java/org/apache/fury/config/FuryBuilder.java | 19 +++- .../main/java/org/apache/fury/meta/ClassDef.java | 48 +++++----- .../java/org/apache/fury/meta/ClassDefDecoder.java | 28 ++---- .../java/org/apache/fury/meta/ClassDefEncoder.java | 53 ++++++----- .../main/java/org/apache/fury/meta/ClassSpec.java | 54 +++++++++++ .../main/java/org/apache/fury/meta/Encoders.java | 66 +++++++++++++ .../org/apache/fury/reflect/ReflectionUtils.java | 5 +- .../java/org/apache/fury/resolver/ClassInfo.java | 29 +----- .../org/apache/fury/resolver/ClassResolver.java | 103 ++++++++------------- .../apache/fury/resolver/SerializationContext.java | 46 ++++++++- .../apache/fury/serializer/ArraySerializers.java | 4 + .../fury/serializer/MetaSharedSerializer.java | 5 +- .../apache/fury/serializer/NonexistentClass.java | 17 +++- .../serializer/NonexistentClassSerializers.java | 4 +- .../apache/fury/serializer/ObjectSerializer.java | 14 +-- .../collection/AbstractCollectionSerializer.java | 4 +- .../src/test/java/org/apache/fury/CyclicTest.java | 2 +- .../test/java/org/apache/fury/FuryInitPerf.java | 6 +- .../java/org/apache/fury/ThreadSafeFuryTest.java | 3 +- .../org/apache/fury/meta/ClassDefEncoderTest.java | 7 +- .../java/org/apache/fury/meta/ClassDefTest.java | 6 +- .../apache/fury/reflect/ReflectionUtilsTest.java | 31 +++++++ .../org/apache/fury/resolver/MetaContextTest.java | 6 +- .../fury/resolver/SerializationContextTest.java | 6 +- .../fury/serializer/MetaSharedCompatibleTest.java | 34 +++---- .../NonexistentClassSerializersTest.java | 46 +++++---- .../serializer/ProtocolInteroperabilityTest.java | 4 +- .../collection/ChildContainerSerializersTest.java | 3 +- 38 files changed, 472 insertions(+), 271 deletions(-) diff --git a/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/RecordSerializersTest.java b/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/RecordSerializersTest.java index f17daa30..daa133e4 100644 --- a/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/RecordSerializersTest.java +++ b/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/RecordSerializersTest.java @@ -85,7 +85,7 @@ public class RecordSerializersTest { Fury.builder() .requireClassRegistration(false) .withCodegen(codegen) - .withMetaContextShare(true) + .withMetaShare(true) .build(); Foo foo = new Foo(10, "abc", new ArrayList<>(Arrays.asList("a", "b")), 'x'); MetaContext context = new MetaContext(); @@ -153,7 +153,7 @@ public class RecordSerializersTest { .requireClassRegistration(false) .withCodegen(codegen) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withMetaContextShare(true) + .withMetaShare(true) .withClassLoader(cls1.getClassLoader()) .build(); String code2 = @@ -169,7 +169,7 @@ public class RecordSerializersTest { .requireClassRegistration(false) .withCodegen(codegen) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withMetaContextShare(true) + .withMetaShare(true) .withClassLoader(cls2.getClassLoader()) .build(); MetaContext metaContext1 = new MetaContext(); @@ -211,7 +211,7 @@ public class RecordSerializersTest { Fury.builder() .requireClassRegistration(false) .withCodegen(codegen) - .withMetaContextShare(true) + .withMetaShare(true) .build(); Object o1 = Records.createPrivateRecord(11); Object o2 = Records.createPublicRecord(11, o1); diff --git a/java/benchmark/pom.xml b/java/benchmark/pom.xml index 89a2cb6a..d43cd6ee 100644 --- a/java/benchmark/pom.xml +++ b/java/benchmark/pom.xml @@ -149,6 +149,12 @@ <artifactId>flatbuffers-java</artifactId> <version>2.0.3</version> </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>2.0.12</version> + <scope>compile</scope> + </dependency> </dependencies> <profiles> diff --git a/java/benchmark/src/main/java/org/apache/fury/benchmark/UserTypeDeserializeSuite.java b/java/benchmark/src/main/java/org/apache/fury/benchmark/UserTypeDeserializeSuite.java index f18acc5a..e3f37e89 100644 --- a/java/benchmark/src/main/java/org/apache/fury/benchmark/UserTypeDeserializeSuite.java +++ b/java/benchmark/src/main/java/org/apache/fury/benchmark/UserTypeDeserializeSuite.java @@ -59,17 +59,13 @@ public class UserTypeDeserializeSuite { @Benchmark public Object fury_deserialize(FuryState.FuryUserTypeState state) { state.buffer.readerIndex(0); - Object o = state.fury.readRef(state.buffer); - state.fury.resetRead(); - return o; + return state.fury.deserialize(state.buffer); } @Benchmark public Object fury_deserialize_compatible(FuryState.FuryCompatibleState state) { state.buffer.readerIndex(0); - Object o = state.fury.readRef(state.buffer); - state.fury.resetRead(); - return o; + return state.fury.deserialize(state.buffer); } @Benchmark diff --git a/java/benchmark/src/main/java/org/apache/fury/benchmark/UserTypeSerializeSuite.java b/java/benchmark/src/main/java/org/apache/fury/benchmark/UserTypeSerializeSuite.java index b6697025..65578178 100644 --- a/java/benchmark/src/main/java/org/apache/fury/benchmark/UserTypeSerializeSuite.java +++ b/java/benchmark/src/main/java/org/apache/fury/benchmark/UserTypeSerializeSuite.java @@ -62,16 +62,14 @@ public class UserTypeSerializeSuite { @Benchmark public Object fury_serialize(FuryState.FuryUserTypeState state) { state.buffer.writerIndex(0); - state.fury.writeRef(state.buffer, state.object); - state.fury.resetWrite(); + state.fury.serialize(state.buffer, state.object); return state.buffer; } @Benchmark public Object fury_serialize_compatible(FuryState.FuryCompatibleState state) { state.buffer.writerIndex(0); - state.fury.writeRef(state.buffer, state.object); - state.fury.resetWrite(); + state.fury.serialize(state.buffer, state.object); return state.buffer; } diff --git a/java/benchmark/src/main/java/org/apache/fury/benchmark/state/FuryState.java b/java/benchmark/src/main/java/org/apache/fury/benchmark/state/FuryState.java index eaa01914..a730e272 100644 --- a/java/benchmark/src/main/java/org/apache/fury/benchmark/state/FuryState.java +++ b/java/benchmark/src/main/java/org/apache/fury/benchmark/state/FuryState.java @@ -128,7 +128,7 @@ public class FuryState { .withRefTracking(references) .requireClassRegistration(false); if (compatible()) { - furyBuilder.withCompatibleMode(CompatibleMode.COMPATIBLE); + furyBuilder.withCompatibleMode(CompatibleMode.COMPATIBLE).withScopedMetaShare(true); } fury = furyBuilder.build(); switch (objectType) { @@ -152,7 +152,7 @@ public class FuryState { break; } - fury.writeRef(buffer, object); + fury.serialize(buffer, object); serializedLength = buffer.writerIndex(); LOG.info( "======> Fury | {} | {} | {} | {} |", @@ -161,7 +161,7 @@ public class FuryState { bufferType, serializedLength); buffer.writerIndex(0); - Preconditions.checkArgument(object.equals(fury.readRef(buffer))); + Preconditions.checkArgument(object.equals(fury.deserialize(buffer))); buffer.readerIndex(0); } @@ -192,7 +192,7 @@ public class FuryState { .withClassVersionCheck(false) .withRefTracking(references) .requireClassRegistration(false) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .build(); // share meta first time. diff --git a/java/benchmark/src/main/java/org/apache/fury/benchmark/state/KryoState.java b/java/benchmark/src/main/java/org/apache/fury/benchmark/state/KryoState.java index a75d60d7..b773caeb 100644 --- a/java/benchmark/src/main/java/org/apache/fury/benchmark/state/KryoState.java +++ b/java/benchmark/src/main/java/org/apache/fury/benchmark/state/KryoState.java @@ -82,6 +82,9 @@ public class KryoState { } kryo.setReferences(references); + kryo.setRegistrationRequired(registerClass); + kryo.register(int[].class); + kryo.register(long[].class); } } @@ -134,6 +137,7 @@ public class KryoState { break; } output.setPosition(0); + kryo.setRegistrationRequired(registerClass); kryo.writeClassAndObject(output, object); serializedLength = output.position(); LOG.info( diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 8115ed5a..ccef2aaa 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -133,7 +133,7 @@ public final class Fury implements BaseFury { metaStringResolver = new MetaStringResolver(); classResolver = new ClassResolver(this); classResolver.initialize(); - serializationContext = new SerializationContext(); + serializationContext = new SerializationContext(config); this.classLoader = classLoader; nativeObjects = new ArrayList<>(); generics = new Generics(this); @@ -305,8 +305,8 @@ public final class Fury implements BaseFury { private void write(MemoryBuffer buffer, Object obj) { int startOffset = buffer.writerIndex(); - boolean shareMetaContext = config.shareMetaContext(); - if (shareMetaContext) { + boolean shareMeta = config.isMetaShareEnabled(); + if (shareMeta) { buffer.writeInt32(-1); // preserve 4-byte for nativeObjects start offsets. } // reduce caller stack @@ -315,7 +315,7 @@ public final class Fury implements BaseFury { classResolver.writeClass(buffer, classInfo); writeData(buffer, classInfo, obj); } - if (shareMetaContext) { + if (shareMeta) { buffer.putInt32(startOffset, buffer.writerIndex()); classResolver.writeClassDefs(buffer); } @@ -754,7 +754,7 @@ public final class Fury implements BaseFury { if (isTargetXLang) { obj = xdeserializeInternal(buffer); } else { - if (config.shareMetaContext()) { + if (config.isMetaShareEnabled()) { classResolver.readClassDefs(buffer); } obj = readRef(buffer); @@ -1025,7 +1025,7 @@ public final class Fury implements BaseFury { if (depth != 0) { throwDepthSerializationException(); } - if (config.shareMetaContext()) { + if (config.isMetaShareEnabled()) { int startOffset = buffer.writerIndex(); buffer.writeInt32(-1); // preserve 4-byte for nativeObjects start offsets. if (!refResolver.writeRefOrNull(buffer, obj)) { @@ -1070,7 +1070,7 @@ public final class Fury implements BaseFury { if (depth != 0) { throwDepthDeserializationException(); } - if (config.shareMetaContext()) { + if (config.isMetaShareEnabled()) { classResolver.readClassDefs(buffer); } T obj; @@ -1184,7 +1184,7 @@ public final class Fury implements BaseFury { if (depth != 0) { throwDepthDeserializationException(); } - if (config.shareMetaContext()) { + if (config.isMetaShareEnabled()) { classResolver.readClassDefs(buffer); } return readRef(buffer); @@ -1263,7 +1263,7 @@ public final class Fury implements BaseFury { refResolver.resetWrite(); classResolver.resetWrite(); metaStringResolver.resetWrite(); - serializationContext.reset(); + serializationContext.resetWrite(); nativeObjects.clear(); bufferCallback = null; depth = 0; @@ -1273,7 +1273,7 @@ public final class Fury implements BaseFury { refResolver.resetRead(); classResolver.resetRead(); metaStringResolver.resetRead(); - serializationContext.reset(); + serializationContext.resetRead(); nativeObjects.clear(); peerOutOfBandEnabled = false; depth = 0; diff --git a/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java b/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java index 580f2284..4a85a7bc 100644 --- a/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java +++ b/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java @@ -58,7 +58,7 @@ import org.apache.fury.util.record.RecordUtils; * info for those types. * * @see CompatibleMode - * @see FuryBuilder#withMetaContextShare + * @see FuryBuilder#withMetaShare * @see GeneratedMetaSharedSerializer * @see MetaSharedSerializer */ diff --git a/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java b/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java index d1d8bc57..ded5065f 100644 --- a/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java +++ b/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java @@ -87,7 +87,7 @@ public class ObjectCodecBuilder extends BaseObjectCodecBuilder { public ObjectCodecBuilder(Class<?> beanClass, Fury fury) { super(TypeRef.of(beanClass), fury, Generated.GeneratedObjectSerializer.class); Collection<Descriptor> descriptors; - boolean shareMeta = fury.getConfig().shareMetaContext(); + boolean shareMeta = fury.getConfig().isMetaShareEnabled(); if (shareMeta) { descriptors = visitFury( diff --git a/java/fury-core/src/main/java/org/apache/fury/config/Config.java b/java/fury-core/src/main/java/org/apache/fury/config/Config.java index cb563853..1a2c95c2 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/Config.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/Config.java @@ -49,7 +49,8 @@ public class Config implements Serializable { private final boolean requireClassRegistration; private final boolean suppressClassRegistrationWarnings; private final boolean registerGuavaTypes; - private final boolean shareMetaContext; + private final boolean metaShareEnabled; + private final boolean scopedMetaShareEnabled; private final boolean asyncCompilationEnabled; private final boolean deserializeNonexistentClass; private final boolean scalaOptimizationEnabled; @@ -74,12 +75,13 @@ public class Config implements Serializable { compatibleMode = builder.compatibleMode; checkJdkClassSerializable = builder.checkJdkClassSerializable; defaultJDKStreamSerializerType = builder.defaultJDKStreamSerializerType; - shareMetaContext = builder.shareMetaContext; + metaShareEnabled = builder.metaShareEnabled; + scopedMetaShareEnabled = builder.scopedMetaShareEnabled; deserializeNonexistentClass = builder.deserializeNonexistentClass; if (deserializeNonexistentClass) { // Only in meta share mode or compatibleMode, fury knows how to deserialize // unexisted class by type info in data. - Preconditions.checkArgument(shareMetaContext || compatibleMode == CompatibleMode.COMPATIBLE); + Preconditions.checkArgument(metaShareEnabled || compatibleMode == CompatibleMode.COMPATIBLE); } asyncCompilationEnabled = builder.asyncCompilationEnabled; scalaOptimizationEnabled = builder.scalaOptimizationEnabled; @@ -178,8 +180,16 @@ public class Config implements Serializable { return defaultJDKStreamSerializerType; } - public boolean shareMetaContext() { - return shareMetaContext; + public boolean isMetaShareEnabled() { + return metaShareEnabled; + } + + /** + * Scoped meta share focuses on a single serialization process. Metadata created or identified + * during this process is exclusive to it and is not shared with by other serializations. + */ + public boolean isScopedMetaShareEnabled() { + return scopedMetaShareEnabled; } /** @@ -235,7 +245,8 @@ public class Config implements Serializable { && requireClassRegistration == config.requireClassRegistration && suppressClassRegistrationWarnings == config.suppressClassRegistrationWarnings && registerGuavaTypes == config.registerGuavaTypes - && shareMetaContext == config.shareMetaContext + && metaShareEnabled == config.metaShareEnabled + && scopedMetaShareEnabled == config.scopedMetaShareEnabled && asyncCompilationEnabled == config.asyncCompilationEnabled && deserializeNonexistentClass == config.deserializeNonexistentClass && scalaOptimizationEnabled == config.scalaOptimizationEnabled @@ -265,7 +276,8 @@ public class Config implements Serializable { requireClassRegistration, suppressClassRegistrationWarnings, registerGuavaTypes, - shareMetaContext, + metaShareEnabled, + scopedMetaShareEnabled, asyncCompilationEnabled, deserializeNonexistentClass, scalaOptimizationEnabled); diff --git a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java index 8b582682..419e3212 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java @@ -68,7 +68,8 @@ public final class FuryBuilder { boolean checkJdkClassSerializable = true; Class<? extends Serializer> defaultJDKStreamSerializerType = ObjectStreamSerializer.class; boolean requireClassRegistration = true; - boolean shareMetaContext = false; + boolean metaShareEnabled = false; + boolean scopedMetaShareEnabled = false; boolean codeGenEnabled = true; Boolean deserializeNonexistentClass; boolean asyncCompilationEnabled = false; @@ -233,8 +234,20 @@ public final class FuryBuilder { } /** Whether to enable meta share mode. */ - public FuryBuilder withMetaContextShare(boolean shareMetaContext) { - this.shareMetaContext = shareMetaContext; + public FuryBuilder withMetaShare(boolean shareMeta) { + this.metaShareEnabled = shareMeta; + return this; + } + + /** + * Scoped meta share focuses on a single serialization process. Metadata created or identified + * during this process is exclusive to it and is not shared with by other serializations. + */ + public FuryBuilder withScopedMetaShare(boolean scoped) { + scopedMetaShareEnabled = scoped; + if (scoped) { + metaShareEnabled = true; + } return this; } diff --git a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDef.java b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDef.java index ccea625e..899890bf 100644 --- a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDef.java +++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDef.java @@ -70,7 +70,7 @@ import org.apache.fury.util.Preconditions; * @see MetaSharedCodecBuilder * @see CompatibleMode#COMPATIBLE * @see CompatibleSerializer - * @see FuryBuilder#withMetaContextShare + * @see FuryBuilder#withMetaShare * @see ReflectionUtils#getFieldOffset */ public class ClassDef implements Serializable { @@ -78,7 +78,7 @@ public class ClassDef implements Serializable { static final int SCHEMA_COMPATIBLE_FLAG = 0b10000; public static final int SIZE_TWO_BYTES_FLAG = 0b100000; - static final int EXT_FLAG = 0b1000000; + static final int OBJECT_TYPE_FLAG = 0b1000000; // TODO use field offset to sort field, which will hit l1-cache more. Since // `objectFieldOffset` is not part of jvm-specification, it may change between different jdk // vendor. But the deserialization peer use the class definition to create deserializer, it's OK @@ -103,9 +103,9 @@ public class ClassDef implements Serializable { } }; - private final String className; + private final ClassSpec classSpec; private final List<FieldInfo> fieldsInfo; - private final Map<String, String> extMeta; + private final boolean isObjectType; // Unique id for class def. If class def are same between processes, then the id will // be same too. private final long id; @@ -113,14 +113,14 @@ public class ClassDef implements Serializable { private transient List<Descriptor> descriptors; ClassDef( - String className, + ClassSpec classSpec, List<FieldInfo> fieldsInfo, - Map<String, String> extMeta, + boolean isObjectType, long id, byte[] encoded) { - this.className = className; + this.classSpec = classSpec; this.fieldsInfo = fieldsInfo; - this.extMeta = extMeta; + this.isObjectType = isObjectType; this.id = id; this.encoded = encoded; } @@ -131,7 +131,11 @@ public class ClassDef implements Serializable { * @see Class#getName() */ public String getClassName() { - return className; + return classSpec.entireClassName; + } + + public ClassSpec getClassSpec() { + return classSpec; } /** Contain all fields info including all parent classes. */ @@ -140,8 +144,8 @@ public class ClassDef implements Serializable { } /** Returns ext meta for the class. */ - public Map<String, String> getExtMeta() { - return extMeta; + public boolean isObjectType() { + return isObjectType; } /** @@ -165,26 +169,26 @@ public class ClassDef implements Serializable { return false; } ClassDef classDef = (ClassDef) o; - return Objects.equals(className, classDef.className) + return Objects.equals(classSpec.entireClassName, classDef.classSpec.entireClassName) && Objects.equals(fieldsInfo, classDef.fieldsInfo) - && Objects.equals(extMeta, classDef.extMeta); + && Objects.equals(id, classDef.id); } @Override public int hashCode() { - return Objects.hash(className, fieldsInfo, extMeta); + return Objects.hash(classSpec.entireClassName, fieldsInfo, id); } @Override public String toString() { return "ClassDef{" + "className='" - + className + + classSpec.entireClassName + '\'' + ", fieldsInfo=" + fieldsInfo - + ", extMeta=" - + extMeta + + ", isObjectType=" + + isObjectType + ", id=" + id + '}'; @@ -630,7 +634,7 @@ public class ClassDef implements Serializable { return TypeRef.of( // We embed `isMonomorphic` flag in ObjectArraySerializer, so this flag can be ignored // here. - NonexistentClass.getUnexistentClass( + NonexistentClass.getNonexistentClass( componentType instanceof EnumFieldType, dimensions, true)); } else { return TypeRef.of(Array.newInstance(componentRawType, new int[dimensions]).getClass()); @@ -758,17 +762,17 @@ public class ClassDef implements Serializable { public static ClassDef buildClassDef(Fury fury, Class<?> cls, boolean resolveParent) { return ClassDefEncoder.buildClassDef( - fury.getClassResolver(), cls, buildFields(fury, cls, resolveParent), new HashMap<>()); + fury.getClassResolver(), cls, buildFields(fury, cls, resolveParent), true); } /** Build class definition from fields of class. */ public static ClassDef buildClassDef( ClassResolver classResolver, Class<?> type, List<Field> fields) { - return buildClassDef(classResolver, type, fields, new HashMap<>()); + return buildClassDef(classResolver, type, fields, true); } public static ClassDef buildClassDef( - ClassResolver classResolver, Class<?> type, List<Field> fields, Map<String, String> extMeta) { - return ClassDefEncoder.buildClassDef(classResolver, type, fields, extMeta); + ClassResolver classResolver, Class<?> type, List<Field> fields, boolean isObjectType) { + return ClassDefEncoder.buildClassDef(classResolver, type, fields, isObjectType); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java index 48d24912..88629e49 100644 --- a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java +++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java @@ -24,15 +24,11 @@ import static org.apache.fury.meta.Encoders.fieldNameEncodings; import static org.apache.fury.meta.Encoders.pkgEncodings; import static org.apache.fury.meta.Encoders.typeNameEncodings; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.meta.ClassDef.FieldType; import org.apache.fury.meta.MetaString.Encoding; -import org.apache.fury.reflect.ReflectionUtils; import org.apache.fury.resolver.ClassResolver; import org.apache.fury.util.Preconditions; @@ -64,37 +60,29 @@ class ClassDefDecoder { numClasses += 1; String className = null; List<ClassDef.FieldInfo> classFields = new ArrayList<>(); + ClassSpec classSpec = null; for (int i = 0; i < numClasses; i++) { // | num fields + register flag | header + package name | header + class name // | header + type id + field name | next field info | ... | int currentClassHeader = buffer.readVarUint32Small7(); boolean isRegistered = (currentClassHeader & 0b1) != 0; int numFields = currentClassHeader >>> 1; - String fullClassName; if (isRegistered) { int registeredId = buffer.readVarUint32Small7(); - fullClassName = classResolver.getClassInfo((short) registeredId).getCls().getName(); + className = classResolver.getClassInfo((short) registeredId).getCls().getName(); } else { String pkg = readPkgName(buffer); String typeName = readTypeName(buffer); - fullClassName = ReflectionUtils.getFullClassName(pkg, typeName); + classSpec = Encoders.decodePkgAndClass(pkg, typeName); + className = classSpec.entireClassName; } - className = fullClassName; - List<ClassDef.FieldInfo> fieldInfos = readFieldsInfo(buffer, fullClassName, numFields); + List<ClassDef.FieldInfo> fieldInfos = readFieldsInfo(buffer, className, numFields); classFields.addAll(fieldInfos); } - boolean hasExtMeta = (header & 0b1000000) != 0; - Map<String, String> extMeta = new HashMap<>(); - if (hasExtMeta) { - int extMetaSize = buffer.readVarUint32Small7(); - for (int i = 0; i < extMetaSize; i++) { - extMeta.put( - new String(buffer.readBytesAndSize(), StandardCharsets.UTF_8), - new String(buffer.readBytesAndSize(), StandardCharsets.UTF_8)); - } - } + Preconditions.checkNotNull(classSpec); + boolean isObjectType = (header & ClassDef.OBJECT_TYPE_FLAG) != 0; return new ClassDef( - className, classFields, extMeta, id, encoded.getBytes(0, encoded.writerIndex())); + classSpec, classFields, isObjectType, id, encoded.getBytes(0, encoded.writerIndex())); } private static List<ClassDef.FieldInfo> readFieldsInfo( diff --git a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java index 652d96b0..078f85cf 100644 --- a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java +++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java @@ -19,7 +19,7 @@ package org.apache.fury.meta; -import static org.apache.fury.meta.ClassDef.EXT_FLAG; +import static org.apache.fury.meta.ClassDef.OBJECT_TYPE_FLAG; import static org.apache.fury.meta.ClassDef.SCHEMA_COMPATIBLE_FLAG; import static org.apache.fury.meta.ClassDef.SIZE_TWO_BYTES_FLAG; import static org.apache.fury.meta.Encoders.fieldNameEncodingsList; @@ -27,7 +27,6 @@ import static org.apache.fury.meta.Encoders.pkgEncodingsList; import static org.apache.fury.meta.Encoders.typeNameEncodingsList; import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -36,9 +35,9 @@ import java.util.List; import java.util.Map; import java.util.function.Function; import org.apache.fury.Fury; +import org.apache.fury.collection.Tuple2; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.MemoryUtils; -import org.apache.fury.memory.Platform; import org.apache.fury.meta.ClassDef.FieldInfo; import org.apache.fury.meta.ClassDef.FieldType; import org.apache.fury.reflect.ReflectionUtils; @@ -103,15 +102,19 @@ class ClassDefEncoder { /** Build class definition from fields of class. */ static ClassDef buildClassDef( - ClassResolver classResolver, Class<?> type, List<Field> fields, Map<String, String> extMeta) { + ClassResolver classResolver, Class<?> type, List<Field> fields, boolean isObjectType) { List<FieldInfo> fieldInfos = buildFieldsInfo(classResolver, fields); Map<String, List<FieldInfo>> classLayers = getClassFields(type, fieldInfos); fieldInfos = new ArrayList<>(fieldInfos.size()); classLayers.values().forEach(fieldInfos::addAll); - MemoryBuffer encodeClassDef = encodeClassDef(classResolver, type, classLayers, extMeta); + MemoryBuffer encodeClassDef = encodeClassDef(classResolver, type, classLayers, isObjectType); byte[] classDefBytes = encodeClassDef.getBytes(0, encodeClassDef.writerIndex()); return new ClassDef( - type.getName(), fieldInfos, extMeta, encodeClassDef.getInt64(0), classDefBytes); + Encoders.buildClassSpec(type), + fieldInfos, + isObjectType, + encodeClassDef.getInt64(0), + classDefBytes); } // see spec documentation: docs/specification/java_serialization_spec.md @@ -120,7 +123,7 @@ class ClassDefEncoder { ClassResolver classResolver, Class<?> type, Map<String, List<FieldInfo>> classLayers, - Map<String, String> extMeta) { + boolean isObjectType) { MemoryBuffer buffer = MemoryUtils.buffer(32); buffer.increaseWriterIndex(9); // header + one byte size long header; @@ -132,8 +135,8 @@ class ClassDefEncoder { header = encodedSize; } header |= SCHEMA_COMPATIBLE_FLAG; - if (!extMeta.isEmpty()) { - header |= EXT_FLAG; + if (isObjectType) { + header |= OBJECT_TYPE_FLAG; } for (Map.Entry<String, List<FieldInfo>> entry : classLayers.entrySet()) { String className = entry.getKey(); @@ -147,25 +150,13 @@ class ClassDefEncoder { buffer.writeVarUint32Small7(classResolver.getRegisteredClassId(type)); } else { buffer.writeVarUint32Small7(currentClassHeader); - String pkg = ReflectionUtils.getPackage(className); - String typeName = ReflectionUtils.getSimpleClassName(className); - writePkgName(buffer, pkg); - writeTypeName(buffer, typeName); + Class<?> currentType = getType(type, className); + Tuple2<String, String> encoded = Encoders.encodePkgAndClass(currentType); + writePkgName(buffer, encoded.f0); + writeTypeName(buffer, encoded.f1); } writeFieldsInfo(buffer, fields); } - if (!extMeta.isEmpty()) { - buffer.writeVarUint32Small7(extMeta.size()); - for (Map.Entry<String, String> entry : extMeta.entrySet()) { - String k = entry.getKey(); - String v = entry.getValue(); - byte[] keyBytes = k.getBytes(StandardCharsets.UTF_8); - byte[] valueBytes = v.getBytes(StandardCharsets.UTF_8); - buffer.writePrimitiveArrayWithSize(keyBytes, Platform.BYTE_ARRAY_OFFSET, keyBytes.length); - buffer.writePrimitiveArrayWithSize( - valueBytes, Platform.BYTE_ARRAY_OFFSET, valueBytes.length); - } - } byte[] encodedClassDef = buffer.getBytes(0, buffer.writerIndex()); long hash = MurmurHash3.murmurhash3_x64_128(encodedClassDef, 0, encodedClassDef.length, 47)[0]; // this id will be part of generated codec, a negative number won't be allowed in class name. @@ -188,6 +179,18 @@ class ClassDefEncoder { return buffer; } + private static Class<?> getType(Class<?> cls, String type) { + Class<?> c = cls; + while (cls != null) { + if (type.equals(cls.getName())) { + return cls; + } + cls = cls.getSuperclass(); + } + throw new IllegalStateException( + String.format("Class %s doesn't have %s as super class", c, type)); + } + static Map<String, List<FieldInfo>> getClassFields(Class<?> type, List<FieldInfo> fieldsInfo) { Map<String, List<FieldInfo>> sortedClassFields = new LinkedHashMap<>(); if (fieldsInfo.isEmpty()) { diff --git a/java/fury-core/src/main/java/org/apache/fury/meta/ClassSpec.java b/java/fury-core/src/main/java/org/apache/fury/meta/ClassSpec.java new file mode 100644 index 00000000..6e09d046 --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassSpec.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fury.meta; + +public class ClassSpec { + public final String entireClassName; + + /** Whether current class is enum of component is enum if current class is array. */ + public final boolean isEnum; + + public final boolean isArray; + public final int dimension; + + public ClassSpec(String entireClassName, boolean isEnum, boolean isArray, int dimension) { + this.entireClassName = entireClassName; + this.isEnum = isEnum; + this.isArray = isArray; + this.dimension = dimension; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ClassSpec classSpec = (ClassSpec) o; + return entireClassName.equals(classSpec.entireClassName); + } + + @Override + public int hashCode() { + return entireClassName.hashCode(); + } +} diff --git a/java/fury-core/src/main/java/org/apache/fury/meta/Encoders.java b/java/fury-core/src/main/java/org/apache/fury/meta/Encoders.java index a16c8f98..11434221 100644 --- a/java/fury-core/src/main/java/org/apache/fury/meta/Encoders.java +++ b/java/fury-core/src/main/java/org/apache/fury/meta/Encoders.java @@ -28,7 +28,11 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import org.apache.fury.collection.Tuple2; import org.apache.fury.meta.MetaString.Encoding; +import org.apache.fury.reflect.ReflectionUtils; +import org.apache.fury.type.TypeUtils; +import org.apache.fury.util.StringUtils; /** A class used to encode package/class/field name. */ public class Encoders { @@ -36,6 +40,8 @@ public class Encoders { public static final MetaStringDecoder PACKAGE_DECODER = new MetaStringDecoder('.', '_'); public static final MetaStringEncoder TYPE_NAME_ENCODER = new MetaStringEncoder('$', '_'); public static final MetaStringDecoder TYPE_NAME_DECODER = new MetaStringDecoder('$', '_'); + public static final String ARRAY_PREFIX = "1"; + public static final String ENUM_PREFIX = "2"; static final MetaStringEncoder FIELD_NAME_ENCODER = new MetaStringEncoder('$', '_'); static final MetaStringDecoder FIELD_NAME_DECODER = new MetaStringDecoder('$', '_'); private static final ConcurrentMap<String, MetaString> pgkMetaStringCache = @@ -67,6 +73,66 @@ public class Encoders { typeName, k -> TYPE_NAME_ENCODER.encode(typeName, typeNameEncodings)); } + public static Tuple2<String, String> encodePkgAndClass(Class<?> cls) { + String packageName = ReflectionUtils.getPackage(cls); + String className = ReflectionUtils.getClassNameWithoutPackage(cls); + if (cls.isArray()) { + Tuple2<Class<?>, Integer> componentInfo = TypeUtils.getArrayComponentInfo(cls); + Class<?> ctype = componentInfo.f0; + if (!ctype.isPrimitive()) { // primitive array has special format like [[[III. + String componentName = ctype.getName(); + packageName = ReflectionUtils.getPackage(componentName); + String componentSimpleName = ReflectionUtils.getClassNameWithoutPackage(componentName); + String prefix = StringUtils.repeat(Encoders.ARRAY_PREFIX, componentInfo.f1); + if (ctype.isEnum()) { + className = prefix + Encoders.ENUM_PREFIX + componentSimpleName; + } else { + className = prefix + componentSimpleName; + } + } + } else if (cls.isEnum()) { + className = Encoders.ENUM_PREFIX + className; + } + return Tuple2.of(packageName, className); + } + + public static ClassSpec buildClassSpec(Class<?> cls) { + if (cls.isArray()) { + Tuple2<Class<?>, Integer> info = TypeUtils.getArrayComponentInfo(cls); + return new ClassSpec(cls.getName(), info.f0.isEnum(), true, info.f1); + } else { + return new ClassSpec(cls.getName(), cls.isEnum(), false, 0); + } + } + + public static ClassSpec decodePkgAndClass(String packageName, String className) { + String rawPkg = packageName; + boolean isArray = className.startsWith(Encoders.ARRAY_PREFIX); + int dimension = 0; + if (isArray) { + while (className.charAt(dimension) == Encoders.ARRAY_PREFIX.charAt(0)) { + dimension++; + } + packageName = StringUtils.repeat("[", dimension) + "L" + packageName; + className = className.substring(dimension) + ";"; + } + boolean isEnum = className.startsWith(Encoders.ENUM_PREFIX); + if (isEnum) { + className = className.substring(1); + } + String entireClassName; + if (StringUtils.isBlank(rawPkg)) { + if (isArray) { + entireClassName = packageName + className; + } else { + entireClassName = className; + } + } else { + entireClassName = packageName + "." + className; + } + return new ClassSpec(entireClassName, isEnum, isArray, dimension); + } + public static MetaString encodeFieldName(String fieldName) { return fieldMetaStringCache.computeIfAbsent( fieldName, k -> FIELD_NAME_ENCODER.encode(fieldName, fieldNameEncodings)); diff --git a/java/fury-core/src/main/java/org/apache/fury/reflect/ReflectionUtils.java b/java/fury-core/src/main/java/org/apache/fury/reflect/ReflectionUtils.java index f641f14a..18357000 100644 --- a/java/fury-core/src/main/java/org/apache/fury/reflect/ReflectionUtils.java +++ b/java/fury-core/src/main/java/org/apache/fury/reflect/ReflectionUtils.java @@ -506,7 +506,10 @@ public class ReflectionUtils { if (targetType.isArray()) { return isMonomorphic(targetType.getComponentType()); } - return Modifier.isFinal(targetType.getModifiers()); + // Although enum itself can be non-final, but it's subclass can only be anonymous + // inner class, which is final, and we serialize enum by value, which already + // identify the actual class, so we can take enum as monomorphic. + return targetType.isEnum() || Modifier.isFinal(targetType.getModifiers()); } public static TypeRef getPublicSuperType(TypeRef typeRef) { diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java index 070adb2b..b5aac593 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java @@ -24,12 +24,11 @@ import static org.apache.fury.meta.Encoders.TYPE_NAME_ENCODER; import org.apache.fury.collection.Tuple2; import org.apache.fury.config.Language; +import org.apache.fury.meta.Encoders; import org.apache.fury.meta.MetaString.Encoding; import org.apache.fury.reflect.ReflectionUtils; import org.apache.fury.serializer.Serializer; -import org.apache.fury.type.TypeUtils; import org.apache.fury.util.Preconditions; -import org.apache.fury.util.StringUtils; import org.apache.fury.util.function.Functions; /** @@ -37,8 +36,6 @@ import org.apache.fury.util.function.Functions; * serialization. */ public class ClassInfo { - static final String ARRAY_PREFIX = "1"; - static final String ENUM_PREFIX = "2"; final Class<?> cls; final MetaStringBytes fullClassNameBytes; @@ -92,29 +89,11 @@ public class ClassInfo { if (cls != null && (classId == ClassResolver.NO_CLASS_ID || classId == ClassResolver.REPLACE_STUB_ID)) { // REPLACE_STUB_ID for write replace class in `ClassSerializer`. - String packageName = ReflectionUtils.getPackage(cls); - String className = ReflectionUtils.getClassNameWithoutPackage(cls); - if (cls.isArray()) { - Tuple2<Class<?>, Integer> componentInfo = TypeUtils.getArrayComponentInfo(cls); - Class<?> ctype = componentInfo.f0; - if (!ctype.isPrimitive()) { // primitive array has special format like [[[III. - String componentName = ctype.getName(); - packageName = ReflectionUtils.getPackage(componentName); - String componentSimpleName = ReflectionUtils.getClassNameWithoutPackage(componentName); - String prefix = StringUtils.repeat(ARRAY_PREFIX, componentInfo.f1); - if (ctype.isEnum()) { - className = prefix + ENUM_PREFIX + componentSimpleName; - } else { - className = prefix + componentSimpleName; - } - } - } else if (cls.isEnum()) { - className = ENUM_PREFIX + className; - } + Tuple2<String, String> tuple2 = Encoders.encodePkgAndClass(cls); this.packageNameBytes = - metaStringResolver.getOrCreateMetaStringBytes(PACKAGE_ENCODER.encode(packageName)); + metaStringResolver.getOrCreateMetaStringBytes(PACKAGE_ENCODER.encode(tuple2.f0)); this.classNameBytes = - metaStringResolver.getOrCreateMetaStringBytes(TYPE_NAME_ENCODER.encode(className)); + metaStringResolver.getOrCreateMetaStringBytes(TYPE_NAME_ENCODER.encode(tuple2.f1)); } else { this.packageNameBytes = null; this.classNameBytes = null; diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java index 23e3ccdf..4fce1cce 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java @@ -19,7 +19,6 @@ package org.apache.fury.resolver; -import static org.apache.fury.collection.Collections.ofHashMap; import static org.apache.fury.meta.ClassDef.SIZE_TWO_BYTES_FLAG; import static org.apache.fury.meta.Encoders.PACKAGE_DECODER; import static org.apache.fury.meta.Encoders.PACKAGE_ENCODER; @@ -88,6 +87,7 @@ import org.apache.fury.codegen.Expression.Invoke; import org.apache.fury.codegen.Expression.Literal; import org.apache.fury.collection.IdentityMap; import org.apache.fury.collection.IdentityObjectIntMap; +import org.apache.fury.collection.LongMap; import org.apache.fury.collection.ObjectMap; import org.apache.fury.collection.Tuple2; import org.apache.fury.config.CompatibleMode; @@ -98,6 +98,8 @@ import org.apache.fury.logging.LoggerFactory; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; import org.apache.fury.meta.ClassDef; +import org.apache.fury.meta.ClassSpec; +import org.apache.fury.meta.Encoders; import org.apache.fury.meta.MetaString; import org.apache.fury.reflect.ReflectionUtils; import org.apache.fury.reflect.TypeRef; @@ -146,7 +148,6 @@ import org.apache.fury.type.ScalaTypes; import org.apache.fury.type.TypeUtils; import org.apache.fury.util.GraalvmSupport; import org.apache.fury.util.Preconditions; -import org.apache.fury.util.StringUtils; import org.apache.fury.util.function.Functions; /** @@ -242,7 +243,7 @@ public class ClassResolver { // ex. A->field1: B, B.field1: A private final Set<Class<?>> getClassCtx = new HashSet<>(); private final Map<Class<?>, FieldResolver> fieldResolverMap = new HashMap<>(); - private final Map<Long, Tuple2<ClassDef, ClassInfo>> classIdToDef = new HashMap<>(); + private final LongMap<Tuple2<ClassDef, ClassInfo>> classIdToDef = new LongMap<>(); private final Map<Class<?>, ClassDef> currentLayerClassDef = new HashMap<>(); // TODO(chaokunyang) Better to use soft reference, see ObjectStreamClass. private final ConcurrentHashMap<Tuple2<Class<?>, Boolean>, SortedMap<Field, Descriptor>> @@ -256,7 +257,7 @@ public class ClassResolver { this.fury = fury; metaStringResolver = fury.getMetaStringResolver(); classInfoCache = NIL_CLASS_INFO; - metaContextShareEnabled = fury.getConfig().shareMetaContext(); + metaContextShareEnabled = fury.getConfig().isMetaShareEnabled(); extRegistry = new ExtRegistry(); extRegistry.objectGenericType = buildGenericType(OBJECT_TYPE); shimDispatcher = new ShimDispatcher(fury); @@ -512,7 +513,7 @@ public class ClassResolver { * non-final to write class def, so that it can be deserialized by the peer still. */ public boolean isMonomorphic(Class<?> clz) { - if (fury.getConfig().shareMetaContext()) { + if (fury.getConfig().isMetaShareEnabled()) { // can't create final map/collection type using TypeUtils.mapOf(TypeToken<K>, // TypeToken<V>) if (!ReflectionUtils.isMonomorphic(clz)) { @@ -811,7 +812,12 @@ public class ClassResolver { // serialized, which will create a class info with serializer null, see `#writeClassInternal` return classInfo.serializer.getClass(); } else { - if (cls.isEnum()) { + if (NonexistentClass.isNonexistent(cls)) { + return NonexistentClassSerializers.getSerializer(fury, "Unknown", cls).getClass(); + } + if (cls.isArray()) { + return ArraySerializers.ObjectArraySerializer.class; + } else if (cls.isEnum()) { return EnumSerializer.class; } else if (Enum.class.isAssignableFrom(cls) && cls != Enum.class) { // handles an enum value that is an inner class. Eg: enum A {b{}}; @@ -820,9 +826,6 @@ public class ClassResolver { return CollectionSerializers.EnumSetSerializer.class; } else if (Charset.class.isAssignableFrom(cls)) { return Serializers.CharsetSerializer.class; - } else if (cls.isArray()) { - Preconditions.checkArgument(!cls.getComponentType().isPrimitive()); - return ArraySerializers.ObjectArraySerializer.class; } else if (Functions.isLambda(cls)) { return LambdaSerializer.class; } else if (ReflectionUtils.isJdkProxy(cls)) { @@ -1293,16 +1296,10 @@ public class ClassResolver { classDef = classDefMap.computeIfAbsent(classInfo.cls, cls -> ClassDef.buildClassDef(fury, cls)); } else { - // TODO(chaokunyang) support more types meta-share serialization + // Some type will use other serializers such MapSerializer and so on. classDef = classDefMap.computeIfAbsent( - classInfo.cls, - cls -> - ClassDef.buildClassDef( - this, - cls, - new ArrayList<>(), - ofHashMap(META_SHARE_FIELDS_INFO_KEY, "false"))); + classInfo.cls, cls -> ClassDef.buildClassDef(this, cls, new ArrayList<>(), false)); } metaContext.writingClassDefs.add(classDef); } @@ -1320,7 +1317,7 @@ public class ClassResolver { if (classInfo == null) { List<ClassDef> readClassDefs = metaContext.readClassDefs; ClassDef classDef = readClassDefs.get(id); - Class<?> cls = loadClass(classDef.getClassName()); + Class<?> cls = loadClass(classDef.getClassSpec()); classInfo = getClassInfo(cls, false); if (classInfo == null) { Short classId = extRegistry.registeredClassIdMap.get(cls); @@ -1343,23 +1340,22 @@ public class ClassResolver { if (classInfo == null) { List<ClassDef> readClassDefs = metaContext.readClassDefs; ClassDef classDef = readClassDefs.get(id); - if ("false".equals(classDef.getExtMeta().getOrDefault(META_SHARE_FIELDS_INFO_KEY, ""))) { - Class<?> cls = loadClass(classDef.getClassName()); - classInfo = getClassInfo(cls); - } else { - Tuple2<ClassDef, ClassInfo> classDefTuple = extRegistry.classIdToDef.get(classDef.getId()); - if (classDefTuple == null || classDefTuple.f1 == null) { - if (classDefTuple != null) { - classDef = classDefTuple.f0; - } - Class<?> cls = loadClass(classDef.getClassName()); - classInfo = getMetaSharedClassInfo(classDef, cls); - // Share serializer for same version class def to avoid too much different meta - // context take up too much memory. - putClassDef(classDef, classInfo); + Tuple2<ClassDef, ClassInfo> classDefTuple = extRegistry.classIdToDef.get(classDef.getId()); + if (classDefTuple == null || classDefTuple.f1 == null) { + if (classDefTuple != null) { + classDef = classDefTuple.f0; + } + Class<?> cls = loadClass(classDef.getClassSpec()); + if (!classDef.isObjectType()) { + classInfo = getClassInfo(cls); } else { - classInfo = classDefTuple.f1; + classInfo = getMetaSharedClassInfo(classDef, cls); } + // Share serializer for same version class def to avoid too much different meta + // context take up too much memory. + putClassDef(classDef, classInfo); + } else { + classInfo = classDefTuple.f1; } readClassInfos.set(id, classInfo); } @@ -1432,8 +1428,7 @@ public class ClassResolver { int numClassDefs = buffer.readVarUint32Small14(); for (int i = 0; i < numClassDefs; i++) { long id = buffer.readInt64(); - long hash = id >>> 8; - Tuple2<ClassDef, ClassInfo> tuple2 = extRegistry.classIdToDef.get(hash); + Tuple2<ClassDef, ClassInfo> tuple2 = extRegistry.classIdToDef.get(id); if (tuple2 != null) { int size = (id & SIZE_TWO_BYTES_FLAG) == 0 @@ -1663,35 +1658,12 @@ public class ClassResolver { MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { String packageName = packageBytes.decode(PACKAGE_DECODER); - String rawPkg = packageName; String className = simpleClassNameBytes.decode(TYPE_NAME_DECODER); - boolean isArray = className.startsWith(ClassInfo.ARRAY_PREFIX); - int dimension = 0; - if (isArray) { - while (className.charAt(dimension) == ClassInfo.ARRAY_PREFIX.charAt(0)) { - dimension++; - } - packageName = StringUtils.repeat("[", dimension) + "L" + packageName; - className = className.substring(dimension) + ";"; - } - boolean isEnum = className.startsWith(ClassInfo.ENUM_PREFIX); - if (isEnum) { - className = className.substring(1); - } - String entireClassName; - if (StringUtils.isBlank(rawPkg)) { - if (isArray) { - entireClassName = packageName + className; - } else { - entireClassName = className; - } - } else { - entireClassName = packageName + "." + className; - } + ClassSpec classSpec = Encoders.decodePkgAndClass(packageName, className); MetaStringBytes fullClassNameBytes = metaStringResolver.getOrCreateMetaStringBytes( - PACKAGE_ENCODER.encode(entireClassName, MetaString.Encoding.UTF_8)); - Class<?> cls = loadClass(entireClassName, isEnum, dimension); + PACKAGE_ENCODER.encode(classSpec.entireClassName, MetaString.Encoding.UTF_8)); + Class<?> cls = loadClass(classSpec.entireClassName, classSpec.isEnum, classSpec.dimension); ClassInfo classInfo = new ClassInfo( cls, @@ -1703,7 +1675,8 @@ public class ClassResolver { null, NO_CLASS_ID); if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { - classInfo.serializer = NonexistentClassSerializers.getSerializer(fury, entireClassName, cls); + classInfo.serializer = + NonexistentClassSerializers.getSerializer(fury, classSpec.entireClassName, cls); } else { // don't create serializer here, if the class is an interface, // there won't be serializer since interface has no instance. @@ -1748,6 +1721,10 @@ public class ClassResolver { return loadClass(className, false, 0); } + private Class<?> loadClass(ClassSpec classSpec) { + return loadClass(classSpec.entireClassName, classSpec.isEnum, classSpec.dimension); + } + private Class<?> loadClass(String className, boolean isEnum, int arrayDims) { extRegistry.classChecker.checkClass(this, className); try { @@ -1762,7 +1739,7 @@ public class ClassResolver { className, fury.getClassLoader(), Thread.currentThread().getContextClassLoader()); if (fury.getConfig().deserializeNonexistentClass()) { LOG.warn(msg); - return NonexistentClass.getUnexistentClass( + return NonexistentClass.getNonexistentClass( className, isEnum, arrayDims, metaContextShareEnabled); } throw new IllegalStateException(msg, ex); diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/SerializationContext.java b/java/fury-core/src/main/java/org/apache/fury/resolver/SerializationContext.java index ed5aead6..56d65bdc 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/SerializationContext.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/SerializationContext.java @@ -20,6 +20,7 @@ package org.apache.fury.resolver; import java.util.IdentityHashMap; +import org.apache.fury.config.Config; import org.apache.fury.config.FuryBuilder; /** @@ -29,8 +30,16 @@ import org.apache.fury.config.FuryBuilder; */ public final class SerializationContext { private final IdentityHashMap<Object, Object> objects = new IdentityHashMap<>(); + private final boolean scopedMetaShareEnabled; private MetaContext metaContext; + public SerializationContext(Config config) { + scopedMetaShareEnabled = config.isScopedMetaShareEnabled(); + if (scopedMetaShareEnabled) { + metaContext = new MetaContext(); + } + } + /** Return the previous value associated with <tt>key</tt>, or <tt>null</tt>. */ public Object add(Object key, Object value) { return objects.put(key, value); @@ -52,16 +61,47 @@ public final class SerializationContext { * Set meta context, which can be used to share data across multiple serialization call. Note that * {@code metaContext} will be cleared after the serialization is finished. Please set the context * before every serialization if metaShare is enabled by {@link - * FuryBuilder#withMetaContextShare(boolean)} + * FuryBuilder#withMetaShare(boolean)} */ public void setMetaContext(MetaContext metaContext) { this.metaContext = metaContext; } + public void resetWrite() { + if (!objects.isEmpty()) { + objects.clear(); + } + if (scopedMetaShareEnabled) { + metaContext.classMap.clear(); + metaContext.writingClassDefs.clear(); + } else { + metaContext = null; + } + } + + public void resetRead() { + if (!objects.isEmpty()) { + objects.clear(); + } + if (scopedMetaShareEnabled) { + metaContext.readClassInfos.clear(); + metaContext.readClassDefs.clear(); + } else { + metaContext = null; + } + } + public void reset() { - if (objects.size() > 0) { + if (!objects.isEmpty()) { objects.clear(); } - metaContext = null; + if (scopedMetaShareEnabled) { + metaContext.classMap.clear(); + metaContext.writingClassDefs.clear(); + metaContext.readClassInfos.clear(); + metaContext.readClassDefs.clear(); + } else { + metaContext = null; + } } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java index 6fc5632a..06f3f88b 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java @@ -826,6 +826,10 @@ public class ArraySerializers { extends AbstractedNonexistentArrayClassSerializer { private final Serializer componentSerializer; + public NonexistentArrayClassSerializer(Fury fury, Class<?> cls) { + this(fury, "Unknown", cls); + } + public NonexistentArrayClassSerializer(Fury fury, String className, Class<?> cls) { super(fury, className, cls); if (TypeUtils.getArrayComponent(cls).isEnum()) { diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/MetaSharedSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/MetaSharedSerializer.java index 6d2c38c6..3fbd3312 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/MetaSharedSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/MetaSharedSerializer.java @@ -57,7 +57,7 @@ import org.apache.fury.util.record.RecordUtils; * info for those types. * * @see CompatibleMode - * @see FuryBuilder#withMetaContextShare + * @see FuryBuilder#withMetaShare * @see MetaSharedCodecBuilder * @see ObjectSerializer */ @@ -85,7 +85,8 @@ public class MetaSharedSerializer<T> extends Serializer<T> { Preconditions.checkArgument( !fury.getConfig().checkClassVersion(), "Class version check should be disabled when compatible mode is enabled."); - Preconditions.checkArgument(fury.getConfig().shareMetaContext(), "Meta share must be enabled."); + Preconditions.checkArgument( + fury.getConfig().isMetaShareEnabled(), "Meta share must be enabled."); Collection<Descriptor> descriptors = consolidateFields(fury.getClassResolver(), type, classDef); DescriptorGrouper descriptorGrouper = DescriptorGrouper.createDescriptorGrouper( diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClass.java b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClass.java index 1fb16ff3..92cd03ef 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClass.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClass.java @@ -23,12 +23,13 @@ import org.apache.fury.collection.LazyMap; import org.apache.fury.config.CompatibleMode; import org.apache.fury.config.Config; import org.apache.fury.meta.ClassDef; +import org.apache.fury.type.TypeUtils; /** * A class for hold deserialization data when the class doesn't exist in this process. When {@link * CompatibleMode#COMPATIBLE} is enabled * - * @see Config#shareMetaContext() + * @see Config#isMetaShareEnabled() */ public interface NonexistentClass { // @formatter:off @@ -122,11 +123,19 @@ public interface NonexistentClass { Class<?> Nonexistent2DArray = NonexistentMetaShared[][].class; Class<?> Nonexistent3DArray = NonexistentMetaShared[][][].class; - static Class<?> getUnexistentClass(boolean isEnum, int arrayDims, boolean shareMeta) { - return getUnexistentClass("Unknown", isEnum, arrayDims, shareMeta); + static boolean isNonexistent(Class<?> cls) { + if (cls.isArray()) { + Class<?> component = TypeUtils.getArrayComponent(cls); + return NonexistentClass.class.isAssignableFrom(component); + } + return NonexistentClass.class.isAssignableFrom(cls); + } + + static Class<?> getNonexistentClass(boolean isEnum, int arrayDims, boolean shareMeta) { + return getNonexistentClass("Unknown", isEnum, arrayDims, shareMeta); } - static Class<?> getUnexistentClass( + static Class<?> getNonexistentClass( String className, boolean isEnum, int arrayDims, boolean shareMeta) { if (arrayDims != 0) { if (isEnum) { diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java index ffe5f6d7..dd92d55d 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java @@ -75,7 +75,7 @@ public final class NonexistentClassSerializers { this.classDef = classDef; classInfoHolder = fury.getClassResolver().nilClassInfoHolder(); fieldsInfoMap = new LongMap<>(); - Preconditions.checkArgument(fury.getConfig().shareMetaContext()); + Preconditions.checkArgument(fury.getConfig().isMetaShareEnabled()); } /** @@ -250,7 +250,7 @@ public final class NonexistentClassSerializers { if (cls.isEnum()) { return new NonexistentEnumClassSerializer(fury); } else { - if (fury.getConfig().shareMetaContext()) { + if (fury.getConfig().isMetaShareEnabled()) { throw new IllegalStateException( String.format( "Serializer of class %s should be set in ClassResolver#getMetaSharedClassInfo", diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java index 8e9a2aac..64618570 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java @@ -98,7 +98,7 @@ public final class ObjectSerializer<T> extends Serializer<T> { // as data serializer. classResolver.setSerializerIfAbsent(cls, this); Collection<Descriptor> descriptors; - boolean shareMeta = fury.getConfig().shareMetaContext(); + boolean shareMeta = fury.getConfig().isMetaShareEnabled(); if (shareMeta) { ClassDef classDef = classResolver.getClassDef(cls, resolveParent); descriptors = classDef.getDescriptors(classResolver, cls); @@ -235,7 +235,7 @@ public final class ObjectSerializer<T> extends Serializer<T> { RefResolver refResolver, ClassResolver classResolver) { FinalTypeField[] finalFields = this.finalFields; - boolean metaContextShareEnabled = fury.getConfig().shareMetaContext(); + boolean metaShareEnabled = fury.getConfig().isMetaShareEnabled(); for (int i = 0; i < finalFields.length; i++) { FinalTypeField fieldInfo = finalFields[i]; FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; @@ -244,7 +244,7 @@ public final class ObjectSerializer<T> extends Serializer<T> { Object fieldValue = fieldAccessor.getObject(value); if (writeBasicObjectFieldValueFailed(fury, buffer, fieldValue, classId)) { Serializer<Object> serializer = fieldInfo.classInfo.getSerializer(); - if (!metaContextShareEnabled || isFinal[i]) { + if (!metaShareEnabled || isFinal[i]) { // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still // consistent with jit serializer. fury.writeRef(buffer, fieldValue, serializer); @@ -341,10 +341,10 @@ public final class ObjectSerializer<T> extends Serializer<T> { int counter = 0; // read order: primitive,boxed,final,other,collection,map FinalTypeField[] finalFields = this.finalFields; - boolean metaContextShareEnabled = fury.getConfig().shareMetaContext(); + boolean metaShareEnabled = fury.getConfig().isMetaShareEnabled(); for (int i = 0; i < finalFields.length; i++) { FinalTypeField fieldInfo = finalFields[i]; - boolean isFinal = !metaContextShareEnabled || this.isFinal[i]; + boolean isFinal = !metaShareEnabled || this.isFinal[i]; short classId = fieldInfo.classId; if (classId >= ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID && classId <= ClassResolver.PRIMITIVE_DOUBLE_CLASS_ID) { @@ -377,10 +377,10 @@ public final class ObjectSerializer<T> extends Serializer<T> { } // read order: primitive,boxed,final,other,collection,map FinalTypeField[] finalFields = this.finalFields; - boolean metaContextShareEnabled = fury.getConfig().shareMetaContext(); + boolean metaShareEnabled = fury.getConfig().isMetaShareEnabled(); for (int i = 0; i < finalFields.length; i++) { FinalTypeField fieldInfo = finalFields[i]; - boolean isFinal = !metaContextShareEnabled || this.isFinal[i]; + boolean isFinal = !metaShareEnabled || this.isFinal[i]; FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; short classId = fieldInfo.classId; if (readPrimitiveFieldValueFailed(fury, buffer, obj, fieldAccessor, classId) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java index 1b0a9af3..15669489 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java @@ -186,7 +186,7 @@ public abstract class AbstractCollectionSerializer<T> extends Serializer<T> { buffer.writeByte(bitmap); } else { // Write class in case peer doesn't have this class. - if (!fury.getConfig().shareMetaContext() && elemClass == declareElementType) { + if (!fury.getConfig().isMetaShareEnabled() && elemClass == declareElementType) { buffer.writeByte(bitmap); } else { bitmap |= CollectionFlags.NOT_DECL_ELEMENT_TYPE; @@ -277,7 +277,7 @@ public abstract class AbstractCollectionSerializer<T> extends Serializer<T> { elemClass = Object.class; } // Write class in case peer doesn't have this class. - if (!fury.getConfig().shareMetaContext() && elemClass == declareElementType) { + if (!fury.getConfig().isMetaShareEnabled() && elemClass == declareElementType) { buffer.writeByte(bitmap); } else { bitmap |= CollectionFlags.NOT_DECL_ELEMENT_TYPE; diff --git a/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java b/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java index 58ca1778..00f226b5 100644 --- a/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java @@ -70,7 +70,7 @@ public class CyclicTest extends FuryTestBase { @Test(dataProvider = "fury") public void testBean(FuryBuilder builder) { - Fury fury = builder.withMetaContextShare(false).withRefTracking(true).build(); + Fury fury = builder.withMetaShare(false).withRefTracking(true).build(); for (Object[] objects : beans()) { Object notCyclic = objects[0]; Object cyclic = objects[1]; diff --git a/java/fury-core/src/test/java/org/apache/fury/FuryInitPerf.java b/java/fury-core/src/test/java/org/apache/fury/FuryInitPerf.java index 55055c1c..bc467286 100644 --- a/java/fury-core/src/test/java/org/apache/fury/FuryInitPerf.java +++ b/java/fury-core/src/test/java/org/apache/fury/FuryInitPerf.java @@ -50,7 +50,7 @@ public class FuryInitPerf { Fury.builder() .withLanguage(Language.JAVA) .withNumberCompressed(true) - .withMetaContextShare(true) + .withMetaShare(true) .requireClassRegistration(false) .withCompatibleMode(CompatibleMode.COMPATIBLE) .withAsyncCompilation(true) @@ -77,7 +77,7 @@ public class FuryInitPerf { Fury.builder() .withLanguage(Language.JAVA) .withNumberCompressed(true) - .withMetaContextShare(true) + .withMetaShare(true) .requireClassRegistration(false) .withCompatibleMode(CompatibleMode.COMPATIBLE) .withAsyncCompilation(true) @@ -98,7 +98,7 @@ public class FuryInitPerf { Fury.builder() .withLanguage(Language.JAVA) .withNumberCompressed(true) - .withMetaContextShare(true) + .withMetaShare(true) .requireClassRegistration(false) .withCompatibleMode(CompatibleMode.COMPATIBLE) .withAsyncCompilation(true) diff --git a/java/fury-core/src/test/java/org/apache/fury/ThreadSafeFuryTest.java b/java/fury-core/src/test/java/org/apache/fury/ThreadSafeFuryTest.java index e604bb74..8c35243b 100644 --- a/java/fury-core/src/test/java/org/apache/fury/ThreadSafeFuryTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/ThreadSafeFuryTest.java @@ -37,7 +37,6 @@ import org.apache.fury.resolver.MetaContext; import org.apache.fury.serializer.Serializer; import org.apache.fury.test.bean.BeanA; import org.apache.fury.test.bean.BeanB; -import org.apache.fury.test.bean.Foo; import org.apache.fury.test.bean.Struct; import org.apache.fury.util.LoaderBinding.StagingType; import org.testng.Assert; @@ -151,7 +150,7 @@ public class ThreadSafeFuryTest extends FuryTestBase { ThreadSafeFury fury2 = Fury.builder() .withLanguage(Language.JAVA) - .withMetaContextShare(true) + .withMetaShare(true) .requireClassRegistration(false) .buildThreadSafeFury(); BeanA beanA = BeanA.createBeanA(2); diff --git a/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java b/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java index 918916d6..45d3075f 100644 --- a/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java @@ -22,7 +22,6 @@ package org.apache.fury.meta; import static org.apache.fury.meta.ClassDefEncoder.buildFieldsInfo; import static org.apache.fury.meta.ClassDefEncoder.getClassFields; -import java.util.HashMap; import java.util.List; import lombok.Data; import org.apache.fury.Fury; @@ -38,12 +37,12 @@ public class ClassDefEncoderTest { @Test public void testBasicClassDef() throws Exception { - Fury fury = Fury.builder().withMetaContextShare(true).build(); + Fury fury = Fury.builder().withMetaShare(true).build(); Class<ClassDefTest.TestFieldsOrderClass1> type = ClassDefTest.TestFieldsOrderClass1.class; List<ClassDef.FieldInfo> fieldsInfo = buildFieldsInfo(fury.getClassResolver(), type); MemoryBuffer buffer = ClassDefEncoder.encodeClassDef( - fury.getClassResolver(), type, getClassFields(type, fieldsInfo), new HashMap<>()); + fury.getClassResolver(), type, getClassFields(type, fieldsInfo), true); ClassDef classDef = ClassDef.readClassDef(fury.getClassResolver(), buffer); Assert.assertEquals(classDef.getClassName(), type.getName()); Assert.assertEquals(classDef.getFieldsInfo().size(), type.getDeclaredFields().length); @@ -56,7 +55,7 @@ public class ClassDefEncoderTest { new Class[] { MapFields.class, BeanA.class, Struct.createStructClass("TestBigMetaEncoding", 5) }) { - Fury fury = Fury.builder().withMetaContextShare(true).build(); + Fury fury = Fury.builder().withMetaShare(true).build(); ClassDef classDef = ClassDef.buildClassDef(fury, type); ClassDef classDef1 = ClassDef.readClassDef( diff --git a/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java b/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java index 45425d78..c1a50ea8 100644 --- a/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java @@ -85,7 +85,7 @@ public class ClassDefTest extends FuryTestBase { @Test public void testClassDefSerialization() throws NoSuchFieldException { - Fury fury = Fury.builder().withMetaContextShare(true).build(); + Fury fury = Fury.builder().withMetaShare(true).build(); { ClassDef classDef = ClassDef.buildClassDef( @@ -134,7 +134,7 @@ public class ClassDefTest extends FuryTestBase { @Test public void testDuplicateFieldsClass() { - Fury fury = Fury.builder().withMetaContextShare(true).build(); + Fury fury = Fury.builder().withMetaShare(true).build(); { ClassDef classDef = ClassDef.buildClassDef( @@ -155,7 +155,7 @@ public class ClassDefTest extends FuryTestBase { @Test public void testContainerClass() { - Fury fury = Fury.builder().withMetaContextShare(true).build(); + Fury fury = Fury.builder().withMetaShare(true).build(); List<Field> fields = ReflectionUtils.getFields(ContainerClass.class, true); ClassDef classDef = ClassDef.buildClassDef(fury.getClassResolver(), ContainerClass.class, fields); diff --git a/java/fury-core/src/test/java/org/apache/fury/reflect/ReflectionUtilsTest.java b/java/fury-core/src/test/java/org/apache/fury/reflect/ReflectionUtilsTest.java index 91a401d9..d55795e8 100644 --- a/java/fury-core/src/test/java/org/apache/fury/reflect/ReflectionUtilsTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/reflect/ReflectionUtilsTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; import org.apache.fury.test.bean.BeanA; import org.apache.fury.type.Descriptor; +import org.testng.Assert; import org.testng.annotations.Test; public class ReflectionUtilsTest { @@ -70,4 +71,34 @@ public class ReflectionUtilsTest { ReflectionUtils.getFieldValues(Descriptor.getFields(GetFieldValuesTestClass.class), o); assertEquals(fieldValues, ImmutableList.of("str", 10)); } + + enum MonomorphicTestEnum1 { + A, + B + } + + enum MonomorphicTestEnum2 { + A { + @Override + int f() { + return 0; + } + }, + B { + @Override + int f() { + return 1; + } + }; + + abstract int f(); + } + + @Test + public void testMonomorphic() { + Assert.assertTrue(ReflectionUtils.isMonomorphic(MonomorphicTestEnum1.class)); + Assert.assertTrue(ReflectionUtils.isMonomorphic(MonomorphicTestEnum2.class)); + Assert.assertTrue(ReflectionUtils.isMonomorphic(MonomorphicTestEnum1[].class)); + Assert.assertTrue(ReflectionUtils.isMonomorphic(MonomorphicTestEnum2[].class)); + } } diff --git a/java/fury-core/src/test/java/org/apache/fury/resolver/MetaContextTest.java b/java/fury-core/src/test/java/org/apache/fury/resolver/MetaContextTest.java index f39f339f..645c4a5a 100644 --- a/java/fury-core/src/test/java/org/apache/fury/resolver/MetaContextTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/resolver/MetaContextTest.java @@ -41,7 +41,7 @@ public class MetaContextTest extends FuryTestBase { Fury.builder() .withLanguage(Language.JAVA) .withRefTracking(true) - .withMetaContextShare(true) + .withMetaShare(true) .requireClassRegistration(false) .build(); for (Object o : new Object[] {Foo.create(), BeanB.createBeanB(2), BeanA.createBeanA(2)}) { @@ -55,7 +55,7 @@ public class MetaContextTest extends FuryTestBase { Fury.builder() .withLanguage(Language.JAVA) .withRefTracking(true) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .withCodegen(enableCodegen) .requireClassRegistration(false) @@ -100,7 +100,7 @@ public class MetaContextTest extends FuryTestBase { Fury.builder() .withLanguage(Language.JAVA) .withRefTracking(true) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .withCodegen(enableCodegen) .requireClassRegistration(false) diff --git a/java/fury-core/src/test/java/org/apache/fury/resolver/SerializationContextTest.java b/java/fury-core/src/test/java/org/apache/fury/resolver/SerializationContextTest.java index f67d25f4..2f0f42fc 100644 --- a/java/fury-core/src/test/java/org/apache/fury/resolver/SerializationContextTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/resolver/SerializationContextTest.java @@ -21,13 +21,17 @@ package org.apache.fury.resolver; import static org.testng.Assert.*; +import org.apache.fury.config.Config; +import org.apache.fury.config.FuryBuilder; import org.testng.annotations.Test; public class SerializationContextTest { @Test public void testSerializationContext() { - SerializationContext context = new SerializationContext(); + SerializationContext context = + new SerializationContext( + new Config(new FuryBuilder().withDeserializeNonexistentClass(false))); assertFalse(context.containsKey("A")); context.add("A", 1); assertTrue(context.containsKey("A")); diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java index d68ca016..760272c6 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java @@ -103,7 +103,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withNumberCompressed(compressNumber) .withRefTracking(referenceTracking) .withCodegen(enableCodegen) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .build(); @@ -125,7 +125,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .build(); @@ -143,7 +143,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .withClassLoader(fooClass.getClassLoader()) @@ -210,7 +210,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { Fury fury1 = Fury.builder() .withCodegen(false) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .withClassLoader(cls1.getClassLoader()) @@ -236,7 +236,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { Fury fury2 = Fury.builder() .withCodegen(false) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .withClassLoader(cls2.getClassLoader()) @@ -284,7 +284,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .withClassLoader(cls1.getClassLoader()) @@ -313,7 +313,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen3) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .withClassLoader(cls2.getClassLoader()) @@ -344,7 +344,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .build(); @@ -369,7 +369,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .build(); @@ -385,7 +385,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .withClassLoader(cls.getClassLoader()) @@ -427,7 +427,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .build(); @@ -449,7 +449,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .withClassLoader(cls2.getClassLoader()) @@ -504,7 +504,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .build(); @@ -525,7 +525,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .build(); @@ -597,7 +597,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .withClassLoader(cls1.getClassLoader()) @@ -637,7 +637,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .withClassLoader(cls2.getClassLoader()) @@ -696,7 +696,7 @@ public class MetaSharedCompatibleTest extends FuryTestBase { .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .withClassLoader(cls1.getClassLoader()) diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java index 3f146e7a..d4012412 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java @@ -70,6 +70,7 @@ public class NonexistentClassSerializersTest extends FuryTestBase { .withLanguage(Language.JAVA) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) + .withCodegen(false) .withDeserializeNonexistentClass(true); } @@ -101,23 +102,35 @@ public class NonexistentClassSerializersTest extends FuryTestBase { } } - @Test - public void testSkipNonexistentEnum() { - Fury fury1 = furyBuilder().withDeserializeNonexistentClass(true).build(); - String enumCode = ("enum TestEnum {" + " A, B" + "}"); + @DataProvider + public static Object[][] scopedMetaShare() { + return new Object[][] {{false}, {true}}; + } + @Test(dataProvider = "scopedMetaShare") + public void testNonexistentEnum(boolean scopedMetaShare) { + FuryBuilder builder = furyBuilder(); + if (scopedMetaShare) { + builder.withMetaShare(true).withScopedMetaShare(true); + } + Fury fury = builder.withDeserializeNonexistentClass(true).build(); + String enumCode = ("enum TestEnum {" + " A, B" + "}"); Class<?> cls = JaninoUtils.compileClass(getClass().getClassLoader(), "", "TestEnum", enumCode); Object c = cls.getEnumConstants()[1]; assertEquals(c.toString(), "B"); - byte[] bytes = fury1.serialize(c); + byte[] bytes = fury.serialize(c); Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - Fury fury2 = furyBuilder().withDeserializeNonexistentClass(true).build(); + Fury fury2 = builder.withDeserializeNonexistentClass(true).build(); Object o = fury2.deserialize(bytes); assertEquals(o, NonexistentClass.NonexistentEnum.V1); } - @Test - public void testSkipNonexistentEnumAndArrayField() throws Exception { + @Test(dataProvider = "scopedMetaShare") + public void testNonexistentEnumAndArrayField(boolean scopedMetaShare) throws Exception { + FuryBuilder builder = furyBuilder(); + if (scopedMetaShare) { + builder.withMetaShare(true).withScopedMetaShare(true); + } String enumStructCode1 = ("public class TestEnumStruct {\n" + " public enum TestEnum {\n" @@ -144,7 +157,7 @@ public class NonexistentClassSerializersTest extends FuryTestBase { enumArray2[1] = enumArray; ReflectionUtils.setObjectFieldValue(o, "f4", enumArray2); Fury fury1 = - furyBuilder() + builder .withDeserializeNonexistentClass(true) .withClassLoader(cls1.getClassLoader()) .build(); @@ -161,8 +174,7 @@ public class NonexistentClassSerializersTest extends FuryTestBase { "", "TestEnumStruct", ("public class TestEnumStruct {" + " public String f1;" + "}"))); - Fury fury2 = - furyBuilder().withDeserializeNonexistentClass(true).withClassLoader(classLoader).build(); + Fury fury2 = builder.withDeserializeNonexistentClass(true).withClassLoader(classLoader).build(); Object o1 = fury2.deserialize(bytes); Assert.assertEquals(ReflectionUtils.getObjectFieldValue(o1, "f1"), "str"); } @@ -236,7 +248,7 @@ public class NonexistentClassSerializersTest extends FuryTestBase { furyBuilder() .withRefTracking(referenceTracking) .withCodegen(enableCodegen1) - .withMetaContextShare(true) + .withMetaShare(true) .build(); ClassLoader classLoader = getClass().getClassLoader(); for (Class<?> structClass : @@ -252,7 +264,7 @@ public class NonexistentClassSerializersTest extends FuryTestBase { furyBuilder() .withRefTracking(referenceTracking) .withCodegen(enableCodegen2) - .withMetaContextShare(true) + .withMetaShare(true) .withClassLoader(classLoader) .build(); MetaContext context2 = new MetaContext(); @@ -265,7 +277,7 @@ public class NonexistentClassSerializersTest extends FuryTestBase { furyBuilder() .withRefTracking(referenceTracking) .withCodegen(enableCodegen3) - .withMetaContextShare(true) + .withMetaShare(true) .withClassLoader(pojo.getClass().getClassLoader()) .build(); MetaContext context3 = new MetaContext(); @@ -286,7 +298,7 @@ public class NonexistentClassSerializersTest extends FuryTestBase { furyBuilder() .withRefTracking(referenceTracking) .withCodegen(enableCodegen1) - .withMetaContextShare(true) + .withMetaShare(true) .build(); MetaContext context1 = new MetaContext(); MetaContext context2 = new MetaContext(); @@ -301,14 +313,14 @@ public class NonexistentClassSerializersTest extends FuryTestBase { furyBuilder() .withRefTracking(referenceTracking) .withCodegen(enableCodegen2) - .withMetaContextShare(true) + .withMetaShare(true) .withClassLoader(classLoader) .build(); Fury fury3 = furyBuilder() .withRefTracking(referenceTracking) .withCodegen(enableCodegen3) - .withMetaContextShare(true) + .withMetaShare(true) .withClassLoader(structClass.getClassLoader()) .build(); for (int i = 0; i < 2; i++) { diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/ProtocolInteroperabilityTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/ProtocolInteroperabilityTest.java index 76de5e56..09f5646e 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/ProtocolInteroperabilityTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/ProtocolInteroperabilityTest.java @@ -235,14 +235,14 @@ public class ProtocolInteroperabilityTest extends FuryTestBase { new Object[] { Fury.builder() .withLanguage(Language.JAVA) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode((CompatibleMode) c[0]) .withCodegen(false) .requireClassRegistration(false) .build(), Fury.builder() .withLanguage(Language.JAVA) - .withMetaContextShare(true) + .withMetaShare(true) .withCompatibleMode((CompatibleMode) c[0]) .withCodegen(true) .requireClassRegistration(false) diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/ChildContainerSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/ChildContainerSerializersTest.java index 042ba0dc..57e906bb 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/ChildContainerSerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/ChildContainerSerializersTest.java @@ -180,8 +180,7 @@ public class ChildContainerSerializersTest extends FuryTestBase { .withLanguage(Language.JAVA) .withCompatibleMode(CompatibleMode.COMPATIBLE) .withDeserializeNonexistentClass(true) - .withMetaContextShare(true) - .requireClassRegistration(false) + .withMetaShare(true) .requireClassRegistration(false) .withCodegen(enableCodegen) .build(); --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org