This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fury.git
The following commit(s) were added to refs/heads/main by this push:
new b3196e39 feat(java): unify java and xlang object serialization (#2146)
b3196e39 is described below
commit b3196e39a46c26b1e49f144283b1f22a398e2b7c
Author: Shawn Yang <shawn.ck.y...@gmail.com>
AuthorDate: Sat Apr 19 09:16:45 2025 +0800
feat(java): unify java and xlang object serialization (#2146)
## What does this PR do?
This PR unify the java and xlang object serialization in java:
- Remove StructSerializer
- Unify struct hash compute between java and python
- Align fields sort between java and python
- unify the java and xlang object serialization in java
## Related issues
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fury/issues/new/choose) describing the
need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
ci/format.sh | 2 +-
.../src/main/java/org/apache/fury/Fury.java | 2 +-
.../fury/builder/MetaSharedCodecBuilder.java | 8 +-
.../apache/fury/builder/ObjectCodecBuilder.java | 11 +-
.../java/org/apache/fury/config/FuryBuilder.java | 5 +
.../java/org/apache/fury/meta/ClassDefEncoder.java | 16 +-
.../org/apache/fury/reflect/FieldAccessor.java | 5 +
.../java/org/apache/fury/resolver/ClassInfo.java | 4 +
.../org/apache/fury/resolver/ClassResolver.java | 78 ++++++
.../org/apache/fury/resolver/TypeResolver.java | 4 +
.../org/apache/fury/resolver/XtypeResolver.java | 50 +++-
.../fury/serializer/AbstractObjectSerializer.java | 51 ++--
.../apache/fury/serializer/ArraySerializers.java | 6 +-
.../apache/fury/serializer/CodegenSerializer.java | 3 +
.../org/apache/fury/serializer/LazySerializer.java | 96 +++++++
.../fury/serializer/MetaSharedSerializer.java | 8 +-
.../serializer/NonexistentClassSerializers.java | 10 +-
.../apache/fury/serializer/ObjectSerializer.java | 83 ++++--
.../fury/serializer/SerializationBinding.java | 114 +++++++-
.../apache/fury/serializer/StructSerializer.java | 286 ---------------------
.../main/java/org/apache/fury/type/Descriptor.java | 8 +
.../org/apache/fury/type/DescriptorGrouper.java | 43 ++--
.../src/main/java/org/apache/fury/type/Types.java | 20 ++
.../fury-core/native-image.properties | 5 +
.../java/org/apache/fury/CrossLanguageTest.java | 18 +-
.../apache/fury/type/DescriptorGrouperTest.java | 41 +--
python/pyfury/_registry.py | 6 +
python/pyfury/_struct.py | 71 ++++-
python/pyfury/resolver.py | 2 +
python/pyfury/type.py | 73 ++++++
30 files changed, 701 insertions(+), 428 deletions(-)
diff --git a/ci/format.sh b/ci/format.sh
index 4f931dda..49cdec85 100755
--- a/ci/format.sh
+++ b/ci/format.sh
@@ -326,7 +326,7 @@ if ! git diff --quiet &>/dev/null; then
echo 'Files updated:'
echo
- git --no-pager diff --name-only
+ git --no-pager diff
exit 1
fi
diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java
b/java/fury-core/src/main/java/org/apache/fury/Fury.java
index b564fc99..81d437d7 100644
--- a/java/fury-core/src/main/java/org/apache/fury/Fury.java
+++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java
@@ -563,7 +563,7 @@ public final class Fury implements BaseFury {
xwriteData(buffer, classInfo, obj);
}
- private void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object
obj) {
+ public void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object obj)
{
switch (classInfo.getXtypeId()) {
case Types.BOOL:
buffer.writeBoolean((Boolean) obj);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java
b/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java
index eb980c02..91a6413e 100644
---
a/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java
+++
b/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java
@@ -77,13 +77,7 @@ public class MetaSharedCodecBuilder extends
ObjectCodecBuilder {
Collection<Descriptor> descriptors =
visitFury(
f -> MetaSharedSerializer.consolidateFields(f.getClassResolver(),
beanClass, classDef));
- DescriptorGrouper grouper =
- DescriptorGrouper.createDescriptorGrouper(
- fury.getClassResolver()::isMonomorphic,
- descriptors,
- false,
- fury.compressInt(),
- fury.compressLong());
+ DescriptorGrouper grouper =
fury.getClassResolver().createDescriptorGrouper(descriptors, false);
objectCodecOptimizer =
new ObjectCodecOptimizer(beanClass, grouper,
!fury.isBasicTypesRefIgnored(), ctx);
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java
b/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java
index a0febb30..c32a49e1 100644
---
a/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java
+++
b/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java
@@ -98,15 +98,10 @@ public class ObjectCodecBuilder extends
BaseObjectCodecBuilder {
} else {
descriptors = fury.getClassResolver().getAllDescriptorsMap(beanClass,
true).values();
}
+ DescriptorGrouper grouper =
classResolver.createDescriptorGrouper(descriptors, false);
+ descriptors = grouper.getSortedDescriptors();
classVersionHash =
- new Literal(ObjectSerializer.computeVersionHash(descriptors),
PRIMITIVE_INT_TYPE);
- DescriptorGrouper grouper =
- DescriptorGrouper.createDescriptorGrouper(
- fury.getClassResolver()::isMonomorphic,
- descriptors,
- false,
- fury.compressInt(),
- fury.compressLong());
+ new Literal(ObjectSerializer.computeStructHash(fury, descriptors),
PRIMITIVE_INT_TYPE);
objectCodecOptimizer =
new ObjectCodecOptimizer(beanClass, grouper,
!fury.isBasicTypesRefIgnored(), ctx);
if (isRecord) {
diff --git
a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java
b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java
index 3b994e1e..4d5bbb22 100644
--- a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java
@@ -376,6 +376,8 @@ public final class FuryBuilder {
}
if (language != Language.JAVA) {
stringRefIgnored = true;
+ longEncoding = LongEncoding.PVL;
+ compressInt = true;
}
if (ENABLE_CLASS_REGISTRATION_FORCIBLY) {
if (!requireClassRegistration) {
@@ -422,6 +424,9 @@ public final class FuryBuilder {
if (metaShareEnabled == null) {
metaShareEnabled = false;
}
+ if (language != Language.JAVA) {
+ checkClassVersion = true;
+ }
}
if (!requireClassRegistration) {
LOG.warn(
diff --git
a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
index 1579b931..a6b2fa91 100644
--- a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
@@ -30,7 +30,6 @@ import static org.apache.fury.util.MathUtils.toInt;
import java.lang.reflect.Field;
import java.util.ArrayList;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -44,7 +43,6 @@ import org.apache.fury.meta.ClassDef.FieldInfo;
import org.apache.fury.meta.ClassDef.FieldType;
import org.apache.fury.reflect.ReflectionUtils;
import org.apache.fury.resolver.ClassResolver;
-import org.apache.fury.type.Descriptor;
import org.apache.fury.type.DescriptorGrouper;
import org.apache.fury.util.MurmurHash3;
@@ -55,16 +53,12 @@ import org.apache.fury.util.MurmurHash3;
*/
class ClassDefEncoder {
static List<Field> buildFields(Fury fury, Class<?> cls, boolean
resolveParent) {
- Comparator<Descriptor> comparator =
- DescriptorGrouper.getPrimitiveComparator(fury.compressInt(),
fury.compressLong());
DescriptorGrouper descriptorGrouper =
- new DescriptorGrouper(
- fury.getClassResolver()::isMonomorphic,
- fury.getClassResolver().getAllDescriptorsMap(cls,
resolveParent).values(),
- false,
- Function.identity(),
- comparator,
- DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME);
+ fury.getClassResolver()
+ .createDescriptorGrouper(
+ fury.getClassResolver().getAllDescriptorsMap(cls,
resolveParent).values(),
+ false,
+ Function.identity());
List<Field> fields = new ArrayList<>();
descriptorGrouper
.getPrimitiveDescriptors()
diff --git
a/java/fury-core/src/main/java/org/apache/fury/reflect/FieldAccessor.java
b/java/fury-core/src/main/java/org/apache/fury/reflect/FieldAccessor.java
index 7b1c450b..038a9329 100644
--- a/java/fury-core/src/main/java/org/apache/fury/reflect/FieldAccessor.java
+++ b/java/fury-core/src/main/java/org/apache/fury/reflect/FieldAccessor.java
@@ -104,6 +104,11 @@ public abstract class FieldAccessor {
}
}
+ @Override
+ public String toString() {
+ return field.toString();
+ }
+
public abstract static class FieldGetter extends FieldAccessor {
private final Object getter;
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
index 0f34b4c3..5e14ba5f 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
@@ -139,6 +139,10 @@ public class ClassInfo {
return (Serializer<T>) serializer;
}
+ public void setSerializer(Serializer<?> serializer) {
+ this.serializer = serializer;
+ }
+
void setSerializer(ClassResolver resolver, Serializer<?> serializer) {
this.serializer = serializer;
needToWriteClassDef = serializer != null &&
resolver.needToWriteClassDef(serializer);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
index 83d62876..3d24ecc6 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
@@ -79,6 +79,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.fury.Fury;
@@ -154,9 +155,11 @@ import
org.apache.fury.serializer.scala.SingletonMapSerializer;
import org.apache.fury.serializer.scala.SingletonObjectSerializer;
import org.apache.fury.serializer.shim.ShimDispatcher;
import org.apache.fury.type.Descriptor;
+import org.apache.fury.type.DescriptorGrouper;
import org.apache.fury.type.GenericType;
import org.apache.fury.type.ScalaTypes;
import org.apache.fury.type.TypeUtils;
+import org.apache.fury.type.Types;
import org.apache.fury.util.GraalvmSupport;
import org.apache.fury.util.Preconditions;
import org.apache.fury.util.StringUtils;
@@ -537,6 +540,7 @@ public class ClassResolver implements TypeResolver {
}
}
+ @Override
public boolean isRegistered(Class<?> cls) {
return extRegistry.registeredClassIdMap.containsKey(cls)
|| extRegistry.registeredClasses.inverse().containsKey(cls);
@@ -611,6 +615,9 @@ public class ClassResolver implements TypeResolver {
* non-final to write class def, so that it can be deserialized by the peer
still.
*/
public boolean isMonomorphic(Class<?> clz) {
+ if (fury.isCrossLanguage()) {
+ return TypeUtils.unwrap(clz).isPrimitive();
+ }
if (fury.getConfig().isMetaShareEnabled()) {
// can't create final map/collection type using
TypeUtils.mapOf(TypeToken<K>,
// TypeToken<V>)
@@ -1198,6 +1205,7 @@ public class ClassResolver implements TypeResolver {
}
// Invoked by fury JIT.
+ @Override
public ClassInfo getClassInfo(Class<?> cls) {
ClassInfo classInfo = classInfoMap.get(cls);
if (classInfo == null || classInfo.serializer == null) {
@@ -2032,6 +2040,76 @@ public class ClassResolver implements TypeResolver {
extRegistry.codeGeneratorMap.put(Arrays.asList(loaders), codeGenerator);
}
+ public DescriptorGrouper createDescriptorGrouper(
+ Collection<Descriptor> descriptors, boolean descriptorsGroupedOrdered) {
+ return createDescriptorGrouper(descriptors, descriptorsGroupedOrdered,
null);
+ }
+
+ public DescriptorGrouper createDescriptorGrouper(
+ Collection<Descriptor> descriptors,
+ boolean descriptorsGroupedOrdered,
+ Function<Descriptor, Descriptor> descriptorUpdator) {
+ if (fury.isCrossLanguage()) {
+ return DescriptorGrouper.createDescriptorGrouper(
+ c -> {
+ if (TypeUtils.unwrap(c).isPrimitive()) {
+ return true;
+ } else if (c == String.class) {
+ return true;
+ }
+ if (c.isArray() && TypeUtils.getArrayComponent(c).isPrimitive()) {
+ return true;
+ }
+ // TODO(chaokunyang) add more types.
+ return false;
+ },
+ descriptors,
+ descriptorsGroupedOrdered,
+ descriptorUpdator,
+ fury.compressInt(),
+ fury.compressLong(),
+ (o1, o2) -> {
+ int xtypeId = getXtypeId(o1.getRawType());
+ int xtypeId2 = getXtypeId(o2.getRawType());
+ if (xtypeId == xtypeId2) {
+ return o1.getSnakeCaseName().compareTo(o2.getSnakeCaseName());
+ } else {
+ return xtypeId - xtypeId2;
+ }
+ });
+ }
+ return DescriptorGrouper.createDescriptorGrouper(
+ fury.getClassResolver()::isMonomorphic,
+ descriptors,
+ descriptorsGroupedOrdered,
+ descriptorUpdator,
+ fury.compressInt(),
+ fury.compressLong(),
+ DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME);
+ }
+
+ private static final int UNKNOWN_TYPE_ID = -1;
+
+ private int getXtypeId(Class<?> cls) {
+ if (isCollection(cls)) {
+ return Types.LIST;
+ }
+ if (cls.isArray() && !cls.getComponentType().isPrimitive()) {
+ return Types.LIST;
+ }
+ if (isMap(cls)) {
+ return Types.MAP;
+ }
+ if (fury.getXtypeResolver().isRegistered(cls)) {
+ return fury.getXtypeResolver().getClassInfo(cls).getXtypeId();
+ } else {
+ if (ReflectionUtils.isMonomorphic(cls)) {
+ throw new UnsupportedOperationException(cls + " is not supported for
xlang serialization");
+ }
+ return UNKNOWN_TYPE_ID;
+ }
+ }
+
public Fury getFury() {
return fury;
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/TypeResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/TypeResolver.java
index 6c57e9cf..df4d2016 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/TypeResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/TypeResolver.java
@@ -32,6 +32,10 @@ import org.apache.fury.type.GenericType;
public interface TypeResolver {
boolean needToWriteRef(TypeRef<?> typeRef);
+ boolean isRegistered(Class<?> cls);
+
+ ClassInfo getClassInfo(Class<?> cls);
+
ClassInfo getClassInfo(Class<?> cls, ClassInfoHolder classInfoHolder);
void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java
index fe68a9ef..51bcae8e 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java
@@ -20,6 +20,7 @@
package org.apache.fury.resolver;
import static org.apache.fury.Fury.NOT_SUPPORT_XLANG;
+import static org.apache.fury.builder.Generated.GeneratedSerializer;
import static org.apache.fury.meta.Encoders.GENERIC_ENCODER;
import static org.apache.fury.meta.Encoders.PACKAGE_DECODER;
import static org.apache.fury.meta.Encoders.PACKAGE_ENCODER;
@@ -60,12 +61,14 @@ import org.apache.fury.meta.Encoders;
import org.apache.fury.meta.MetaString;
import org.apache.fury.reflect.ReflectionUtils;
import org.apache.fury.reflect.TypeRef;
+import org.apache.fury.serializer.ArraySerializers;
import org.apache.fury.serializer.EnumSerializer;
+import org.apache.fury.serializer.LazySerializer;
import org.apache.fury.serializer.NonexistentClass;
import org.apache.fury.serializer.NonexistentClassSerializers;
+import org.apache.fury.serializer.ObjectSerializer;
import org.apache.fury.serializer.Serializer;
import org.apache.fury.serializer.Serializers;
-import org.apache.fury.serializer.StructSerializer;
import org.apache.fury.serializer.collection.CollectionSerializer;
import org.apache.fury.serializer.collection.MapSerializer;
import org.apache.fury.type.GenericType;
@@ -150,7 +153,7 @@ public class XtypeResolver implements TypeResolver {
xtypeId = (xtypeId << 8) + Types.ENUM;
} else {
if (serializer != null) {
- if (serializer instanceof StructSerializer) {
+ if (isStructType(serializer)) {
xtypeId = (xtypeId << 8) + Types.STRUCT;
} else {
xtypeId = (xtypeId << 8) + Types.EXT;
@@ -187,7 +190,7 @@ public class XtypeResolver implements TypeResolver {
}
short xtypeId;
if (serializer != null) {
- if (serializer instanceof StructSerializer) {
+ if (isStructType(serializer)) {
xtypeId = Types.NAMED_STRUCT;
} else if (serializer instanceof EnumSerializer) {
xtypeId = Types.NAMED_ENUM;
@@ -212,7 +215,9 @@ public class XtypeResolver implements TypeResolver {
if (type.isEnum()) {
classInfo.serializer = new EnumSerializer(fury, (Class<Enum>) type);
} else {
- classInfo.serializer = new StructSerializer(fury, type);
+ classInfo.serializer =
+ new LazySerializer.LazyObjectSerializer(
+ fury, type, () -> new ObjectSerializer<>(fury, type));
}
}
classInfoMap.put(type, classInfo);
@@ -220,6 +225,13 @@ public class XtypeResolver implements TypeResolver {
xtypeIdToClassMap.put(xtypeId, classInfo);
}
+ private boolean isStructType(Serializer serializer) {
+ if (serializer instanceof ObjectSerializer || serializer instanceof
GeneratedSerializer) {
+ return true;
+ }
+ return serializer instanceof LazySerializer.LazyObjectSerializer;
+ }
+
private ClassInfo newClassInfo(Class<?> type, Serializer<?> serializer,
short xtypeId) {
return newClassInfo(
type,
@@ -262,6 +274,12 @@ public class XtypeResolver implements TypeResolver {
return classInfo;
}
+ @Override
+ public boolean isRegistered(Class<?> cls) {
+ return classInfoMap.get(cls) != null;
+ }
+
+ @Override
public ClassInfo getClassInfo(Class<?> cls) {
ClassInfo classInfo = classInfoMap.get(cls);
if (classInfo == null) {
@@ -305,12 +323,18 @@ public class XtypeResolver implements TypeResolver {
serializer = new CollectionSerializer(fury, cls);
xtypeId = Types.SET;
} else if (classResolver.isCollection(cls)) {
+ if (cls.isAssignableFrom(ArrayList.class)) {
+ cls = ArrayList.class;
+ }
serializer = new CollectionSerializer(fury, cls);
xtypeId = Types.LIST;
} else if (cls.isArray() &&
!TypeUtils.getArrayComponent(cls).isPrimitive()) {
- serializer = classResolver.getSerializer(cls);
+ serializer = new ArraySerializers.ObjectArraySerializer(fury, cls);
xtypeId = Types.LIST;
} else if (classResolver.isMap(cls)) {
+ if (cls.isAssignableFrom(HashMap.class)) {
+ cls = HashMap.class;
+ }
serializer = new MapSerializer(fury, cls);
xtypeId = Types.MAP;
} else {
@@ -458,10 +482,7 @@ public class XtypeResolver implements TypeResolver {
GenericType genericType = generics.nextGenericType();
fury.incDepth(-1);
if (genericType != null) {
- Class<?> cls = genericType.getCls();
- if (cls.isArray()) {
- return classResolver.getClassInfo(cls);
- }
+ return getOrBuildClassInfo(genericType.getCls());
}
return xtypeIdToClassMap.get(Types.LIST);
}
@@ -471,11 +492,20 @@ public class XtypeResolver implements TypeResolver {
GenericType genericType = generics.nextGenericType();
fury.incDepth(-1);
if (genericType != null) {
- return classResolver.getClassInfo(genericType.getCls());
+ return getOrBuildClassInfo(genericType.getCls());
}
return xtypeIdToClassMap.get(Types.TIMESTAMP);
}
+ private ClassInfo getOrBuildClassInfo(Class<?> cls) {
+ ClassInfo classInfo = classInfoMap.get(cls);
+ if (classInfo == null) {
+ classInfo = buildClassInfo(cls);
+ classInfoMap.put(cls, classInfo);
+ }
+ return classInfo;
+ }
+
private ClassInfo loadBytesToClassInfo(
int internalTypeId, MetaStringBytes packageBytes, MetaStringBytes
simpleClassNameBytes) {
TypeNameBytes typeNameBytes =
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/AbstractObjectSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/AbstractObjectSerializer.java
index 3c23e1a2..317df9e8 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/AbstractObjectSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/AbstractObjectSerializer.java
@@ -19,7 +19,6 @@
package org.apache.fury.serializer;
-import static org.apache.fury.type.DescriptorGrouper.createDescriptorGrouper;
import static org.apache.fury.type.TypeUtils.getRawType;
import java.lang.invoke.MethodHandle;
@@ -124,13 +123,13 @@ public abstract class AbstractObjectSerializer<T> extends
Serializer<T> {
SerializationBinding binding, GenericTypeField fieldInfo, MemoryBuffer
buffer) {
Object fieldValue;
if (fieldInfo.trackingRef) {
- fieldValue = binding.readRef(buffer, fieldInfo.classInfoHolder);
+ fieldValue = binding.readRef(buffer, fieldInfo);
} else {
byte headFlag = buffer.readByte();
if (headFlag == Fury.NULL_FLAG) {
fieldValue = null;
} else {
- fieldValue = binding.readNonRef(buffer, fieldInfo.classInfoHolder);
+ fieldValue = binding.readNonRef(buffer, fieldInfo);
}
}
return fieldValue;
@@ -144,7 +143,7 @@ public abstract class AbstractObjectSerializer<T> extends
Serializer<T> {
Object fieldValue;
if (fieldInfo.trackingRef) {
generics.pushGenericType(fieldInfo.genericType);
- fieldValue = binding.readRef(buffer, fieldInfo.classInfoHolder);
+ fieldValue = binding.readContainerFieldValueRef(buffer, fieldInfo);
generics.popGenericType();
} else {
byte headFlag = buffer.readByte();
@@ -152,7 +151,7 @@ public abstract class AbstractObjectSerializer<T> extends
Serializer<T> {
fieldValue = null;
} else {
generics.pushGenericType(fieldInfo.genericType);
- fieldValue = binding.readNonRef(buffer, fieldInfo.classInfoHolder);
+ fieldValue = binding.readContainerFieldValue(buffer, fieldInfo);
generics.popGenericType();
}
}
@@ -752,12 +751,7 @@ public abstract class AbstractObjectSerializer<T> extends
Serializer<T> {
}
}
DescriptorGrouper descriptorGrouper =
- createDescriptorGrouper(
- fury.getClassResolver()::isMonomorphic,
- descriptors,
- false,
- fury.compressInt(),
- fury.compressLong());
+ fury.getClassResolver().createDescriptorGrouper(descriptors, false);
Tuple3<Tuple2<FinalTypeField[], boolean[]>, GenericTypeField[],
GenericTypeField[]> infos =
buildFieldInfos(fury, descriptorGrouper);
fieldInfos = new InternalFieldInfo[descriptors.size()];
@@ -782,12 +776,7 @@ public abstract class AbstractObjectSerializer<T> extends
Serializer<T> {
}
}
DescriptorGrouper descriptorGrouper =
- createDescriptorGrouper(
- fury.getClassResolver()::isMonomorphic,
- descriptors,
- false,
- fury.compressInt(),
- fury.compressLong());
+ fury.getClassResolver().createDescriptorGrouper(descriptors, false);
Tuple3<Tuple2<FinalTypeField[], boolean[]>, GenericTypeField[],
GenericTypeField[]> infos =
buildFieldInfos(fury, descriptorGrouper);
InternalFieldInfo[] fieldInfos = new InternalFieldInfo[descriptors.size()];
@@ -926,14 +915,36 @@ public abstract class AbstractObjectSerializer<T> extends
Serializer<T> {
final GenericType genericType;
final ClassInfoHolder classInfoHolder;
final boolean trackingRef;
+ final boolean isArray;
+ final ClassInfo containerClassInfo;
private GenericTypeField(
TypeRef<?> typeRef, String qualifiedFieldName, FieldAccessor accessor,
Fury fury) {
super(typeRef, getRegisteredClassId(fury, getRawType(typeRef)),
qualifiedFieldName, accessor);
// TODO support generics <T> in Pojo<T>, see
ComplexObjectSerializer.getGenericTypes
- genericType = fury.getClassResolver().buildGenericType(typeRef);
- classInfoHolder = fury.getClassResolver().nilClassInfoHolder();
- trackingRef = fury.getClassResolver().needToWriteRef(typeRef);
+ ClassResolver classResolver = fury.getClassResolver();
+ GenericType t = classResolver.buildGenericType(typeRef);
+ Class<?> cls = t.getCls();
+ if (t.getTypeParametersCount() > 0) {
+ boolean skip =
+ Arrays.stream(t.getTypeParameters()).allMatch(p -> p.getCls() ==
Object.class);
+ if (skip) {
+ t = new GenericType(t.getTypeRef(), t.isMonomorphic());
+ }
+ }
+ genericType = t;
+ classInfoHolder = classResolver.nilClassInfoHolder();
+ trackingRef = classResolver.needToWriteRef(typeRef);
+ isArray = cls.isArray();
+ if (!fury.isCrossLanguage()) {
+ containerClassInfo = null;
+ } else {
+ if (classResolver.isMap(cls) || classResolver.isCollection(cls)) {
+ containerClassInfo = fury.getXtypeResolver().getClassInfo(cls);
+ } else {
+ containerClassInfo = null;
+ }
+ }
}
@Override
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java
index d21de509..4306cbab 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java
@@ -67,7 +67,11 @@ public class ArraySerializers {
Class<?> componentType = cls.getComponentType();
componentGenericType =
fury.getClassResolver().buildGenericType(componentType);
if (fury.getClassResolver().isMonomorphic(componentType)) {
- this.componentTypeSerializer =
fury.getClassResolver().getSerializer(componentType);
+ if (fury.isCrossLanguage()) {
+ this.componentTypeSerializer = null;
+ } else {
+ this.componentTypeSerializer =
fury.getClassResolver().getSerializer(componentType);
+ }
} else {
// TODO add ClassInfo cache for non-final component type.
this.componentTypeSerializer = null;
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/CodegenSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/CodegenSerializer.java
index 4017171e..0698b23c 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/CodegenSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/CodegenSerializer.java
@@ -99,13 +99,16 @@ public final class CodegenSerializer {
if (interpreterSerializer != null) {
return interpreterSerializer;
}
+ fury.getClassResolver().getClassInfo(type).setSerializer(null);
if (fury.getConfig().isAsyncCompilationEnabled()) {
// jit not finished, avoid recursive call current serializer.
Class<? extends Serializer> sc =
fury.getClassResolver().getSerializerClass(type, false);
+ fury.getClassResolver().getClassInfo(type).setSerializer(this);
return interpreterSerializer = Serializers.newSerializer(fury,
type, sc);
} else {
Class<? extends Serializer> sc =
fury.getClassResolver().getSerializerClass(type);
+ fury.getClassResolver().getClassInfo(type).setSerializer(this);
checkArgument(
Generated.GeneratedSerializer.class.isAssignableFrom(sc),
"Expect jit serializer but got %s for class %s",
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/LazySerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/LazySerializer.java
new file mode 100644
index 00000000..7f0b0811
--- /dev/null
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/LazySerializer.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fury.serializer;
+
+import java.util.function.Supplier;
+import org.apache.fury.Fury;
+import org.apache.fury.memory.MemoryBuffer;
+
+@SuppressWarnings({"rawtypes", "unchecked"})
+public class LazySerializer extends Serializer {
+ private final Supplier<Serializer> serializerSupplier;
+ private Serializer serializer;
+
+ public LazySerializer(Fury fury, Class type, Supplier<Serializer>
serializerSupplier) {
+ super(fury, type);
+ this.serializerSupplier = serializerSupplier;
+ }
+
+ @Override
+ public void write(MemoryBuffer buffer, Object value) {
+ if (serializer == null) {
+ serializer = serializerSupplier.get();
+ fury.getClassResolver().setSerializer(value.getClass(), serializer);
+ }
+ serializer.write(buffer, value);
+ }
+
+ @Override
+ public Object read(MemoryBuffer buffer) {
+ boolean unInit = serializer == null;
+ if (unInit) {
+ serializer = serializerSupplier.get();
+ }
+ Object value = serializer.read(buffer);
+ if (unInit) {
+ fury.getClassResolver().setSerializer(value.getClass(), serializer);
+ }
+ return value;
+ }
+
+ @Override
+ public void xwrite(MemoryBuffer buffer, Object value) {
+ if (serializer == null) {
+ serializer = serializerSupplier.get();
+ fury.getClassResolver().setSerializer(value.getClass(), serializer);
+
fury.getXtypeResolver().getClassInfo(value.getClass()).setSerializer(serializer);
+ }
+ serializer.xwrite(buffer, value);
+ }
+
+ @Override
+ public Object xread(MemoryBuffer buffer) {
+ boolean unInit = serializer == null;
+ if (unInit) {
+ serializer = serializerSupplier.get();
+ }
+ Object value = serializer.xread(buffer);
+ if (unInit) {
+ fury.getClassResolver().setSerializer(value.getClass(), serializer);
+
fury.getXtypeResolver().getClassInfo(value.getClass()).setSerializer(serializer);
+ }
+ return value;
+ }
+
+ @Override
+ public Object copy(Object value) {
+ if (serializer == null) {
+ serializer = serializerSupplier.get();
+ fury.getClassResolver().setSerializer(value.getClass(), serializer);
+ }
+ return serializer.copy(value);
+ }
+
+ public static class LazyObjectSerializer extends LazySerializer {
+ public LazyObjectSerializer(Fury fury, Class type, Supplier<Serializer>
serializerSupplier) {
+ super(fury, type, serializerSupplier);
+ }
+ }
+}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/MetaSharedSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/MetaSharedSerializer.java
index 1d46bfbf..41ae4677 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/MetaSharedSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/MetaSharedSerializer.java
@@ -85,13 +85,7 @@ public class MetaSharedSerializer<T> extends
AbstractObjectSerializer<T> {
Preconditions.checkArgument(
fury.getConfig().isMetaShareEnabled(), "Meta share must be enabled.");
Collection<Descriptor> descriptors = consolidateFields(this.classResolver,
type, classDef);
- DescriptorGrouper descriptorGrouper =
- DescriptorGrouper.createDescriptorGrouper(
- this.classResolver::isMonomorphic,
- descriptors,
- false,
- fury.compressInt(),
- fury.getConfig().compressLong());
+ DescriptorGrouper descriptorGrouper =
classResolver.createDescriptorGrouper(descriptors, false);
// d.getField() may be null if not exists in this class when meta share
enabled.
Tuple3<
Tuple2<ObjectSerializer.FinalTypeField[], boolean[]>,
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java
index 9ec243ab..4d7ccf48 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java
@@ -160,20 +160,16 @@ public final class NonexistentClassSerializers {
MetaSharedSerializer.consolidateFields(
fury.getClassResolver(),
NonexistentClass.NonexistentSkip.class, classDef);
DescriptorGrouper descriptorGrouper =
- DescriptorGrouper.createDescriptorGrouper(
- fury.getClassResolver()::isMonomorphic,
- descriptors,
- false,
- fury.compressInt(),
- fury.compressLong());
+ fury.getClassResolver().createDescriptorGrouper(descriptors,
false);
Tuple3<
Tuple2<ObjectSerializer.FinalTypeField[], boolean[]>,
ObjectSerializer.GenericTypeField[],
ObjectSerializer.GenericTypeField[]>
tuple = AbstractObjectSerializer.buildFieldInfos(fury,
descriptorGrouper);
+ descriptors = descriptorGrouper.getSortedDescriptors();
int classVersionHash = 0;
if (fury.checkClassVersion()) {
- classVersionHash = ObjectSerializer.computeVersionHash(descriptors);
+ classVersionHash = ObjectSerializer.computeStructHash(fury,
descriptors);
}
fieldsInfo =
new ClassFieldsInfo(tuple.f0.f0, tuple.f0.f1, tuple.f1, tuple.f2,
classVersionHash);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
index f3bc5185..b1a996b4 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
@@ -19,13 +19,10 @@
package org.apache.fury.serializer;
-import static org.apache.fury.type.DescriptorGrouper.createDescriptorGrouper;
-
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
-import java.util.Objects;
+import java.util.Map;
import java.util.stream.Collectors;
import org.apache.fury.Fury;
import org.apache.fury.collection.Tuple2;
@@ -35,6 +32,7 @@ import org.apache.fury.memory.MemoryBuffer;
import org.apache.fury.memory.Platform;
import org.apache.fury.meta.ClassDef;
import org.apache.fury.reflect.FieldAccessor;
+import org.apache.fury.reflect.TypeRef;
import org.apache.fury.resolver.ClassInfo;
import org.apache.fury.resolver.ClassResolver;
import org.apache.fury.resolver.RefResolver;
@@ -42,6 +40,9 @@ import org.apache.fury.resolver.TypeResolver;
import org.apache.fury.type.Descriptor;
import org.apache.fury.type.DescriptorGrouper;
import org.apache.fury.type.Generics;
+import org.apache.fury.type.TypeUtils;
+import org.apache.fury.type.Types;
+import org.apache.fury.util.Preconditions;
import org.apache.fury.util.record.RecordInfo;
import org.apache.fury.util.record.RecordUtils;
@@ -99,25 +100,17 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
} else {
descriptors = fury.getClassResolver().getAllDescriptorsMap(cls,
resolveParent).values();
}
- DescriptorGrouper descriptorGrouper =
- createDescriptorGrouper(
- fury.getClassResolver()::isMonomorphic,
- descriptors,
- false,
- fury.compressInt(),
- fury.compressLong());
-
+ DescriptorGrouper descriptorGrouper =
classResolver.createDescriptorGrouper(descriptors, false);
+ descriptors = descriptorGrouper.getSortedDescriptors();
if (isRecord) {
List<String> fieldNames =
- descriptorGrouper.getSortedDescriptors().stream()
- .map(Descriptor::getName)
- .collect(Collectors.toList());
+
descriptors.stream().map(Descriptor::getName).collect(Collectors.toList());
recordInfo = new RecordInfo(cls, fieldNames);
} else {
recordInfo = null;
}
if (fury.checkClassVersion()) {
- classVersionHash = computeVersionHash(descriptors);
+ classVersionHash = computeStructHash(fury, descriptors);
} else {
classVersionHash = 0;
}
@@ -150,6 +143,11 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
writeContainerFields(buffer, value, fury, refResolver, typeResolver);
}
+ @Override
+ public void xwrite(MemoryBuffer buffer, T value) {
+ write(buffer, value);
+ }
+
private void writeFinalFields(
MemoryBuffer buffer, T value, Fury fury, RefResolver refResolver,
TypeResolver typeResolver) {
FinalTypeField[] finalFields = this.finalFields;
@@ -210,7 +208,7 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
ClassInfo classInfo =
typeResolver.getClassInfo(fieldValue.getClass(),
fieldInfo.classInfoHolder);
generics.pushGenericType(fieldInfo.genericType);
- binding.writeNonRef(buffer, fieldValue, classInfo);
+ binding.writeContainerFieldValue(buffer, fieldValue, classInfo);
generics.popGenericType();
}
} else {
@@ -219,7 +217,7 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
} else {
buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG);
generics.pushGenericType(fieldInfo.genericType);
- binding.writeNonRef(
+ binding.writeContainerFieldValue(
buffer,
fieldValue,
typeResolver.getClassInfo(fieldValue.getClass(),
fieldInfo.classInfoHolder));
@@ -246,6 +244,11 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
return readAndSetFields(buffer, obj);
}
+ @Override
+ public T xread(MemoryBuffer buffer) {
+ return read(buffer);
+ }
+
public Object[] readFields(MemoryBuffer buffer) {
Fury fury = this.fury;
RefResolver refResolver = this.refResolver;
@@ -324,14 +327,44 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
return obj;
}
- public static int computeVersionHash(Collection<Descriptor> descriptors) {
- // TODO(chaokunyang) use murmurhash
- List<Integer> list = new ArrayList<>();
- for (Descriptor d : descriptors) {
- Integer integer = Objects.hash(d.getName(), d.getRawType().getName(),
d.getDeclaringClass());
- list.add(integer);
+ public static int computeStructHash(Fury fury, Collection<Descriptor>
descriptors) {
+ int hash = 17;
+ for (Descriptor descriptor : descriptors) {
+ hash = computeFieldHash(hash, fury, descriptor.getTypeRef());
+ }
+ Preconditions.checkState(hash != 0);
+ return hash;
+ }
+
+ private static int computeFieldHash(int hash, Fury fury, TypeRef<?> typeRef)
{
+ int id;
+ if (typeRef.isSubtypeOf(List.class)) {
+ // TODO(chaokunyang) add list element type into schema hash
+ id = Types.LIST;
+ } else if (typeRef.isSubtypeOf(Map.class)) {
+ // TODO(chaokunyang) add map key&value type into schema hash
+ id = Types.MAP;
+ } else {
+ try {
+ TypeResolver resolver =
+ fury.isCrossLanguage() ? fury.getXtypeResolver() :
fury.getClassResolver();
+ ClassInfo classInfo = resolver.getClassInfo(typeRef.getRawType());
+ int xtypeId = classInfo.getXtypeId();
+ if (Types.isStructType((byte) xtypeId)) {
+ id =
+ TypeUtils.computeStringHash(classInfo.decodeNamespace() +
classInfo.decodeTypeName());
+ } else {
+ id = Math.abs(xtypeId);
+ }
+ } catch (Exception e) {
+ id = 0;
+ }
+ }
+ long newHash = ((long) hash) * 31 + id;
+ while (newHash >= Integer.MAX_VALUE) {
+ newHash /= 7;
}
- return list.hashCode();
+ return (int) newHash;
}
public static void checkClassVersion(Fury fury, int readHash, int
classVersionHash) {
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/SerializationBinding.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/SerializationBinding.java
index d21fa404..3db2ef31 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/SerializationBinding.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/SerializationBinding.java
@@ -19,11 +19,15 @@
package org.apache.fury.serializer;
+import static org.apache.fury.Fury.NOT_NULL_VALUE_FLAG;
+import static
org.apache.fury.serializer.AbstractObjectSerializer.GenericTypeField;
+
import org.apache.fury.Fury;
import org.apache.fury.memory.MemoryBuffer;
import org.apache.fury.resolver.ClassInfo;
import org.apache.fury.resolver.ClassInfoHolder;
import org.apache.fury.resolver.ClassResolver;
+import org.apache.fury.resolver.RefResolver;
import org.apache.fury.resolver.XtypeResolver;
// This polymorphic interface has cost, do not expose it as a public class
@@ -49,12 +53,16 @@ interface SerializationBinding {
void writeNullable(MemoryBuffer buffer, Object obj, ClassInfo classInfo);
+ void writeContainerFieldValue(MemoryBuffer buffer, Object fieldValue,
ClassInfo classInfo);
+
void write(MemoryBuffer buffer, Serializer serializer, Object value);
Object read(MemoryBuffer buffer, Serializer serializer);
<T> T readRef(MemoryBuffer buffer, Serializer<T> serializer);
+ Object readRef(MemoryBuffer buffer, GenericTypeField field);
+
Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder);
Object readRef(MemoryBuffer buffer);
@@ -63,8 +71,14 @@ interface SerializationBinding {
Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder);
+ Object readNonRef(MemoryBuffer buffer, GenericTypeField field);
+
Object readNullable(MemoryBuffer buffer, Serializer<Object> serializer);
+ Object readContainerFieldValue(MemoryBuffer buffer, GenericTypeField field);
+
+ Object readContainerFieldValueRef(MemoryBuffer buffer, GenericTypeField
fieldInfo);
+
static SerializationBinding createBinding(Fury fury) {
if (fury.isCrossLanguage()) {
return new XlangSerializationBinding(fury);
@@ -102,6 +116,11 @@ interface SerializationBinding {
return fury.readRef(buffer, serializer);
}
+ @Override
+ public Object readRef(MemoryBuffer buffer, GenericTypeField field) {
+ return fury.readRef(buffer, field.classInfoHolder);
+ }
+
@Override
public Object readRef(MemoryBuffer buffer, ClassInfoHolder
classInfoHolder) {
return fury.readRef(buffer, classInfoHolder);
@@ -122,11 +141,35 @@ interface SerializationBinding {
return fury.readNonRef(buffer, classInfoHolder);
}
+ @Override
+ public Object readNonRef(MemoryBuffer buffer, GenericTypeField field) {
+ return fury.readNonRef(buffer, field.classInfoHolder);
+ }
+
@Override
public Object readNullable(MemoryBuffer buffer, Serializer<Object>
serializer) {
return fury.readNullable(buffer, serializer);
}
+ @Override
+ public Object readContainerFieldValue(MemoryBuffer buffer,
GenericTypeField field) {
+ return fury.readNonRef(buffer, field.classInfoHolder);
+ }
+
+ @Override
+ public Object readContainerFieldValueRef(MemoryBuffer buffer,
GenericTypeField fieldInfo) {
+ RefResolver refResolver = fury.getRefResolver();
+ int nextReadRefId = refResolver.tryPreserveRefId(buffer);
+ if (nextReadRefId >= NOT_NULL_VALUE_FLAG) {
+ // ref value or not-null value
+ Object o = fury.readData(buffer, classResolver.readClassInfo(buffer));
+ refResolver.setReadObject(nextReadRefId, o);
+ return o;
+ } else {
+ return refResolver.getReadObject();
+ }
+ }
+
@Override
public void write(MemoryBuffer buffer, Serializer serializer, Object
value) {
serializer.write(buffer, value);
@@ -152,7 +195,7 @@ interface SerializationBinding {
if (obj == null) {
buffer.writeByte(Fury.NULL_FLAG);
} else {
- buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG);
+ buffer.writeByte(NOT_NULL_VALUE_FLAG);
writeNonRef(buffer, obj);
}
}
@@ -162,7 +205,7 @@ interface SerializationBinding {
if (obj == null) {
buffer.writeByte(Fury.NULL_FLAG);
} else {
- buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG);
+ buffer.writeByte(NOT_NULL_VALUE_FLAG);
serializer.write(buffer, obj);
}
}
@@ -172,7 +215,7 @@ interface SerializationBinding {
if (obj == null) {
buffer.writeByte(Fury.NULL_FLAG);
} else {
- buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG);
+ buffer.writeByte(NOT_NULL_VALUE_FLAG);
fury.writeNonRef(buffer, obj,
classResolver.getClassInfo(obj.getClass(), classInfoHolder));
}
}
@@ -182,20 +225,28 @@ interface SerializationBinding {
if (obj == null) {
buffer.writeByte(Fury.NULL_FLAG);
} else {
- buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG);
+ buffer.writeByte(NOT_NULL_VALUE_FLAG);
fury.writeNonRef(buffer, obj, classInfo);
}
}
+
+ @Override
+ public void writeContainerFieldValue(
+ MemoryBuffer buffer, Object fieldValue, ClassInfo classInfo) {
+ fury.writeNonRef(buffer, fieldValue, classInfo);
+ }
}
final class XlangSerializationBinding implements SerializationBinding {
private final Fury fury;
private final XtypeResolver xtypeResolver;
+ private final RefResolver refResolver;
XlangSerializationBinding(Fury fury) {
this.fury = fury;
xtypeResolver = fury.getXtypeResolver();
+ refResolver = fury.getRefResolver();
}
@Override
@@ -218,6 +269,18 @@ interface SerializationBinding {
return (T) fury.xreadRef(buffer, serializer);
}
+ @Override
+ public Object readRef(MemoryBuffer buffer, GenericTypeField field) {
+ if (field.isArray) {
+ fury.getGenerics().pushGenericType(field.genericType);
+ Object o = fury.xreadRef(buffer);
+ fury.getGenerics().popGenericType();
+ return o;
+ } else {
+ return fury.xreadRef(buffer);
+ }
+ }
+
@Override
public Object readRef(MemoryBuffer buffer, ClassInfoHolder
classInfoHolder) {
return fury.xreadRef(buffer);
@@ -238,11 +301,40 @@ interface SerializationBinding {
return fury.xreadNonRef(buffer, xtypeResolver.readClassInfo(buffer,
classInfoHolder));
}
+ @Override
+ public Object readNonRef(MemoryBuffer buffer, GenericTypeField field) {
+ if (field.isArray) {
+ fury.getGenerics().pushGenericType(field.genericType);
+ Object o = fury.xreadNonRef(buffer);
+ fury.getGenerics().popGenericType();
+ return o;
+ } else {
+ return fury.xreadNonRef(buffer);
+ }
+ }
+
@Override
public Object readNullable(MemoryBuffer buffer, Serializer<Object>
serializer) {
return fury.xreadNullable(buffer, serializer);
}
+ @Override
+ public Object readContainerFieldValue(MemoryBuffer buffer,
GenericTypeField field) {
+ return fury.xreadNonRef(buffer, field.containerClassInfo);
+ }
+
+ @Override
+ public Object readContainerFieldValueRef(MemoryBuffer buffer,
GenericTypeField field) {
+ int nextReadRefId = refResolver.tryPreserveRefId(buffer);
+ if (nextReadRefId >= NOT_NULL_VALUE_FLAG) {
+ Object o = fury.xreadNonRef(buffer, field.containerClassInfo);
+ refResolver.setReadObject(nextReadRefId, o);
+ return o;
+ } else {
+ return refResolver.getReadObject();
+ }
+ }
+
@Override
public void write(MemoryBuffer buffer, Serializer serializer, Object
value) {
serializer.xwrite(buffer, value);
@@ -268,7 +360,7 @@ interface SerializationBinding {
if (obj == null) {
buffer.writeByte(Fury.NULL_FLAG);
} else {
- buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG);
+ buffer.writeByte(NOT_NULL_VALUE_FLAG);
fury.xwriteNonRef(buffer, obj);
}
}
@@ -278,7 +370,7 @@ interface SerializationBinding {
if (obj == null) {
buffer.writeByte(Fury.NULL_FLAG);
} else {
- buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG);
+ buffer.writeByte(NOT_NULL_VALUE_FLAG);
serializer.xwrite(buffer, obj);
}
}
@@ -288,7 +380,7 @@ interface SerializationBinding {
if (obj == null) {
buffer.writeByte(Fury.NULL_FLAG);
} else {
- buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG);
+ buffer.writeByte(NOT_NULL_VALUE_FLAG);
fury.xwriteNonRef(buffer, obj,
xtypeResolver.getClassInfo(obj.getClass(), classInfoHolder));
}
}
@@ -298,9 +390,15 @@ interface SerializationBinding {
if (obj == null) {
buffer.writeByte(Fury.NULL_FLAG);
} else {
- buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG);
+ buffer.writeByte(NOT_NULL_VALUE_FLAG);
fury.xwriteNonRef(buffer, obj, classInfo);
}
}
+
+ @Override
+ public void writeContainerFieldValue(
+ MemoryBuffer buffer, Object fieldValue, ClassInfo classInfo) {
+ fury.xwriteData(buffer, classInfo, fieldValue);
+ }
}
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java
deleted file mode 100644
index d209d8ef..00000000
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.fury.serializer;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.AbstractMap;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.fury.Fury;
-import org.apache.fury.config.Language;
-import org.apache.fury.exception.ClassNotCompatibleException;
-import org.apache.fury.logging.Logger;
-import org.apache.fury.logging.LoggerFactory;
-import org.apache.fury.memory.MemoryBuffer;
-import org.apache.fury.memory.Platform;
-import org.apache.fury.reflect.FieldAccessor;
-import org.apache.fury.reflect.ReflectionUtils;
-import org.apache.fury.reflect.TypeRef;
-import org.apache.fury.resolver.ClassInfo;
-import org.apache.fury.resolver.ClassResolver;
-import org.apache.fury.serializer.collection.CollectionSerializer;
-import org.apache.fury.serializer.collection.MapSerializer;
-import org.apache.fury.type.Descriptor;
-import org.apache.fury.type.GenericType;
-import org.apache.fury.type.Generics;
-import org.apache.fury.type.TypeUtils;
-import org.apache.fury.type.Types;
-import org.apache.fury.util.ExceptionUtils;
-import org.apache.fury.util.Preconditions;
-import org.apache.fury.util.StringUtils;
-
-/**
- * A serializer used for cross-language serialization for custom objects.
- *
- * <p>TODO(chaokunyang) support generics optimization for {@code SomeClass<T>}.
- */
-@SuppressWarnings({"unchecked", "rawtypes"})
-public class StructSerializer<T> extends Serializer<T> {
- private static final Logger LOG =
LoggerFactory.getLogger(StructSerializer.class);
- private final Constructor<T> constructor;
- private final FieldAccessor[] fieldAccessors;
- private GenericType[] fieldGenerics;
- private GenericType genericType;
- private final IdentityHashMap<GenericType, GenericType[]> genericTypesCache;
- private int typeHash;
-
- public StructSerializer(Fury fury, Class<T> cls) {
- super(fury, cls);
- if (fury.getLanguage() == Language.JAVA) {
- LOG.warn("Type of class {} shouldn't be serialized using cross-language
serializer", cls);
- }
- Constructor<T> ctr = null;
- try {
- ctr = cls.getConstructor();
- if (!ctr.isAccessible()) {
- ctr.setAccessible(true);
- }
- } catch (Exception e) {
- ExceptionUtils.ignore(e);
- }
- this.constructor = ctr;
- fieldAccessors =
- Descriptor.getFields(cls).stream()
- .map(
- f ->
- new AbstractMap.SimpleEntry<>(
- f,
StringUtils.lowerCamelToLowerUnderscore(f.getName())))
- .sorted(Map.Entry.comparingByValue())
- .map(Map.Entry::getKey)
- .map(FieldAccessor::createAccessor)
- .toArray(FieldAccessor[]::new);
- fieldGenerics = buildFieldGenerics(fury, TypeRef.of(cls), fieldAccessors);
- genericTypesCache = new IdentityHashMap<>();
- genericTypesCache.put(null, fieldGenerics);
- }
-
- private <T> GenericType[] buildFieldGenerics(
- Fury fury, TypeRef<T> type, FieldAccessor[] fieldAccessors) {
- return Arrays.stream(fieldAccessors)
- .map(fieldAccessor -> getGenericType(fury, type, fieldAccessor))
- .toArray(GenericType[]::new);
- }
-
- private static <T> GenericType getGenericType(
- Fury fury, TypeRef<T> type, FieldAccessor fieldAccessor) {
- GenericType t = GenericType.build(type,
fieldAccessor.getField().getGenericType());
- if (t.getTypeParametersCount() > 0) {
- boolean skip = Arrays.stream(t.getTypeParameters()).allMatch(p ->
p.getCls() == Object.class);
- if (skip) {
- t = new GenericType(t.getTypeRef(), t.isMonomorphic());
- }
- }
- ClassResolver resolver = fury.getClassResolver();
- Class cls = t.getCls();
- if (resolver.isMonomorphic(cls)) {
-
t.setSerializer(fury.getXtypeResolver().getClassInfo(cls).getSerializer());
- return t;
- }
- // We have one type id for map, there is no map polymorphic support.
- // If one want to deserialize map data into different map type, he should
- // use the concrete map subclass type to declare the field type or use
some hints
- // such as field annotation.
- if (resolver.isMap(cls)) {
- t.setSerializer(
- ReflectionUtils.isAbstract(cls)
- ? new MapSerializer(fury, HashMap.class)
- : resolver.getSerializer(cls));
- } else if (resolver.isCollection(cls)) {
- t.setSerializer(
- ReflectionUtils.isAbstract(cls)
- ? new CollectionSerializer(fury, ArrayList.class)
- : resolver.getSerializer(cls));
- } else if (cls.isArray()) {
- t.setSerializer(new ArraySerializers.ObjectArraySerializer(fury, cls));
- }
- return t;
- }
-
- @Override
- public void write(MemoryBuffer buffer, T value) {
- xwrite(buffer, value);
- }
-
- @Override
- public T read(MemoryBuffer buffer) {
- return xread(buffer);
- }
-
- @Override
- public void xwrite(MemoryBuffer buffer, T value) {
- // TODO(chaokunyang) support fields back and forward compatible.
- // Maybe need to serialize fields name too.
- int typeHash = this.typeHash;
- if (typeHash == 0) {
- typeHash = computeStructHash();
- this.typeHash = typeHash;
- }
- buffer.writeInt32(typeHash);
- Generics generics = fury.getGenerics();
- GenericType[] fieldGenerics = getGenericTypes(generics);
- for (int i = 0; i < fieldAccessors.length; i++) {
- FieldAccessor fieldAccessor = fieldAccessors[i];
- GenericType fieldGeneric = fieldGenerics[i];
- boolean hasGenerics = fieldGeneric.hasGenericParameters();
- if (hasGenerics) {
- generics.pushGenericType(fieldGeneric);
- }
- Serializer serializer = fieldGeneric.getSerializer();
- if (serializer != null) {
- fury.xwriteRef(buffer, fieldAccessor.get(value), serializer);
- } else {
- fury.xwriteRef(buffer, fieldAccessor.get(value));
- }
- if (hasGenerics) {
- generics.popGenericType();
- }
- }
- }
-
- private GenericType[] getGenericTypes(Generics generics) {
- GenericType[] fieldGenerics = this.fieldGenerics;
- // support generics <T> in Pojo<T>
- GenericType genericType = generics.nextGenericType();
- if (genericType != this.genericType) {
- this.genericType = genericType;
- fieldGenerics = genericTypesCache.get(genericType);
- if (fieldGenerics == null) {
- fieldGenerics = buildFieldGenerics(fury, genericType.getTypeRef(),
fieldAccessors);
- genericTypesCache.put(genericType, fieldGenerics);
- }
- this.fieldGenerics = fieldGenerics;
- }
- return fieldGenerics;
- }
-
- @Override
- public T xread(MemoryBuffer buffer) {
- int typeHash = this.typeHash;
- if (typeHash == 0) {
- typeHash = computeStructHash();
- this.typeHash = typeHash;
- }
- int newHash = buffer.readInt32();
- if (newHash != typeHash) {
- throw new ClassNotCompatibleException(
- String.format(
- "Hash %d is not consistent with %s for class %s",
- newHash, typeHash,
fury.getClassResolver().getCurrentReadClass()));
- }
- T obj = newBean();
- fury.getRefResolver().reference(obj);
- Generics generics = fury.getGenerics();
- GenericType[] fieldGenerics = getGenericTypes(generics);
- for (int i = 0; i < fieldAccessors.length; i++) {
- FieldAccessor fieldAccessor = fieldAccessors[i];
- GenericType fieldGeneric = fieldGenerics[i];
- boolean hasGenerics = fieldGeneric.hasGenericParameters();
- if (hasGenerics) {
- generics.pushGenericType(fieldGeneric);
- }
- Object fieldValue;
- Serializer serializer = fieldGeneric.getSerializer();
- if (serializer == null) {
- fieldValue = fury.xreadRef(buffer);
- } else {
- fieldValue = fury.xreadRef(buffer, serializer);
- }
- fieldAccessor.set(obj, fieldValue);
- if (hasGenerics) {
- generics.popGenericType();
- }
- }
- return obj;
- }
-
- private T newBean() {
- if (constructor != null) {
- try {
- return constructor.newInstance();
- } catch (InstantiationException | IllegalAccessException |
InvocationTargetException e) {
- Platform.throwException(e);
- }
- }
- return Platform.newInstance(type);
- }
-
- int computeStructHash() {
- int hash = 17;
- for (GenericType fieldGeneric : fieldGenerics) {
- hash = computeFieldHash(hash, fieldGeneric);
- }
- Preconditions.checkState(hash != 0);
- return hash;
- }
-
- int computeFieldHash(int hash, GenericType fieldGeneric) {
- int id;
- if (fieldGeneric.getTypeRef().isSubtypeOf(List.class)) {
- // TODO(chaokunyang) add list element type into schema hash
- id = Types.LIST;
- } else if (fieldGeneric.getTypeRef().isSubtypeOf(Map.class)) {
- // TODO(chaokunyang) add map key&value type into schema hash
- id = Types.MAP;
- } else {
- try {
- ClassInfo classInfo =
fury.getXtypeResolver().getClassInfo(fieldGeneric.getCls());
- int xtypeId = classInfo.getXtypeId();
- if (Types.isStructType((byte) xtypeId)) {
- id =
- TypeUtils.computeStringHash(classInfo.decodeNamespace() +
classInfo.decodeTypeName());
- } else {
- id = Math.abs(xtypeId);
- }
- } catch (Exception e) {
- id = 0;
- }
- }
- long newHash = ((long) hash) * 31 + id;
- while (newHash >= Integer.MAX_VALUE) {
- newHash /= 7;
- }
- return (int) newHash;
- }
-}
diff --git a/java/fury-core/src/main/java/org/apache/fury/type/Descriptor.java
b/java/fury-core/src/main/java/org/apache/fury/type/Descriptor.java
index 25ba210d..5ff776f2 100644
--- a/java/fury-core/src/main/java/org/apache/fury/type/Descriptor.java
+++ b/java/fury-core/src/main/java/org/apache/fury/type/Descriptor.java
@@ -78,6 +78,7 @@ public class Descriptor {
private Class<?> type;
private final String typeName;
private final String name;
+ private String snakeCaseName;
private final int modifier;
private final String declaringClass;
private final Field field;
@@ -154,6 +155,13 @@ public class Descriptor {
return name;
}
+ public String getSnakeCaseName() {
+ if (snakeCaseName == null) {
+ snakeCaseName = StringUtils.lowerCamelToLowerUnderscore(name);
+ }
+ return snakeCaseName;
+ }
+
public int getModifiers() {
return modifier;
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/type/DescriptorGrouper.java
b/java/fury-core/src/main/java/org/apache/fury/type/DescriptorGrouper.java
index 20dc11c3..b1864d14 100644
--- a/java/fury-core/src/main/java/org/apache/fury/type/DescriptorGrouper.java
+++ b/java/fury-core/src/main/java/org/apache/fury/type/DescriptorGrouper.java
@@ -41,15 +41,17 @@ import org.apache.fury.util.record.RecordUtils;
* <li>other fields
*/
public class DescriptorGrouper {
- // sort primitive descriptors from largest to smallest, if size is the same,
- // sort by field name to fix order.
- private static final Comparator<Descriptor> PRIMITIVE_COMPARATOR =
+ static final Comparator<Descriptor> COMPARATOR_BY_PRIMITIVE_TYPE_ID =
(d1, d2) -> {
int c =
- getSizeOfPrimitiveType(TypeUtils.unwrap(d2.getRawType()))
- - getSizeOfPrimitiveType(TypeUtils.unwrap(d1.getRawType()));
+ Types.getPrimitiveTypeId(TypeUtils.unwrap(d2.getRawType()))
+ - Types.getPrimitiveTypeId(TypeUtils.unwrap(d1.getRawType()));
if (c == 0) {
- c = DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME.compare(d1, d2);
+ c = d1.getSnakeCaseName().compareTo(d2.getSnakeCaseName());
+ if (c == 0) {
+ // Field name duplicate in super/child classes.
+ c = d1.getDeclaringClass().compareTo(d2.getDeclaringClass());
+ }
}
return c;
};
@@ -64,7 +66,17 @@ public class DescriptorGrouper {
public static Comparator<Descriptor> getPrimitiveComparator(
boolean compressInt, boolean compressLong) {
if (!compressInt && !compressLong) {
- return PRIMITIVE_COMPARATOR;
+ // sort primitive descriptors from largest to smallest, if size is the
same,
+ // sort by field name to fix order.
+ return (d1, d2) -> {
+ int c =
+ getSizeOfPrimitiveType(TypeUtils.unwrap(d2.getRawType()))
+ - getSizeOfPrimitiveType(TypeUtils.unwrap(d1.getRawType()));
+ if (c == 0) {
+ c = COMPARATOR_BY_PRIMITIVE_TYPE_ID.compare(d1, d2);
+ }
+ return c;
+ };
}
return (d1, d2) -> {
Class<?> t1 = TypeUtils.unwrap(d1.getRawType());
@@ -74,7 +86,7 @@ public class DescriptorGrouper {
if ((t1Compress && t2Compress) || (!t1Compress && !t2Compress)) {
int c = getSizeOfPrimitiveType(t2) - getSizeOfPrimitiveType(t1);
if (c == 0) {
- c = DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME.compare(d1, d2);
+ c = COMPARATOR_BY_PRIMITIVE_TYPE_ID.compare(d1, d2);
}
return c;
}
@@ -138,7 +150,7 @@ public class DescriptorGrouper {
* @param primitiveComparator comparator for primitive/boxed fields.
* @param comparator comparator for non-primitive fields.
*/
- public DescriptorGrouper(
+ private DescriptorGrouper(
Predicate<Class<?>> isMonomorphic,
Collection<Descriptor> descriptors,
boolean descriptorsGroupedOrdered,
@@ -178,9 +190,9 @@ public class DescriptorGrouper {
descriptors.addAll(getPrimitiveDescriptors());
descriptors.addAll(getBoxedDescriptors());
descriptors.addAll(getFinalDescriptors());
+ descriptors.addAll(getOtherDescriptors());
descriptors.addAll(getCollectionDescriptors());
descriptors.addAll(getMapDescriptors());
- descriptors.addAll(getOtherDescriptors());
return descriptors;
}
@@ -224,16 +236,17 @@ public class DescriptorGrouper {
Predicate<Class<?>> isMonomorphic,
Collection<Descriptor> descriptors,
boolean descriptorsGroupedOrdered,
+ Function<Descriptor, Descriptor> descriptorUpdator,
boolean compressInt,
- boolean compressLong) {
- Comparator<Descriptor> comparator = getPrimitiveComparator(compressInt,
compressLong);
+ boolean compressLong,
+ Comparator<Descriptor> comparator) {
return new DescriptorGrouper(
isMonomorphic,
descriptors,
descriptorsGroupedOrdered,
- DescriptorGrouper::createDescriptor,
- comparator,
- COMPARATOR_BY_TYPE_AND_NAME);
+ descriptorUpdator == null ? DescriptorGrouper::createDescriptor :
descriptorUpdator,
+ getPrimitiveComparator(compressInt, compressLong),
+ comparator);
}
public int getNumDescriptors() {
diff --git a/java/fury-core/src/main/java/org/apache/fury/type/Types.java
b/java/fury-core/src/main/java/org/apache/fury/type/Types.java
index a2da4ac4..a76a2414 100644
--- a/java/fury-core/src/main/java/org/apache/fury/type/Types.java
+++ b/java/fury-core/src/main/java/org/apache/fury/type/Types.java
@@ -19,6 +19,11 @@
package org.apache.fury.type;
+import static org.apache.fury.collection.Collections.ofHashMap;
+
+import java.util.Map;
+import org.apache.fury.util.Preconditions;
+
public class Types {
/** bool: a boolean value (true or false). */
@@ -172,4 +177,19 @@ public class Types {
public static boolean isEnumType(int value) {
return value == ENUM || value == NAMED_ENUM;
}
+
+ private static final Map<Class, Integer> PRIMITIVE_TYPE_ID_MAP =
+ ofHashMap(
+ boolean.class, BOOL,
+ byte.class, INT8,
+ short.class, INT16,
+ int.class, INT32,
+ long.class, INT64,
+ float.class, FLOAT32,
+ double.class, FLOAT64);
+
+ public static int getPrimitiveTypeId(Class<?> cls) {
+ Preconditions.checkArgument(cls.isPrimitive(), "Class %s is not
primitive", cls);
+ return PRIMITIVE_TYPE_ID_MAP.getOrDefault(cls, -1);
+ }
}
diff --git
a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
index fd8b3e86..e0f1d9c4 100644
---
a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
+++
b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
@@ -309,6 +309,8 @@
Args=--initialize-at-build-time=org.apache.fury.memory.MemoryBuffer,\
org.apache.fury.serializer.LambdaSerializer$ReplaceStub,\
org.apache.fury.serializer.LambdaSerializer,\
org.apache.fury.serializer.LocaleSerializer,\
+ org.apache.fury.serializer.LazySerializer,\
+ org.apache.fury.serializer.LazySerializer$LazyObjectSerializer,\
org.apache.fury.serializer.NoneSerializer,\
org.apache.fury.serializer.NonexistentClassSerializers$ClassFieldsInfo,\
org.apache.fury.serializer.NonexistentClassSerializers$NonexistentClassSerializer,\
@@ -421,6 +423,8 @@
Args=--initialize-at-build-time=org.apache.fury.memory.MemoryBuffer,\
org.apache.fury.serializer.collection.UnmodifiableSerializers$UnmodifiableCollectionSerializer,\
org.apache.fury.serializer.collection.UnmodifiableSerializers$UnmodifiableMapSerializer,\
org.apache.fury.serializer.collection.UnmodifiableSerializers,\
+ org.apache.fury.serializer.LazySerializer,\
+ org.apache.fury.serializer.LazySerializer$LazyObjectSerializer,\
org.apache.fury.serializer.shim.ShimDispatcher,\
org.apache.fury.shaded.org.codehaus.janino.IClass$1,\
org.apache.fury.type.Descriptor$1,\
@@ -429,6 +433,7 @@
Args=--initialize-at-build-time=org.apache.fury.memory.MemoryBuffer,\
org.apache.fury.type.GenericType,\
org.apache.fury.type.Generics,\
org.apache.fury.type.Type,\
+ org.apache.fury.type.Types,\
org.apache.fury.type.TypeUtils,\
org.apache.fury.util.ClassLoaderUtils$ByteArrayClassLoader,\
org.apache.fury.util.ClassLoaderUtils$ParentClassLoader,\
diff --git
a/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java
b/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java
index c6a93038..05570fd9 100644
--- a/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java
+++ b/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java
@@ -36,6 +36,7 @@ import java.time.Instant;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
@@ -63,9 +64,10 @@ import org.apache.fury.memory.MemoryUtils;
import org.apache.fury.serializer.ArraySerializersTest;
import org.apache.fury.serializer.BufferObject;
import org.apache.fury.serializer.EnumSerializerTest;
+import org.apache.fury.serializer.ObjectSerializer;
import org.apache.fury.serializer.Serializer;
-import org.apache.fury.serializer.StructSerializer;
import org.apache.fury.test.TestUtils;
+import org.apache.fury.type.Descriptor;
import org.apache.fury.util.DateTimeUtils;
import org.apache.fury.util.MurmurHash3;
import org.testng.Assert;
@@ -459,10 +461,16 @@ public class CrossLanguageTest extends FuryTestBase {
.requireClassRegistration(false)
.build();
fury.register(ComplexObject1.class, "test.ComplexObject1");
- StructSerializer serializer = (StructSerializer)
fury.getSerializer(ComplexObject1.class);
- Method method =
StructSerializer.class.getDeclaredMethod("computeStructHash");
+ fury.serialize(new ComplexObject1()); // trigger serializer update
+ ObjectSerializer serializer = (ObjectSerializer)
fury.getSerializer(ComplexObject1.class);
+ Method method =
+ ObjectSerializer.class.getDeclaredMethod("computeStructHash",
Fury.class, Collection.class);
method.setAccessible(true);
- Integer hash = (Integer) method.invoke(serializer);
+ Collection<Descriptor> descriptors =
+ fury.getClassResolver().getAllDescriptorsMap(ComplexObject1.class,
true).values();
+ descriptors =
+ fury.getClassResolver().createDescriptorGrouper(descriptors,
false).getSortedDescriptors();
+ Integer hash = (Integer) method.invoke(serializer, fury, descriptors);
MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(4);
buffer.writeInt32(hash);
roundBytes("test_struct_hash", buffer.getBytes(0, 4));
@@ -514,7 +522,7 @@ public class CrossLanguageTest extends FuryTestBase {
private void structRoundBack(Fury fury, Object obj, String testName) throws
IOException {
byte[] serialized = fury.serialize(obj);
- // Assert.assertEquals(fury.deserialize(serialized), obj);
+ Assert.assertEquals(fury.deserialize(serialized), obj);
Path dataFile = Paths.get(testName);
System.out.println(dataFile.toAbsolutePath());
Files.deleteIfExists(dataFile);
diff --git
a/java/fury-core/src/test/java/org/apache/fury/type/DescriptorGrouperTest.java
b/java/fury-core/src/test/java/org/apache/fury/type/DescriptorGrouperTest.java
index d8c7be43..30ca99be 100644
---
a/java/fury-core/src/test/java/org/apache/fury/type/DescriptorGrouperTest.java
+++
b/java/fury-core/src/test/java/org/apache/fury/type/DescriptorGrouperTest.java
@@ -35,7 +35,6 @@ import org.apache.fury.reflect.ReflectionUtils;
import org.apache.fury.reflect.TypeRef;
import org.testng.annotations.Test;
-@SuppressWarnings("UnstableApiUsage")
public class DescriptorGrouperTest {
private List<Descriptor> createDescriptors() {
@@ -101,10 +100,10 @@ public class DescriptorGrouperTest {
long.class,
float.class,
int.class,
- char.class,
short.class,
- boolean.class,
+ char.class,
byte.class,
+ boolean.class,
void.class);
assertEquals(classes, expected);
}
@@ -124,10 +123,10 @@ public class DescriptorGrouperTest {
Arrays.asList(
double.class,
float.class,
- char.class,
short.class,
- boolean.class,
+ char.class,
byte.class,
+ boolean.class,
void.class,
long.class,
int.class);
@@ -151,7 +150,13 @@ public class DescriptorGrouperTest {
new Descriptor(new TypeRef<Map<String, String>>() {}, "c" + index++,
-1, "TestClass"));
DescriptorGrouper grouper =
DescriptorGrouper.createDescriptorGrouper(
- ReflectionUtils::isMonomorphic, descriptors, false, false, false);
+ ReflectionUtils::isMonomorphic,
+ descriptors,
+ false,
+ null,
+ false,
+ false,
+ DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME);
{
List<? extends Class<?>> classes =
grouper.getPrimitiveDescriptors().stream()
@@ -163,10 +168,10 @@ public class DescriptorGrouperTest {
long.class,
float.class,
int.class,
- char.class,
short.class,
- boolean.class,
+ char.class,
byte.class,
+ boolean.class,
void.class);
assertEquals(classes, expected);
}
@@ -181,10 +186,10 @@ public class DescriptorGrouperTest {
Long.class,
Float.class,
Integer.class,
- Character.class,
Short.class,
- Boolean.class,
+ Character.class,
Byte.class,
+ Boolean.class,
Void.class);
assertEquals(classes, expected);
}
@@ -227,7 +232,13 @@ public class DescriptorGrouperTest {
public void testCompressedPrimitiveGrouper() {
DescriptorGrouper grouper =
DescriptorGrouper.createDescriptorGrouper(
- ReflectionUtils::isMonomorphic, createDescriptors(), false, true,
true);
+ ReflectionUtils::isMonomorphic,
+ createDescriptors(),
+ false,
+ null,
+ true,
+ true,
+ DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME);
{
List<? extends Class<?>> classes =
grouper.getPrimitiveDescriptors().stream()
@@ -237,10 +248,10 @@ public class DescriptorGrouperTest {
Arrays.asList(
double.class,
float.class,
- char.class,
short.class,
- boolean.class,
+ char.class,
byte.class,
+ boolean.class,
void.class,
long.class,
int.class);
@@ -255,10 +266,10 @@ public class DescriptorGrouperTest {
Arrays.asList(
Double.class,
Float.class,
- Character.class,
Short.class,
- Boolean.class,
+ Character.class,
Byte.class,
+ Boolean.class,
Void.class,
Long.class,
Integer.class);
diff --git a/python/pyfury/_registry.py b/python/pyfury/_registry.py
index be1280c8..1c390597 100644
--- a/python/pyfury/_registry.py
+++ b/python/pyfury/_registry.py
@@ -556,6 +556,12 @@ class ClassResolver:
if typeinfo is None:
ns = ns_metabytes.decode(self.namespace_decoder)
typename = type_metabytes.decode(self.typename_decoder)
+ typeinfo = self._named_type_to_classinfo.get((ns, typename))
+ if typeinfo is not None:
+ self._ns_type_to_classinfo[
+ (ns_metabytes, type_metabytes)
+ ] = typeinfo
+ return typeinfo
# TODO(chaokunyang) generate a dynamic class and serializer
# when meta share is enabled.
name = ns + "." + typename if ns else typename
diff --git a/python/pyfury/_struct.py b/python/pyfury/_struct.py
index 9b639315..d96ef1e4 100644
--- a/python/pyfury/_struct.py
+++ b/python/pyfury/_struct.py
@@ -39,8 +39,17 @@ from pyfury.type import (
Float64Type,
is_py_array_type,
compute_string_hash,
+ is_primitive_type,
)
+from pyfury.type import (
+ is_list_type,
+ is_map_type,
+ get_primitive_type_size,
+ is_primitive_array_type,
+)
+
+
logger = logging.getLogger(__name__)
@@ -100,6 +109,62 @@ def _get_hash(fury, field_names: list, type_hints: dict):
return hash_
+_UNKNOWN_TYPE_ID = -1
+
+
+def _sort_fields(class_resolver, field_names, serializers):
+ boxed_types = []
+ collection_types = []
+ map_types = []
+ final_types = []
+ other_types = []
+ type_ids = []
+ for field_name, serializer in zip(field_names, serializers):
+ if serializer is None:
+ other_types.append((_UNKNOWN_TYPE_ID, serializer, field_name))
+ else:
+ type_ids.append(
+ (
+ class_resolver.get_classinfo(serializer.type_).type_id,
+ serializer,
+ field_name,
+ )
+ )
+ for type_id, serializer, field_name in type_ids:
+ if is_primitive_type(type_id):
+ container = boxed_types
+ elif is_list_type(serializer.type_):
+ container = collection_types
+ elif is_map_type(serializer.type_):
+ container = map_types
+ elif type_id in {TypeId.STRING} or is_primitive_array_type(type_id):
+ container = final_types
+ else:
+ container = other_types
+ container.append((type_id, serializer, field_name))
+
+ def sorter(item):
+ return item[0], item[2]
+
+ def numeric_sorter(item):
+ id_ = item[0]
+ compress = id_ in {
+ TypeId.INT32,
+ TypeId.INT64,
+ TypeId.VAR_INT32,
+ TypeId.VAR_INT64,
+ }
+ return int(compress), -get_primitive_type_size(id_), item[2]
+
+ boxed_types = sorted(boxed_types, key=numeric_sorter)
+ collection_types = sorted(collection_types, key=sorter)
+ final_types = sorted(final_types, key=sorter)
+ map_types = sorted(map_types, key=sorter)
+ other_types = sorted(other_types, key=sorter)
+ all_types = boxed_types + final_types + other_types + collection_types +
map_types
+ return [t[1] for t in all_types], [t[2] for t in all_types]
+
+
class ComplexObjectSerializer(Serializer):
def __init__(self, fury, clz):
super().__init__(fury, clz)
@@ -110,7 +175,11 @@ class ComplexObjectSerializer(Serializer):
for index, key in enumerate(self._field_names):
serializer = infer_field(key, self._type_hints[key], visitor,
types_path=[])
self._serializers[index] = serializer
- from pyfury._fury import Language
+ self._serializers, self._field_names = _sort_fields(
+ fury.class_resolver, self._field_names, self._serializers
+ )
+
+ from pyfury import Language
if self.fury.language == Language.PYTHON:
logger.warning(
diff --git a/python/pyfury/resolver.py b/python/pyfury/resolver.py
index 1b0cbc9d..1dbec953 100644
--- a/python/pyfury/resolver.py
+++ b/python/pyfury/resolver.py
@@ -199,6 +199,8 @@ class MapRefResolver(RefResolver):
def set_read_object(self, id_, obj):
if id_ >= 0:
+ if id_ >= len(self.read_objects):
+ raise RuntimeError(f"Ref id {id_} invalid")
self.read_objects[id_] = obj
def reset(self):
diff --git a/python/pyfury/type.py b/python/pyfury/type.py
index eb3040de..8b4069a3 100644
--- a/python/pyfury/type.py
+++ b/python/pyfury/type.py
@@ -248,14 +248,45 @@ _primitive_types = {
Float64Type,
}
+_primitive_types_ids = {
+ TypeId.BOOL,
+ TypeId.INT8,
+ TypeId.INT16,
+ TypeId.INT32,
+ TypeId.INT64,
+ TypeId.FLOAT16,
+ TypeId.FLOAT32,
+ TypeId.FLOAT64,
+}
+
# `Union[type, TypeVar]` is not supported in py3.6, so skip adding type hints
for `type_` # noqa: E501
# See more at https://github.com/python/typing/issues/492 and
#
https://stackoverflow.com/questions/69427175/how-to-pass-forwardref-as-args-to-typevar-in-python-3-6
# noqa: E501
def is_primitive_type(type_) -> bool:
+ if type(type_) is int:
+ return type_ in _primitive_types_ids
return type_ in _primitive_types
+_primitive_type_sizes = {
+ TypeId.BOOL: 1,
+ TypeId.INT8: 1,
+ TypeId.INT16: 2,
+ TypeId.INT32: 4,
+ TypeId.VAR_INT32: 4,
+ TypeId.INT64: 8,
+ TypeId.VAR_INT64: 8,
+ TypeId.FLOAT16: 2,
+ TypeId.FLOAT32: 4,
+ TypeId.FLOAT64: 8,
+}
+
+
+def get_primitive_type_size(type_id) -> int:
+ return _primitive_type_sizes.get(type_id, -1)
+
+
# Int8ArrayType = TypeVar("Int8ArrayType", bound=array.ArrayType)
BoolArrayType = TypeVar("BoolArrayType")
Int16ArrayType = TypeVar("Int16ArrayType", bound=array.ArrayType)
@@ -279,12 +310,54 @@ _py_array_types = {
Float32ArrayType,
Float64ArrayType,
}
+_np_array_types = {
+ BoolNDArrayType,
+ Int16NDArrayType,
+ Int32NDArrayType,
+ Int64NDArrayType,
+ Float32NDArrayType,
+ Float64NDArrayType,
+}
+_primitive_array_types = _py_array_types.union(_np_array_types)
def is_py_array_type(type_) -> bool:
return type_ in _py_array_types
+_primitive_array_type_ids = {
+ TypeId.BOOL_ARRAY,
+ TypeId.INT8_ARRAY,
+ TypeId.INT16_ARRAY,
+ TypeId.INT32_ARRAY,
+ TypeId.INT64_ARRAY,
+ TypeId.FLOAT32_ARRAY,
+ TypeId.FLOAT64_ARRAY,
+}
+
+
+def is_primitive_array_type(type_) -> bool:
+ if type(type_) is int:
+ return type_ in _primitive_array_type_ids
+ return type_ in _primitive_array_types
+
+
+def is_list_type(type_):
+ try:
+ # type_ may not be a instance of type
+ return issubclass(type_, typing.List)
+ except TypeError:
+ return False
+
+
+def is_map_type(type_):
+ try:
+ # type_ may not be a instance of type
+ return issubclass(type_, typing.Dict)
+ except TypeError:
+ return False
+
+
class TypeVisitor(ABC):
@abstractmethod
def visit_list(self, field_name, elem_type, types_path=None):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
