This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fury.git
The following commit(s) were added to refs/heads/main by this push:
new 681a3088 feat: xlang map chunk serialization between java/python
(#2127)
681a3088 is described below
commit 681a308831e928227e04dcca86b295121f6674ec
Author: Shawn Yang <[email protected]>
AuthorDate: Fri Mar 28 18:54:49 2025 +0800
feat: xlang map chunk serialization between java/python (#2127)
## What does this PR do?
This PR supports xlang map chunk serialization between java/python
## Related issues
Closes #2125
Closes #2126
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fury/issues/new/choose) describing the
need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
.../src/main/java/org/apache/fury/Fury.java | 30 ++-
.../fury/builder/CompatibleCodecBuilder.java | 2 +-
.../org/apache/fury/resolver/ClassResolver.java | 24 +-
.../org/apache/fury/resolver/TypeResolver.java | 52 ++++
.../org/apache/fury/resolver/XtypeResolver.java | 50 +++-
.../fury/serializer/CompatibleSerializer.java | 18 +-
.../apache/fury/serializer/ObjectSerializer.java | 2 +-
.../collection/AbstractCollectionSerializer.java | 6 +-
.../collection/AbstractMapSerializer.java | 277 +++++++--------------
.../collection/CollectionSerializers.java | 2 +-
.../collection/GuavaCollectionSerializers.java | 2 +-
.../collection/SerializationBinding.java | 128 ++++++++++
.../java/org/apache/fury/type/GenericType.java | 6 +-
.../fury-core/native-image.properties | 3 +
python/pyfury/_serialization.pyx | 128 ++++++----
.../fury/serializer/scala/RangeSerializer.scala | 2 +-
16 files changed, 452 insertions(+), 280 deletions(-)
diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java
b/java/fury-core/src/main/java/org/apache/fury/Fury.java
index e15ad6db..68059c25 100644
--- a/java/fury-core/src/main/java/org/apache/fury/Fury.java
+++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java
@@ -113,6 +113,7 @@ public final class Fury implements BaseFury {
private final ArrayListSerializer arrayListSerializer;
private final HashMapSerializer hashMapSerializer;
private final Language language;
+ private final boolean crossLanguage;
private final boolean compressInt;
private final LongEncoding longEncoding;
private final Generics generics;
@@ -131,6 +132,7 @@ public final class Fury implements BaseFury {
// `org.apache.fury.ThreadSafeFury.clearClassLoader` is called.
config = new Config(builder);
this.language = config.getLanguage();
+ crossLanguage = language != Language.JAVA;
this.refTracking = config.trackingRef();
this.copyRefTracking = config.copyRef();
this.shareMeta = config.isMetaShareEnabled();
@@ -145,12 +147,15 @@ public final class Fury implements BaseFury {
generics = new Generics(this);
metaStringResolver = new MetaStringResolver();
classResolver = new ClassResolver(this);
- classResolver.initialize();
if (language != Language.JAVA) {
xtypeResolver = new XtypeResolver(this);
} else {
xtypeResolver = null;
}
+ classResolver.initialize();
+ if (xtypeResolver != null) {
+ xtypeResolver.initialize();
+ }
serializationContext = new SerializationContext(config);
this.classLoader = classLoader;
stringSerializer = new StringSerializer(this);
@@ -396,7 +401,7 @@ public final class Fury implements BaseFury {
// reduce caller stack
if (!refResolver.writeRefOrNull(buffer, obj)) {
ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass());
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
writeData(buffer, classInfo, obj);
}
MetaContext metaContext = serializationContext.getMetaContext();
@@ -410,7 +415,7 @@ public final class Fury implements BaseFury {
public void writeRef(MemoryBuffer buffer, Object obj) {
if (!refResolver.writeRefOrNull(buffer, obj)) {
ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass());
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
writeData(buffer, classInfo, obj);
}
}
@@ -418,7 +423,7 @@ public final class Fury implements BaseFury {
public void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder
classInfoHolder) {
if (!refResolver.writeRefOrNull(buffer, obj)) {
ClassInfo classInfo = classResolver.getClassInfo(obj.getClass(),
classInfoHolder);
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
writeData(buffer, classInfo, obj);
}
}
@@ -427,7 +432,7 @@ public final class Fury implements BaseFury {
Serializer<Object> serializer = classInfo.getSerializer();
if (serializer.needToWriteRef()) {
if (!refResolver.writeRefOrNull(buffer, obj)) {
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
depth++;
serializer.write(buffer, obj);
depth--;
@@ -437,7 +442,7 @@ public final class Fury implements BaseFury {
buffer.writeByte(Fury.NULL_FLAG);
} else {
buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG);
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
depth++;
serializer.write(buffer, obj);
depth--;
@@ -510,12 +515,12 @@ public final class Fury implements BaseFury {
*/
public void writeNonRef(MemoryBuffer buffer, Object obj) {
ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass());
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
writeData(buffer, classInfo, obj);
}
public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo
classInfo) {
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
Serializer serializer = classInfo.getSerializer();
depth++;
serializer.write(buffer, obj);
@@ -1119,7 +1124,7 @@ public final class Fury implements BaseFury {
buffer.writeInt32(-1); // preserve 4-byte for meta start offsets.
if (!refResolver.writeRefOrNull(buffer, obj)) {
ClassInfo classInfo =
classResolver.getOrUpdateClassInfo(obj.getClass());
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
writeData(buffer, classInfo, obj);
MetaContext metaContext = serializationContext.getMetaContext();
if (metaContext != null && !metaContext.writingClassDefs.isEmpty()) {
@@ -1589,6 +1594,9 @@ public final class Fury implements BaseFury {
return xtypeResolver;
}
+ // don't provide getTypeResolver directly to avoid users use this
+ // API too much since it has polymorphic invoke cost.
+
public MetaStringResolver getMetaStringResolver() {
return metaStringResolver;
}
@@ -1630,6 +1638,10 @@ public final class Fury implements BaseFury {
return language;
}
+ public boolean isCrossLanguage() {
+ return crossLanguage;
+ }
+
public boolean trackingRef() {
return refTracking;
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/builder/CompatibleCodecBuilder.java
b/java/fury-core/src/main/java/org/apache/fury/builder/CompatibleCodecBuilder.java
index 92c79527..d49ffc61 100644
---
a/java/fury-core/src/main/java/org/apache/fury/builder/CompatibleCodecBuilder.java
+++
b/java/fury-core/src/main/java/org/apache/fury/builder/CompatibleCodecBuilder.java
@@ -877,7 +877,7 @@ public class CompatibleCodecBuilder extends
BaseObjectCodecBuilder {
return classResolver.writeClassExpr(buffer, classInfo.getClassId());
}
Expression classInfoExpr = getFinalClassInfo(cls);
- return new Invoke(classResolverRef, "writeClass", buffer, classInfoExpr);
+ return new Invoke(classResolverRef, "writeClassInfo", buffer,
classInfoExpr);
}
protected Expression skipFinalClassInfo(Class<?> cls, Expression buffer) {
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
index 2917b8a3..d864dfc9 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
@@ -167,7 +167,7 @@ import org.apache.fury.util.function.Functions;
* up relations between serializer and types.
*/
@SuppressWarnings({"rawtypes", "unchecked"})
-public class ClassResolver {
+public class ClassResolver implements TypeResolver {
private static final Logger LOG =
LoggerFactory.getLogger(ClassResolver.class);
// bit 0 unset indicates class is written as an id.
@@ -277,6 +277,7 @@ public class ClassResolver {
ClassResolver._addGraalvmClassRegistry(fury.getConfig().getConfigHash(),
this);
}
+ @Override
public void initialize() {
register(LambdaSerializer.ReplaceStub.class, LAMBDA_STUB_ID);
register(JdkProxySerializer.ReplaceStub.class, JDK_PROXY_STUB_ID);
@@ -1379,7 +1380,7 @@ public class ClassResolver {
/**
* Write class info to <code>buffer</code>. TODO(chaokunyang): The method
should try to write
- * aligned data to reduce cpu instruction overhead. `writeClass` is the last
step before
+ * aligned data to reduce cpu instruction overhead. `writeClassInfo` is the
last step before
* serializing object, if this writes are aligned, then later serialization
will be more
* efficient.
*/
@@ -1390,26 +1391,27 @@ public class ClassResolver {
} else if (cls == Long.class) {
buffer.writeVarUint32Small7(LONG_CLASS_ID << 1);
} else {
- writeClass(buffer, getOrUpdateClassInfo(cls));
+ writeClassInfo(buffer, getOrUpdateClassInfo(cls));
}
}
// The jit-compiled native code fot this method will be too big for inline,
so we generated
// `getClassInfo`
// in fury-jit, see `BaseSeqCodecBuilder#writeAndGetClassInfo`
- // public ClassInfo writeClass(MemoryBuffer buffer, Class<?> cls,
ClassInfoHolder classInfoHolder)
+ // public ClassInfo writeClassInfo(MemoryBuffer buffer, Class<?> cls,
ClassInfoHolder
+ // classInfoHolder)
// {
// ClassInfo classInfo = getClassInfo(cls, classInfoHolder);
- // writeClass(buffer, classInfo);
+ // writeClassInfo(buffer, classInfo);
// return classInfo;
// }
/** Write classname for java serialization. */
- public void writeClass(MemoryBuffer buffer, ClassInfo classInfo) {
+ public void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) {
if (metaContextShareEnabled) {
// FIXME(chaokunyang) Register class but not register serializer can't
be used with
// meta share mode, because no class def are sent to peer.
- writeClassWithMetaShare(buffer, classInfo);
+ writeClassInfoWithMetaShare(buffer, classInfo);
} else {
if (classInfo.classId == NO_CLASS_ID) { // no class id provided.
// use classname
@@ -1425,7 +1427,7 @@ public class ClassResolver {
}
}
- public void writeClassWithMetaShare(MemoryBuffer buffer, ClassInfo
classInfo) {
+ public void writeClassInfoWithMetaShare(MemoryBuffer buffer, ClassInfo
classInfo) {
if (classInfo.classId != NO_CLASS_ID && !classInfo.needToWriteClassDef) {
buffer.writeVarUint32(classInfo.classId << 1);
return;
@@ -1680,7 +1682,7 @@ public class ClassResolver {
}
/**
- * Native code for ClassResolver.writeClass is too big to inline, so inline
it manually.
+ * Native code for ClassResolver.writeClassInfo is too big to inline, so
inline it manually.
*
* <p>See `already compiled into a big method` in <a
*
href="https://wiki.openjdk.org/display/HotSpot/Server+Compiler+Inlining+Messages";>Server+Compiler+Inlining+Messages</a>
@@ -1688,7 +1690,7 @@ public class ClassResolver {
// Note: Thread safe fot jit thread to call.
public Expression writeClassExpr(
Expression classResolverRef, Expression buffer, Expression classInfo) {
- return new Invoke(classResolverRef, "writeClass", buffer, classInfo);
+ return new Invoke(classResolverRef, "writeClassInfo", buffer, classInfo);
}
// Note: Thread safe fot jit thread to call.
@@ -1710,7 +1712,7 @@ public class ClassResolver {
/**
* Write classname for java serialization. Note that the object of provided
class can be
* non-serializable, and class with writeReplace/readResolve defined won't
be skipped. For
- * serializable object, {@link #writeClass(MemoryBuffer, ClassInfo)} should
be invoked.
+ * serializable object, {@link #writeClassInfo(MemoryBuffer, ClassInfo)}
should be invoked.
*/
public void writeClassInternal(MemoryBuffer buffer, Class<?> cls) {
ClassInfo classInfo = classInfoMap.get(cls);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/TypeResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/TypeResolver.java
new file mode 100644
index 00000000..98b453a3
--- /dev/null
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/TypeResolver.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fury.resolver;
+
+import java.lang.reflect.Type;
+import org.apache.fury.annotation.Internal;
+import org.apache.fury.memory.MemoryBuffer;
+import org.apache.fury.reflect.TypeRef;
+import org.apache.fury.serializer.Serializer;
+import org.apache.fury.type.GenericType;
+
+// Internal type dispatcher.
+// Do not use this interface outside of fury package
+@Internal
+public interface TypeResolver {
+ boolean needToWriteRef(TypeRef<?> typeRef);
+
+ ClassInfo getClassInfo(Class<?> cls, ClassInfoHolder classInfoHolder);
+
+ void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo);
+
+ ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder
classInfoHolder);
+
+ <T> Serializer<T> getSerializer(Class<T> cls);
+
+ ClassInfo nilClassInfo();
+
+ ClassInfoHolder nilClassInfoHolder();
+
+ GenericType buildGenericType(TypeRef<?> typeRef);
+
+ GenericType buildGenericType(Type type);
+
+ void initialize();
+}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java
index b0ef00f0..bd8bb478 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java
@@ -27,6 +27,7 @@ import static org.apache.fury.meta.Encoders.TYPE_NAME_DECODER;
import static org.apache.fury.resolver.ClassResolver.NO_CLASS_ID;
import static org.apache.fury.type.TypeUtils.qualifiedName;
+import java.lang.reflect.Type;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Timestamp;
@@ -58,6 +59,7 @@ import org.apache.fury.memory.Platform;
import org.apache.fury.meta.Encoders;
import org.apache.fury.meta.MetaString;
import org.apache.fury.reflect.ReflectionUtils;
+import org.apache.fury.reflect.TypeRef;
import org.apache.fury.serializer.EnumSerializer;
import org.apache.fury.serializer.NonexistentClass;
import org.apache.fury.serializer.NonexistentClassSerializers;
@@ -74,7 +76,7 @@ import org.apache.fury.util.Preconditions;
@SuppressWarnings({"unchecked", "rawtypes"})
// TODO(chaokunyang) Abstract type resolver for java/xlang type resolution.
-public class XtypeResolver {
+public class XtypeResolver implements TypeResolver {
private static final Logger LOG =
LoggerFactory.getLogger(XtypeResolver.class);
private static final float loadFactor = 0.5f;
@@ -107,6 +109,10 @@ public class XtypeResolver {
this.classResolver = fury.getClassResolver();
this.generics = fury.getGenerics();
this.metaStringResolver = fury.getMetaStringResolver();
+ }
+
+ @Override
+ public void initialize() {
registerDefaultTypes();
}
@@ -277,6 +283,21 @@ public class XtypeResolver {
return classInfo;
}
+ @Override
+ public boolean needToWriteRef(TypeRef<?> typeRef) {
+ return getClassInfo(typeRef.getRawType()).serializer.needToWriteRef();
+ }
+
+ @Override
+ public GenericType buildGenericType(TypeRef<?> typeRef) {
+ return classResolver.buildGenericType(typeRef);
+ }
+
+ @Override
+ public GenericType buildGenericType(Type type) {
+ return classResolver.buildGenericType(type);
+ }
+
private ClassInfo buildClassInfo(Class<?> cls) {
Serializer serializer;
int xtypeId;
@@ -354,6 +375,12 @@ public class XtypeResolver {
public ClassInfo writeClassInfo(MemoryBuffer buffer, Object obj) {
ClassInfo classInfo = getClassInfo(obj.getClass(), classInfoCache);
+ writeClassInfo(buffer, classInfo);
+ return classInfo;
+ }
+
+ @Override
+ public void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) {
int xtypeId = classInfo.getXtypeId();
byte internalTypeId = (byte) xtypeId;
buffer.writeVarUint32Small7(xtypeId);
@@ -370,7 +397,26 @@ public class XtypeResolver {
default:
break;
}
- return classInfo;
+ }
+
+ @Override
+ public <T> Serializer<T> getSerializer(Class<T> cls) {
+ return (Serializer) getClassInfo(cls).serializer;
+ }
+
+ @Override
+ public ClassInfo nilClassInfo() {
+ return classResolver.nilClassInfo();
+ }
+
+ @Override
+ public ClassInfoHolder nilClassInfoHolder() {
+ return classResolver.nilClassInfoHolder();
+ }
+
+ @Override
+ public ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder
classInfoHolder) {
+ return readClassInfo(buffer);
}
public ClassInfo readClassInfo(MemoryBuffer buffer) {
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/CompatibleSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/CompatibleSerializer.java
index 433093d8..0638ff3b 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/CompatibleSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/CompatibleSerializer.java
@@ -221,10 +221,10 @@ public final class CompatibleSerializer<T> extends
CompatibleSerializerBase<T> {
private void writeCollectionField(
MemoryBuffer buffer, FieldResolver.CollectionFieldInfo fieldInfo,
Collection fieldValue) {
ClassInfo elementClassInfo = fieldInfo.getElementClassInfo();
- classResolver.writeClass(buffer, elementClassInfo);
+ classResolver.writeClassInfo(buffer, elementClassInfo);
// following write is consistent with
`BaseSeqCodecBuilder.serializeForCollection`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
AbstractCollectionSerializer collectionSerializer =
(AbstractCollectionSerializer) classInfo.getSerializer();
try {
@@ -241,11 +241,11 @@ public final class CompatibleSerializer<T> extends
CompatibleSerializerBase<T> {
MemoryBuffer buffer, FieldResolver.MapFieldInfo fieldInfo, Map
fieldValue) {
ClassInfo keyClassInfo = fieldInfo.getKeyClassInfo();
ClassInfo valueClassInfo = fieldInfo.getValueClassInfo();
- classResolver.writeClass(buffer, keyClassInfo);
- classResolver.writeClass(buffer, valueClassInfo);
+ classResolver.writeClassInfo(buffer, keyClassInfo);
+ classResolver.writeClassInfo(buffer, valueClassInfo);
// following write is consistent with `BaseSeqCodecBuilder.serializeForMap`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
@@ -262,10 +262,10 @@ public final class CompatibleSerializer<T> extends
CompatibleSerializerBase<T> {
private void writeMapKeyFinal(
MemoryBuffer buffer, FieldResolver.MapFieldInfo fieldInfo, Map
fieldValue) {
ClassInfo keyClassInfo = fieldInfo.getKeyClassInfo();
- classResolver.writeClass(buffer, keyClassInfo);
+ classResolver.writeClassInfo(buffer, keyClassInfo);
// following write is consistent with `BaseSeqCodecBuilder.serializeForMap`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
@@ -280,10 +280,10 @@ public final class CompatibleSerializer<T> extends
CompatibleSerializerBase<T> {
private void writeMapValueFinal(
MemoryBuffer buffer, FieldResolver.MapFieldInfo fieldInfo, Map
fieldValue) {
ClassInfo valueClassInfo = fieldInfo.getValueClassInfo();
- classResolver.writeClass(buffer, valueClassInfo);
+ classResolver.writeClassInfo(buffer, valueClassInfo);
// following write is consistent with `BaseSeqCodecBuilder.serializeForMap`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
index 320d8afc..fd640ebf 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
@@ -173,7 +173,7 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
} else {
if (fieldInfo.trackingRef && serializer.needToWriteRef()) {
if (!refResolver.writeRefOrNull(buffer, fieldValue)) {
- classResolver.writeClass(buffer, fieldInfo.classInfo);
+ classResolver.writeClassInfo(buffer, fieldInfo.classInfo);
// No generics for field, no need to update `depth`.
serializer.write(buffer, fieldValue);
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java
index ee0cab84..d63f3762 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java
@@ -204,7 +204,7 @@ public abstract class AbstractCollectionSerializer<T>
extends Serializer<T> {
// Update classinfo, the caller will use it.
ClassResolver classResolver = fury.getClassResolver();
ClassInfo classInfo = classResolver.getClassInfo(elemClass, cache);
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
}
}
return bitmap;
@@ -246,7 +246,7 @@ public abstract class AbstractCollectionSerializer<T>
extends Serializer<T> {
bitmap |= CollectionFlags.TRACKING_REF;
}
buffer.writeByte(bitmap);
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
}
return bitmap;
}
@@ -294,7 +294,7 @@ public abstract class AbstractCollectionSerializer<T>
extends Serializer<T> {
buffer.writeByte(bitmap);
ClassResolver classResolver = fury.getClassResolver();
ClassInfo classInfo = classResolver.getClassInfo(elemClass, cache);
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
}
}
return bitmap;
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java
index d9741139..0e39420f 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java
@@ -46,8 +46,8 @@ import org.apache.fury.reflect.ReflectionUtils;
import org.apache.fury.reflect.TypeRef;
import org.apache.fury.resolver.ClassInfo;
import org.apache.fury.resolver.ClassInfoHolder;
-import org.apache.fury.resolver.ClassResolver;
import org.apache.fury.resolver.RefResolver;
+import org.apache.fury.resolver.TypeResolver;
import org.apache.fury.serializer.Serializer;
import org.apache.fury.type.GenericType;
import org.apache.fury.type.Generics;
@@ -70,7 +70,7 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
// or one of types is already instantiated in subclass, ex: `Subclass<T>
implements Map<String,
// T>`
private final IdentityMap<GenericType, GenericType> partialGenericKVTypeMap;
- private final GenericType objType =
fury.getClassResolver().buildGenericType(Object.class);
+ private final GenericType objType;
// For subclass whose kv type are instantiated already, such as
// `Subclass implements Map<String, Long>`. If declared `Map` doesn't specify
// instantiated kv type, then the serialization will need to write those kv
@@ -80,33 +80,29 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
// interpreter and jit mode although it seems unnecessary.
// With kv header in future, we can write this kv classes only once, the
cost won't be too much.
private int numElements;
- private final ClassResolver classResolver;
+ private final TypeResolver typeResolver;
+ protected final SerializationBinding binding;
public AbstractMapSerializer(Fury fury, Class<T> cls) {
this(fury, cls, !ReflectionUtils.isDynamicGeneratedCLass(cls));
}
public AbstractMapSerializer(Fury fury, Class<T> cls, boolean
supportCodegenHook) {
- super(fury, cls);
- this.classResolver = fury.getClassResolver();
- this.supportCodegenHook = supportCodegenHook;
- keyClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder();
- keyClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder();
- valueClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder();
- valueClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder();
- partialGenericKVTypeMap = new IdentityMap<>();
+ this(fury, cls, supportCodegenHook, false);
}
public AbstractMapSerializer(
Fury fury, Class<T> cls, boolean supportCodegenHook, boolean immutable) {
super(fury, cls, immutable);
- this.classResolver = fury.getClassResolver();
+ this.typeResolver = fury.isCrossLanguage() ? fury.getXtypeResolver() :
fury.getClassResolver();
this.supportCodegenHook = supportCodegenHook;
- keyClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder();
- keyClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder();
- valueClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder();
- valueClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder();
+ keyClassInfoWriteCache = typeResolver.nilClassInfoHolder();
+ keyClassInfoReadCache = typeResolver.nilClassInfoHolder();
+ valueClassInfoWriteCache = typeResolver.nilClassInfoHolder();
+ valueClassInfoReadCache = typeResolver.nilClassInfoHolder();
partialGenericKVTypeMap = new IdentityMap<>();
+ objType = typeResolver.buildGenericType(Object.class);
+ binding = SerializationBinding.createBinding(fury);
}
/**
@@ -138,7 +134,7 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
if (map.isEmpty()) {
return;
}
- ClassResolver classResolver = fury.getClassResolver();
+ TypeResolver classResolver = typeResolver;
Iterator<Entry<Object, Object>> iterator = map.entrySet().iterator();
Entry<Object, Object> entry = iterator.next();
while (entry != null) {
@@ -165,8 +161,7 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
@Override
public void xwrite(MemoryBuffer buffer, T value) {
- Map map = onMapWrite(buffer, value);
- xwriteElements(fury, buffer, map);
+ write(buffer, value);
}
public final Entry writeJavaNullChunk(
@@ -199,14 +194,14 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
if (keySerializer != null) {
if (keySerializer.needToWriteRef()) {
buffer.writeByte(NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF);
- fury.writeRef(buffer, key, keySerializer);
+ binding.writeRef(buffer, key, keySerializer);
} else {
buffer.writeByte(NULL_VALUE_KEY_DECL_TYPE);
- keySerializer.write(buffer, key);
+ binding.write(buffer, keySerializer, key);
}
} else {
buffer.writeByte(VALUE_HAS_NULL | TRACKING_KEY_REF);
- fury.writeRef(buffer, key, keyClassInfoWriteCache);
+ binding.writeRef(buffer, key, keyClassInfoWriteCache);
}
}
@@ -221,14 +216,14 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
if (valueSerializer != null) {
if (valueSerializer.needToWriteRef()) {
buffer.writeByte(NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF);
- fury.writeRef(buffer, value, valueSerializer);
+ binding.writeRef(buffer, value, valueSerializer);
} else {
buffer.writeByte(NULL_KEY_VALUE_DECL_TYPE);
- valueSerializer.write(buffer, value);
+ binding.write(buffer, valueSerializer, value);
}
} else {
buffer.writeByte(KEY_HAS_NULL | TRACKING_VALUE_REF);
- fury.writeRef(buffer, value, valueClassInfoWriteCache);
+ binding.writeRef(buffer, value, valueClassInfoWriteCache);
}
} else {
buffer.writeByte(KV_NULL);
@@ -250,7 +245,7 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
return entry;
}
buffer.writeByte(NULL_VALUE_KEY_DECL_TYPE);
- keySerializer.write(buffer, key);
+ binding.write(buffer, keySerializer, key);
} else {
writeNullKeyChunk(buffer, valueSerializer, value);
}
@@ -264,7 +259,7 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
// Make byte code of this method smaller than 325 for better jit inline
private Entry writeJavaChunk(
- ClassResolver classResolver,
+ TypeResolver classResolver,
MemoryBuffer buffer,
Entry<Object, Object> entry,
Iterator<Entry<Object, Object>> iterator,
@@ -309,10 +304,10 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
break;
}
if (!keyWriteRef || !refResolver.writeRefOrNull(buffer, key)) {
- keySerializer.write(buffer, key);
+ binding.write(buffer, keySerializer, key);
}
if (!valueWriteRef || !refResolver.writeRefOrNull(buffer, value)) {
- valueSerializer.write(buffer, value);
+ binding.write(buffer, valueSerializer, value);
}
// noinspection Duplicates
++chunkSize;
@@ -333,21 +328,21 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
}
private Serializer writeKeyClassInfo(
- ClassResolver classResolver, Class keyType, MemoryBuffer buffer) {
+ TypeResolver classResolver, Class keyType, MemoryBuffer buffer) {
ClassInfo classInfo = classResolver.getClassInfo(keyType,
keyClassInfoWriteCache);
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
return classInfo.getSerializer();
}
private Serializer writeValueClassInfo(
- ClassResolver classResolver, Class valueType, MemoryBuffer buffer) {
+ TypeResolver classResolver, Class valueType, MemoryBuffer buffer) {
ClassInfo classInfo = classResolver.getClassInfo(valueType,
valueClassInfoWriteCache);
- classResolver.writeClass(buffer, classInfo);
+ classResolver.writeClassInfo(buffer, classInfo);
return classInfo.getSerializer();
}
private Entry writeJavaChunkGeneric(
- ClassResolver classResolver,
+ TypeResolver classResolver,
Generics generics,
GenericType genericType,
MemoryBuffer buffer,
@@ -423,14 +418,14 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
generics.pushGenericType(keyGenericType);
if (!keyWriteRef || !refResolver.writeRefOrNull(buffer, key)) {
fury.incDepth(1);
- keySerializer.write(buffer, key);
+ binding.write(buffer, keySerializer, key);
fury.incDepth(-1);
}
generics.popGenericType();
generics.pushGenericType(valueGenericType);
if (!valueWriteRef || !refResolver.writeRefOrNull(buffer, value)) {
fury.incDepth(1);
- valueSerializer.write(buffer, value);
+ binding.write(buffer, valueSerializer, value);
fury.incDepth(-1);
}
generics.popGenericType();
@@ -452,66 +447,6 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
return entry;
}
- public static void xwriteElements(Fury fury, MemoryBuffer buffer, Map value)
{
- Generics generics = fury.getGenerics();
- GenericType genericType = generics.nextGenericType();
- // TODO(chaokunyang) support map subclass whose key or value generics only
are available.
- if (genericType == null || genericType.getTypeParametersCount() != 2) {
- for (Object object : value.entrySet()) {
- Map.Entry entry = (Map.Entry) object;
- fury.xwriteRef(buffer, entry.getKey());
- fury.xwriteRef(buffer, entry.getValue());
- }
- } else {
- // TODO(chaokunyang) use codegen to remove all branches.
- GenericType keyGenericType = genericType.getTypeParameter0();
- GenericType valueGenericType = genericType.getTypeParameter1();
- Serializer keySerializer =
keyGenericType.getSerializer(fury.getClassResolver());
- Serializer valueSerializer =
valueGenericType.getSerializer(fury.getClassResolver());
- if (!keyGenericType.hasGenericParameters() &&
!valueGenericType.hasGenericParameters()) {
- for (Object object : value.entrySet()) {
- Map.Entry entry = (Map.Entry) object;
- xwriteRefByNullableSerializer(fury, buffer, entry.getKey(),
keySerializer);
- xwriteRefByNullableSerializer(fury, buffer, entry.getValue(),
valueSerializer);
- }
- } else if (valueGenericType.hasGenericParameters()) {
- for (Object object : value.entrySet()) {
- Map.Entry entry = (Map.Entry) object;
- xwriteRefByNullableSerializer(fury, buffer, entry.getKey(),
keySerializer);
- generics.pushGenericType(valueGenericType);
- xwriteRefByNullableSerializer(fury, buffer, entry.getValue(),
valueSerializer);
- generics.popGenericType();
- }
- } else if (keyGenericType.hasGenericParameters()) {
- for (Object object : value.entrySet()) {
- Map.Entry entry = (Map.Entry) object;
- generics.pushGenericType(keyGenericType);
- xwriteRefByNullableSerializer(fury, buffer, entry.getKey(),
keySerializer);
- generics.popGenericType();
- xwriteRefByNullableSerializer(fury, buffer, entry.getValue(),
valueSerializer);
- }
- } else {
- for (Object object : value.entrySet()) {
- Map.Entry entry = (Map.Entry) object;
- generics.pushGenericType(keyGenericType);
- xwriteRefByNullableSerializer(fury, buffer, entry.getKey(),
keySerializer);
- generics.pushGenericType(valueGenericType);
- xwriteRefByNullableSerializer(fury, buffer, entry.getValue(),
valueSerializer);
- }
- }
- generics.popGenericType();
- }
- }
-
- public static <T> void xwriteRefByNullableSerializer(
- Fury fury, MemoryBuffer buffer, T obj, Serializer<T> serializer) {
- if (serializer == null) {
- fury.xwriteRef(buffer, obj);
- } else {
- fury.xwriteRef(buffer, obj, serializer);
- }
- }
-
private GenericType getKVGenericType(GenericType genericType) {
GenericType mapGenericType = partialGenericKVTypeMap.get(genericType);
if (mapGenericType == null) {
@@ -530,13 +465,11 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
@Override
public T xread(MemoryBuffer buffer) {
- Map map = newMap(buffer);
- xreadElements(fury, buffer, map, numElements);
- return onMapRead(map);
+ return read(buffer);
}
protected <K, V> void copyEntry(Map<K, V> originMap, Map<K, V> newMap) {
- ClassResolver classResolver = fury.getClassResolver();
+ TypeResolver classResolver = typeResolver;
for (Map.Entry<K, V> entry : originMap.entrySet()) {
K key = entry.getKey();
if (key != null) {
@@ -558,7 +491,7 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
}
protected <K, V> void copyEntry(Map<K, V> originMap, Builder<K, V> builder) {
- ClassResolver classResolver = fury.getClassResolver();
+ TypeResolver classResolver = typeResolver;
for (Entry<K, V> entry : originMap.entrySet()) {
K key = entry.getKey();
if (key != null) {
@@ -580,7 +513,7 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
}
protected <K, V> void copyEntry(Map<K, V> originMap, Object[] elements) {
- ClassResolver classResolver = fury.getClassResolver();
+ TypeResolver classResolver = typeResolver;
int index = 0;
for (Entry<K, V> entry : originMap.entrySet()) {
K key = entry.getKey();
@@ -667,12 +600,12 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
Object key;
if ((chunkHeader & KEY_DECL_TYPE) != 0) {
if (trackKeyRef) {
- key = fury.readRef(buffer, keySerializer);
+ key = binding.readRef(buffer, keySerializer);
} else {
- key = keySerializer.read(buffer);
+ key = binding.read(buffer, keySerializer);
}
} else {
- key = fury.readRef(buffer, keyClassInfoReadCache);
+ key = binding.readRef(buffer, keyClassInfoReadCache);
}
map.put(key, null);
}
@@ -703,12 +636,12 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
boolean trackValueRef = (chunkHeader & TRACKING_VALUE_REF) != 0;
if ((chunkHeader & VALUE_DECL_TYPE) != 0) {
if (trackValueRef) {
- value = fury.readRef(buffer, valueSerializer);
+ value = binding.readRef(buffer, valueSerializer);
} else {
- value = valueSerializer.read(buffer);
+ value = binding.read(buffer, valueSerializer);
}
} else {
- value = fury.readRef(buffer, valueClassInfoReadCache);
+ value = binding.readRef(buffer, valueClassInfoReadCache);
}
map.put(null, value);
} else {
@@ -731,7 +664,7 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
if (!valueHasNull) {
return (size << 8) | chunkHeader;
} else {
- Object key = keySerializer.read(buffer);
+ Object key = binding.read(buffer, keySerializer);
map.put(key, null);
}
} else {
@@ -760,16 +693,20 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
boolean valueIsDeclaredType = (chunkHeader & VALUE_DECL_TYPE) != 0;
int chunkSize = buffer.readUnsignedByte();
if (!keyIsDeclaredType) {
- keySerializer = classResolver.readClassInfo(buffer,
keyClassInfoReadCache).getSerializer();
+ keySerializer = typeResolver.readClassInfo(buffer,
keyClassInfoReadCache).getSerializer();
}
if (!valueIsDeclaredType) {
- valueSerializer =
- classResolver.readClassInfo(buffer,
valueClassInfoReadCache).getSerializer();
+ valueSerializer = typeResolver.readClassInfo(buffer,
valueClassInfoReadCache).getSerializer();
}
for (int i = 0; i < chunkSize; i++) {
- Object key = trackKeyRef ? fury.readRef(buffer, keySerializer) :
keySerializer.read(buffer);
+ Object key =
+ trackKeyRef
+ ? binding.readRef(buffer, keySerializer)
+ : binding.read(buffer, keySerializer);
Object value =
- trackValueRef ? fury.readRef(buffer, valueSerializer) :
valueSerializer.read(buffer);
+ trackValueRef
+ ? binding.readRef(buffer, valueSerializer)
+ : binding.read(buffer, valueSerializer);
map.put(key, value);
size--;
}
@@ -800,93 +737,51 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
int chunkSize = buffer.readUnsignedByte();
Serializer keySerializer, valueSerializer;
if (!keyIsDeclaredType) {
- keySerializer = classResolver.readClassInfo(buffer,
keyClassInfoReadCache).getSerializer();
+ keySerializer = typeResolver.readClassInfo(buffer,
keyClassInfoReadCache).getSerializer();
} else {
- keySerializer = keyGenericType.getSerializer(classResolver);
+ keySerializer = keyGenericType.getSerializer(typeResolver);
}
if (!valueIsDeclaredType) {
- valueSerializer =
- classResolver.readClassInfo(buffer,
valueClassInfoReadCache).getSerializer();
+ valueSerializer = typeResolver.readClassInfo(buffer,
valueClassInfoReadCache).getSerializer();
} else {
- valueSerializer = valueGenericType.getSerializer(classResolver);
+ valueSerializer = valueGenericType.getSerializer(typeResolver);
}
- for (int i = 0; i < chunkSize; i++) {
- generics.pushGenericType(keyGenericType);
- fury.incDepth(1);
- Object key = trackKeyRef ? fury.readRef(buffer, keySerializer) :
keySerializer.read(buffer);
- fury.incDepth(-1);
- generics.popGenericType();
- generics.pushGenericType(valueGenericType);
- fury.incDepth(1);
- Object value =
- trackValueRef ? fury.readRef(buffer, valueSerializer) :
valueSerializer.read(buffer);
- fury.incDepth(-1);
- generics.popGenericType();
- map.put(key, value);
- size--;
- }
- return size > 0 ? (size << 8) | buffer.readUnsignedByte() : 0;
- }
-
- @SuppressWarnings("unchecked")
- public static void xreadElements(Fury fury, MemoryBuffer buffer, Map map,
int size) {
- Generics generics = fury.getGenerics();
- GenericType genericType = generics.nextGenericType();
- if (genericType == null || genericType.getTypeParametersCount() != 2) {
- for (int i = 0; i < size; i++) {
- Object key = fury.xreadRef(buffer);
- Object value = fury.xreadRef(buffer);
+ if (keyGenericType.hasGenericParameters() ||
valueGenericType.hasGenericParameters()) {
+ for (int i = 0; i < chunkSize; i++) {
+ generics.pushGenericType(keyGenericType);
+ fury.incDepth(1);
+ Object key =
+ trackKeyRef
+ ? binding.readRef(buffer, keySerializer)
+ : binding.read(buffer, keySerializer);
+ fury.incDepth(-1);
+ generics.popGenericType();
+ generics.pushGenericType(valueGenericType);
+ fury.incDepth(1);
+ Object value =
+ trackValueRef
+ ? binding.readRef(buffer, valueSerializer)
+ : binding.read(buffer, valueSerializer);
+ fury.incDepth(-1);
+ generics.popGenericType();
map.put(key, value);
+ size--;
}
} else {
- // TODO(chaokunyang) use codegen to remove all branches.
- GenericType keyGenericType = genericType.getTypeParameter0();
- GenericType valueGenericType = genericType.getTypeParameter1();
- Serializer keySerializer =
keyGenericType.getSerializer(fury.getClassResolver());
- Serializer valueSerializer =
valueGenericType.getSerializer(fury.getClassResolver());
- if (!keyGenericType.hasGenericParameters() &&
!valueGenericType.hasGenericParameters()) {
- for (int i = 0; i < size; i++) {
- Object key = xreadRefByNullableSerializer(fury, buffer,
keySerializer);
- Object value = xreadRefByNullableSerializer(fury, buffer,
valueSerializer);
- map.put(key, value);
- }
- } else if (valueGenericType.hasGenericParameters()) {
- for (int i = 0; i < size; i++) {
- Object key = xreadRefByNullableSerializer(fury, buffer,
keySerializer);
- generics.pushGenericType(valueGenericType);
- Object value = xreadRefByNullableSerializer(fury, buffer,
valueSerializer);
- generics.popGenericType();
- map.put(key, value);
- }
- } else if (keyGenericType.hasGenericParameters()) {
- for (int i = 0; i < size; i++) {
- generics.pushGenericType(keyGenericType);
- Object key = xreadRefByNullableSerializer(fury, buffer,
keySerializer);
- generics.popGenericType();
- Object value = xreadRefByNullableSerializer(fury, buffer,
valueSerializer);
- map.put(key, value);
- }
- } else {
- for (int i = 0; i < size; i++) {
- // FIXME(chaokunyang) nested generics may be get by mistake.
- generics.pushGenericType(keyGenericType);
- Object key = xreadRefByNullableSerializer(fury, buffer,
keySerializer);
- generics.pushGenericType(valueGenericType);
- Object value = xreadRefByNullableSerializer(fury, buffer,
valueSerializer);
- map.put(key, value);
- }
+ for (int i = 0; i < chunkSize; i++) {
+ Object key =
+ trackKeyRef
+ ? binding.readRef(buffer, keySerializer)
+ : binding.read(buffer, keySerializer);
+ Object value =
+ trackValueRef
+ ? binding.readRef(buffer, valueSerializer)
+ : binding.read(buffer, valueSerializer);
+ map.put(key, value);
+ size--;
}
- generics.popGenericType();
- }
- }
-
- public static Object xreadRefByNullableSerializer(
- Fury fury, MemoryBuffer buffer, Serializer<?> serializer) {
- if (serializer == null) {
- return fury.xreadRef(buffer);
- } else {
- return fury.xreadRef(buffer, serializer);
}
+ return size > 0 ? (size << 8) | buffer.readUnsignedByte() : 0;
}
/**
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java
index a8f78a1d..80ae21ce 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java
@@ -488,7 +488,7 @@ public class CollectionSerializers {
final Map<?, Boolean> map = (Map<?, Boolean>) Platform.getObject(value,
MAP_FIELD_OFFSET);
final ClassInfo classInfo =
fury.getClassResolver().getClassInfo(map.getClass());
AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
- fury.getClassResolver().writeClass(buffer, classInfo);
+ fury.getClassResolver().writeClassInfo(buffer, classInfo);
if (mapSerializer.supportCodegenHook) {
buffer.writeBoolean(true);
mapSerializer.onMapWrite(buffer, map);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java
index 44740cc9..b08ca0a6 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java
@@ -260,7 +260,7 @@ public class GuavaCollectionSerializers {
public T xread(MemoryBuffer buffer) {
int size = buffer.readVarUint32Small7();
Map map = new HashMap();
- xreadElements(fury, buffer, map, size);
+ readElements(buffer, size, map);
return xnewInstance(map);
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SerializationBinding.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SerializationBinding.java
new file mode 100644
index 00000000..e5b629f6
--- /dev/null
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SerializationBinding.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fury.serializer.collection;
+
+import org.apache.fury.Fury;
+import org.apache.fury.memory.MemoryBuffer;
+import org.apache.fury.resolver.ClassInfoHolder;
+import org.apache.fury.serializer.Serializer;
+
+// This polymorphic interface has cost, do not expose it as a public class
+// If it's used in other packages in fury, duplicate it in those packages.
+@SuppressWarnings({"rawtypes", "unchecked"})
+// noinspection Duplicates
+interface SerializationBinding {
+ <T> void writeRef(MemoryBuffer buffer, T obj, Serializer<T> serializer);
+
+ void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder
classInfoHolder);
+
+ <T> T readRef(MemoryBuffer buffer, Serializer<T> serializer);
+
+ Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder);
+
+ void write(MemoryBuffer buffer, Serializer serializer, Object value);
+
+ Object read(MemoryBuffer buffer, Serializer serializer);
+
+ static SerializationBinding createBinding(Fury fury) {
+ if (fury.isCrossLanguage()) {
+ return new XlangSerializationBinding(fury);
+ } else {
+ return new JavaSerializationBinding(fury);
+ }
+ }
+
+ final class JavaSerializationBinding implements SerializationBinding {
+ private final Fury fury;
+
+ JavaSerializationBinding(Fury fury) {
+ this.fury = fury;
+ }
+
+ @Override
+ public <T> void writeRef(MemoryBuffer buffer, T obj, Serializer<T>
serializer) {
+ fury.writeRef(buffer, obj, serializer);
+ }
+
+ @Override
+ public void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder
classInfoHolder) {
+ fury.writeRef(buffer, obj, classInfoHolder);
+ }
+
+ @Override
+ public <T> T readRef(MemoryBuffer buffer, Serializer<T> serializer) {
+ return fury.readRef(buffer, serializer);
+ }
+
+ @Override
+ public Object readRef(MemoryBuffer buffer, ClassInfoHolder
classInfoHolder) {
+ return fury.readRef(buffer, classInfoHolder);
+ }
+
+ @Override
+ public void write(MemoryBuffer buffer, Serializer serializer, Object
value) {
+ serializer.write(buffer, value);
+ }
+
+ @Override
+ public Object read(MemoryBuffer buffer, Serializer serializer) {
+ return serializer.read(buffer);
+ }
+ }
+
+ final class XlangSerializationBinding implements SerializationBinding {
+
+ private final Fury fury;
+
+ XlangSerializationBinding(Fury fury) {
+ this.fury = fury;
+ }
+
+ @Override
+ public <T> void writeRef(MemoryBuffer buffer, T obj, Serializer<T>
serializer) {
+ fury.xwriteRef(buffer, obj, serializer);
+ }
+
+ @Override
+ public void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder
classInfoHolder) {
+ fury.xwriteRef(buffer, obj);
+ }
+
+ @Override
+ public <T> T readRef(MemoryBuffer buffer, Serializer<T> serializer) {
+ return (T) fury.xreadRef(buffer, serializer);
+ }
+
+ @Override
+ public Object readRef(MemoryBuffer buffer, ClassInfoHolder
classInfoHolder) {
+ return fury.xreadRef(buffer);
+ }
+
+ @Override
+ public void write(MemoryBuffer buffer, Serializer serializer, Object
value) {
+ serializer.xwrite(buffer, value);
+ }
+
+ @Override
+ public Object read(MemoryBuffer buffer, Serializer serializer) {
+ return serializer.xread(buffer);
+ }
+ }
+}
diff --git a/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java
b/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java
index 3e9cea6c..8a6bb6f6 100644
--- a/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java
+++ b/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java
@@ -31,7 +31,7 @@ import java.util.List;
import java.util.function.Predicate;
import org.apache.fury.reflect.ReflectionUtils;
import org.apache.fury.reflect.TypeRef;
-import org.apache.fury.resolver.ClassResolver;
+import org.apache.fury.resolver.TypeResolver;
import org.apache.fury.serializer.Serializer;
/** GenericType for building java generics as a tree and binding with fury
serializers. */
@@ -186,7 +186,7 @@ public class GenericType {
this.serializer = serializer;
}
- public Serializer<?> getSerializer(ClassResolver classResolver) {
+ public Serializer<?> getSerializer(TypeResolver classResolver) {
Serializer<?> serializer = this.serializer;
if (serializer == null) {
serializer = classResolver.getSerializer(cls);
@@ -203,7 +203,7 @@ public class GenericType {
return isMonomorphic;
}
- public boolean trackingRef(ClassResolver classResolver) {
+ public boolean trackingRef(TypeResolver classResolver) {
Boolean trackingRef = this.trackingRef;
if (trackingRef == null) {
trackingRef = this.trackingRef = classResolver.needToWriteRef(typeRef);
diff --git
a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
index a54bfa97..18960e4c 100644
---
a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
+++
b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
@@ -279,6 +279,9 @@
Args=--initialize-at-build-time=org.apache.fury.memory.MemoryBuffer,\
org.apache.fury.resolver.NoRefResolver,\
org.apache.fury.resolver.SerializationContext,\
org.apache.fury.resolver.TypeNameBytes,\
+ org.apache.fury.serializer.collection.SerializationBinding,\
+
org.apache.fury.serializer.collection.SerializationBinding$XlangSerializationBinding,\
+
org.apache.fury.serializer.collection.SerializationBinding$JavaSerializationBinding,\
org.apache.fury.serializer.ArraySerializers$BooleanArraySerializer,\
org.apache.fury.serializer.ArraySerializers$ByteArraySerializer,\
org.apache.fury.serializer.ArraySerializers$CharArraySerializer,\
diff --git a/python/pyfury/_serialization.pyx b/python/pyfury/_serialization.pyx
index 9b2a5e1a..caaf0302 100644
--- a/python/pyfury/_serialization.pyx
+++ b/python/pyfury/_serialization.pyx
@@ -1618,6 +1618,7 @@ cdef class MapSerializer(Serializer):
cdef MapRefResolver ref_resolver
cdef Serializer key_serializer
cdef Serializer value_serializer
+ cdef c_bool is_py
def __init__(self, fury, type_, key_serializer=None,
value_serializer=None):
super().__init__(fury, type_)
@@ -1625,6 +1626,7 @@ cdef class MapSerializer(Serializer):
self.ref_resolver = fury.ref_resolver
self.key_serializer = key_serializer
self.value_serializer = value_serializer
+ self.is_py = fury.language == Language.PYTHON
cpdef inline write(self, Buffer buffer, o):
cdef dict obj = o
@@ -1644,6 +1646,7 @@ cdef class MapSerializer(Serializer):
cdef int32_t chunk_size_offset, chunk_header, chunk_size
cdef c_bool key_write_ref, value_write_ref
cdef int has_next = PyDict_Next(obj, &pos, <PyObject **>&key_addr,
<PyObject **>&value_addr)
+ cdef c_bool is_py = self.is_py
while has_next != 0:
key = int2obj(key_addr)
Py_INCREF(key)
@@ -1657,28 +1660,49 @@ cdef class MapSerializer(Serializer):
if key_serializer.need_to_write_ref:
buffer.write_int8(NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF)
if not self.ref_resolver.write_ref_or_null(buffer,
key):
- key_serializer.write(buffer, key)
+ if is_py:
+ key_serializer.write(buffer, key)
+ else:
+ key_serializer.xwrite(buffer, key)
else:
buffer.write_int8(NULL_VALUE_KEY_DECL_TYPE)
- key_serializer.write(buffer, key)
+ if is_py:
+ key_serializer.write(buffer, key)
+ else:
+ key_serializer.xwrite(buffer, key)
else:
buffer.write_int8(VALUE_HAS_NULL | TRACKING_KEY_REF)
- fury.serialize_ref(buffer, key)
+ if is_py:
+ fury.serialize_ref(buffer, key)
+ else:
+ fury.xserialize_ref(buffer, key)
else:
if value is not None:
if value_serializer is not None:
if value_serializer.need_to_write_ref:
buffer.write_int8(NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF)
- if not
self.ref_resolver.write_ref_or_null(buffer, key):
- value_serializer.write(buffer, key)
if not
self.ref_resolver.write_ref_or_null(buffer, value):
- value_serializer.write(buffer, value)
+ if is_py:
+ value_serializer.write(buffer, value)
+ else:
+ value_serializer.xwrite(buffer, value)
+ if not
self.ref_resolver.write_ref_or_null(buffer, value):
+ if is_py:
+ value_serializer.write(buffer, value)
+ else:
+ value_serializer.xwrite(buffer, value)
else:
buffer.write_int8(NULL_KEY_VALUE_DECL_TYPE)
- value_serializer.write(buffer, value)
+ if is_py:
+ value_serializer.write(buffer, value)
+ else:
+ value_serializer.xwrite(buffer, value)
else:
buffer.write_int8(KEY_HAS_NULL |
TRACKING_VALUE_REF)
- fury.serialize_ref(buffer, value)
+ if is_py:
+ fury.serialize_ref(buffer, value)
+ else:
+ fury.xserialize_ref(buffer, value)
else:
buffer.write_int8(KV_NULL)
has_next = PyDict_Next(obj, &pos, <PyObject **>&key_addr,
<PyObject **>&value_addr)
@@ -1731,7 +1755,10 @@ cdef class MapSerializer(Serializer):
elif key_serializer_type is Float32Serializer:
buffer.write_float(key)
else:
- key_serializer.write(buffer, key)
+ if is_py:
+ key_serializer.write(buffer, key)
+ else:
+ key_serializer.xwrite(buffer, key)
if not value_write_ref or not
ref_resolver.write_ref_or_null(buffer, value):
if value_cls is str:
buffer.write_string(value)
@@ -1746,7 +1773,10 @@ cdef class MapSerializer(Serializer):
elif value_serializer_type is BooleanSerializer:
buffer.write_bool(value)
else:
- value_serializer.write(buffer, value)
+ if is_py:
+ value_serializer.write(buffer, value)
+ else:
+ value_serializer.xwrite(buffer, value)
chunk_size += 1
has_next = PyDict_Next(obj, &pos, <PyObject **>&key_addr,
<PyObject **>&value_addr)
if has_next == 0:
@@ -1773,11 +1803,13 @@ cdef class MapSerializer(Serializer):
cdef int32_t chunk_header = 0
if size != 0:
chunk_header = buffer.read_uint8()
- cdef Serializer key_serializer, value_serializer
+ cdef Serializer key_serializer = self.key_serializer
+ cdef Serializer value_serializer = self.value_serializer
cdef c_bool key_has_null, value_has_null, track_key_ref,
track_value_ref
cdef c_bool key_is_declared_type, value_is_declared_type
cdef type key_serializer_type, value_serializer_type
cdef int32_t chunk_size
+ cdef c_bool is_py = self.is_py
while size > 0:
while True:
key_has_null = (chunk_header & KEY_HAS_NULL) != 0
@@ -1793,12 +1825,21 @@ cdef class MapSerializer(Serializer):
if ref_id < NOT_NULL_VALUE_FLAG:
key = ref_resolver.get_read_object()
else:
- key = key_serializer.read(buffer)
+ if is_py:
+ key = key_serializer.read(buffer)
+ else:
+ key = key_serializer.xread(buffer)
ref_resolver.set_read_object(ref_id, key)
else:
- key = key_serializer.read(buffer)
+ if is_py:
+ key = key_serializer.read(buffer)
+ else:
+ key = key_serializer.xread(buffer)
else:
- key = fury.deserialize_ref(buffer)
+ if is_py:
+ key = fury.deserialize_ref(buffer)
+ else:
+ key = fury.xdeserialize_ref(buffer)
map_[key] = None
else:
if not value_has_null:
@@ -1809,10 +1850,16 @@ cdef class MapSerializer(Serializer):
if ref_id < NOT_NULL_VALUE_FLAG:
value = ref_resolver.get_read_object()
else:
- value = value_serializer.read(buffer)
+ if is_py:
+ value = value_serializer.read(buffer)
+ else:
+ value = value_serializer.xread(buffer)
ref_resolver.set_read_object(ref_id, value)
else:
- value = fury.deserialize_ref(buffer)
+ if is_py:
+ value = fury.deserialize_ref(buffer)
+ else:
+ value = fury.xdeserialize_ref(buffer)
map_[None] = value
else:
map_[None] = None
@@ -1838,7 +1885,10 @@ cdef class MapSerializer(Serializer):
if ref_id < NOT_NULL_VALUE_FLAG:
key = ref_resolver.get_read_object()
else:
- key = key_serializer.read(buffer)
+ if is_py:
+ key = key_serializer.read(buffer)
+ else:
+ key = key_serializer.xread(buffer)
ref_resolver.set_read_object(ref_id, key)
else:
if key_serializer_type is StringSerializer:
@@ -1852,13 +1902,19 @@ cdef class MapSerializer(Serializer):
elif key_serializer_type is Float32Serializer:
key = buffer.read_float()
else:
- key = key_serializer.read(buffer)
+ if is_py:
+ key = key_serializer.read(buffer)
+ else:
+ key = key_serializer.xread(buffer)
if track_value_ref:
ref_id = ref_resolver.try_preserve_ref_id(buffer)
if ref_id < NOT_NULL_VALUE_FLAG:
value = ref_resolver.get_read_object()
else:
- value = value_serializer.read(buffer)
+ if is_py:
+ value = value_serializer.read(buffer)
+ else:
+ value = value_serializer.xread(buffer)
ref_resolver.set_read_object(ref_id, value)
else:
if value_serializer_type is StringSerializer:
@@ -1874,7 +1930,10 @@ cdef class MapSerializer(Serializer):
elif value_serializer_type is BooleanSerializer:
value = buffer.read_bool()
else:
- value = value_serializer.read(buffer)
+ if is_py:
+ value = value_serializer.read(buffer)
+ else:
+ value = value_serializer.xread(buffer)
map_[key] = value
size -= 1
if size != 0:
@@ -1882,35 +1941,10 @@ cdef class MapSerializer(Serializer):
return map_
cpdef inline xwrite(self, Buffer buffer, o):
- cdef dict value = o
- buffer.write_varuint32(len(value))
- cdef int64_t key_addr, value_addr
- cdef Py_ssize_t pos = 0
- while PyDict_Next(value, &pos, <PyObject **>&key_addr, <PyObject
**>&value_addr) != 0:
- k = int2obj(key_addr)
- Py_INCREF(k)
- v = int2obj(value_addr)
- Py_INCREF(v)
- self.fury.xserialize_ref(
- buffer, k, serializer=self.key_serializer
- )
- self.fury.xserialize_ref(
- buffer, v, serializer=self.value_serializer
- )
+ self.write(buffer, o)
cpdef inline xread(self, Buffer buffer):
- cdef int32_t len_ = buffer.read_varuint32()
- cdef dict map_ = _PyDict_NewPresized(len_)
- self.fury.ref_resolver.reference(map_)
- for i in range(len_):
- k = self.fury.xdeserialize_ref(
- buffer, serializer=self.key_serializer
- )
- v = self.fury.xdeserialize_ref(
- buffer, serializer=self.value_serializer
- )
- map_[k] = v
- return map_
+ return self.read(buffer)
@cython.final
diff --git
a/scala/src/main/scala/org/apache/fury/serializer/scala/RangeSerializer.scala
b/scala/src/main/scala/org/apache/fury/serializer/scala/RangeSerializer.scala
index 7e40b66e..06a10df6 100644
---
a/scala/src/main/scala/org/apache/fury/serializer/scala/RangeSerializer.scala
+++
b/scala/src/main/scala/org/apache/fury/serializer/scala/RangeSerializer.scala
@@ -74,7 +74,7 @@ class NumericRangeSerializer[A, T <: NumericRange[A]](fury:
Fury, cls: Class[T])
val cls = value.start.getClass
val resolver = fury.getClassResolver
val classInfo = resolver.getClassInfo(cls)
- resolver.writeClass(buffer, classInfo)
+ resolver.writeClassInfo(buffer, classInfo)
val serializer = classInfo.getSerializer.asInstanceOf[Serializer[A]]
serializer.write(buffer, value.start)
serializer.write(buffer, value.end)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
