This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch releases-0.11.0
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/releases-0.11.0 by this push:
new cc91f02d chore: cp more commits to 0.11.0 (#2319)
cc91f02d is described below
commit cc91f02d95945c3534920eb075dcff60ce0b435c
Author: Shawn Yang <[email protected]>
AuthorDate: Mon Jun 9 23:08:06 2025 +0800
chore: cp more commits to 0.11.0 (#2319)
<!--
**Thanks for contributing to Fory.**
**If this is your first time opening a PR on fory, you can refer to
[CONTRIBUTING.md](https://github.com/apache/fory/blob/main/CONTRIBUTING.md).**
Contribution Checklist
- The **Apache Fory (incubating)** community has restrictions on the
naming of pr titles. You can also find instructions in
[CONTRIBUTING.md](https://github.com/apache/fory/blob/main/CONTRIBUTING.md).
- Fory has a strong focus on performance. If the PR you submit will have
an impact on performance, please benchmark it first and provide the
benchmark result here.
-->
## What does this PR do?
<!-- Describe the purpose of this PR. -->
## Related issues
<!--
Is there any related issue? Please attach here.
- #xxxx0
- #xxxx1
- #xxxx2
-->
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fory/issues/new/choose) describing the
need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---------
Co-authored-by: hongwen.chw <[email protected]>
Co-authored-by: Steven Schlansker <[email protected]>
Co-authored-by: penguin_wwy <[email protected]>
Co-authored-by: wenyangwang <[email protected]>
Co-authored-by: OmCheeLin <[email protected]>
---
bazel/fory_deps_setup.bzl | 4 +-
.../cpython_benchmark/fory_benchmark.py | 28 +++-
.../main/java/org/apache/fory/meta/ClassDef.java | 6 +
.../org/apache/fory/resolver/ClassResolver.java | 34 ++++-
.../org/apache/fory/resolver/XtypeResolver.java | 2 +-
.../org/apache/fory/type/CustomTypeRegistry.java | 14 +-
.../main/java/org/apache/fory/type/TypeUtils.java | 62 +++++----
.../apache/fory/resolver/ClassResolverTest.java | 146 +++++++++++++++++++++
.../org/apache/fory/xlang/MetaSharedXlangTest.java | 3 +-
.../fory/format/encoder/ArrayDataForEach.java | 2 +-
.../fory/format/encoder/ArrayEncoderBuilder.java | 4 +-
.../format/encoder/BaseBinaryEncoderBuilder.java | 126 +++++++++++-------
.../apache/fory/format/encoder/CustomCodec.java | 47 ++++++-
.../fory/format/encoder/RowEncoderBuilder.java | 95 ++++++++------
.../format/type/CustomTypeEncoderRegistry.java | 4 +-
.../apache/fory/format/type/CustomTypeHandler.java | 14 +-
.../org/apache/fory/format/type/TypeInference.java | 29 ++--
.../fory/format/encoder/CustomCodecTest.java | 92 +++++++++++--
python/pyproject.toml | 2 +-
19 files changed, 542 insertions(+), 172 deletions(-)
diff --git a/bazel/fory_deps_setup.bzl b/bazel/fory_deps_setup.bzl
index ca73e266..a8131bd3 100644
--- a/bazel/fory_deps_setup.bzl
+++ b/bazel/fory_deps_setup.bzl
@@ -145,8 +145,8 @@ def setup_deps():
auto_http_archive(
name = "cython",
build_file = "@com_github_grpc_grpc//third_party:cython.BUILD",
- url =
"https://github.com/cython/cython/releases/download/3.1.0a1/cython-3.1.0a1.tar.gz";,
- sha256 =
"35b53f6947c3133452b84f0f9703f222deb9b02874861427a45e63c891379440",
+ url =
"https://github.com/cython/cython/releases/download/3.1.1/cython-3.1.1.tar.gz";,
+ sha256 =
"505ccd413669d5132a53834d792c707974248088c4f60c497deb1b416e366397",
)
auto_http_archive(
name = "com_google_googletest",
diff --git a/integration_tests/cpython_benchmark/fory_benchmark.py
b/integration_tests/cpython_benchmark/fory_benchmark.py
index eb4f7cd1..e2e5ae0d 100644
--- a/integration_tests/cpython_benchmark/fory_benchmark.py
+++ b/integration_tests/cpython_benchmark/fory_benchmark.py
@@ -16,7 +16,6 @@
# under the License.
import argparse
-import array
from dataclasses import dataclass
import datetime
import os
@@ -146,8 +145,7 @@ COMPLEX_OBJECT = ComplexObject1(
f8=2**63 - 1,
f9=1.0 / 2,
f10=1 / 3.0,
- f11=array.array("h", [1, 2]),
- f12=[-1, 4],
+ f11=[-1, 4],
)
@@ -157,6 +155,13 @@ def fory_object(language, ref_tracking, obj):
fory.deserialize(binary)
+def fory_data_class(language, ref_tracking, obj, register_callable):
+ fory = pyfory.Fory(language=language, ref_tracking=ref_tracking)
+ register_callable(fory)
+ binary = fory.serialize(obj)
+ fory.deserialize(binary)
+
+
def benchmark_args():
parser = argparse.ArgumentParser(description="Fory Benchmark")
parser.add_argument("--xlang", action="store_true", default=False)
@@ -190,7 +195,6 @@ def micro_benchmark():
runner.bench_func(
"fory_large_tuple", fory_object, language, not args.no_ref, LARGE_TUPLE
)
- runner.bench_func("fory_list", fory_object, language, not args.no_ref,
LIST)
runner.bench_func(
"fory_large_float_tuple",
fory_object,
@@ -209,8 +213,22 @@ def micro_benchmark():
runner.bench_func(
"fory_large_list", fory_object, language, not args.no_ref, LARGE_LIST
)
+
+ def register_complex(fory):
+ if args.xlang:
+ fory.register_type(ComplexObject1,
typename="example.ComplexObject1")
+ fory.register_type(ComplexObject2,
typename="example.ComplexObject2")
+ else:
+ fory.register_type(ComplexObject1)
+ fory.register_type(ComplexObject2)
+
runner.bench_func(
- "fory_complex", fory_object, language, not args.no_ref, COMPLEX_OBJECT
+ "fory_complex",
+ fory_data_class,
+ language,
+ not args.no_ref,
+ COMPLEX_OBJECT,
+ register_complex,
)
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
index f5c9e41f..ff3a46ff 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
@@ -843,6 +843,9 @@ public class ClassDef implements Serializable {
@Override
public TypeRef<?> toTypeToken(TypeResolver classResolver, TypeRef<?>
declared) {
+ while (declared != null && declared.isArray()) {
+ declared = declared.getComponentType();
+ }
TypeRef<?> componentTypeRef = componentType.toTypeToken(classResolver,
declared);
Class<?> componentRawType = componentTypeRef.getRawType();
if (NonexistentClass.class.isAssignableFrom(componentRawType)) {
@@ -993,6 +996,9 @@ public class ClassDef implements Serializable {
}
if (rawType.isArray()) {
Class<?> elemType = rawType.getComponentType();
+ while (elemType.isArray()) {
+ elemType = elemType.getComponentType();
+ }
if (isXlang && !elemType.isPrimitive()) {
return new CollectionFieldType(
xtypeId,
diff --git
a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
index 852340ef..69db915f 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
@@ -255,6 +255,9 @@ public class ClassResolver implements TypeResolver {
new IdentityMap<>(estimatedNumRegistered);
private final BiMap<String, Class<?>> registeredClasses =
HashBiMap.create(estimatedNumRegistered);
+ // cache absClassInfo, support customized serializer for abstract or
interface.
+ private final IdentityMap<Class<?>, ClassInfo> absClassInfo =
+ new IdentityMap<>(estimatedNumRegistered, foryMapLoadFactor);
// avoid potential recursive call for seq codec generation.
// ex. A->field1: B, B.field1: A
private final Set<Class<?>> getClassCtx = new HashSet<>();
@@ -466,7 +469,7 @@ public class ClassResolver implements TypeResolver {
classInfo = new ClassInfo(this, cls, null, id, NOT_SUPPORT_XLANG);
// make `extRegistry.registeredClassIdMap` and `classInfoMap` share same
classInfo
// instances.
- classInfoMap.put(cls, classInfo);
+ setClassInfo(cls, classInfo);
}
// serializer will be set lazily in `addSerializer` method if it's null.
registeredId2ClassInfo[id] = classInfo;
@@ -514,7 +517,7 @@ public class ClassResolver implements TypeResolver {
MetaStringBytes nameBytes =
metaStringResolver.getOrCreateMetaStringBytes(encodeTypeName(name));
ClassInfo classInfo =
new ClassInfo(cls, fullNameBytes, nsBytes, nameBytes, false, null,
NO_CLASS_ID, (short) -1);
- classInfoMap.put(cls, classInfo);
+ setClassInfo(cls, classInfo);
compositeNameBytes2ClassInfo.put(
new TypeNameBytes(nsBytes.hashCode, nameBytes.hashCode), classInfo);
extRegistry.registeredClasses.put(fullname, cls);
@@ -857,7 +860,7 @@ public class ClassResolver implements TypeResolver {
if (classInfo == null || classId != classInfo.classId) {
classInfo = new ClassInfo(this, type, null, classId, (short) 0);
- classInfoMap.put(type, classInfo);
+ setClassInfo(type, classInfo);
if (registered) {
registeredId2ClassInfo[classId] = classInfo;
}
@@ -1294,6 +1297,10 @@ public class ClassResolver implements TypeResolver {
void setClassInfo(Class<?> cls, ClassInfo classInfo) {
classInfoMap.put(cls, classInfo);
+ // in order to support customized serializer for abstract or interface.
+ if (!cls.isPrimitive() && (ReflectionUtils.isAbstract(cls) ||
cls.isInterface())) {
+ extRegistry.absClassInfo.put(cls, classInfo);
+ }
}
@Internal
@@ -1363,6 +1370,23 @@ public class ClassResolver implements TypeResolver {
return shimSerializer;
}
+ // support customized serializer for abstract or interface.
+ if (!extRegistry.absClassInfo.isEmpty()) {
+ Class<?> tmpCls = cls;
+ while (tmpCls != null && tmpCls != Object.class) {
+ ClassInfo absClass = null;
+ if ((absClass = extRegistry.absClassInfo.get(tmpCls.getSuperclass()))
!= null) {
+ return absClass.serializer;
+ }
+ for (Class<?> tmpI : tmpCls.getInterfaces()) {
+ if ((absClass = extRegistry.absClassInfo.get(tmpI)) != null) {
+ return absClass.serializer;
+ }
+ }
+ tmpCls = tmpCls.getSuperclass();
+ }
+ }
+
Class<? extends Serializer> serializerClass = getSerializerClass(cls);
Serializer serializer = Serializers.newSerializer(fory, cls,
serializerClass);
if (ForyCopyable.class.isAssignableFrom(cls)) {
@@ -1772,7 +1796,7 @@ public class ClassResolver implements TypeResolver {
classInfo =
new ClassInfo(
this, cls, null, classId == null ? NO_CLASS_ID : classId,
NOT_SUPPORT_XLANG);
- classInfoMap.put(cls, classInfo);
+ setClassInfo(cls, classInfo);
}
writeClassInternal(buffer, classInfo);
}
@@ -1953,7 +1977,7 @@ public class ClassResolver implements TypeResolver {
// don't create serializer here, if the class is an interface,
// there won't be serializer since interface has no instance.
if (!classInfoMap.containsKey(cls)) {
- classInfoMap.put(cls, classInfo);
+ setClassInfo(cls, classInfo);
}
}
compositeNameBytes2ClassInfo.put(typeNameBytes, classInfo);
diff --git
a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
index 28e25d8d..4cd2bfdf 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
@@ -431,7 +431,7 @@ public class XtypeResolver implements TypeResolver {
serializer = getCollectionSerializer(cls);
}
xtypeId = Types.LIST;
- } else if (cls.isArray() &&
!TypeUtils.getArrayComponent(cls).isPrimitive()) {
+ } else if (cls.isArray() && !cls.getComponentType().isPrimitive()) {
serializer = new ArraySerializers.ObjectArraySerializer(fory, cls);
xtypeId = Types.LIST;
} else if (classResolver.isMap(cls)) {
diff --git
a/java/fory-core/src/main/java/org/apache/fory/type/CustomTypeRegistry.java
b/java/fory-core/src/main/java/org/apache/fory/type/CustomTypeRegistry.java
index 69edb5f7..bfbe4cf2 100644
--- a/java/fory-core/src/main/java/org/apache/fory/type/CustomTypeRegistry.java
+++ b/java/fory-core/src/main/java/org/apache/fory/type/CustomTypeRegistry.java
@@ -20,14 +20,15 @@
package org.apache.fory.type;
import org.apache.fory.annotation.Internal;
+import org.apache.fory.reflect.TypeRef;
@Internal
public interface CustomTypeRegistry {
CustomTypeRegistry EMPTY =
new CustomTypeRegistry() {
@Override
- public boolean hasCodec(final Class<?> beanType, final Class<?>
fieldType) {
- return false;
+ public TypeRef<?> replacementTypeFor(final Class<?> beanType, final
Class<?> fieldType) {
+ return null;
}
@Override
@@ -35,9 +36,16 @@ public interface CustomTypeRegistry {
final Class<?> collectionType, final Class<?> elementType) {
return false;
}
+
+ @Override
+ public boolean isExtraSupportedType(final TypeRef<?> type) {
+ return false;
+ }
};
- boolean hasCodec(Class<?> beanType, Class<?> fieldType);
+ TypeRef<?> replacementTypeFor(Class<?> beanType, Class<?> fieldType);
boolean canConstructCollection(Class<?> collectionType, Class<?>
elementType);
+
+ boolean isExtraSupportedType(TypeRef<?> type);
}
diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java
b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java
index 23785a5c..1c7b3946 100644
--- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java
+++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java
@@ -605,6 +605,10 @@ public class TypeUtils {
if (Modifier.isAbstract(cls.getModifiers()) ||
Modifier.isInterface(cls.getModifiers())) {
return false;
}
+ if (ctx.getWalkedTypePath().contains(typeRef)
+ || ctx.getCustomTypeRegistry().isExtraSupportedType(typeRef)) {
+ return false;
+ }
// since we need to access class in generated code in our package, the
class must be public
// if ReflectionUtils.hasNoArgConstructor(cls) return false, we use Unsafe
to create object.
if (Modifier.isPublic(cls.getModifiers())) {
@@ -624,17 +628,21 @@ public class TypeUtils {
&& !ITERABLE_TYPE.isSupertypeOf(typeRef)
&& !MAP_TYPE.isSupertypeOf(typeRef);
if (maybe) {
- return Descriptor.getDescriptors(cls).stream()
- .allMatch(
- d -> {
- TypeRef<?> t = d.getTypeRef();
- // do field modifiers and getter/setter validation here, not
in getDescriptors.
- // If Modifier.isFinal(d.getModifiers()), use reflection
- // private field that doesn't have getter/setter will be
handled by reflection.
- return ctx.getCustomTypeRegistry().hasCodec(cls,
t.getRawType())
- || isSupported(t, newTypePath)
- || isBean(t, newTypePath);
- });
+ for (Descriptor d : Descriptor.getDescriptors(cls)) {
+ TypeRef<?> t = d.getTypeRef();
+ // do field modifiers and getter/setter validation here, not in
getDescriptors.
+ // If Modifier.isFinal(d.getModifiers()), use reflection
+ // private field that doesn't have getter/setter will be handled by
reflection.
+ TypeRef<?> replacementType =
+ ctx.getCustomTypeRegistry().replacementTypeFor(cls,
t.getRawType());
+ if (replacementType != null) {
+ t = replacementType;
+ }
+ if (!isSupported(t, newTypePath)) {
+ return false;
+ }
+ }
+ return true;
} else {
return false;
}
@@ -658,10 +666,16 @@ public class TypeUtils {
// box.
return true;
}
- if (SUPPORTED_TYPES.contains(typeRef)) {
+ TypeRef<?> replacementType =
+ ctx.getCustomTypeRegistry()
+ .replacementTypeFor(ctx.getEnclosingType().getRawType(),
typeRef.getRawType());
+ if (replacementType != null) {
+ return isSupported(replacementType, ctx);
+ } else if (SUPPORTED_TYPES.contains(typeRef)
+ || ctx.getCustomTypeRegistry().isExtraSupportedType(typeRef)) {
return true;
} else if (typeRef.isArray()) {
- return isSupported(Objects.requireNonNull(typeRef.getComponentType()));
+ return isSupported(Objects.requireNonNull(typeRef.getComponentType()),
ctx);
} else if (ITERABLE_TYPE.isSupertypeOf(typeRef)) {
TypeRef<?> elementType = getElementType(typeRef);
boolean isSuperOfArrayList = cls.isAssignableFrom(ArrayList.class);
@@ -672,7 +686,7 @@ public class TypeUtils {
.canConstructCollection(typeRef.getRawType(),
elementType.getRawType())) {
return false;
}
- return isSupported(getElementType(typeRef));
+ return isSupported(elementType, ctx);
} else if (MAP_TYPE.isSupertypeOf(typeRef)) {
boolean isSuperOfHashMap = cls.isAssignableFrom(HashMap.class);
if (!isSuperOfHashMap && (cls.isInterface() ||
Modifier.isAbstract(cls.getModifiers()))) {
@@ -684,7 +698,7 @@ public class TypeUtils {
return true;
} else {
ctx.checkNoCycle(typeRef);
- return isBean(typeRef, ctx.appendTypePath(typeRef));
+ return isBean(typeRef, ctx);
}
}
@@ -715,30 +729,32 @@ public class TypeUtils {
LinkedHashSet<Class<?>> beans = new LinkedHashSet<>();
Class<?> enclosingType = ctx.getEnclosingType().getRawType();
Class<?> type = typeRef.getRawType();
- TypeResolutionContext newCtx = ctx;
- if (ctx.getCustomTypeRegistry().hasCodec(enclosingType, type)) {
+ TypeRef<?> replacementType =
+ ctx.getCustomTypeRegistry().replacementTypeFor(enclosingType, type);
+ if (replacementType != null && !replacementType.equals(typeRef)) {
+ beans.addAll(listBeansRecursiveInclusive(replacementType, ctx));
return beans;
} else if (type == Optional.class) {
TypeRef<?> elemType = getTypeArguments(typeRef).get(0);
- beans.addAll(listBeansRecursiveInclusive(elemType, newCtx));
+ beans.addAll(listBeansRecursiveInclusive(elemType, ctx));
} else if (isCollection(type) || Iterable.class == type) {
TypeRef<?> elementType = getElementType(typeRef);
- beans.addAll(listBeansRecursiveInclusive(elementType, newCtx));
+ beans.addAll(listBeansRecursiveInclusive(elementType, ctx));
} else if (isMap(type)) {
Tuple2<TypeRef<?>, TypeRef<?>> mapKeyValueType =
getMapKeyValueType(typeRef);
- TypeResolutionContext mapCtx = newCtx;
+ TypeResolutionContext mapCtx = ctx;
beans.addAll(listBeansRecursiveInclusive(mapKeyValueType.f0, mapCtx));
beans.addAll(listBeansRecursiveInclusive(mapKeyValueType.f1, mapCtx));
} else if (type.isArray()) {
Class<?> arrayComponent = getArrayComponent(type);
- beans.addAll(listBeansRecursiveInclusive(TypeRef.of(arrayComponent),
newCtx));
- } else if (isBean(type, newCtx)) {
+ beans.addAll(listBeansRecursiveInclusive(TypeRef.of(arrayComponent),
ctx));
+ } else if (isBean(type, ctx)) {
List<Descriptor> descriptors = Descriptor.getDescriptors(type);
beans.add(type);
for (Descriptor descriptor : descriptors) {
ctx.checkNoCycle(typeRef);
beans.addAll(
- listBeansRecursiveInclusive(descriptor.getTypeRef(),
newCtx.appendTypePath(typeRef)));
+ listBeansRecursiveInclusive(descriptor.getTypeRef(),
ctx.appendTypePath(typeRef)));
}
}
return beans;
diff --git
a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java
b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java
index 4559e579..61d0251d 100644
---
a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java
+++
b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java
@@ -423,4 +423,150 @@ public class ClassResolverTest extends ForyTestBase {
return null;
});
}
+
+ interface ITest {
+ int getF1();
+
+ void setF1(int f1);
+ }
+
+ @ToString
+ @EqualsAndHashCode
+ static class ImplTest implements ITest {
+ int f1;
+
+ @Override
+ public int getF1() {
+ return f1;
+ }
+
+ @Override
+ public void setF1(int f1) {
+ this.f1 = f1;
+ }
+ }
+
+ static class InterfaceCustomSerializer extends Serializer<ITest> {
+
+ public InterfaceCustomSerializer(Fory fory, Class<ITest> type) {
+ super(fory, type);
+ }
+
+ @Override
+ public void write(MemoryBuffer buffer, ITest value) {
+ buffer.writeInt32(value.getF1());
+ }
+
+ @Override
+ public ITest read(MemoryBuffer buffer) {
+ final ITest iTest = new ImplTest();
+ iTest.setF1(buffer.readInt32());
+ return iTest;
+ }
+ }
+
+ @Test
+ public void testInterfaceCustomSerializer() {
+ ThreadSafeFory threadSafeFory =
+ Fory.builder()
+ .withLanguage(Language.JAVA)
+ .requireClassRegistration(false)
+ .buildThreadSafeFory();
+ threadSafeFory.registerSerializer(
+ ITest.class, f -> new InterfaceCustomSerializer(f, ITest.class));
+ final ITest iTest = new ImplTest();
+ iTest.setF1(100);
+
+ threadSafeFory.execute(
+ fory -> {
+ Assert.assertEquals(iTest, serDe(fory, iTest));
+ return null;
+ });
+ threadSafeFory.execute(
+ fory -> {
+ Assert.assertEquals(
+
fory.getClassResolver().getSerializer(iTest.getClass()).getClass(),
+ InterfaceCustomSerializer.class);
+ return null;
+ });
+ }
+
+ @Data
+ abstract static class AbsTest {
+ int f1;
+ }
+
+ @EqualsAndHashCode(callSuper = true)
+ @ToString
+ static class SubAbsTest extends AbsTest {
+ long f2;
+ }
+
+ @EqualsAndHashCode(callSuper = true)
+ @ToString
+ static class Sub2AbsTest extends SubAbsTest {
+ Object f3;
+ }
+
+ static class AbstractCustomSerializer extends Serializer<AbsTest> {
+
+ public AbstractCustomSerializer(Fory fory, Class<AbsTest> type) {
+ super(fory, type);
+ }
+
+ @Override
+ public void write(MemoryBuffer buffer, AbsTest value) {
+ buffer.writeInt32(value.getF1());
+ }
+
+ @Override
+ public AbsTest read(MemoryBuffer buffer) {
+ // TODO maybe new SubAbsTest or Sub2AbsTest
+ final AbsTest absTest = new SubAbsTest();
+ absTest.setF1(buffer.readInt32());
+ return absTest;
+ }
+ }
+
+ @Test
+ public void testAbstractCustomSerializer() {
+ ThreadSafeFory threadSafeFory =
+ Fory.builder()
+ .withLanguage(Language.JAVA)
+ .requireClassRegistration(false)
+ .buildThreadSafeFory();
+ threadSafeFory.registerSerializer(
+ AbsTest.class, f -> new AbstractCustomSerializer(f, AbsTest.class));
+ final AbsTest absTest = new SubAbsTest();
+ absTest.setF1(100);
+
+ threadSafeFory.execute(
+ fory -> {
+ Assert.assertEquals(absTest, serDe(fory, absTest));
+ return null;
+ });
+ threadSafeFory.execute(
+ fory -> {
+ Assert.assertEquals(
+
fory.getClassResolver().getSerializer(absTest.getClass()).getClass(),
+ AbstractCustomSerializer.class);
+ return null;
+ });
+
+ final AbsTest abs2Test = new Sub2AbsTest();
+ abs2Test.setF1(100);
+
+ threadSafeFory.execute(
+ fory -> {
+ Assert.assertEquals(abs2Test.getF1(), serDe(fory, abs2Test).getF1());
+ return null;
+ });
+ threadSafeFory.execute(
+ fory -> {
+ Assert.assertEquals(
+
fory.getClassResolver().getSerializer(abs2Test.getClass()).getClass(),
+ AbstractCustomSerializer.class);
+ return null;
+ });
+ }
}
diff --git
a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java
b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java
index 0528c834..903f63f6 100644
---
a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java
+++
b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java
@@ -62,7 +62,7 @@ public class MetaSharedXlangTest extends ForyTestBase {
int[][] arr;
}
- // @Test
+ @Test
public void testMDArrayField() {
Fory fory =
Fory.builder()
@@ -70,7 +70,6 @@ public class MetaSharedXlangTest extends ForyTestBase {
.withCodegen(false)
.withCompatibleMode(CompatibleMode.COMPATIBLE)
.build();
- // TODO support multi-dimensional array serialization
fory.register(MDArrayFieldStruct.class, "example.a");
MDArrayFieldStruct s = new MDArrayFieldStruct();
s.arr = new int[][] {{1, 2}, {3, 4}};
diff --git
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayDataForEach.java
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayDataForEach.java
index ed98ef3e..7451aeb2 100644
---
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayDataForEach.java
+++
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayDataForEach.java
@@ -85,7 +85,7 @@ public class ArrayDataForEach extends AbstractExpression {
if (customEncoder == null) {
accessType = elemType;
} else {
- accessType = TypeRef.of(customEncoder.encodedType());
+ accessType = customEncoder.encodedType();
}
CustomTypeHandler customTypeHandler =
CustomTypeEncoderRegistry.customTypeHandler();
TypeResolutionContext ctx = new TypeResolutionContext(customTypeHandler,
true);
diff --git
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java
index 164c8d04..8937a66e 100644
---
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java
+++
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java
@@ -23,6 +23,7 @@ import static org.apache.fory.type.TypeUtils.CLASS_TYPE;
import static org.apache.fory.type.TypeUtils.getRawType;
import java.lang.reflect.Array;
+import java.util.HashSet;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.fory.Fory;
import org.apache.fory.codegen.CodeGenerator;
@@ -181,7 +182,8 @@ public class ArrayEncoderBuilder extends
BaseBinaryEncoderBuilder {
arrayData,
elemType,
(i, value) ->
- new Expression.Invoke(collection, "add", deserializeFor(value,
elemType, typeCtx)),
+ new Expression.Invoke(
+ collection, "add", deserializeFor(value, elemType,
typeCtx, new HashSet<>())),
i -> new Expression.Invoke(collection, "add",
ExpressionUtils.nullValue(elemType)));
return new Expression.ListExpression(collection, addElemsOp, collection);
}
diff --git
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java
index 79f369e9..332a2bcc 100644
---
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java
+++
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java
@@ -30,6 +30,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.fory.builder.CodecBuilder;
@@ -150,22 +151,18 @@ public abstract class BaseBinaryEncoderBuilder extends
CodecBuilder {
Expression inputObject,
Expression writer,
TypeRef<?> typeRef,
- Expression arrowField) {
+ Expression arrowField,
+ Set<TypeRef<?>> visitedCustomTypes) {
Class<?> rawType = getRawType(typeRef);
- CustomCodec<?, ?> customHandler =
customTypeHandler.findCodec(beanType.getRawType(), rawType);
- if (customHandler != null) {
- TypeRef<?> rewrittenType = TypeRef.of(customHandler.encodedType());
- Expression newInputObject =
- new Expression.StaticInvoke(
- customTypeHandler.getClass(),
- "encode",
- "rewrittenValue",
- rewrittenType,
- true,
- new Expression.Null(beanType, true),
- inputObject);
+ TypeRef<?> rewrittenType =
customTypeHandler.replacementTypeFor(beanType.getRawType(), rawType);
+ if (rewrittenType != null
+ && !visitedCustomTypes.contains(typeRef)
+ && !typeRef.equals(rewrittenType)) {
+ Expression newInputObject = customEncode(inputObject, rewrittenType);
+ visitedCustomTypes.add(typeRef);
Expression doSerialize =
- serializeFor(ordinal, newInputObject, writer, rewrittenType,
arrowField);
+ serializeFor(
+ ordinal, newInputObject, writer, rewrittenType, arrowField,
visitedCustomTypes);
return new If(
ExpressionUtils.eqNull(inputObject),
new Invoke(writer, "setNullAt", ordinal),
@@ -177,7 +174,12 @@ public abstract class BaseBinaryEncoderBuilder extends
CodecBuilder {
Expression unwrapped =
new If(ExpressionUtils.eqNull(inputObject), new
Expression.Null(elemType), orNull);
return serializeFor(
- ordinal, new Expression.Cast(unwrapped, elemType), writer, elemType,
arrowField);
+ ordinal,
+ new Expression.Cast(unwrapped, elemType),
+ writer,
+ elemType,
+ arrowField,
+ visitedCustomTypes);
} else if (TypeUtils.isPrimitive(rawType)) {
return new ListExpression(
// notNull is by default, no need to call setNotNullAt
@@ -310,7 +312,8 @@ public abstract class BaseBinaryEncoderBuilder extends
CodecBuilder {
value,
arrayWriter,
Objects.requireNonNull(typeRef.getComponentType()),
- arrayElementField));
+ arrayElementField,
+ new HashSet<>()));
return new ListExpression(reset, forEach, arrayWriter);
}
} else if (getRawType(typeRef) == Iterable.class) {
@@ -322,7 +325,12 @@ public abstract class BaseBinaryEncoderBuilder extends
CodecBuilder {
listFromIterable,
(i, value) ->
serializeFor(
- i, value, arrayWriter,
TypeUtils.getElementType(typeRef), arrayElementField));
+ i,
+ value,
+ arrayWriter,
+ TypeUtils.getElementType(typeRef),
+ arrayElementField,
+ new HashSet<>()));
return new ListExpression(reset, forEach, arrayWriter);
} else { // collection
Invoke size = new Invoke(inputObject, "size",
TypeUtils.PRIMITIVE_INT_TYPE);
@@ -332,7 +340,12 @@ public abstract class BaseBinaryEncoderBuilder extends
CodecBuilder {
inputObject,
(i, value) ->
serializeFor(
- i, value, arrayWriter,
TypeUtils.getElementType(typeRef), arrayElementField));
+ i,
+ value,
+ arrayWriter,
+ TypeUtils.getElementType(typeRef),
+ arrayElementField,
+ new HashSet<>()));
return new ListExpression(reset, forEach, arrayWriter);
}
}
@@ -517,36 +530,20 @@ public abstract class BaseBinaryEncoderBuilder extends
CodecBuilder {
* typeToken</code>.
*/
protected Expression deserializeFor(
- Expression value, TypeRef<?> typeRef, TypeResolutionContext ctx) {
+ Expression value,
+ TypeRef<?> typeRef,
+ TypeResolutionContext ctx,
+ Set<TypeRef<?>> visitedCustomTypes) {
Class<?> rawType = getRawType(typeRef);
- CustomCodec<?, ?> customHandler =
customTypeHandler.findCodec(beanType.getRawType(), rawType);
- if (customHandler != null) {
- TypeRef<?> rewrittenType = TypeRef.of(customHandler.encodedType());
- Class<?> rawRewrittenType = rewrittenType.getRawType();
- Expression inputValue;
- if (rawRewrittenType == byte[].class) {
- inputValue = Invoke.inlineInvoke(value, "toByteArray",
TypeRef.of(byte[].class));
- } else {
- inputValue = value;
- }
- Expression newValue =
- new Expression.StaticInvoke(
- customTypeHandler.getClass(),
- "decode",
- "decodedValue",
- typeRef,
- true,
- new Expression.Null(beanType, true),
- new Expression.Null(typeRef, true),
- inputValue);
- if (rawRewrittenType == MemoryBuffer.class) {
- return newValue;
- } else if (rawRewrittenType == BinaryArray.class) {
- return newValue;
- } else if (rawRewrittenType == byte[].class) {
- return newValue;
- }
- return deserializeFor(newValue, rewrittenType, ctx);
+ TypeRef<?> rewrittenType =
customTypeHandler.replacementTypeFor(beanType.getRawType(), rawType);
+ ;
+ if (rewrittenType != null
+ && !visitedCustomTypes.contains(typeRef)
+ && !typeRef.equals(rewrittenType)) {
+ visitedCustomTypes.add(typeRef);
+ final Expression deserializedValue =
+ deserializeFor(value, rewrittenType, ctx, visitedCustomTypes);
+ return customDecode(typeRef, deserializedValue);
} else if (rawType == Optional.class) {
TypeRef<?> elemType = TypeUtils.getTypeArguments(typeRef).get(0);
return new Expression.StaticInvoke(
@@ -555,7 +552,7 @@ public abstract class BaseBinaryEncoderBuilder extends
CodecBuilder {
"optional",
typeRef,
true,
- deserializeFor(value, elemType, ctx));
+ deserializeFor(value, elemType, ctx, visitedCustomTypes));
} else if (TypeUtils.isPrimitive(rawType) || TypeUtils.isBoxed(rawType)) {
return value;
} else if (rawType == BigDecimal.class) {
@@ -575,6 +572,10 @@ public abstract class BaseBinaryEncoderBuilder extends
CodecBuilder {
DateTimeUtils.class, "microsToInstant", TypeUtils.INSTANT_TYPE,
false, value);
} else if (rawType == String.class) {
return value;
+ } else if (rawType == MemoryBuffer.class) {
+ return value;
+ } else if (rawType == BinaryArray.class) {
+ return value;
} else if (rawType.isEnum()) {
return ExpressionUtils.valueOf(typeRef, value);
} else if (rawType.isArray()) {
@@ -644,7 +645,9 @@ public abstract class BaseBinaryEncoderBuilder extends
CodecBuilder {
new ArrayDataForEach(
arrayData,
elemType,
- (i, value) -> new Invoke(collection, "add",
deserializeFor(value, elemType, typeCtx)),
+ (i, value) ->
+ new Invoke(
+ collection, "add", deserializeFor(value, elemType,
typeCtx, new HashSet<>())),
i -> new Invoke(collection, "add",
ExpressionUtils.nullValue(elemType)));
return new ListExpression(collection, addElemsOp, collection);
} catch (Exception e) {
@@ -800,7 +803,7 @@ public abstract class BaseBinaryEncoderBuilder extends
CodecBuilder {
arrayData,
elemType,
(i, value) -> {
- Expression elemValue = deserializeFor(value, elemType,
typeCtx);
+ Expression elemValue = deserializeFor(value, elemType,
typeCtx, new HashSet<>());
return new AssignArrayElem(javaArray, elemValue, i);
});
// add javaArray at last as expression value
@@ -816,4 +819,27 @@ public abstract class BaseBinaryEncoderBuilder extends
CodecBuilder {
protected Expression deserializeForObject(Expression value, TypeRef<?>
typeRef) {
return new Invoke(foryRef, "deserialize", typeRef, value);
}
+
+ protected Expression customEncode(Expression inputObject, TypeRef<?>
rewrittenType) {
+ return new Expression.StaticInvoke(
+ customTypeHandler.getClass(),
+ "encode",
+ "rewrittenValue",
+ rewrittenType,
+ true,
+ new Expression.Null(beanType, true),
+ inputObject);
+ }
+
+ protected Expression customDecode(TypeRef<?> typeRef, final Expression
deserializedValue) {
+ return new Expression.StaticInvoke(
+ customTypeHandler.getClass(),
+ "decode",
+ "decodedValue",
+ typeRef,
+ true,
+ new Expression.Null(beanType, true),
+ new Expression.Null(typeRef, true),
+ deserializedValue);
+ }
}
diff --git
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CustomCodec.java
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CustomCodec.java
index 75dcb111..09811234 100644
---
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CustomCodec.java
+++
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CustomCodec.java
@@ -24,20 +24,29 @@ import org.apache.arrow.vector.types.pojo.Field;
import org.apache.fory.format.row.binary.BinaryArray;
import org.apache.fory.format.type.DataTypes;
import org.apache.fory.memory.MemoryBuffer;
+import org.apache.fory.reflect.TypeRef;
+/**
+ * Extension point to customize Fory row codec behavior. Supports intercepting
types to be written
+ * ({@code encode}) and read ({@code decode}).
+ *
+ * @param <T> the type the codec decodes to (used in Java)
+ * @param <E> the type the codec encodes to (byte representation)
+ */
public interface CustomCodec<T, E> {
Field getField(String fieldName);
- Class<E> encodedType();
+ TypeRef<E> encodedType();
E encode(T value);
T decode(E value);
+ /** Specialized codec base for encoding and decoding to/from {@link
MemoryBuffer}. */
interface MemoryBufferCodec<T> extends CustomCodec<T, MemoryBuffer> {
@Override
- default Class<MemoryBuffer> encodedType() {
- return MemoryBuffer.class;
+ default TypeRef<MemoryBuffer> encodedType() {
+ return TypeRef.of(MemoryBuffer.class);
}
@Override
@@ -46,10 +55,11 @@ public interface CustomCodec<T, E> {
}
}
+ /** Specialized codec base for encoding and decoding to/from {@code byte[]}.
*/
interface ByteArrayCodec<T> extends CustomCodec<T, byte[]> {
@Override
- default Class<byte[]> encodedType() {
- return byte[].class;
+ default TypeRef<byte[]> encodedType() {
+ return TypeRef.of(byte[].class);
}
@Override
@@ -58,10 +68,11 @@ public interface CustomCodec<T, E> {
}
}
+ /** Specialized codec base for encoding and decoding to/from {@link
BinaryArray}. */
interface BinaryArrayCodec<T> extends CustomCodec<T, BinaryArray> {
@Override
- default Class<BinaryArray> encodedType() {
- return BinaryArray.class;
+ default TypeRef<BinaryArray> encodedType() {
+ return TypeRef.of(BinaryArray.class);
}
@Override
@@ -69,4 +80,26 @@ public interface CustomCodec<T, E> {
return DataTypes.primitiveArrayField(fieldName, DataTypes.int8());
}
}
+
+ /**
+ * Specialized codec base for read and write replace of a value, without
changing its type.
+ * Example use: converting Fory generated implementation into a standard
user-provided
+ * implementation.
+ */
+ interface InterceptingCodec<T> extends CustomCodec<T, T> {
+ @Override
+ default Field getField(final String fieldName) {
+ return null;
+ }
+
+ @Override
+ default T decode(final T value) {
+ return value;
+ }
+
+ @Override
+ default T encode(final T value) {
+ return value;
+ }
+ }
}
diff --git
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java
index a82f368c..a1026052 100644
---
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java
+++
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java
@@ -24,6 +24,7 @@ import static org.apache.fory.type.TypeUtils.getRawType;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.SortedMap;
@@ -175,12 +176,17 @@ public class RowEncoderBuilder extends
BaseBinaryEncoderBuilder {
@Override
public Expression buildEncodeExpression() {
Reference inputObject = new Reference(ROOT_OBJECT_NAME,
TypeUtils.OBJECT_TYPE, false);
+ Expression bean = new Expression.Cast(inputObject, beanType,
ctx.newName(beanClass));
Reference writer = new Reference(ROOT_ROW_WRITER_NAME, rowWriterTypeToken,
false);
Reference schemaExpr = new Reference(SCHEMA_NAME, schemaTypeToken, false);
+ CustomCodec<?, ?> customCodec = customTypeHandler.findCodec(beanClass,
beanClass);
+ if (customCodec != null && customCodec.encodedType().equals(beanType)) {
+ bean = customEncode(bean, beanType);
+ }
+
int numFields = schema.getFields().size();
Expression.ListExpression expressions = new Expression.ListExpression();
- Expression.Cast bean = new Expression.Cast(inputObject, beanType,
ctx.newName(beanClass));
// schema field's name must correspond to descriptor's name.
for (int i = 0; i < numFields; i++) {
Descriptor d =
getDescriptorByFieldName(schema.getFields().get(i).getName());
@@ -191,7 +197,8 @@ public class RowEncoderBuilder extends
BaseBinaryEncoderBuilder {
Expression.StaticInvoke field =
new Expression.StaticInvoke(
DataTypes.class, "fieldOfSchema", ARROW_FIELD_TYPE, false,
schemaExpr, ordinal);
- Expression fieldExpr = serializeFor(ordinal, fieldValue, writer,
fieldType, field);
+ Expression fieldExpr =
+ serializeFor(ordinal, fieldValue, writer, fieldType, field, new
HashSet<>());
expressions.add(fieldExpr);
}
expressions.add(
@@ -210,51 +217,54 @@ public class RowEncoderBuilder extends
BaseBinaryEncoderBuilder {
addDecoderMethods();
- if (generatedBeanImpl != null) {
- return new Expression.Return(
- new Expression.Reference("new " + generatedBeanImplName + "(row)"));
- }
-
- int numFields = schema.getFields().size();
- List<String> fieldNames = new ArrayList<>(numFields);
- Expression[] values = new Expression[numFields];
- Descriptor[] descriptors = new Descriptor[numFields];
Expression.ListExpression expressions = new Expression.ListExpression();
- // schema field's name must correspond to descriptor's name.
- for (int i = 0; i < numFields; i++) {
- Literal ordinal = Literal.ofInt(i);
- Descriptor d =
getDescriptorByFieldName(schema.getFields().get(i).getName());
- fieldNames.add(d.getName());
- descriptors[i] = d;
- TypeRef<?> fieldType = d.getTypeRef();
- Expression.Variable value = new Expression.Variable(d.getName(),
nullValue(fieldType));
- values[i] = value;
- expressions.add(value);
- Expression.Invoke isNullAt =
- new Expression.Invoke(row, "isNullAt",
TypeUtils.PRIMITIVE_BOOLEAN_TYPE, ordinal);
- Expression decode =
- new Expression.If(
- ExpressionUtils.not(isNullAt),
- new Expression.Assign(
- value, new Expression.Reference(decodeMethodName(i) +
"(row)", fieldType)));
- expressions.add(decode);
- }
Expression bean;
- if (RecordUtils.isRecord(beanClass)) {
- int[] map = RecordUtils.buildRecordComponentMapping(beanClass,
fieldNames);
- Expression[] args = new Expression[numFields];
+ if (generatedBeanImpl != null) {
+ bean = new Expression.Reference("new " + generatedBeanImplName +
"(row)");
+ } else {
+ int numFields = schema.getFields().size();
+ List<String> fieldNames = new ArrayList<>(numFields);
+ Expression[] values = new Expression[numFields];
+ Descriptor[] descriptors = new Descriptor[numFields];
+ // schema field's name must correspond to descriptor's name.
for (int i = 0; i < numFields; i++) {
- args[i] = values[map[i]];
+ Literal ordinal = Literal.ofInt(i);
+ Descriptor d =
getDescriptorByFieldName(schema.getFields().get(i).getName());
+ fieldNames.add(d.getName());
+ descriptors[i] = d;
+ TypeRef<?> fieldType = d.getTypeRef();
+ Expression.Variable value = new Expression.Variable(d.getName(),
nullValue(fieldType));
+ values[i] = value;
+ expressions.add(value);
+ Expression.Invoke isNullAt =
+ new Expression.Invoke(row, "isNullAt",
TypeUtils.PRIMITIVE_BOOLEAN_TYPE, ordinal);
+ Expression decode =
+ new Expression.If(
+ ExpressionUtils.not(isNullAt),
+ new Expression.Assign(
+ value, new Expression.Reference(decodeMethodName(i) +
"(row)", fieldType)));
+ expressions.add(decode);
}
- bean = new Expression.NewInstance(beanType,
beanType.getRawType().getName(), args);
- } else {
- bean = newBean();
- expressions.add(bean);
- for (int i = 0; i < values.length; i++) {
- expressions.add(setFieldValue(bean, descriptors[i], values[i]));
+ if (RecordUtils.isRecord(beanClass)) {
+ int[] map = RecordUtils.buildRecordComponentMapping(beanClass,
fieldNames);
+ Expression[] args = new Expression[numFields];
+ for (int i = 0; i < numFields; i++) {
+ args[i] = values[map[i]];
+ }
+ bean = new Expression.NewInstance(beanType,
beanType.getRawType().getName(), args);
+ } else {
+ bean = newBean();
+ expressions.add(bean);
+ for (int i = 0; i < values.length; i++) {
+ expressions.add(setFieldValue(bean, descriptors[i], values[i]));
+ }
}
}
+ CustomCodec<?, ?> customCodec = customTypeHandler.findCodec(beanClass,
beanClass);
+ if (customCodec != null && customCodec.encodedType().equals(beanType)) {
+ bean = customDecode(beanType, bean);
+ }
expressions.add(new Expression.Return(bean));
return expressions;
}
@@ -284,7 +294,7 @@ public class RowEncoderBuilder extends
BaseBinaryEncoderBuilder {
if (customEncoder == null) {
columnAccessType = fieldType;
} else {
- columnAccessType = TypeRef.of(customEncoder.encodedType());
+ columnAccessType = customEncoder.encodedType();
}
}
String columnAccessMethodName =
@@ -298,7 +308,8 @@ public class RowEncoderBuilder extends
BaseBinaryEncoderBuilder {
colType,
false,
ordinal);
- Expression value = new Expression.Return(deserializeFor(columnValue,
fieldType, typeCtx));
+ Expression value =
+ new Expression.Return(deserializeFor(columnValue, fieldType,
typeCtx, new HashSet<>()));
ctx.addMethod(
decodeMethodName(i),
value.doGenCode(ctx).code(),
diff --git
a/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeEncoderRegistry.java
b/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeEncoderRegistry.java
index 09945593..b9520879 100644
---
a/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeEncoderRegistry.java
+++
b/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeEncoderRegistry.java
@@ -139,7 +139,7 @@ public class CustomTypeEncoderRegistry {
+ ")"
+ codecFieldName
+ ".encode(fieldValue);",
- enc.encodedType(),
+ enc.encodedType().getRawType(),
reg.getBeanType(),
"bean",
reg.getFieldType(),
@@ -156,7 +156,7 @@ public class CustomTypeEncoderRegistry {
"bean",
reg.getFieldType(),
"fieldNull",
- enc.encodedType(),
+ enc.encodedType().getRawType(),
"encodedValue");
ctx.addField(
true,
diff --git
a/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeHandler.java
b/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeHandler.java
index c3cad271..28aa0e14 100644
---
a/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeHandler.java
+++
b/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeHandler.java
@@ -22,6 +22,9 @@ package org.apache.fory.format.type;
import org.apache.fory.annotation.Internal;
import org.apache.fory.format.encoder.CustomCodec;
import org.apache.fory.format.encoder.CustomCollectionFactory;
+import org.apache.fory.format.row.binary.BinaryArray;
+import org.apache.fory.memory.MemoryBuffer;
+import org.apache.fory.reflect.TypeRef;
import org.apache.fory.type.CustomTypeRegistry;
@Internal
@@ -46,8 +49,9 @@ public interface CustomTypeHandler extends CustomTypeRegistry
{
Class<?> collectionType, Class<?> elementType);
@Override
- default boolean hasCodec(final Class<?> beanType, final Class<?> fieldType) {
- return findCodec(beanType, fieldType) != null;
+ default TypeRef<?> replacementTypeFor(final Class<?> beanType, final
Class<?> fieldType) {
+ final CustomCodec<?, ?> codec = findCodec(beanType, fieldType);
+ return codec == null ? null : codec.encodedType();
}
@Override
@@ -55,4 +59,10 @@ public interface CustomTypeHandler extends
CustomTypeRegistry {
final Class<?> collectionType, final Class<?> elementType) {
return findCollectionFactory(collectionType, elementType) != null;
}
+
+ @Override
+ default boolean isExtraSupportedType(final TypeRef<?> type) {
+ final Class<?> cls = type.getRawType();
+ return cls == BinaryArray.class || cls == MemoryBuffer.class;
+ }
}
diff --git
a/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java
b/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java
index a3196017..e7a67ce4 100644
---
a/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java
+++
b/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java
@@ -22,12 +22,12 @@ package org.apache.fory.format.type;
import static org.apache.fory.format.type.DataTypes.field;
import static org.apache.fory.type.TypeUtils.getRawType;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.stream.Collectors;
import org.apache.arrow.vector.complex.MapVector;
import org.apache.arrow.vector.types.DateUnit;
import org.apache.arrow.vector.types.FloatingPointPrecision;
@@ -39,6 +39,7 @@ import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.fory.collection.Tuple2;
import org.apache.fory.format.encoder.CustomCodec;
import org.apache.fory.format.encoder.CustomCollectionFactory;
+import org.apache.fory.format.row.binary.BinaryArray;
import org.apache.fory.reflect.TypeRef;
import org.apache.fory.type.Descriptor;
import org.apache.fory.type.TypeResolutionContext;
@@ -155,8 +156,12 @@ public class TypeInference {
true, fieldType.getType(), fieldType.getDictionary(),
fieldType.getMetadata()),
result.getChildren());
} else if (customEncoder != null) {
- return customEncoder.getField(name);
- } else if (rawType == boolean.class) {
+ Field replacementField = customEncoder.getField(name);
+ if (replacementField != null) {
+ return replacementField;
+ }
+ }
+ if (rawType == boolean.class) {
return field(name, DataTypes.notNullFieldType(ArrowType.Bool.INSTANCE));
} else if (rawType == byte.class) {
return field(name, DataTypes.notNullFieldType(new ArrowType.Int(8,
true)));
@@ -209,6 +214,8 @@ public class TypeInference {
return field(name, FieldType.nullable(ArrowType.Utf8.INSTANCE));
} else if (rawType.isEnum()) {
return field(name, FieldType.nullable(ArrowType.Utf8.INSTANCE));
+ } else if (rawType == BinaryArray.class) {
+ return field(name, FieldType.nullable(ArrowType.Binary.INSTANCE));
} else if (rawType.isArray()) { // array
Field f =
inferField(
@@ -230,15 +237,13 @@ public class TypeInference {
return DataTypes.mapField(name, keyField, valueField);
} else if (TypeUtils.isBean(rawType, ctx)) { // bean field
ctx.checkNoCycle(rawType);
- List<Field> fields =
- Descriptor.getDescriptors(rawType).stream()
- .map(
- descriptor -> {
- String n =
StringUtils.lowerCamelToLowerUnderscore(descriptor.getName());
- TypeRef<?> fieldType = descriptor.getTypeRef();
- return inferField(n, fieldType,
ctx.appendTypePath(rawType));
- })
- .collect(Collectors.toList());
+ List<Descriptor> descriptors = Descriptor.getDescriptors(rawType);
+ List<Field> fields = new ArrayList<>(descriptors.size());
+ for (Descriptor descriptor : descriptors) {
+ String n =
StringUtils.lowerCamelToLowerUnderscore(descriptor.getName());
+ TypeRef<?> fieldType = descriptor.getTypeRef();
+ fields.add(inferField(n, fieldType, ctx.appendTypePath(rawType)));
+ }
return DataTypes.structField(name, true, fields);
} else {
throw new UnsupportedOperationException(
diff --git
a/java/fory-format/src/test/java/org/apache/fory/format/encoder/CustomCodecTest.java
b/java/fory-format/src/test/java/org/apache/fory/format/encoder/CustomCodecTest.java
index bfda9e42..33b0faa4 100644
---
a/java/fory-format/src/test/java/org/apache/fory/format/encoder/CustomCodecTest.java
+++
b/java/fory-format/src/test/java/org/apache/fory/format/encoder/CustomCodecTest.java
@@ -32,9 +32,9 @@ import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.fory.format.row.binary.BinaryArray;
import org.apache.fory.format.row.binary.BinaryRow;
-import org.apache.fory.format.type.DataTypes;
import org.apache.fory.memory.MemoryBuffer;
import org.apache.fory.memory.MemoryUtils;
+import org.apache.fory.reflect.TypeRef;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -46,6 +46,7 @@ public class CustomCodecTest {
Encoders.registerCustomCodec(CustomByteBuf2.class, new
CustomByteBuf2Encoder());
Encoders.registerCustomCodec(CustomByteBuf3.class, new
CustomByteBuf3Encoder());
Encoders.registerCustomCodec(UUID.class, new UuidEncoder());
+ Encoders.registerCustomCodec(InterceptedType.class, new
InterceptedTypeEncoder());
Encoders.registerCustomCollectionFactory(
SortedSet.class, UUID.class, new SortedSetOfUuidDecoder());
}
@@ -154,8 +155,8 @@ public class CustomCodecTest {
}
@Override
- public Class<String> encodedType() {
- return String.class;
+ public TypeRef<String> encodedType() {
+ return TypeRef.of(String.class);
}
}
@@ -184,16 +185,6 @@ public class CustomCodecTest {
}
static class CustomByteBuf3Encoder implements
CustomCodec.BinaryArrayCodec<CustomByteBuf3> {
- @Override
- public Field getField(final String fieldName) {
- return DataTypes.primitiveArrayField(fieldName, DataTypes.int8());
- }
-
- @Override
- public Class<BinaryArray> encodedType() {
- return BinaryArray.class;
- }
-
@Override
public BinaryArray encode(final CustomByteBuf3 value) {
return BinaryArray.fromPrimitiveArray(value.buf);
@@ -240,4 +231,79 @@ public class CustomCodecTest {
return Long.compareUnsigned(o1.getLeastSignificantBits(),
o2.getLeastSignificantBits());
}
}
+
+ public interface InterceptedType {
+ int f1();
+ }
+
+ public static class InterceptedTypeImpl implements InterceptedType {
+ private final int f1;
+
+ public InterceptedTypeImpl(final int f1) {
+ this.f1 = f1;
+ }
+
+ @Override
+ public int f1() {
+ return f1;
+ }
+ }
+
+ static class InterceptedTypeEncoder implements
CustomCodec.InterceptingCodec<InterceptedType> {
+ @Override
+ public TypeRef<InterceptedType> encodedType() {
+ return TypeRef.of(InterceptedType.class);
+ }
+
+ @Override
+ public InterceptedType encode(final InterceptedType value) {
+ return new InterceptedTypeImpl(value.f1() + 2);
+ }
+
+ @Override
+ public InterceptedType decode(final InterceptedType value) {
+ return new InterceptedTypeImpl(value.f1() + 3);
+ }
+ }
+
+ @Test
+ public void testCodecTypeInterception() {
+ final InterceptedType bean = new InterceptedTypeImpl(42);
+ final RowEncoder<InterceptedType> encoder =
Encoders.bean(InterceptedType.class);
+ final BinaryRow row = encoder.toRow(bean);
+ final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes());
+ row.pointTo(buffer, 0, buffer.size());
+ final InterceptedType deserializedBean = encoder.fromRow(row);
+ Assert.assertEquals(deserializedBean.f1(), bean.f1() + 5);
+ Assert.assertEquals(deserializedBean.getClass(),
InterceptedTypeImpl.class);
+ }
+
+ public interface WrapInterceptedType {
+ InterceptedType f1();
+ }
+
+ public static class WrapInterceptedTypeImpl implements WrapInterceptedType {
+ private final InterceptedType f1;
+
+ public WrapInterceptedTypeImpl(final InterceptedType f1) {
+ this.f1 = f1;
+ }
+
+ @Override
+ public InterceptedType f1() {
+ return f1;
+ }
+ }
+
+ @Test
+ public void testNestedCodecTypeInterception() {
+ final WrapInterceptedType bean = new WrapInterceptedTypeImpl(new
InterceptedTypeImpl(42));
+ final RowEncoder<WrapInterceptedType> encoder =
Encoders.bean(WrapInterceptedType.class);
+ final BinaryRow row = encoder.toRow(bean);
+ final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes());
+ row.pointTo(buffer, 0, buffer.size());
+ final WrapInterceptedType deserializedBean = encoder.fromRow(row);
+ Assert.assertEquals(deserializedBean.f1().f1(), bean.f1().f1() + 5);
+ Assert.assertEquals(deserializedBean.f1().getClass(),
InterceptedTypeImpl.class);
+ }
}
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 4891b7b8..c7aa3f07 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -58,7 +58,7 @@ format = ["pyarrow"]
all = ["pyarrow"]
[tool.setuptools]
-packages = ["pyfory", "pyfory.format", "pyfory.lib.mmh3"]
+packages = ["pyfory", "pyfory.format", "pyfory.lib", "pyfory.meta"]
include-package-data = true
zip-safe = false
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
