This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch releases-0.10
in repository https://gitbox.apache.org/repos/asf/fury.git
The following commit(s) were added to refs/heads/releases-0.10 by this push:
new 673ada01 chore: cherry-pick commits to 0.10 for 0.10.1 release (#2117)
673ada01 is described below
commit 673ada01495c2a843a3015a03f8fd3acd903924d
Author: Shawn Yang <[email protected]>
AuthorDate: Mon Mar 24 02:55:46 2025 +0800
chore: cherry-pick commits to 0.10 for 0.10.1 release (#2117)
## What does this PR do?
<!-- Describe the purpose of this PR. -->
## Related issues
<!--
Is there any related issue? Please attach here.
- #xxxx0
- #xxxx1
- #xxxx2
-->
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fury/issues/new/choose) describing the
need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---------
Signed-off-by: Seppe Volkaerts <[email protected]>
Co-authored-by: Leo <[email protected]>
Co-authored-by: monk <[email protected]>
Co-authored-by: w-yfan <[email protected]>
Co-authored-by: 吴怡帆 <[email protected]>
Co-authored-by: hn <[email protected]>
Co-authored-by: hening <[email protected]>
Co-authored-by: PJ Fanning <[email protected]>
Co-authored-by: Seppe Volkaerts <[email protected]>
---
.../integration_tests/JDKCompatibilityTest.java | 61 +++++++++++
.../apache/fury/benchmark/ThreadPoolFurySuite.java | 92 ++++++++++++++++
.../src/main/java/org/apache/fury/Fury.java | 18 ++++
.../fury/builder/BaseObjectCodecBuilder.java | 35 +++---
.../apache/fury/builder/ObjectCodecBuilder.java | 17 +--
.../java/org/apache/fury/io/FuryInputStream.java | 7 +-
.../java/org/apache/fury/memory/MemoryBuffer.java | 63 ++++++-----
.../main/java/org/apache/fury/meta/ClassDef.java | 118 +++++++++++++--------
.../java/org/apache/fury/meta/ClassDefDecoder.java | 3 +-
.../java/org/apache/fury/meta/ClassDefEncoder.java | 1 +
.../org/apache/fury/meta/MetaStringEncoder.java | 51 ++++++---
.../TypeExtMeta.java} | 25 ++---
.../apache/fury/pool/ClassLoaderFuryPooled.java | 88 ++++++++-------
.../main/java/org/apache/fury/reflect/TypeRef.java | 22 ++++
.../org/apache/fury/resolver/ClassResolver.java | 13 ++-
.../org/apache/fury/resolver/DisallowedList.java | 48 ++++++++-
.../fury/serializer/AbstractObjectSerializer.java | 31 +++---
.../fury/serializer/ImmutableSerializer.java | 2 +-
...mmutableSerializer.java => NoneSerializer.java} | 30 +++---
.../apache/fury/serializer/ObjectSerializer.java | 15 ++-
.../apache/fury/serializer/TimeSerializers.java | 82 +++++++++++---
.../collection/AbstractCollectionSerializer.java | 3 +
.../collection/AbstractMapSerializer.java | 2 +-
.../serializer/collection/SubListSerializers.java | 16 +--
.../java/org/apache/fury/type/GenericType.java | 17 +--
.../main/java/org/apache/fury/type/TypeUtils.java | 56 +++++++---
.../java/org/apache/fury/util/LoaderBinding.java | 3 +-
.../fury-core/native-image.properties | 2 +
.../src/main/resources/fury/disallowed.txt | 2 +
.../test/java/org/apache/fury/FuryCopyTest.java | 16 ++-
.../src/test/java/org/apache/fury/StreamTest.java | 35 ++++++
.../java/org/apache/fury/meta/ClassDefTest.java | 12 +++
.../apache/fury/resolver/ClassResolverTest.java | 4 +-
.../collection/CollectionSerializersTest.java | 29 +++++
.../java/org/apache/fury/type/TypeUtilsTest.java | 21 ++++
rust/fury-core/src/buffer.rs | 14 +--
rust/fury-core/src/meta/meta_string.rs | 14 +--
rust/fury-core/src/util.rs | 12 +--
38 files changed, 790 insertions(+), 290 deletions(-)
diff --git
a/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fury/integration_tests/JDKCompatibilityTest.java
b/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fury/integration_tests/JDKCompatibilityTest.java
index ceef88d0..b31afd89 100644
---
a/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fury/integration_tests/JDKCompatibilityTest.java
+++
b/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fury/integration_tests/JDKCompatibilityTest.java
@@ -61,26 +61,54 @@ public class JDKCompatibilityTest {
Assert.assertEquals(fury.deserialize(serialized), object);
write("object_schema_compatible" + Platform.JAVA_VERSION, serialized);
}
+ // 11Test the case for the user registration class
+ {
+ Fury fury = builder().build();
+ fury.register(CustomObject.class);
+ CustomObject customObject = createCustomObject();
+ Assert.assertEquals(createCustomObject(), customObject);
+ byte[] serialized = fury.serialize(customObject);
+ Assert.assertEquals(fury.deserialize(serialized), customObject);
+ write("custom_object_schema_consistent" + Platform.JAVA_VERSION,
serialized);
+ }
+ {
+ Fury fury =
builder().withCompatibleMode(CompatibleMode.COMPATIBLE).build();
+ fury.register(CustomObject.class);
+ CustomObject customObject = createCustomObject();
+ byte[] serialized = fury.serialize(customObject);
+ Assert.assertEquals(fury.deserialize(serialized), customObject);
+ write("custom_object_schema_compatible" + Platform.JAVA_VERSION,
serialized);
+ }
}
@Test
public void testSchemaConsist() throws IOException {
Object object = createObject();
Fury fury = builder().build();
+ fury.register(CustomObject.class);
File dir = new File(".");
File[] files = dir.listFiles((d, name) ->
name.startsWith("object_schema_consistent"));
assert files != null;
check(object, fury, files);
+ CustomObject customObject = createCustomObject();
+ File[] files1 = dir.listFiles((d, name) ->
name.startsWith("custom_object_schema_consistent"));
+ assert files1 != null;
+ check(customObject, fury, files1);
}
@Test
public void testSchemaCompatible() throws IOException {
Object object = createObject();
Fury fury =
builder().withCompatibleMode(CompatibleMode.COMPATIBLE).build();
+ fury.register(CustomObject.class);
File dir = new File(".");
File[] files = dir.listFiles((d, name) ->
name.startsWith("object_schema_compatible"));
assert files != null;
check(object, fury, files);
+ CustomObject customObject = createCustomObject();
+ File[] files1 = dir.listFiles((d, name) ->
name.startsWith("custom_object_schema_compatible"));
+ assert files1 != null;
+ check(customObject, fury, files1);
}
private static void check(Object object, Fury fury, File[] files) throws
IOException {
@@ -106,4 +134,37 @@ public class JDKCompatibilityTest {
throw new RuntimeException(e);
}
}
+
+ static class CustomObject {
+ private String str;
+
+ public String getStr() {
+ return str;
+ }
+
+ public void setStr(String str) {
+ this.str = str;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ CustomObject entity = (CustomObject) o;
+ return str.equals(entity.str);
+ }
+
+ @Override
+ public int hashCode() {
+ return str.hashCode();
+ }
+ }
+
+ CustomObject createCustomObject() {
+ CustomObject customObject = new CustomObject();
+ customObject.setStr("hello");
+ return customObject;
+ }
}
diff --git
a/java/benchmark/src/main/java/org/apache/fury/benchmark/ThreadPoolFurySuite.java
b/java/benchmark/src/main/java/org/apache/fury/benchmark/ThreadPoolFurySuite.java
new file mode 100644
index 00000000..3911e0d1
--- /dev/null
+++
b/java/benchmark/src/main/java/org/apache/fury/benchmark/ThreadPoolFurySuite.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fury.benchmark;
+
+import java.io.IOException;
+import org.apache.fury.Fury;
+import org.apache.fury.ThreadSafeFury;
+import org.apache.fury.config.CompatibleMode;
+import org.apache.fury.config.Language;
+import org.apache.fury.logging.Logger;
+import org.apache.fury.logging.LoggerFactory;
+import org.openjdk.jmh.Main;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.infra.Blackhole;
+
+@BenchmarkMode(Mode.SingleShotTime)
+@CompilerControl(value = CompilerControl.Mode.INLINE)
+@State(Scope.Benchmark)
+@OutputTimeUnit(java.util.concurrent.TimeUnit.MILLISECONDS)
+public class ThreadPoolFurySuite {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(ThreadPoolFurySuite.class);
+
+ private ThreadSafeFury fury =
+ Fury.builder()
+ .withLanguage(Language.JAVA)
+ .requireClassRegistration(false)
+ .withJdkClassSerializableCheck(false)
+ .withRefTracking(false)
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .withAsyncCompilation(true)
+ .withRefTracking(true)
+ .buildThreadSafeFuryPool(10, 60);
+
+ private static StructBenchmark.NumericStruct struct;
+
+ static {
+ struct = StructBenchmark.NumericStruct.build();
+ struct.f1 = 1;
+ struct.f2 = 2;
+ struct.f3 = 3;
+ struct.f4 = 4;
+ struct.f5 = 5;
+ struct.f6 = 6;
+ struct.f7 = 7;
+ struct.f8 = 8;
+ }
+
+ @Benchmark()
+ @Threads(10000)
+ public void testObjectPool(Blackhole bh) {
+ bh.consume(fury.serialize(struct));
+ }
+
+ @TearDown
+ public void tearDown() {}
+
+ public static void main(String[] args) throws IOException {
+ if (args.length == 0) {
+ String commandLine =
+ "org.apache.fury.*ObjectPoolBenchmark.* -f 1 -wi 0 -i 5 -w 2s -r 2s
-rf csv";
+ System.out.println(commandLine);
+ args = commandLine.split(" ");
+ }
+ Main.main(args);
+ }
+}
diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java
b/java/fury-core/src/main/java/org/apache/fury/Fury.java
index cfc01656..8de0796b 100644
--- a/java/fury-core/src/main/java/org/apache/fury/Fury.java
+++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java
@@ -441,6 +441,15 @@ public final class Fury implements BaseFury {
}
}
+ public void writeNullable(MemoryBuffer buffer, Object obj, Serializer
serializer) {
+ if (obj == null) {
+ buffer.writeByte(Fury.NULL_FLAG);
+ } else {
+ buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG);
+ serializer.write(buffer, obj);
+ }
+ }
+
/** Write object class and data without tracking ref. */
public void writeNullable(MemoryBuffer buffer, Object obj, ClassInfoHolder
classInfoHolder) {
if (obj == null) {
@@ -924,6 +933,15 @@ public final class Fury implements BaseFury {
}
}
+ public Object readNullable(MemoryBuffer buffer, Serializer serializer) {
+ byte headFlag = buffer.readByte();
+ if (headFlag == Fury.NULL_FLAG) {
+ return null;
+ } else {
+ return serializer.read(buffer);
+ }
+ }
+
public Object readNullable(MemoryBuffer buffer, ClassInfoHolder
classInfoHolder) {
byte headFlag = buffer.readByte();
if (headFlag == Fury.NULL_FLAG) {
diff --git
a/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
b/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
index ae044628..744ac711 100644
---
a/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
+++
b/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
@@ -224,8 +224,8 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
return fury.getJITContext().asyncVisitFury(function);
}
- private boolean needWriteRef(Class<?> cls) {
- return visitFury(fury -> fury.getClassResolver().needToWriteRef(cls));
+ private boolean needWriteRef(TypeRef<?> type) {
+ return visitFury(fury -> fury.getClassResolver().needToWriteRef(type));
}
@Override
@@ -354,7 +354,7 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
boolean generateNewMethod) {
// access rawType without jit lock to reduce lock competition.
Class<?> rawType = getRawType(typeRef);
- if (needWriteRef(rawType)) {
+ if (needWriteRef(typeRef)) {
return new If(
not(writeRefOrNull(buffer, inputObject)),
serializeForNotNull(inputObject, buffer, typeRef, serializer,
generateNewMethod));
@@ -366,7 +366,7 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
Expression action =
new ListExpression(
new Invoke(
- buffer, "writeByte", new Literal(Fury.REF_VALUE_FLAG,
PRIMITIVE_BYTE_TYPE)),
+ buffer, "writeByte", new Literal(Fury.NOT_NULL_VALUE_FLAG,
PRIMITIVE_BYTE_TYPE)),
serializeForNotNull(inputObject, buffer, typeRef, serializer,
generateNewMethod));
return new If(
eqNull(inputObject),
@@ -791,7 +791,7 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
walkPath.add(elementType.toString());
ListExpression builder = new ListExpression();
Class<?> elemClass = TypeUtils.getRawType(elementType);
- boolean trackingRef = needWriteRef(elemClass);
+ boolean trackingRef = needWriteRef(elementType);
Tuple2<Expression, Invoke> writeElementsHeader =
writeElementsHeader(elemClass, trackingRef, serializer, buffer,
collection);
Expression flags = writeElementsHeader.f0;
@@ -1084,10 +1084,8 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
boolean inline = keyMonomorphic && valueMonomorphic;
Class<?> keyTypeRawType = keyType.getRawType();
Class<?> valueTypeRawType = valueType.getRawType();
- boolean trackingKeyRef =
- visitFury(fury ->
fury.getClassResolver().needToWriteRef(keyTypeRawType));
- boolean trackingValueRef =
- visitFury(fury ->
fury.getClassResolver().needToWriteRef(valueTypeRawType));
+ boolean trackingKeyRef = visitFury(fury ->
fury.getClassResolver().needToWriteRef(keyType));
+ boolean trackingValueRef = visitFury(fury ->
fury.getClassResolver().needToWriteRef(valueType));
Tuple2<Expression, Expression> mapKVSerializer =
getMapKVSerializer(keyTypeRawType, valueTypeRawType);
Expression keySerializer = mapKVSerializer.f0;
@@ -1179,10 +1177,8 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
Expression chunkHeader;
Expression keySerializer, valueSerializer;
- boolean trackingKeyRef =
- visitFury(fury ->
fury.getClassResolver().needToWriteRef(keyTypeRawType));
- boolean trackingValueRef =
- visitFury(fury ->
fury.getClassResolver().needToWriteRef(valueTypeRawType));
+ boolean trackingKeyRef = visitFury(fury ->
fury.getClassResolver().needToWriteRef(keyType));
+ boolean trackingValueRef = visitFury(fury ->
fury.getClassResolver().needToWriteRef(valueType));
Expression keyWriteRef = Literal.ofBoolean(trackingKeyRef);
Expression valueWriteRef = Literal.ofBoolean(trackingValueRef);
boolean inline = keyMonomorphic && valueMonomorphic;
@@ -1392,8 +1388,7 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
TypeRef<?> typeRef,
Function<Expression, Expression> callback,
InvokeHint invokeHint) {
- Class<?> rawType = getRawType(typeRef);
- if (visitFury(f -> f.getClassResolver().needToWriteRef(rawType))) {
+ if (visitFury(f -> f.getClassResolver().needToWriteRef(typeRef))) {
return readRef(buffer, callback, () -> deserializeForNotNull(buffer,
typeRef, invokeHint));
} else {
if (typeRef.isPrimitive()) {
@@ -1566,7 +1561,7 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
Class<?> elemClass = TypeUtils.getRawType(elementType);
walkPath.add(elementType.toString());
boolean finalType = isMonomorphic(elemClass);
- boolean trackingRef = visitFury(fury ->
fury.getClassResolver().needToWriteRef(elemClass));
+ boolean trackingRef = visitFury(fury ->
fury.getClassResolver().needToWriteRef(elementType));
if (finalType) {
if (trackingRef) {
builder.add(readContainerElements(elementType, true, null, null,
buffer, collection, size));
@@ -1770,8 +1765,8 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
Class<?> valueCls = valueType.getRawType();
boolean keyMonomorphic = isMonomorphic(keyCls);
boolean valueMonomorphic = isMonomorphic(valueCls);
- boolean refKey = needWriteRef(keyCls);
- boolean refValue = needWriteRef(valueCls);
+ boolean refKey = needWriteRef(keyType);
+ boolean refValue = needWriteRef(valueType);
boolean inline = keyMonomorphic && valueMonomorphic && (!refKey ||
!refValue);
Tuple2<Expression, Expression> mapKVSerializer =
getMapKVSerializer(keyCls, valueCls);
Expression keySerializer = mapKVSerializer.f0;
@@ -1845,8 +1840,8 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
boolean valueMonomorphic = isMonomorphic(valueType);
Class<?> keyTypeRawType = keyType.getRawType();
Class<?> valueTypeRawType = valueType.getRawType();
- boolean trackingKeyRef = needWriteRef(keyTypeRawType);
- boolean trackingValueRef = needWriteRef(valueTypeRawType);
+ boolean trackingKeyRef = needWriteRef(keyType);
+ boolean trackingValueRef = needWriteRef(valueType);
boolean inline = keyMonomorphic && valueMonomorphic && (!trackingKeyRef ||
!trackingValueRef);
ListExpression expressions = new ListExpression(buffer);
Expression trackKeyRef = neq(bitand(chunkHeader, ofInt(TRACKING_KEY_REF)),
ofInt(0));
diff --git
a/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java
b/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java
index 11407791..a0febb30 100644
---
a/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java
+++
b/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java
@@ -625,14 +625,14 @@ public class ObjectCodecBuilder extends
BaseObjectCodecBuilder {
List<Expression> expressions = new ArrayList<>();
int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups);
Literal totalSizeLiteral = Literal.ofInt(totalSize);
+ // After this check, following read can be totally unsafe without checks
+ expressions.add(new Invoke(buffer, "checkReadableBytes",
totalSizeLiteral));
Expression heapBuffer =
new Invoke(buffer, "getHeapMemory", "heapBuffer",
PRIMITIVE_BYTE_ARRAY_TYPE);
Expression readerAddr =
new Invoke(buffer, "getUnsafeReaderAddress", "readerAddr",
PRIMITIVE_LONG_TYPE);
expressions.add(heapBuffer);
expressions.add(readerAddr);
- // After this check, following read can be totally unsafe without checks
- expressions.add(new Invoke(buffer, "checkReadableBytes",
totalSizeLiteral));
int acc = 0;
for (List<Descriptor> group : primitiveGroups) {
ListExpression groupExpressions = new ListExpression();
@@ -690,16 +690,17 @@ public class ObjectCodecBuilder extends
BaseObjectCodecBuilder {
Expression bean, Expression buffer, List<List<Descriptor>>
primitiveGroups) {
List<Expression> expressions = new ArrayList<>();
int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups);
- Expression heapBuffer =
- new Invoke(buffer, "getHeapMemory", "heapBuffer",
PRIMITIVE_BYTE_ARRAY_TYPE);
- expressions.add(heapBuffer);
for (List<Descriptor> group : primitiveGroups) {
- ListExpression groupExpressions = new ListExpression();
- Expression readerAddr =
- new Invoke(buffer, "getUnsafeReaderAddress", "readerAddr",
PRIMITIVE_LONG_TYPE);
// After this check, following read can be totally unsafe without checks.
+ // checkReadableBytes first, `fillBuffer` may create a new heap buffer.
ReplaceStub checkReadableBytesStub = new ReplaceStub();
expressions.add(checkReadableBytesStub);
+ Expression heapBuffer =
+ new Invoke(buffer, "getHeapMemory", "heapBuffer",
PRIMITIVE_BYTE_ARRAY_TYPE);
+ expressions.add(heapBuffer);
+ ListExpression groupExpressions = new ListExpression();
+ Expression readerAddr =
+ new Invoke(buffer, "getUnsafeReaderAddress", "readerAddr",
PRIMITIVE_LONG_TYPE);
int acc = 0;
boolean compressStarted = false;
for (Descriptor descriptor : group) {
diff --git
a/java/fury-core/src/main/java/org/apache/fury/io/FuryInputStream.java
b/java/fury-core/src/main/java/org/apache/fury/io/FuryInputStream.java
index 82711447..0d06e67e 100644
--- a/java/fury-core/src/main/java/org/apache/fury/io/FuryInputStream.java
+++ b/java/fury-core/src/main/java/org/apache/fury/io/FuryInputStream.java
@@ -76,15 +76,16 @@ public class FuryInputStream extends InputStream implements
FuryStreamReader {
private static byte[] growBuffer(int minFillSize, MemoryBuffer buffer) {
int newSize;
- int targetSize = buffer.size() + minFillSize;
+ int size = buffer.size();
+ int targetSize = size + minFillSize;
newSize =
targetSize < MemoryBuffer.BUFFER_GROW_STEP_THRESHOLD
? targetSize << 2
: (int) Math.min(targetSize * 1.5d, Integer.MAX_VALUE - 8);
byte[] newBuffer = new byte[newSize];
byte[] heapMemory = buffer.getHeapMemory();
- System.arraycopy(heapMemory, 0, newBuffer, 0, buffer.size());
- buffer.initHeapBuffer(newBuffer, 0, buffer.size());
+ System.arraycopy(heapMemory, 0, newBuffer, 0, size);
+ buffer.initHeapBuffer(newBuffer, 0, size);
heapMemory = newBuffer;
return heapMemory;
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
b/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
index 6379b6ed..e9f4ba11 100644
--- a/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
+++ b/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
@@ -313,17 +313,23 @@ public final class MemoryBuffer {
}
public void get(int index, byte[] dst, int offset, int length) {
- final long pos = address + index;
- if ((index
- | offset
- | length
- | (offset + length)
- | (dst.length - (offset + length))
- | addressLimit - length - pos)
- < 0) {
- throwOOBException();
+ final byte[] heapMemory = this.heapMemory;
+ if (heapMemory != null) {
+ // System.arraycopy faster for some jdk than Unsafe.
+ System.arraycopy(heapMemory, heapOffset + index, dst, offset, length);
+ } else {
+ final long pos = address + index;
+ if ((index
+ | offset
+ | length
+ | (offset + length)
+ | (dst.length - (offset + length))
+ | addressLimit - length - pos)
+ < 0) {
+ throwOOBException();
+ }
+ Platform.copyMemory(null, pos, dst, Platform.BYTE_ARRAY_OFFSET + offset,
length);
}
- Platform.copyMemory(heapMemory, pos, dst, Platform.BYTE_ARRAY_OFFSET +
offset, length);
}
public void get(int offset, ByteBuffer target, int numBytes) {
@@ -378,19 +384,25 @@ public final class MemoryBuffer {
}
public void put(int index, byte[] src, int offset, int length) {
- final long pos = address + index;
- // check the byte array offset and length
- if ((index
- | offset
- | length
- | (offset + length)
- | (src.length - (offset + length))
- | addressLimit - length - pos)
- < 0) {
- throwOOBException();
+ final byte[] heapMemory = this.heapMemory;
+ if (heapMemory != null) {
+ // System.arraycopy faster for some jdk than Unsafe.
+ System.arraycopy(src, offset, heapMemory, heapOffset + index, length);
+ } else {
+ final long pos = address + index;
+ // check the byte array offset and length
+ if ((index
+ | offset
+ | length
+ | (offset + length)
+ | (src.length - (offset + length))
+ | addressLimit - length - pos)
+ < 0) {
+ throwOOBException();
+ }
+ final long arrayAddress = Platform.BYTE_ARRAY_OFFSET + offset;
+ Platform.copyMemory(src, arrayAddress, null, pos, length);
}
- final long arrayAddress = Platform.BYTE_ARRAY_OFFSET + offset;
- Platform.copyMemory(src, arrayAddress, heapMemory, pos, length);
}
public byte getByte(int index) {
@@ -2198,10 +2210,9 @@ public final class MemoryBuffer {
int remaining = size - readerIdx;
if (remaining >= 8) {
readerIndex = readerIdx + len;
- long v =
- UNSAFE.getLong(heapMemory, address + readerIdx)
- & (0xffffffffffffffffL >>> ((8 - len) * 8));
- return LITTLE_ENDIAN ? v : Long.reverseBytes(v);
+ long v = UNSAFE.getLong(heapMemory, address + readerIdx);
+ v = (LITTLE_ENDIAN ? v : Long.reverseBytes(v)) & (0xffffffffffffffffL
>>> ((8 - len) * 8));
+ return v;
}
return slowReadBytesAsInt64(remaining, len);
}
diff --git a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDef.java
b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDef.java
index 695d552b..6f213436 100644
--- a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDef.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDef.java
@@ -348,16 +348,22 @@ public class ClassDef implements Serializable {
}
public abstract static class FieldType implements Serializable {
- public FieldType(boolean isMonomorphic) {
+ public FieldType(boolean isMonomorphic, boolean trackingRef) {
this.isMonomorphic = isMonomorphic;
+ this.trackingRef = trackingRef;
}
protected final boolean isMonomorphic;
+ protected final boolean trackingRef;
public boolean isMonomorphic() {
return isMonomorphic;
}
+ public boolean trackingRef() {
+ return trackingRef;
+ }
+
/**
* Convert a serializable field type to type token. If field type is a
generic type with
* generics, the generics will be built up recursively. The final leaf
object type will be built
@@ -369,48 +375,46 @@ public class ClassDef implements Serializable {
@Override
public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
if (o == null || getClass() != o.getClass()) {
return false;
}
FieldType fieldType = (FieldType) o;
- return isMonomorphic == fieldType.isMonomorphic;
+ return isMonomorphic == fieldType.isMonomorphic && trackingRef ==
fieldType.trackingRef;
}
@Override
public int hashCode() {
- return Objects.hash(isMonomorphic);
+ return Objects.hash(isMonomorphic, trackingRef);
}
/** Write field type info. */
- public void write(MemoryBuffer buffer, boolean writeMonomorphicFlag) {
- byte header = (byte) (isMonomorphic ? 1 : 0);
+ public void write(MemoryBuffer buffer, boolean writeHeader) {
+ byte header = (byte) ((isMonomorphic ? 1 : 0) << 1);
+ // header of nested generic fields in collection/map will be written
independently
+ header |= (byte) (trackingRef ? 1 : 0);
if (this instanceof ClassDef.RegisteredFieldType) {
short classId = ((ClassDef.RegisteredFieldType) this).getClassId();
- buffer.writeVarUint32Small7(
- writeMonomorphicFlag ? ((5 + classId) << 1) | header : 5 +
classId);
+ buffer.writeVarUint32Small7(writeHeader ? ((5 + classId) << 2) |
header : 5 + classId);
} else if (this instanceof ClassDef.EnumFieldType) {
- buffer.writeVarUint32Small7(writeMonomorphicFlag ? ((4) << 1) | header
: 4);
+ buffer.writeVarUint32Small7(writeHeader ? ((4) << 2) | header : 4);
} else if (this instanceof ClassDef.ArrayFieldType) {
ClassDef.ArrayFieldType arrayFieldType = (ClassDef.ArrayFieldType)
this;
- buffer.writeVarUint32Small7(writeMonomorphicFlag ? ((3) << 1) | header
: 3);
+ buffer.writeVarUint32Small7(writeHeader ? ((3) << 2) | header : 3);
buffer.writeVarUint32Small7(arrayFieldType.getDimensions());
(arrayFieldType).getComponentType().write(buffer);
} else if (this instanceof ClassDef.CollectionFieldType) {
- buffer.writeVarUint32Small7(writeMonomorphicFlag ? ((2) << 1) | header
: 2);
+ buffer.writeVarUint32Small7(writeHeader ? ((2) << 2) | header : 2);
// TODO remove it when new collection deserialization jit finished.
((ClassDef.CollectionFieldType) this).getElementType().write(buffer);
} else if (this instanceof ClassDef.MapFieldType) {
- buffer.writeVarUint32Small7(writeMonomorphicFlag ? ((1) << 1) | header
: 1);
+ buffer.writeVarUint32Small7(writeHeader ? ((1) << 2) | header : 1);
// TODO remove it when new map deserialization jit finished.
ClassDef.MapFieldType mapFieldType = (ClassDef.MapFieldType) this;
mapFieldType.getKeyType().write(buffer);
mapFieldType.getValueType().write(buffer);
} else {
Preconditions.checkArgument(this instanceof ClassDef.ObjectFieldType);
- buffer.writeVarUint32Small7(writeMonomorphicFlag ? header : 0);
+ buffer.writeVarUint32Small7(writeHeader ? header : 0);
}
}
@@ -420,25 +424,27 @@ public class ClassDef implements Serializable {
public static FieldType read(MemoryBuffer buffer) {
int header = buffer.readVarUint32Small7();
- boolean isMonomorphic = (header & 0b1) != 0;
- return read(buffer, isMonomorphic, header >>> 1);
+ boolean isMonomorphic = (header & 0b10) != 0;
+ boolean trackingRef = (header & 0b1) != 0;
+ return read(buffer, isMonomorphic, trackingRef, header >>> 2);
}
/** Read field type info. */
- public static FieldType read(MemoryBuffer buffer, boolean isFinal, int
typeId) {
+ public static FieldType read(
+ MemoryBuffer buffer, boolean isFinal, boolean trackingRef, int typeId)
{
if (typeId == 0) {
- return new ObjectFieldType(isFinal);
+ return new ObjectFieldType(isFinal, trackingRef);
} else if (typeId == 1) {
- return new MapFieldType(isFinal, read(buffer), read(buffer));
+ return new MapFieldType(isFinal, trackingRef, read(buffer),
read(buffer));
} else if (typeId == 2) {
- return new CollectionFieldType(isFinal, read(buffer));
+ return new CollectionFieldType(isFinal, trackingRef, read(buffer));
} else if (typeId == 3) {
int dims = buffer.readVarUint32Small7();
- return new ArrayFieldType(isFinal, read(buffer), dims);
+ return new ArrayFieldType(isFinal, trackingRef, read(buffer), dims);
} else if (typeId == 4) {
return EnumFieldType.getInstance();
} else {
- return new RegisteredFieldType(isFinal, (short) (typeId - 5));
+ return new RegisteredFieldType(isFinal, trackingRef, (short) (typeId -
5));
}
}
}
@@ -447,8 +453,8 @@ public class ClassDef implements Serializable {
public static class RegisteredFieldType extends FieldType {
private final short classId;
- public RegisteredFieldType(boolean isFinal, short classId) {
- super(isFinal);
+ public RegisteredFieldType(boolean isFinal, boolean trackingRef, short
classId) {
+ super(isFinal, trackingRef);
this.classId = classId;
}
@@ -458,7 +464,7 @@ public class ClassDef implements Serializable {
@Override
public TypeRef<?> toTypeToken(ClassResolver classResolver) {
- return TypeRef.of(classResolver.getRegisteredClass(classId));
+ return TypeRef.of(classResolver.getRegisteredClass(classId), new
TypeExtMeta(trackingRef));
}
@Override
@@ -486,6 +492,8 @@ public class ClassDef implements Serializable {
return "RegisteredFieldType{"
+ "isMonomorphic="
+ isMonomorphic()
+ + ", trackingRef="
+ + trackingRef()
+ ", classId="
+ classId
+ '}';
@@ -503,8 +511,8 @@ public class ClassDef implements Serializable {
public static class CollectionFieldType extends FieldType {
private final FieldType elementType;
- public CollectionFieldType(boolean isFinal, FieldType elementType) {
- super(isFinal);
+ public CollectionFieldType(boolean isFinal, boolean trackingRef, FieldType
elementType) {
+ super(isFinal, trackingRef);
this.elementType = elementType;
}
@@ -514,7 +522,8 @@ public class ClassDef implements Serializable {
@Override
public TypeRef<?> toTypeToken(ClassResolver classResolver) {
- return collectionOf(elementType.toTypeToken(classResolver));
+ // TODO support preserve element TypeExtMeta
+ return collectionOf(elementType.toTypeToken(classResolver), new
TypeExtMeta(trackingRef));
}
@Override
@@ -544,6 +553,8 @@ public class ClassDef implements Serializable {
+ elementType
+ ", isFinal="
+ isMonomorphic()
+ + ", trackingRef="
+ + trackingRef()
+ '}';
}
}
@@ -560,8 +571,9 @@ public class ClassDef implements Serializable {
private final FieldType keyType;
private final FieldType valueType;
- public MapFieldType(boolean isFinal, FieldType keyType, FieldType
valueType) {
- super(isFinal);
+ public MapFieldType(
+ boolean isFinal, boolean trackingRef, FieldType keyType, FieldType
valueType) {
+ super(isFinal, trackingRef);
this.keyType = keyType;
this.valueType = valueType;
}
@@ -576,7 +588,11 @@ public class ClassDef implements Serializable {
@Override
public TypeRef<?> toTypeToken(ClassResolver classResolver) {
- return mapOf(keyType.toTypeToken(classResolver),
valueType.toTypeToken(classResolver));
+ // TODO support preserve element TypeExtMeta, it will be lost when
building other TypeRef
+ return mapOf(
+ keyType.toTypeToken(classResolver),
+ valueType.toTypeToken(classResolver),
+ new TypeExtMeta(trackingRef));
}
@Override
@@ -608,6 +624,8 @@ public class ClassDef implements Serializable {
+ valueType
+ ", isFinal="
+ isMonomorphic()
+ + ", trackingRef="
+ + trackingRef()
+ '}';
}
}
@@ -616,7 +634,7 @@ public class ClassDef implements Serializable {
private static final EnumFieldType INSTANCE = new EnumFieldType();
private EnumFieldType() {
- super(true);
+ super(true, false);
}
@Override
@@ -633,8 +651,9 @@ public class ClassDef implements Serializable {
private final FieldType componentType;
private final int dimensions;
- public ArrayFieldType(boolean isMonomorphic, FieldType componentType, int
dimensions) {
- super(isMonomorphic);
+ public ArrayFieldType(
+ boolean isMonomorphic, boolean trackingRef, FieldType componentType,
int dimensions) {
+ super(isMonomorphic, trackingRef);
this.componentType = componentType;
this.dimensions = dimensions;
}
@@ -648,9 +667,12 @@ public class ClassDef implements Serializable {
// We embed `isMonomorphic` flag in ObjectArraySerializer, so this
flag can be ignored
// here.
NonexistentClass.getNonexistentClass(
- componentType instanceof EnumFieldType, dimensions, true));
+ componentType instanceof EnumFieldType, dimensions, true),
+ new TypeExtMeta(trackingRef));
} else {
- return TypeRef.of(Array.newInstance(componentRawType, new
int[dimensions]).getClass());
+ return TypeRef.of(
+ Array.newInstance(componentRawType, new
int[dimensions]).getClass(),
+ new TypeExtMeta(trackingRef));
}
}
@@ -691,6 +713,8 @@ public class ClassDef implements Serializable {
+ dimensions
+ ", isMonomorphic="
+ isMonomorphic
+ + ", trackingRef="
+ + trackingRef
+ '}';
}
}
@@ -698,13 +722,15 @@ public class ClassDef implements Serializable {
/** Class for field type which isn't registered and not collection/map type
too. */
public static class ObjectFieldType extends FieldType {
- public ObjectFieldType(boolean isFinal) {
- super(isFinal);
+ public ObjectFieldType(boolean isFinal, boolean trackingRef) {
+ super(isFinal, trackingRef);
}
@Override
public TypeRef<?> toTypeToken(ClassResolver classResolver) {
- return isMonomorphic() ? TypeRef.of(FinalObjectTypeStub.class) :
TypeRef.of(Object.class);
+ return isMonomorphic()
+ ? TypeRef.of(FinalObjectTypeStub.class, new TypeExtMeta(trackingRef))
+ : TypeRef.of(Object.class, new TypeExtMeta(trackingRef));
}
@Override
@@ -730,9 +756,11 @@ public class ClassDef implements Serializable {
Preconditions.checkNotNull(genericType);
Class<?> rawType = genericType.getCls();
boolean isMonomorphic = genericType.isMonomorphic();
+ boolean trackingRef = genericType.trackingRef(classResolver);
if (COLLECTION_TYPE.isSupertypeOf(genericType.getTypeRef())) {
return new CollectionFieldType(
isMonomorphic,
+ trackingRef,
buildFieldType(
classResolver,
genericType.getTypeParameter0() == null
@@ -741,6 +769,7 @@ public class ClassDef implements Serializable {
} else if (MAP_TYPE.isSupertypeOf(genericType.getTypeRef())) {
return new MapFieldType(
isMonomorphic,
+ trackingRef,
buildFieldType(
classResolver,
genericType.getTypeParameter0() == null
@@ -754,7 +783,7 @@ public class ClassDef implements Serializable {
} else {
Short classId = classResolver.getRegisteredClassId(rawType);
if (classId != null && classId != ClassResolver.NO_CLASS_ID) {
- return new RegisteredFieldType(isMonomorphic, classId);
+ return new RegisteredFieldType(isMonomorphic, trackingRef, classId);
} else {
if (rawType.isEnum()) {
return EnumFieldType.getInstance();
@@ -762,9 +791,12 @@ public class ClassDef implements Serializable {
if (rawType.isArray()) {
Tuple2<Class<?>, Integer> info =
TypeUtils.getArrayComponentInfo(rawType);
return new ArrayFieldType(
- isMonomorphic, buildFieldType(classResolver,
GenericType.build(info.f0)), info.f1);
+ isMonomorphic,
+ trackingRef,
+ buildFieldType(classResolver, GenericType.build(info.f0)),
+ info.f1);
}
- return new ObjectFieldType(isMonomorphic);
+ return new ObjectFieldType(isMonomorphic, trackingRef);
}
}
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java
b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java
index c5d90d94..5567e2a6 100644
--- a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java
@@ -118,8 +118,9 @@ class ClassDefDecoder {
Encoding encoding = fieldNameEncodings[encodingFlags];
String fieldName =
Encoders.FIELD_NAME_DECODER.decode(buffer.readBytes(size), encoding);
boolean isMonomorphic = (header & 0b100) != 0;
+ boolean trackingRef = (header & 0b001) != 0;
int typeId = buffer.readVarUint32Small14();
- FieldType fieldType = FieldType.read(buffer, isMonomorphic, typeId);
+ FieldType fieldType = FieldType.read(buffer, isMonomorphic, trackingRef,
typeId);
fieldInfos.add(new ClassDef.FieldInfo(className, fieldName, fieldType));
}
return fieldInfos;
diff --git
a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
index d40dd53d..9a592cd2 100644
--- a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
@@ -250,6 +250,7 @@ class ClassDefEncoder {
// `3 bits size + 2 bits field name encoding + polymorphism flag +
nullability flag + ref
// tracking flag`
int header = ((fieldType.isMonomorphic() ? 1 : 0) << 2);
+ header |= ((fieldType.trackingRef() ? 1 : 0));
// Encoding `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL/TAG_ID`
MetaString metaString =
Encoders.encodeFieldName(fieldInfo.getFieldName());
int encodingFlags =
fieldNameEncodingsList.indexOf(metaString.getEncoding());
diff --git
a/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
b/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
index 90298e8e..396721f6 100644
--- a/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
@@ -246,25 +246,50 @@ public class MetaStringEncoder {
private byte[] encodeGeneric(char[] chars, int bitsPerChar) {
int totalBits = chars.length * bitsPerChar + 1;
- int byteLength = (totalBits + 7) / 8; // Calculate number of needed bytes
+ int byteLength = (totalBits + 7) / 8;
byte[] bytes = new byte[byteLength];
- int currentBit = 1;
- for (char c : chars) {
- int value =
- (bitsPerChar == 5) ? charToValueLowerSpecial(c) :
charToValueLowerUpperDigitSpecial(c);
- // Encode the value in bitsPerChar bits
- for (int i = bitsPerChar - 1; i >= 0; i--) {
- if ((value & (1 << i)) != 0) {
- // Set the bit in the byte array
- int bytePos = currentBit / 8;
- int bitPos = currentBit % 8;
- bytes[bytePos] |= (byte) (1 << (7 - bitPos));
+ int byteInd = 0;
+ int bitInd = 1; // Start from the second bit (the first is reserved for
the flag)
+ int charInd = 0;
+ int charBitRemain = bitsPerChar; // Remaining bits to process for the
current character
+ int mask;
+ while (charInd < chars.length) {
+ int charVal =
+ (bitsPerChar == 5)
+ ? charToValueLowerSpecial(chars[charInd])
+ : charToValueLowerUpperDigitSpecial(chars[charInd]);
+ // Calculate how many bits are remaining in the current byte
+ int nowByteRemain = 8 - bitInd;
+ if (nowByteRemain >= charBitRemain) {
+ // If the remaining bits in the current byte can fit the whole
character value
+ mask = (1 << charBitRemain) - 1; // Create a mask for the bits of the
character
+ bytes[byteInd] |=
+ (byte)
+ ((charVal & mask)
+ << (nowByteRemain - charBitRemain)); // Place the
character bits into the byte
+ bitInd += charBitRemain;
+ if (bitInd == 8) {
+ ++byteInd;
+ bitInd = 0;
}
- currentBit++;
+ ++charInd;
+ charBitRemain = bitsPerChar; // Reset the remaining bits for the next
character
+ } else {
+ // If the remaining bits in the current byte are not enough to hold
the whole character
+ mask = (1 << nowByteRemain) - 1; // Create a mask for the current
available bits in the byte
+ bytes[byteInd] |=
+ (byte)
+ ((charVal >> (charBitRemain - nowByteRemain))
+ & mask); // Place part of the character bits into the byte
+ ++byteInd; // Move to the next byte
+ bitInd = 0; // Reset bit index for the new byte
+ charBitRemain -= nowByteRemain; // Decrease the remaining bits for the
character
}
}
+
boolean stripLastChar = bytes.length * 8 >= totalBits + bitsPerChar;
if (stripLastChar) {
+ // Mark the first byte as indicating a stripped character
bytes[0] = (byte) (bytes[0] | 0x80);
}
return bytes;
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/ImmutableSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/meta/TypeExtMeta.java
similarity index 57%
copy from
java/fury-core/src/main/java/org/apache/fury/serializer/ImmutableSerializer.java
copy to java/fury-core/src/main/java/org/apache/fury/meta/TypeExtMeta.java
index c8cec37c..37df8273 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/ImmutableSerializer.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/TypeExtMeta.java
@@ -17,26 +17,21 @@
* under the License.
*/
-package org.apache.fury.serializer;
+package org.apache.fury.meta;
-import org.apache.fury.Fury;
+public class TypeExtMeta {
+ private final boolean trackingRef;
-/**
- * Serializer for immutable objects.
- *
- * @param <T> type of objects being serializing/deserializing
- */
-public abstract class ImmutableSerializer<T> extends Serializer<T> {
-
- public ImmutableSerializer(Fury fury, Class<T> type) {
- super(fury, type, true);
+ TypeExtMeta(boolean trackingRef) {
+ this.trackingRef = trackingRef;
}
- public ImmutableSerializer(Fury fury, Class<T> type, boolean needToWriteRef)
{
- super(fury, type, needToWriteRef, true);
+ public boolean trackingRef() {
+ return trackingRef;
}
- public ImmutableSerializer(Fury fury, Class<T> type, boolean needToWriteRef,
boolean immutable) {
- super(fury, type, needToWriteRef, true);
+ @Override
+ public String toString() {
+ return "TypeExtMeta{" + "trackingRef=" + trackingRef + '}';
}
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/pool/ClassLoaderFuryPooled.java
b/java/fury-core/src/main/java/org/apache/fury/pool/ClassLoaderFuryPooled.java
index ececce99..40d4d9f1 100644
---
a/java/fury-core/src/main/java/org/apache/fury/pool/ClassLoaderFuryPooled.java
+++
b/java/fury-core/src/main/java/org/apache/fury/pool/ClassLoaderFuryPooled.java
@@ -20,11 +20,10 @@
package org.apache.fury.pool;
import java.util.Objects;
-import java.util.Queue;
import java.util.WeakHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
@@ -43,15 +42,15 @@ public class ClassLoaderFuryPooled {
private final ClassLoader classLoader;
- /**
- * idle Fury cache change. by : 1. getLoaderBind() 2.
returnObject(LoaderBinding) 3.
- * addObjAndWarp()
- */
- private final Queue<Fury> idleCacheQueue;
+ /** idle Fury cache change. by : 1. init() 2. getFury() 3.returnFury() */
+ private final BlockingQueue<Fury> idleCacheQueue;
final WeakHashMap<Fury, Object> allFury = new WeakHashMap<>();
- /** active cache size's number change by : 1. getLoaderBind() 2.
returnObject(LoaderBinding). */
+ /**
+ * The number of active Fury objects in the cache.Make sure it does not
exceed the maximum number
+ * of object pools.
+ */
private final AtomicInteger activeCacheNumber = new AtomicInteger(0);
/**
@@ -61,7 +60,6 @@ public class ClassLoaderFuryPooled {
private final int maxPoolSize;
private final Lock lock = new ReentrantLock();
- private final Condition furyCondition = lock.newCondition();
public ClassLoaderFuryPooled(
ClassLoader classLoader,
@@ -72,58 +70,66 @@ public class ClassLoaderFuryPooled {
this.maxPoolSize = maxPoolSize;
this.furyFactory = furyFactory;
this.classLoader = classLoader;
- idleCacheQueue = new ConcurrentLinkedQueue<>();
+ idleCacheQueue = new LinkedBlockingQueue<>(maxPoolSize);
while (idleCacheQueue.size() < minPoolSize) {
- addFury();
+ addFury(true);
}
}
public Fury getFury() {
- try {
- lock.lock();
+ if (activeCacheNumber.get() < maxPoolSize) {
Fury fury = idleCacheQueue.poll();
- while (fury == null) {
- if (activeCacheNumber.get() < maxPoolSize) {
- addFury();
- } else {
- furyCondition.await();
+ if (fury != null) {
+ return fury;
+ } else {
+ // new Fury return directly, no need to add to queue, it will be added
by returnFury()
+ fury = addFury(false);
+ if (fury != null) {
+ return fury;
}
- fury = idleCacheQueue.poll();
}
- activeCacheNumber.incrementAndGet();
- return fury;
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
+ }
+ try {
+ return idleCacheQueue.take();
+ } catch (InterruptedException e) {
throw new RuntimeException(e);
- } finally {
- lock.unlock();
}
}
public void returnFury(Fury fury) {
Objects.requireNonNull(fury);
+ idleCacheQueue.offer(fury);
+ }
+
+ private Fury addFury(boolean addQueue) {
+ // only activeCacheNumber increment success, can lock and create new Fury,
otherwise return
+ // null, and block in getFury(), wait for other thread to release
idleCacheQueue.
+ int after = activeCacheNumber.incrementAndGet();
+ if (after > maxPoolSize) {
+ activeCacheNumber.decrementAndGet();
+ return null;
+ }
try {
lock.lock();
- idleCacheQueue.add(fury);
- activeCacheNumber.decrementAndGet();
- furyCondition.signalAll();
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw new RuntimeException(e);
+ Fury fury = furyFactory.apply(classLoader);
+ factoryCallback.accept(fury);
+ allFury.put(fury, null);
+ if (addQueue) {
+ idleCacheQueue.add(fury);
+ }
+ return fury;
} finally {
lock.unlock();
}
}
- private void addFury() {
- Fury fury = furyFactory.apply(classLoader);
- factoryCallback.accept(fury);
- idleCacheQueue.add(fury);
- allFury.put(fury, null);
- }
-
void setFactoryCallback(Consumer<Fury> factoryCallback) {
- this.factoryCallback = this.factoryCallback.andThen(factoryCallback);
- allFury.keySet().forEach(factoryCallback);
+ try {
+ lock.lock();
+ this.factoryCallback = this.factoryCallback.andThen(factoryCallback);
+ allFury.keySet().forEach(factoryCallback);
+ } finally {
+ lock.unlock();
+ }
}
}
diff --git a/java/fury-core/src/main/java/org/apache/fury/reflect/TypeRef.java
b/java/fury-core/src/main/java/org/apache/fury/reflect/TypeRef.java
index 16869879..c521096d 100644
--- a/java/fury-core/src/main/java/org/apache/fury/reflect/TypeRef.java
+++ b/java/fury-core/src/main/java/org/apache/fury/reflect/TypeRef.java
@@ -40,6 +40,7 @@ import org.apache.fury.type.TypeUtils;
//
https://github.com/google/guava/blob/9f6a3840/guava/src/com/google/common/reflect/TypeToken.java
public class TypeRef<T> {
private final Type type;
+ private final Object extInfo;
private transient Class<? super T> rawType;
private transient Map<TypeVariableKey, Type> typeMappings;
@@ -57,14 +58,27 @@ public class TypeRef<T> {
*/
protected TypeRef() {
this.type = capture();
+ this.extInfo = null;
+ }
+
+ protected TypeRef(Object extInfo) {
+ this.type = capture();
+ this.extInfo = extInfo;
}
private TypeRef(Class<T> declaringClass) {
this.type = declaringClass;
+ this.extInfo = null;
+ }
+
+ private TypeRef(Class<T> declaringClass, Object extInfo) {
+ this.type = declaringClass;
+ this.extInfo = extInfo;
}
private TypeRef(Type type) {
this.type = type;
+ this.extInfo = null;
}
/** Returns an instance of type token that wraps {@code type}. */
@@ -72,6 +86,10 @@ public class TypeRef<T> {
return new TypeRef<>(clazz);
}
+ public static <T> TypeRef<T> of(Class<T> clazz, Object extInfo) {
+ return new TypeRef<>(clazz, extInfo);
+ }
+
/** Returns an instance of type token that wraps {@code type}. */
public static <T> TypeRef<T> of(Type type) {
return new TypeRef<>(type);
@@ -140,6 +158,10 @@ public class TypeRef<T> {
});
}
+ public Object getExtInfo() {
+ return extInfo;
+ }
+
/** Returns true if this type is one of the primitive types (including
{@code void}). */
public boolean isPrimitive() {
return type instanceof Class && ((Class<?>) type).isPrimitive();
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
index b6cf5157..1c4874c2 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
@@ -103,6 +103,7 @@ import org.apache.fury.meta.ClassDef;
import org.apache.fury.meta.ClassSpec;
import org.apache.fury.meta.Encoders;
import org.apache.fury.meta.MetaString;
+import org.apache.fury.meta.TypeExtMeta;
import org.apache.fury.reflect.ReflectionUtils;
import org.apache.fury.reflect.TypeRef;
import org.apache.fury.serializer.ArraySerializers;
@@ -117,6 +118,7 @@ import org.apache.fury.serializer.JdkProxySerializer;
import org.apache.fury.serializer.LambdaSerializer;
import org.apache.fury.serializer.LocaleSerializer;
import org.apache.fury.serializer.MetaSharedSerializer;
+import org.apache.fury.serializer.NoneSerializer;
import org.apache.fury.serializer.NonexistentClass;
import org.apache.fury.serializer.NonexistentClass.NonexistentMetaShared;
import org.apache.fury.serializer.NonexistentClass.NonexistentSkip;
@@ -319,6 +321,7 @@ public class ClassResolver {
private void addDefaultSerializers() {
// primitive types will be boxed.
+ addDefaultSerializer(void.class, NoneSerializer.class);
addDefaultSerializer(String.class, new StringSerializer(fury));
PrimitiveSerializers.registerDefaultSerializers(fury);
Serializers.registerDefaultSerializers(fury);
@@ -1086,7 +1089,13 @@ public class ClassResolver {
* Whether to track reference for this type. If false, reference tracing of
subclasses may be
* ignored too.
*/
- public boolean needToWriteRef(Class<?> cls) {
+ public boolean needToWriteRef(TypeRef<?> typeRef) {
+ Object extInfo = typeRef.getExtInfo();
+ if (extInfo instanceof TypeExtMeta) {
+ TypeExtMeta meta = (TypeExtMeta) extInfo;
+ return meta.trackingRef();
+ }
+ Class<?> cls = typeRef.getRawType();
if (fury.trackingRef()) {
ClassInfo classInfo = getClassInfo(cls, false);
if (classInfo == null || classInfo.serializer == null) {
@@ -1923,7 +1932,7 @@ public class ClassResolver {
public GenericType buildGenericType(TypeRef<?> typeRef) {
return GenericType.build(
- typeRef.getType(),
+ typeRef,
t -> {
if (t.getClass() == Class.class) {
return isMonomorphic((Class<?>) t);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/DisallowedList.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/DisallowedList.java
index e26402e9..b6ea8840 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/DisallowedList.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/DisallowedList.java
@@ -19,11 +19,13 @@
package org.apache.fury.resolver;
-import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.fury.exception.InsecureException;
@@ -31,15 +33,24 @@ import org.apache.fury.exception.InsecureException;
/** A class to record which classes are not allowed for serialization. */
class DisallowedList {
private static final String DISALLOWED_LIST_TXT_PATH = "fury/disallowed.txt";
+ // when disallowed.txt changed, update this hash by result of `sha256sum
disallowed.txt`
+ private static final String SHA256_HASH =
+ "30dc5228f52b02f61aff35a94d29ccd903abbf490d8231810c5e1c0321c56557";
private static final Set<String> DEFAULT_DISALLOWED_LIST_SET;
static {
try (InputStream is =
DisallowedList.class.getClassLoader().getResourceAsStream(DISALLOWED_LIST_TXT_PATH))
{
if (is != null) {
+ byte[] fileBytes = readAllBytes(is);
+ String calculatedHash = calculateSHA256(fileBytes);
+ if (!SHA256_HASH.equals(calculatedHash)) {
+ // add a check to avoid some malicious overwrite disallowed.txt
+ throw new SecurityException("Disallowed list has been tampered");
+ }
DEFAULT_DISALLOWED_LIST_SET =
- new BufferedReader(new InputStreamReader(is,
StandardCharsets.UTF_8))
- .lines()
+ Arrays.stream(
+ new String(fileBytes,
StandardCharsets.UTF_8).split(System.lineSeparator()))
.filter(line -> !line.isEmpty() && !line.startsWith("#"))
.collect(Collectors.toSet());
} else {
@@ -52,6 +63,35 @@ class DisallowedList {
}
}
+ private static byte[] readAllBytes(InputStream inputStream) throws
IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ int numbytesRead;
+ byte[] data = new byte[1024];
+ while ((numbytesRead = inputStream.read(data, 0, data.length)) != -1) {
+ buffer.write(data, 0, numbytesRead);
+ }
+ buffer.flush();
+ return buffer.toByteArray();
+ }
+
+ private static String calculateSHA256(byte[] input) {
+ try {
+ MessageDigest digest = MessageDigest.getInstance("SHA-256");
+ byte[] hashBytes = digest.digest(input);
+ StringBuilder hexString = new StringBuilder();
+ for (byte b : hashBytes) {
+ String hex = Integer.toHexString(0xff & b);
+ if (hex.length() == 1) {
+ hexString.append('0');
+ }
+ hexString.append(hex);
+ }
+ return hexString.toString();
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException("SHA-256 algorithm not available", e);
+ }
+ }
+
/**
* Determine whether the current Class is in the default disallowed list.
*
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/AbstractObjectSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/AbstractObjectSerializer.java
index a6e3af86..f7540407 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/AbstractObjectSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/AbstractObjectSerializer.java
@@ -371,7 +371,7 @@ public abstract class AbstractObjectSerializer<T> extends
Serializer<T> {
for (Descriptor descriptor : grouper.getOtherDescriptors()) {
GenericTypeField genericTypeField =
new GenericTypeField(
- descriptor.getRawType(),
+ descriptor.getTypeRef(),
descriptor.getDeclaringClass() + "." + descriptor.getName(),
descriptor.getField() != null
? FieldAccessor.createAccessor(descriptor.getField())
@@ -394,7 +394,7 @@ public abstract class AbstractObjectSerializer<T> extends
Serializer<T> {
private static FinalTypeField buildFinalTypeField(Fury fury, Descriptor d) {
return new FinalTypeField(
- d.getRawType(),
+ d.getTypeRef(),
d.getDeclaringClass() + "." + d.getName(),
// `d.getField()` will be null when peer class doesn't have this field.
d.getField() != null ? FieldAccessor.createAccessor(d.getField()) :
null,
@@ -410,12 +410,14 @@ public abstract class AbstractObjectSerializer<T> extends
Serializer<T> {
}
public static class InternalFieldInfo {
+ private final TypeRef<?> typeRef;
protected final short classId;
protected final String qualifiedFieldName;
protected final FieldAccessor fieldAccessor;
private InternalFieldInfo(
- short classId, String qualifiedFieldName, FieldAccessor fieldAccessor)
{
+ TypeRef<?> typeRef, short classId, String qualifiedFieldName,
FieldAccessor fieldAccessor) {
+ this.typeRef = typeRef;
this.classId = classId;
this.qualifiedFieldName = qualifiedFieldName;
this.fieldAccessor = fieldAccessor;
@@ -436,17 +438,19 @@ public abstract class AbstractObjectSerializer<T> extends
Serializer<T> {
static final class FinalTypeField extends InternalFieldInfo {
final ClassInfo classInfo;
+ final boolean trackingRef;
- private FinalTypeField(Class<?> type, String fieldName, FieldAccessor
accessor, Fury fury) {
- super(getRegisteredClassId(fury, type), fieldName, accessor);
+ private FinalTypeField(TypeRef<?> type, String fieldName, FieldAccessor
accessor, Fury fury) {
+ super(type, getRegisteredClassId(fury, type.getRawType()), fieldName,
accessor);
// invoke `copy` to avoid ObjectSerializer construct clear serializer by
`clearSerializer`.
- if (type == FinalObjectTypeStub.class) {
+ if (type.getRawType() == FinalObjectTypeStub.class) {
// `FinalObjectTypeStub` has no fields, using its `classInfo`
// will make deserialization failed.
classInfo = null;
} else {
- classInfo = fury.getClassResolver().getClassInfo(type);
+ classInfo = fury.getClassResolver().getClassInfo(type.getRawType());
}
+ trackingRef = fury.getClassResolver().needToWriteRef(type);
}
}
@@ -455,22 +459,13 @@ public abstract class AbstractObjectSerializer<T> extends
Serializer<T> {
final ClassInfoHolder classInfoHolder;
final boolean trackingRef;
- private GenericTypeField(
- Class<?> cls, String qualifiedFieldName, FieldAccessor accessor, Fury
fury) {
- super(getRegisteredClassId(fury, cls), qualifiedFieldName, accessor);
- // TODO support generics <T> in Pojo<T>, see
ComplexObjectSerializer.getGenericTypes
- genericType = fury.getClassResolver().buildGenericType(cls);
- classInfoHolder = fury.getClassResolver().nilClassInfoHolder();
- trackingRef = fury.getClassResolver().needToWriteRef(cls);
- }
-
private GenericTypeField(
TypeRef<?> typeRef, String qualifiedFieldName, FieldAccessor accessor,
Fury fury) {
- super(getRegisteredClassId(fury, getRawType(typeRef)),
qualifiedFieldName, accessor);
+ super(typeRef, getRegisteredClassId(fury, getRawType(typeRef)),
qualifiedFieldName, accessor);
// TODO support generics <T> in Pojo<T>, see
ComplexObjectSerializer.getGenericTypes
genericType = fury.getClassResolver().buildGenericType(typeRef);
classInfoHolder = fury.getClassResolver().nilClassInfoHolder();
- trackingRef =
fury.getClassResolver().needToWriteRef(getRawType(typeRef));
+ trackingRef = fury.getClassResolver().needToWriteRef(typeRef);
}
@Override
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/ImmutableSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/ImmutableSerializer.java
index c8cec37c..939f4703 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/ImmutableSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/ImmutableSerializer.java
@@ -37,6 +37,6 @@ public abstract class ImmutableSerializer<T> extends
Serializer<T> {
}
public ImmutableSerializer(Fury fury, Class<T> type, boolean needToWriteRef,
boolean immutable) {
- super(fury, type, needToWriteRef, true);
+ super(fury, type, needToWriteRef, immutable);
}
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/ImmutableSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/NoneSerializer.java
similarity index 62%
copy from
java/fury-core/src/main/java/org/apache/fury/serializer/ImmutableSerializer.java
copy to
java/fury-core/src/main/java/org/apache/fury/serializer/NoneSerializer.java
index c8cec37c..c7cc3e81 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/ImmutableSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/NoneSerializer.java
@@ -20,23 +20,27 @@
package org.apache.fury.serializer;
import org.apache.fury.Fury;
+import org.apache.fury.memory.MemoryBuffer;
-/**
- * Serializer for immutable objects.
- *
- * @param <T> type of objects being serializing/deserializing
- */
-public abstract class ImmutableSerializer<T> extends Serializer<T> {
-
- public ImmutableSerializer(Fury fury, Class<T> type) {
- super(fury, type, true);
+@SuppressWarnings("rawtypes")
+public class NoneSerializer extends Serializer {
+ public NoneSerializer(Fury fury, Class type) {
+ super(fury, type);
}
- public ImmutableSerializer(Fury fury, Class<T> type, boolean needToWriteRef)
{
- super(fury, type, needToWriteRef, true);
+ @Override
+ public void write(MemoryBuffer buffer, Object value) {}
+
+ @Override
+ public Object read(MemoryBuffer buffer) {
+ return null;
}
- public ImmutableSerializer(Fury fury, Class<T> type, boolean needToWriteRef,
boolean immutable) {
- super(fury, type, needToWriteRef, true);
+ @Override
+ public void xwrite(MemoryBuffer buffer, Object value) {}
+
+ @Override
+ public Object xread(MemoryBuffer buffer) {
+ return null;
}
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
index c6faf3e7..320d8afc 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectSerializer.java
@@ -163,11 +163,15 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
if (writeBasicObjectFieldValueFailed(fury, buffer, fieldValue,
classId)) {
Serializer<Object> serializer = fieldInfo.classInfo.getSerializer();
if (!metaShareEnabled || isFinal[i]) {
- // whether tracking ref is recorded in `fieldInfo.serializer`, so
it's still
- // consistent with jit serializer.
- fury.writeRef(buffer, fieldValue, serializer);
+ if (!fieldInfo.trackingRef) {
+ fury.writeNullable(buffer, fieldValue, serializer);
+ } else {
+ // whether tracking ref is recorded in `fieldInfo.serializer`,
so it's still
+ // consistent with jit serializer.
+ fury.writeRef(buffer, fieldValue, serializer);
+ }
} else {
- if (serializer.needToWriteRef()) {
+ if (fieldInfo.trackingRef && serializer.needToWriteRef()) {
if (!refResolver.writeRefOrNull(buffer, fieldValue)) {
classResolver.writeClass(buffer, fieldInfo.classInfo);
// No generics for field, no need to update `depth`.
@@ -336,6 +340,9 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
Serializer<Object> serializer = fieldInfo.classInfo.getSerializer();
Object fieldValue;
if (isFinal) {
+ if (!fieldInfo.trackingRef) {
+ return fury.readNullable(buffer, serializer);
+ }
// whether tracking ref is recorded in `fieldInfo.serializer`, so it's
still
// consistent with jit serializer.
fieldValue = fury.readRef(buffer, serializer);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java
index cdd4129e..f0629000 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java
@@ -46,13 +46,24 @@ import org.apache.fury.util.DateTimeUtils;
/** Serializers for all time related types. */
public class TimeSerializers {
- public abstract static class TimeSerializer<T> extends
ImmutableSerializer<T> {
+ public abstract static class TimeSerializer<T> extends Serializer<T> {
public TimeSerializer(Fury fury, Class<T> type) {
- super(fury, type, !fury.getConfig().isTimeRefIgnored());
+ super(fury, type, !fury.getConfig().isTimeRefIgnored(), false);
}
public TimeSerializer(Fury fury, Class<T> type, boolean needToWriteRef) {
+ super(fury, type, needToWriteRef, false);
+ }
+ }
+
+ public abstract static class ImmutableTimeSerializer<T> extends
ImmutableSerializer<T> {
+
+ public ImmutableTimeSerializer(Fury fury, Class<T> type) {
+ super(fury, type, !fury.getConfig().isTimeRefIgnored());
+ }
+
+ public ImmutableTimeSerializer(Fury fury, Class<T> type, boolean
needToWriteRef) {
super(fury, type, needToWriteRef);
}
}
@@ -92,6 +103,11 @@ public class TimeSerializers {
protected Date newInstance(long time) {
return new Date(time);
}
+
+ @Override
+ public Date copy(Date value) {
+ return newInstance(value.getTime());
+ }
}
public static final class SqlDateSerializer extends
BaseDateSerializer<java.sql.Date> {
@@ -107,6 +123,11 @@ public class TimeSerializers {
protected java.sql.Date newInstance(long time) {
return new java.sql.Date(time);
}
+
+ @Override
+ public java.sql.Date copy(java.sql.Date value) {
+ return newInstance(value.getTime());
+ }
}
public static final class SqlTimeSerializer extends BaseDateSerializer<Time>
{
@@ -123,6 +144,11 @@ public class TimeSerializers {
protected Time newInstance(long time) {
return new Time(time);
}
+
+ @Override
+ public Time copy(Time value) {
+ return newInstance(value.getTime());
+ }
}
public static final class TimestampSerializer extends
TimeSerializer<Timestamp> {
@@ -167,9 +193,14 @@ public class TimeSerializers {
t.setNanos(buffer.readInt32());
return t;
}
+
+ @Override
+ public Timestamp copy(Timestamp value) {
+ return new Timestamp(value.getTime());
+ }
}
- public static final class LocalDateSerializer extends
TimeSerializer<LocalDate> {
+ public static final class LocalDateSerializer extends
ImmutableTimeSerializer<LocalDate> {
public LocalDateSerializer(Fury fury) {
super(fury, LocalDate.class);
}
@@ -218,7 +249,7 @@ public class TimeSerializers {
}
}
- public static final class InstantSerializer extends TimeSerializer<Instant> {
+ public static final class InstantSerializer extends
ImmutableTimeSerializer<Instant> {
public InstantSerializer(Fury fury) {
super(fury, Instant.class);
}
@@ -257,7 +288,7 @@ public class TimeSerializers {
}
}
- public static class DurationSerializer extends TimeSerializer<Duration> {
+ public static class DurationSerializer extends
ImmutableTimeSerializer<Duration> {
public DurationSerializer(Fury fury) {
super(fury, Duration.class);
}
@@ -280,7 +311,7 @@ public class TimeSerializers {
}
}
- public static class LocalDateTimeSerializer extends
TimeSerializer<LocalDateTime> {
+ public static class LocalDateTimeSerializer extends
ImmutableTimeSerializer<LocalDateTime> {
public LocalDateTimeSerializer(Fury fury) {
super(fury, LocalDateTime.class);
}
@@ -303,7 +334,7 @@ public class TimeSerializers {
}
}
- public static class LocalTimeSerializer extends TimeSerializer<LocalTime> {
+ public static class LocalTimeSerializer extends
ImmutableTimeSerializer<LocalTime> {
public LocalTimeSerializer(Fury fury) {
super(fury, LocalTime.class);
}
@@ -384,6 +415,11 @@ public class TimeSerializers {
public TimeZone read(MemoryBuffer buffer) {
return TimeZone.getTimeZone(fury.readJavaString(buffer));
}
+
+ @Override
+ public TimeZone copy(TimeZone value) {
+ return TimeZone.getTimeZone(value.getID());
+ }
}
public static final class CalendarSerializer extends
TimeSerializer<Calendar> {
@@ -427,9 +463,23 @@ public class TimeSerializers {
}
return result;
}
+
+ @Override
+ public Calendar copy(Calendar value) {
+ Calendar copy = Calendar.getInstance(value.getTimeZone());
+ copy.setTimeInMillis(value.getTimeInMillis());
+ copy.setLenient(value.isLenient());
+ copy.setFirstDayOfWeek(value.getFirstDayOfWeek());
+ copy.setMinimalDaysInFirstWeek(value.getMinimalDaysInFirstWeek());
+ if (value instanceof GregorianCalendar) {
+ ((GregorianCalendar) copy)
+ .setGregorianChange(((GregorianCalendar)
value).getGregorianChange());
+ }
+ return copy;
+ }
}
- public static class ZoneIdSerializer extends TimeSerializer<ZoneId> {
+ public static class ZoneIdSerializer extends ImmutableTimeSerializer<ZoneId>
{
public ZoneIdSerializer(Fury fury, Class<ZoneId> type) {
super(fury, type);
}
@@ -449,7 +499,7 @@ public class TimeSerializers {
}
}
- public static class ZoneOffsetSerializer extends TimeSerializer<ZoneOffset> {
+ public static class ZoneOffsetSerializer extends
ImmutableTimeSerializer<ZoneOffset> {
public ZoneOffsetSerializer(Fury fury) {
super(fury, ZoneOffset.class);
}
@@ -483,7 +533,7 @@ public class TimeSerializers {
}
}
- public static class ZonedDateTimeSerializer extends
TimeSerializer<ZonedDateTime> {
+ public static class ZonedDateTimeSerializer extends
ImmutableTimeSerializer<ZonedDateTime> {
public ZonedDateTimeSerializer(Fury fury) {
super(fury, ZonedDateTime.class);
@@ -507,7 +557,7 @@ public class TimeSerializers {
}
}
- public static class YearSerializer extends TimeSerializer<Year> {
+ public static class YearSerializer extends ImmutableTimeSerializer<Year> {
public YearSerializer(Fury fury) {
super(fury, Year.class);
}
@@ -527,7 +577,7 @@ public class TimeSerializers {
}
}
- public static class YearMonthSerializer extends TimeSerializer<YearMonth> {
+ public static class YearMonthSerializer extends
ImmutableTimeSerializer<YearMonth> {
public YearMonthSerializer(Fury fury) {
super(fury, YearMonth.class);
}
@@ -550,7 +600,7 @@ public class TimeSerializers {
}
}
- public static class MonthDaySerializer extends TimeSerializer<MonthDay> {
+ public static class MonthDaySerializer extends
ImmutableTimeSerializer<MonthDay> {
public MonthDaySerializer(Fury fury) {
super(fury, MonthDay.class);
}
@@ -571,7 +621,7 @@ public class TimeSerializers {
}
}
- public static class PeriodSerializer extends TimeSerializer<Period> {
+ public static class PeriodSerializer extends ImmutableTimeSerializer<Period>
{
public PeriodSerializer(Fury fury) {
super(fury, Period.class);
}
@@ -594,7 +644,7 @@ public class TimeSerializers {
}
}
- public static class OffsetTimeSerializer extends TimeSerializer<OffsetTime> {
+ public static class OffsetTimeSerializer extends
ImmutableTimeSerializer<OffsetTime> {
public OffsetTimeSerializer(Fury fury) {
super(fury, OffsetTime.class);
}
@@ -615,7 +665,7 @@ public class TimeSerializers {
}
}
- public static class OffsetDateTimeSerializer extends
TimeSerializer<OffsetDateTime> {
+ public static class OffsetDateTimeSerializer extends
ImmutableTimeSerializer<OffsetDateTime> {
public OffsetDateTimeSerializer(Fury fury) {
super(fury, OffsetDateTime.class);
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java
index 04a68fc3..e0a25cdb 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java
@@ -192,6 +192,9 @@ public abstract class AbstractCollectionSerializer<T>
extends Serializer<T> {
bitmap |= CollectionFlags.NOT_SAME_TYPE |
CollectionFlags.NOT_DECL_ELEMENT_TYPE;
buffer.writeByte(bitmap);
} else {
+ if (elemClass == null) {
+ elemClass = void.class;
+ }
// Write class in case peer doesn't have this class.
if (!fury.getConfig().isMetaShareEnabled() && elemClass ==
declareElementType) {
buffer.writeByte(bitmap);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java
index 274f36b7..d9741139 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java
@@ -679,7 +679,7 @@ public abstract class AbstractMapSerializer<T> extends
Serializer<T> {
} else {
readNullKeyChunk(buffer, map, chunkHeader, valueSerializer,
valueHasNull);
}
- if (size-- == 0) {
+ if (--size == 0) {
return 0;
} else {
chunkHeader = buffer.readUnsignedByte();
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java
index 60353e78..985c9cc8 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java
@@ -37,7 +37,6 @@ public class SubListSerializers {
private static final Class<?> SubListClass;
private static final Class<?> RandomAccessSubListClass;
private static final Class<?> ArrayListSubListClass;
- private static final Class<?> ImmutableSubListClass;
private interface Stub {}
@@ -62,22 +61,13 @@ public class SubListSerializers {
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
- Class<?> cls;
- try {
- cls = Class.forName("java.util.ImmutableCollections$SubList");
- } catch (ClassNotFoundException e) {
- class ImmutableSubListStub implements Stub {}
-
- cls = ImmutableSubListStub.class;
- }
- ImmutableSubListClass = cls;
}
public static void registerSerializers(Fury fury, boolean preserveView) {
+ // java.util.ImmutableCollections$SubList is already registered in
+ // ImmutableCollectionSerializers
for (Class<?> cls :
- new Class[] {
- SubListClass, RandomAccessSubListClass, ArrayListSubListClass,
ImmutableSubListClass
- }) {
+ new Class[] {SubListClass, RandomAccessSubListClass,
ArrayListSubListClass}) {
if (fury.trackingRef() && preserveView && fury.getConfig().getLanguage()
== Language.JAVA) {
fury.registerSerializer(cls, new SubListViewSerializer(fury, cls));
} else {
diff --git a/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java
b/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java
index 86202fea..677e7104 100644
--- a/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java
+++ b/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java
@@ -36,6 +36,7 @@ import org.apache.fury.serializer.Serializer;
/** GenericType for building java generics as a tree and binding with fury
serializers. */
// TODO(chaokunyang) refine generics which can be inspired by spring
ResolvableType.
+@SuppressWarnings("rawtypes")
public class GenericType {
private static final Predicate<Type> defaultFinalPredicate =
type -> {
@@ -115,11 +116,15 @@ public class GenericType {
}
public static GenericType build(TypeRef<?> context, Type type,
Predicate<Type> finalPredicate) {
- return build(context.resolveType(type).getType(), finalPredicate);
+ return build(context.resolveType(type), finalPredicate);
}
- @SuppressWarnings("rawtypes")
public static GenericType build(Type type, Predicate<Type> finalPredicate) {
+ return build(TypeRef.of(type), finalPredicate);
+ }
+
+ public static GenericType build(TypeRef<?> typeRef, Predicate<Type>
finalPredicate) {
+ Type type = typeRef.getType();
if (type instanceof ParameterizedType) {
// List<String>, List<T>, Map<String, List<String>>, SomeClass<T>
Type[] actualTypeArguments = ((ParameterizedType)
type).getActualTypeArguments();
@@ -129,10 +134,10 @@ public class GenericType {
list.add(build);
}
GenericType[] genericTypes = list.toArray(new GenericType[0]);
- return new GenericType(TypeRef.of(type), finalPredicate.test(type),
genericTypes);
+ return new GenericType(typeRef, finalPredicate.test(type), genericTypes);
} else if (type instanceof GenericArrayType) { // List<String>[] or T[]
Type componentType = ((GenericArrayType) type).getGenericComponentType();
- return new GenericType(TypeRef.of(type), finalPredicate.test(type),
build(componentType));
+ return new GenericType(typeRef, finalPredicate.test(type),
build(componentType));
} else if (type instanceof TypeVariable) { // T
TypeVariable typeVariable = (TypeVariable) type;
Type typeVariableBound =
@@ -149,7 +154,7 @@ public class GenericType {
}
} else {
// Class type: String, Integer
- return new GenericType(TypeRef.of(type), finalPredicate.test(type));
+ return new GenericType(typeRef, finalPredicate.test(type));
}
}
@@ -201,7 +206,7 @@ public class GenericType {
public boolean trackingRef(ClassResolver classResolver) {
Boolean trackingRef = this.trackingRef;
if (trackingRef == null) {
- trackingRef = this.trackingRef = classResolver.needToWriteRef(cls);
+ trackingRef = this.trackingRef = classResolver.needToWriteRef(typeRef);
}
return trackingRef;
}
diff --git a/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
b/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
index 4c38f53a..108c38c8 100644
--- a/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
+++ b/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
@@ -511,6 +511,10 @@ public class TypeUtils {
return new TypeRef<Collection<E>>() {}.where(new TypeParameter<E>() {},
elemType);
}
+ public static <E> TypeRef<Collection<E>> collectionOf(TypeRef<E> elemType,
Object extMeta) {
+ return new TypeRef<Collection<E>>(extMeta) {}.where(new TypeParameter<E>()
{}, elemType);
+ }
+
public static <K, V> TypeRef<Map<K, V>> mapOf(Class<K> keyType, Class<V>
valueType) {
return mapOf(TypeRef.of(keyType), TypeRef.of(valueType));
}
@@ -520,6 +524,12 @@ public class TypeUtils {
.where(new TypeParameter<V>() {}, valueType);
}
+ public static <K, V> TypeRef<Map<K, V>> mapOf(
+ TypeRef<K> keyType, TypeRef<V> valueType, Object extMeta) {
+ return new TypeRef<Map<K, V>>(extMeta) {}.where(new TypeParameter<K>() {},
keyType)
+ .where(new TypeParameter<V>() {}, valueType);
+ }
+
public static <K, V> TypeRef<? extends Map<K, V>> mapOf(
Class<?> mapType, TypeRef<K> keyType, TypeRef<V> valueType) {
TypeRef<Map<K, V>> mapTypeRef = mapOf(keyType, valueType);
@@ -675,21 +685,37 @@ public class TypeUtils {
typeRefs.addAll(getAllTypeArguments(typeRef));
}
- typeRefs.stream()
- .filter(typeToken -> isBean(getRawType(typeToken)))
- .forEach(
- typeToken -> {
- Class<?> cls = getRawType(typeToken);
- beans.add(cls);
- if (walkedTypePath.contains(typeToken)) {
- throw new UnsupportedOperationException(
- "cyclic type is not supported. walkedTypePath: " +
walkedTypePath);
- } else {
- LinkedHashSet<TypeRef<?>> newPath = new
LinkedHashSet<>(walkedTypePath);
- newPath.add(typeToken);
- beans.addAll(listBeansRecursiveInclusive(cls, newPath));
- }
- });
+ for (TypeRef<?> typeToken : typeRefs) {
+ Class<?> type = getRawType(typeToken);
+ if (isBean(type)) {
+ beans.add(type);
+ if (walkedTypePath.contains(typeToken)) {
+ throw new UnsupportedOperationException(
+ "cyclic type is not supported. walkedTypePath: " +
walkedTypePath);
+ } else {
+ LinkedHashSet<TypeRef<?>> newPath = new
LinkedHashSet<>(walkedTypePath);
+ newPath.add(typeToken);
+ beans.addAll(listBeansRecursiveInclusive(type, newPath));
+ }
+ } else if (isCollection(type)) {
+ TypeRef<?> elementType = getElementType(typeToken);
+ LinkedHashSet<TypeRef<?>> newPath = new
LinkedHashSet<>(walkedTypePath);
+ newPath.add(elementType);
+ beans.addAll(listBeansRecursiveInclusive(elementType.getClass(),
newPath));
+ } else if (isMap(type)) {
+ Tuple2<TypeRef<?>, TypeRef<?>> mapKeyValueType =
getMapKeyValueType(typeToken);
+ LinkedHashSet<TypeRef<?>> newPath = new
LinkedHashSet<>(walkedTypePath);
+ newPath.add(mapKeyValueType.f0);
+ newPath.add(mapKeyValueType.f1);
+
beans.addAll(listBeansRecursiveInclusive(mapKeyValueType.f0.getRawType(),
newPath));
+
beans.addAll(listBeansRecursiveInclusive(mapKeyValueType.f1.getRawType(),
newPath));
+ } else if (type.isArray()) {
+ Class<?> arrayComponent = getArrayComponent(type);
+ LinkedHashSet<TypeRef<?>> newPath = new
LinkedHashSet<>(walkedTypePath);
+ newPath.add(TypeRef.of(arrayComponent));
+ beans.addAll(listBeansRecursiveInclusive(arrayComponent, newPath));
+ }
+ }
return beans;
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/util/LoaderBinding.java
b/java/fury-core/src/main/java/org/apache/fury/util/LoaderBinding.java
index 3b747d22..883a33c6 100644
--- a/java/fury-core/src/main/java/org/apache/fury/util/LoaderBinding.java
+++ b/java/fury-core/src/main/java/org/apache/fury/util/LoaderBinding.java
@@ -20,7 +20,6 @@
package org.apache.fury.util;
import java.lang.ref.SoftReference;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
@@ -69,7 +68,7 @@ public final class LoaderBinding {
}
} else {
Set<Fury> furySet = new HashSet<>(furyMap.size());
- Collections.addAll(furyMap.values());
+ furySet.addAll(furyMap.values());
for (SoftReference<Fury> ref : furySoftMap.values()) {
Fury f = ref.get();
if (f != null) {
diff --git
a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
index 7c751490..f45a363d 100644
---
a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
+++
b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
@@ -235,6 +235,7 @@
Args=--initialize-at-build-time=org.apache.fury.memory.MemoryBuffer,\
org.apache.fury.meta.ClassDefDecoder,\
org.apache.fury.meta.ClassDefEncoder,\
org.apache.fury.meta.ClassSpec,\
+ org.apache.fury.meta.TypeExtMeta,\
org.apache.fury.meta.DeflaterMetaCompressor,\
org.apache.fury.meta.Encoders,\
org.apache.fury.meta.MetaCompressor,\
@@ -298,6 +299,7 @@
Args=--initialize-at-build-time=org.apache.fury.memory.MemoryBuffer,\
org.apache.fury.serializer.LambdaSerializer$ReplaceStub,\
org.apache.fury.serializer.LambdaSerializer,\
org.apache.fury.serializer.LocaleSerializer,\
+ org.apache.fury.serializer.NoneSerializer,\
org.apache.fury.serializer.NonexistentClassSerializers$ClassFieldsInfo,\
org.apache.fury.serializer.NonexistentClassSerializers$NonexistentClassSerializer,\
org.apache.fury.serializer.NonexistentClassSerializers$NonexistentEnumClassSerializer,\
diff --git a/java/fury-core/src/main/resources/fury/disallowed.txt
b/java/fury-core/src/main/resources/fury/disallowed.txt
index bebfc692..2a6e5b02 100644
--- a/java/fury-core/src/main/resources/fury/disallowed.txt
+++ b/java/fury-core/src/main/resources/fury/disallowed.txt
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+# Note: update `DisallowedList#SHA256_HASH` when added new classes.
bsh.Interpreter
bsh.XThis
ch.qos.logback.core.db.DriverManagerConnectionSource
@@ -262,6 +263,7 @@ org.thymeleaf.standard.expression.Expression
org.thymeleaf.standard.expression.StandardExpressionParser
org.yaml.snakeyaml.tokens.DirectiveToken
pstore.shaded.org.apache.commons.collections.functors.InvokerTransformer
+sun.print.PrintServiceLookupProvider
sun.print.UnixPrintService
sun.print.UnixPrintServiceLookup
sun.rmi.server.UnicastRef
diff --git a/java/fury-core/src/test/java/org/apache/fury/FuryCopyTest.java
b/java/fury-core/src/test/java/org/apache/fury/FuryCopyTest.java
index 4b1a7107..26a127de 100644
--- a/java/fury-core/src/test/java/org/apache/fury/FuryCopyTest.java
+++ b/java/fury-core/src/test/java/org/apache/fury/FuryCopyTest.java
@@ -45,6 +45,7 @@ import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
+import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.Currency;
@@ -62,6 +63,7 @@ import java.util.OptionalDouble;
import java.util.OptionalInt;
import java.util.OptionalLong;
import java.util.PriorityQueue;
+import java.util.TimeZone;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
@@ -130,6 +132,7 @@ public class FuryCopyTest extends FuryTestBase {
collectionCopyTest();
mapCopyTest();
objectCopyTest();
+ dateCopyTest();
}
@Test
@@ -255,10 +258,6 @@ public class FuryCopyTest extends FuryTestBase {
}
private void timeCopyTest() {
- assertSame(new java.util.Date());
- assertSame(new java.sql.Date(System.currentTimeMillis()));
- assertSame(new Time(System.currentTimeMillis()));
- assertSame(new Timestamp(System.currentTimeMillis()));
assertSame(LocalDate.now());
assertSame(LocalTime.now());
assertSame(LocalDateTime.now());
@@ -274,6 +273,15 @@ public class FuryCopyTest extends FuryTestBase {
assertSame(OffsetDateTime.now());
}
+ private void dateCopyTest() {
+ assetEqualsButNotSame(new java.util.Date());
+ assetEqualsButNotSame(new java.sql.Date(System.currentTimeMillis()));
+ assetEqualsButNotSame(new Time(System.currentTimeMillis()));
+ assetEqualsButNotSame(new Timestamp(System.currentTimeMillis()));
+ assetEqualsButNotSame(Calendar.getInstance());
+ assetEqualsButNotSame(TimeZone.getDefault());
+ }
+
private void assertSame(Object obj) {
Assert.assertSame(obj, fury.copy(obj));
}
diff --git a/java/fury-core/src/test/java/org/apache/fury/StreamTest.java
b/java/fury-core/src/test/java/org/apache/fury/StreamTest.java
index 56e7da3c..efac23ea 100644
--- a/java/fury-core/src/test/java/org/apache/fury/StreamTest.java
+++ b/java/fury-core/src/test/java/org/apache/fury/StreamTest.java
@@ -34,6 +34,7 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.fury.config.CompatibleMode;
import org.apache.fury.io.FuryInputStream;
import org.apache.fury.io.FuryReadableChannel;
@@ -378,4 +379,38 @@ public class StreamTest extends FuryTestBase {
assertEquals(fury.deserialize(stream), new long[5000]);
assertEquals(fury.deserialize(stream), new int[5000]);
}
+
+ @Test
+ public void testReadNullChunkMapOnFillBound() {
+ Fury fury = builder().build();
+ Map<String, String> m = new HashMap<>();
+ m.put("1", null);
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream(100);
+ fury.serialize(outputStream, m);
+ InputStream inputStream = new
ByteArrayInputStream(outputStream.toByteArray());
+ FuryInputStream input = new FuryInputStream(inputStream);
+ assertEquals(fury.deserialize(input), m);
+ }
+
+ public static class SimpleType {
+ public double dVal;
+
+ public SimpleType() {
+ dVal = 0.5;
+ }
+ }
+
+ // For issue https://github.com/apache/fury/issues/2060
+ @Test
+ public void testReadPrimitivesOnBufferFillBound() {
+ Fury fury = builder().build();
+ fury.register(SimpleType.class);
+ SimpleType v = new SimpleType();
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ fury.serialize(outputStream, v);
+ InputStream inputStream = new
ByteArrayInputStream(outputStream.toByteArray());
+ FuryInputStream input = new FuryInputStream(inputStream, 11);
+ SimpleType newValue = (SimpleType) fury.deserialize(input);
+ Assert.assertEquals(v.dVal, newValue.dVal, 0.001);
+ }
}
diff --git
a/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java
b/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java
index 89435b21..7ecf20b0 100644
--- a/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java
+++ b/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java
@@ -20,6 +20,7 @@
package org.apache.fury.meta;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
import com.google.common.collect.ImmutableList;
@@ -34,6 +35,9 @@ import org.apache.fury.Fury;
import org.apache.fury.FuryTestBase;
import org.apache.fury.memory.MemoryBuffer;
import org.apache.fury.reflect.ReflectionUtils;
+import org.apache.fury.reflect.TypeRef;
+import org.apache.fury.resolver.ClassResolver;
+import org.apache.fury.test.bean.Foo;
import org.testng.annotations.Test;
public class ClassDefTest extends FuryTestBase {
@@ -176,4 +180,12 @@ public class ClassDefTest extends FuryTestBase {
assertTrue(classDef.getFieldsInfo().isEmpty());
assertTrue(classDef.isObjectType());
}
+
+ @Test
+ public void testTypeExtInfo() {
+ Fury fury = Fury.builder().withMetaShare(true).build();
+ ClassResolver classResolver = fury.getClassResolver();
+ assertTrue(classResolver.needToWriteRef(TypeRef.of(Foo.class, new
TypeExtMeta(true))));
+ assertFalse(classResolver.needToWriteRef(TypeRef.of(Foo.class, new
TypeExtMeta(false))));
+ }
}
diff --git
a/java/fury-core/src/test/java/org/apache/fury/resolver/ClassResolverTest.java
b/java/fury-core/src/test/java/org/apache/fury/resolver/ClassResolverTest.java
index ecb35305..02cb5b4f 100644
---
a/java/fury-core/src/test/java/org/apache/fury/resolver/ClassResolverTest.java
+++
b/java/fury-core/src/test/java/org/apache/fury/resolver/ClassResolverTest.java
@@ -53,6 +53,7 @@ import org.apache.fury.logging.Logger;
import org.apache.fury.logging.LoggerFactory;
import org.apache.fury.memory.MemoryBuffer;
import org.apache.fury.memory.MemoryUtils;
+import org.apache.fury.reflect.TypeRef;
import org.apache.fury.resolver.longlongpkg.C1;
import org.apache.fury.resolver.longlongpkg.C2;
import org.apache.fury.resolver.longlongpkg.C3;
@@ -293,7 +294,8 @@ public class ClassResolverTest extends FuryTestBase {
.requireClassRegistration(false)
.build();
ClassResolver classResolver = fury.getClassResolver();
-
Assert.assertFalse(classResolver.needToWriteRef(TestNeedToWriteReferenceClass.class));
+ Assert.assertFalse(
+
classResolver.needToWriteRef(TypeRef.of(TestNeedToWriteReferenceClass.class)));
assertNull(classResolver.getClassInfo(TestNeedToWriteReferenceClass.class,
false));
}
diff --git
a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/CollectionSerializersTest.java
b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/CollectionSerializersTest.java
index 68c93173..c13abc91 100644
---
a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/CollectionSerializersTest.java
+++
b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/CollectionSerializersTest.java
@@ -904,4 +904,33 @@ public class CollectionSerializersTest extends
FuryTestBase {
copyCheck(fury, test);
}
}
+
+ @Test(dataProvider = "enableCodegen")
+ public void testCollectionAllNullElements(boolean enableCodegen) {
+ Fury fury =
+ Fury.builder()
+ .withCodegen(true)
+ .withRefTracking(true)
+ .requireClassRegistration(false)
+ .build();
+ List<Foo> fooList = new ArrayList<>();
+ fooList.add(null);
+ // serDeCheck(fury, fooList);
+
+ CollectionAbstractTest obj = new CollectionAbstractTest();
+ // fill elemTypeCache
+ obj.fooList = ofArrayList(new Foo1());
+ serDeCheck(fury, obj);
+
+ obj.fooList = fooList;
+ serDeCheck(fury, obj);
+
+ fury =
+ Fury.builder()
+ .withCodegen(enableCodegen)
+ .withRefTracking(true)
+ .requireClassRegistration(false)
+ .build();
+ serDeCheck(fury, obj);
+ }
}
diff --git
a/java/fury-core/src/test/java/org/apache/fury/type/TypeUtilsTest.java
b/java/fury-core/src/test/java/org/apache/fury/type/TypeUtilsTest.java
index 0dbd5851..848f8f3d 100644
--- a/java/fury-core/src/test/java/org/apache/fury/type/TypeUtilsTest.java
+++ b/java/fury-core/src/test/java/org/apache/fury/type/TypeUtilsTest.java
@@ -132,6 +132,27 @@ public class TypeUtilsTest {
assertEquals(classes.size(), 2);
}
+ public static class ListBeanGenericFoo1 {}
+
+ public static class ListBeanGenericFoo2 {}
+
+ public static class ListBeanGenericFoo3 {}
+
+ public static class ListBeanGenericFoo4 {}
+
+ public static class ListBeanGeneric {
+ List<ListBeanGenericFoo1> f1;
+ ListBeanGenericFoo2[] f2;
+ Map<ListBeanGenericFoo3, ListBeanGenericFoo4> f3;
+ }
+
+ @Test
+ public void listBeanGeneric() {
+ LinkedHashSet<Class<?>> classes =
TypeUtils.listBeansRecursiveInclusive(ListBeanGeneric.class);
+ // System.out.println(classes);
+ assertEquals(classes.size(), 5);
+ }
+
@Test
public void isBean() {
Assert.assertTrue(TypeUtils.isBean(BeanA.class));
diff --git a/rust/fury-core/src/buffer.rs b/rust/fury-core/src/buffer.rs
index 5668034c..78f6aef1 100644
--- a/rust/fury-core/src/buffer.rs
+++ b/rust/fury-core/src/buffer.rs
@@ -96,20 +96,20 @@ impl Writer {
self.u16(((u1 << 8) | u2) as u16);
} else if value >> 21 == 0 {
let u1 = (value & 0x7F) | 0x80;
- let u2 = value >> 7 | 0x80;
+ let u2 = (value >> 7) | 0x80;
self.u16(((u1 << 8) | u2) as u16);
self.u8((value >> 14) as u8);
} else if value >> 28 == 0 {
let u1 = (value & 0x7F) | 0x80;
- let u2 = value >> 7 | 0x80;
- let u3 = value >> 14 | 0x80;
- let u4 = value >> 21 | 0x80;
+ let u2 = (value >> 7) | 0x80;
+ let u3 = (value >> 14) | 0x80;
+ let u4 = (value >> 21) | 0x80;
self.u32(((u1 << 24) | (u2 << 16) | (u3 << 8) | u4) as u32);
} else {
let u1 = (value & 0x7F) | 0x80;
- let u2 = value >> 7 | 0x80;
- let u3 = value >> 14 | 0x80;
- let u4 = value >> 21 | 0x80;
+ let u2 = (value >> 7) | 0x80;
+ let u3 = (value >> 14) | 0x80;
+ let u4 = (value >> 21) | 0x80;
self.u32(((u1 << 24) | (u2 << 16) | (u3 << 8) | u4) as u32);
self.u8((value >> 28) as u8);
}
diff --git a/rust/fury-core/src/meta/meta_string.rs
b/rust/fury-core/src/meta/meta_string.rs
index df74c384..4214fa55 100644
--- a/rust/fury-core/src/meta/meta_string.rs
+++ b/rust/fury-core/src/meta/meta_string.rs
@@ -326,16 +326,16 @@ impl MetaStringDecoder {
let byte_index = bit_index / 8;
let intra_byte_index = bit_index % 8;
let char_value: usize = if intra_byte_index > 3 {
- ((data[byte_index] as usize) << 8
+ ((((data[byte_index] as usize) << 8)
| if byte_index + 1 < data.len() {
- data.get(byte_index + 1).cloned().unwrap() as usize &
0xFF
+ (data.get(byte_index + 1).cloned().unwrap() as usize)
& 0xFF
} else {
0
})
- >> (11 - intra_byte_index)
+ >> (11 - intra_byte_index))
& bit_mask
} else {
- (data[byte_index] as usize) >> (3 - intra_byte_index) &
bit_mask
+ ((data[byte_index] as usize) >> (3 - intra_byte_index)) &
bit_mask
};
bit_index += 5;
decoded.push(self.decode_lower_special_char(char_value as u8)?);
@@ -353,16 +353,16 @@ impl MetaStringDecoder {
let byte_index = bit_index / 8;
let intra_byte_index = bit_index % 8;
let char_value: usize = if intra_byte_index > 2 {
- ((data[byte_index] as usize) << 8
+ ((((data[byte_index] as usize) << 8)
| if byte_index + 1 < data.len() {
data.get(byte_index + 1).cloned().unwrap() as usize &
0xFF
} else {
0
})
- >> (10 - intra_byte_index)
+ >> (10 - intra_byte_index))
& bit_mask
} else {
- (data[byte_index] as usize) >> (2 - intra_byte_index) &
bit_mask
+ ((data[byte_index] as usize) >> (2 - intra_byte_index)) &
bit_mask
};
bit_index += 6;
decoded.push(self.decode_lower_upper_digit_special_char(char_value
as u8)?);
diff --git a/rust/fury-core/src/util.rs b/rust/fury-core/src/util.rs
index 458a7f21..2a468779 100644
--- a/rust/fury-core/src/util.rs
+++ b/rust/fury-core/src/util.rs
@@ -58,7 +58,7 @@ pub fn to_utf8(utf16: &[u16], is_little_endian: bool) ->
Result<Vec<u8>, String>
// 2-byte UTF-8
// [0000|0bbb|bbcc|cccc] => [110|bbbbb], [10|cccccc]
let bytes = [
- (code_point >> 6 & 0b1_1111) as u8 | 0b1100_0000,
+ ((code_point >> 6) & 0b1_1111) as u8 | 0b1100_0000,
(code_point & 0b11_1111) as u8 | 0b1000_0000,
];
unsafe {
@@ -84,9 +84,9 @@ pub fn to_utf8(utf16: &[u16], is_little_endian: bool) ->
Result<Vec<u8>, String>
// 11110??? 10?????? 10?????? 10??????
// Need 21 bit suffix of code_point
let bytes = [
- (code_point >> 18 & 0b111) as u8 | 0b1111_0000,
- (code_point >> 12 & 0b11_1111) as u8 | 0b1000_0000,
- (code_point >> 6 & 0b11_1111) as u8 | 0b1000_0000,
+ ((code_point >> 18) & 0b111) as u8 | 0b1111_0000,
+ ((code_point >> 12) & 0b11_1111) as u8 | 0b1000_0000,
+ ((code_point >> 6) & 0b11_1111) as u8 | 0b1000_0000,
(code_point & 0b11_1111) as u8 | 0b1000_0000,
];
unsafe {
@@ -102,8 +102,8 @@ pub fn to_utf8(utf16: &[u16], is_little_endian: bool) ->
Result<Vec<u8>, String>
// [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10|bbbbbb],
[10|cccccc]
// Need 16 bit suffix of wc, as same as wc itself
let bytes = [
- (wc >> 12 | 0b1110_0000) as u8,
- (wc >> 6 & 0b11_1111) as u8 | 0b1000_0000,
+ ((wc >> 12) | 0b1110_0000) as u8,
+ ((wc >> 6) & 0b11_1111) as u8 | 0b1000_0000,
(wc & 0b11_1111) as u8 | 0b1000_0000,
];
unsafe {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
