This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new bf6eb235 perf(java): optimize fury creation speed (#1511)
bf6eb235 is described below
commit bf6eb23584a3d43bd7256bbae3d8e6426ee872a8
Author: Shawn Yang <[email protected]>
AuthorDate: Sun Apr 14 20:46:47 2024 +0800
perf(java): optimize fury creation speed (#1511)
## What does this PR do?
## Related issues
<!--
Is there any related issue? Please attach here.
- #xxxx0
- #xxxx1
- #xxxx2
-->
Closes #1510
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/incubator-fury/issues/new/choose)
describing the need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
.../org/apache/fury/resolver/ClassResolver.java | 67 +++++++++++-----------
.../main/java/org/apache/fury/type/TypeUtils.java | 4 +-
.../main/java/org/apache/fury/util/Platform.java | 11 +---
3 files changed, 38 insertions(+), 44 deletions(-)
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
index 84654e2a..ad60e9af 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
@@ -195,10 +195,10 @@ public class ClassResolver {
public static final short HASHSET_CLASS_ID = (short)
(PRIMITIVE_DOUBLE_ARRAY_CLASS_ID + 5);
public static final short CLASS_CLASS_ID = (short)
(PRIMITIVE_DOUBLE_ARRAY_CLASS_ID + 6);
public static final short EMPTY_OBJECT_ID = (short)
(PRIMITIVE_DOUBLE_ARRAY_CLASS_ID + 7);
- private static final int initialCapacity = 128;
// use a lower load factor to minimize hash collision
private static final float loadFactor = 0.25f;
private static final float furyMapLoadFactor = 0.25f;
+ private static final int estimatedNumRegistered = 150;
private static final String META_SHARE_FIELDS_INFO_KEY = "shareFieldsInfo";
private static final ClassInfo NIL_CLASS_INFO =
new ClassInfo(null, null, null, null, false, null, null,
ClassResolver.NO_CLASS_ID);
@@ -208,17 +208,15 @@ public class ClassResolver {
// IdentityMap has better lookup performance, when loadFactor is 0.05f,
performance is better
private final IdentityMap<Class<?>, ClassInfo> classInfoMap =
- new IdentityMap<>(initialCapacity, furyMapLoadFactor);
+ new IdentityMap<>(estimatedNumRegistered, furyMapLoadFactor);
private ClassInfo classInfoCache;
private final ObjectMap<EnumStringBytes, Class<?>> classNameBytes2Class =
- new ObjectMap<>(initialCapacity, furyMapLoadFactor);
+ new ObjectMap<>(16, furyMapLoadFactor);
// Every deserialization for unregistered class will query it, performance
is important.
private final ObjectMap<ClassNameBytes, Class<?>>
compositeClassNameBytes2Class =
- new ObjectMap<>(initialCapacity, furyMapLoadFactor);
- private final HashMap<Short, Class<?>> typeIdToClassXLangMap =
- new HashMap<>(initialCapacity, loadFactor);
- private final HashMap<String, Class<?>> typeTagToClassXLangMap =
- new HashMap<>(initialCapacity, loadFactor);
+ new ObjectMap<>(16, furyMapLoadFactor);
+ private final HashMap<Short, Class<?>> typeIdToClassXLangMap = new
HashMap<>(8, loadFactor);
+ private final HashMap<String, Class<?>> typeTagToClassXLangMap = new
HashMap<>(8, loadFactor);
private final EnumStringResolver enumStringResolver;
private final boolean metaContextShareEnabled;
private final Map<Class<?>, ClassDef> classDefMap = new HashMap<>();
@@ -234,8 +232,8 @@ public class ClassResolver {
private short classIdGenerator = 1;
private SerializerFactory serializerFactory;
private final IdentityMap<Class<?>, Short> registeredClassIdMap =
- new IdentityMap<>(initialCapacity);
- private final Map<String, Class<?>> registeredClasses = new
HashMap<>(initialCapacity);
+ new IdentityMap<>(estimatedNumRegistered);
+ private final Map<String, Class<?>> registeredClasses = new
HashMap<>(estimatedNumRegistered);
// avoid potential recursive call for seq codec generation.
// ex. A->field1: B, B.field1: A
private final Set<Class<?>> getClassCtx = new HashSet<>();
@@ -365,8 +363,8 @@ public class ClassResolver {
}
private void registerDefaultClasses() {
- register(ByteBuffer.allocate(1).getClass());
- register(ByteBuffer.allocateDirect(1).getClass());
+ register(Platform.HEAP_BYTE_BUFFER_CLASS);
+ register(Platform.DIRECT_BYTE_BUFFER_CLASS);
register(Comparator.naturalOrder().getClass());
register(Comparator.reverseOrder().getClass());
register(ConcurrentHashMap.class);
@@ -429,17 +427,18 @@ public class ClassResolver {
public void register(Class<?> cls, int classId) {
// class id must be less than Integer.MAX_VALUE/2 since we use bit 0 as
class id flag.
Preconditions.checkArgument(classId >= 0 && classId < Short.MAX_VALUE);
- Preconditions.checkArgument(
- !extRegistry.registeredClassIdMap.containsKey(cls),
- String.format(
- "Class %s already registered with id %s.",
- cls, extRegistry.registeredClassIdMap.get(cls)));
- Preconditions.checkArgument(
- !extRegistry.registeredClasses.containsKey(cls.getName()),
- String.format(
- "Class %s with name %s has been registered, registering class with
same name are not allowed.",
- extRegistry.registeredClasses.get(cls.getName()), cls.getName()));
-
+ if (extRegistry.registeredClassIdMap.containsKey(cls)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Class %s already registered with id %s.",
+ cls, extRegistry.registeredClassIdMap.get(cls)));
+ }
+ if (extRegistry.registeredClasses.containsKey(cls.getName())) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Class %s with name %s has been registered, registering class
with same name are not allowed.",
+ extRegistry.registeredClasses.get(cls.getName()),
cls.getName()));
+ }
short id = (short) classId;
if (id < registeredId2ClassInfo.length && registeredId2ClassInfo[id] !=
null) {
throw new IllegalArgumentException(
@@ -1136,23 +1135,15 @@ public class ClassResolver {
private Serializer createSerializer(Class<?> cls) {
DisallowedList.checkNotInDisallowedList(cls.getName());
- String msg =
- String.format(
- "%s is not registered, please check whether it's the type you want
to serialize or "
- + "a **vulnerability**. If safe, you should invoke
`Fury#register` to register class, "
- + " which will have better performance by skipping classname
serialization. "
- + "If your env is 100%% secure, you can also avoid this
exception by disabling class "
- + "registration check using
`FuryBuilder#requireClassRegistration(false)`",
- cls);
if (!isSecure(cls)) {
- throw new InsecureException(msg);
+ throw new InsecureException(generateSecurityMsg(cls));
} else {
if (!fury.getConfig().suppressClassRegistrationWarnings()
&& !Functions.isLambda(cls)
&& !ReflectionUtils.isJdkProxy(cls)
&& !extRegistry.registeredClassIdMap.containsKey(cls)
&& !shimDispatcher.contains(cls)) {
- LOG.warn(msg);
+ LOG.warn(generateSecurityMsg(cls));
}
}
@@ -1172,6 +1163,16 @@ public class ClassResolver {
return Serializers.newSerializer(fury, cls, serializerClass);
}
+ private String generateSecurityMsg(Class<?> cls) {
+ String tpl =
+ "%s is not registered, please check whether it's the type you want to
serialize or "
+ + "a **vulnerability**. If safe, you should invoke `Fury#register`
to register class, "
+ + " which will have better performance by skipping classname
serialization. "
+ + "If your env is 100%% secure, you can also avoid this exception
by disabling class "
+ + "registration check using
`FuryBuilder#requireClassRegistration(false)`";
+ return String.format(tpl, cls);
+ }
+
private boolean isSecure(Class<?> cls) {
if (extRegistry.registeredClassIdMap.containsKey(cls) ||
shimDispatcher.contains(cls)) {
return true;
diff --git a/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
b/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
index 25f2009b..459ab9e8 100644
--- a/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
+++ b/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
@@ -188,8 +188,8 @@ public class TypeUtils {
Long.class,
Double.class);
private static final int[] sortedSizes = new int[] {0, 1, 1, 2, 2, 4, 4, 8,
8};
- private static final IdentityMap<Class<?>, Class<?>> primToWrap = new
IdentityMap<>(9);
- private static final IdentityMap<Class<?>, Class<?>> wrapToPrim = new
IdentityMap<>(9);
+ private static final IdentityMap<Class<?>, Class<?>> primToWrap = new
IdentityMap<>(18);
+ private static final IdentityMap<Class<?>, Class<?>> wrapToPrim = new
IdentityMap<>(18);
static {
add(primToWrap, wrapToPrim, boolean.class, Boolean.class);
diff --git a/java/fury-core/src/main/java/org/apache/fury/util/Platform.java
b/java/fury-core/src/main/java/org/apache/fury/util/Platform.java
index feb3af2a..4a64bb97 100644
--- a/java/fury-core/src/main/java/org/apache/fury/util/Platform.java
+++ b/java/fury-core/src/main/java/org/apache/fury/util/Platform.java
@@ -36,7 +36,8 @@ public final class Platform {
public static final int JAVA_VERSION = _JDKAccess.JAVA_VERSION;
public static final boolean IS_LITTLE_ENDIAN = ByteOrder.nativeOrder() ==
ByteOrder.LITTLE_ENDIAN;
-
+ public static final Class<?> HEAP_BYTE_BUFFER_CLASS =
ByteBuffer.allocate(1).getClass();
+ public static final Class<?> DIRECT_BYTE_BUFFER_CLASS =
ByteBuffer.allocateDirect(1).getClass();
public static final int BOOLEAN_ARRAY_OFFSET;
public static final int BYTE_ARRAY_OFFSET;
@@ -123,14 +124,6 @@ public final class Platform {
}
}
- private static Class<?>
getClassByName(@SuppressWarnings("SameParameterValue") String className) {
- try {
- return Class.forName(className);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
- }
- }
-
/**
* Returns true when running JVM is having sun's Unsafe package available in
it and underlying
* system having unaligned-access capability.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
