This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch releases-0.10
in repository https://gitbox.apache.org/repos/asf/fury.git
The following commit(s) were added to refs/heads/releases-0.10 by this push:
new 015b852a chore: cherry pick to 0.10 (#2048)
015b852a is described below
commit 015b852a3d7e77e00195e0fd1a4174f53904e915
Author: Shawn Yang <[email protected]>
AuthorDate: Fri Feb 7 15:14:13 2025 +0800
chore: cherry pick to 0.10 (#2048)
<!--
**Thanks for contributing to Fury.**
**If this is your first time opening a PR on fury, you can refer to
[CONTRIBUTING.md](https://github.com/apache/fury/blob/main/CONTRIBUTING.md).**
Contribution Checklist
- The **Apache Fury (incubating)** community has restrictions on the
naming of pr titles. You can also find instructions in
[CONTRIBUTING.md](https://github.com/apache/fury/blob/main/CONTRIBUTING.md).
- Fury has a strong focus on performance. If the PR you submit will have
an impact on performance, please benchmark it first and provide the
benchmark result here.
-->
## What does this PR do?
<!-- Describe the purpose of this PR. -->
## Related issues
<!--
Is there any related issue? Please attach here.
- #xxxx0
- #xxxx1
- #xxxx2
-->
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fury/issues/new/choose) describing the
need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---------
Co-authored-by: weipeng <[email protected]>
Co-authored-by: orisgarno <[email protected]>
Co-authored-by: hn <[email protected]>
Co-authored-by: hening <[email protected]>
---
docs/guide/java_serialization_guide.md | 187 ++--
java/benchmark/pom.xml | 10 -
.../src/main/java/org/apache/fury/Fury.java | 5 +
.../org/apache/fury/resolver/ClassResolver.java | 7 +-
java/fury-extensions/pom.xml | 94 ++
.../org/apache/fury/meta/ZstdMetaCompressor.java | 69 ++
.../org/apache/fury/meta/ClassDefEncoderTest.java | 54 ++
java/pom.xml | 1 +
javascript/.gitignore | 1 +
javascript/benchmark/index.js | 2 +-
javascript/jest.config.js | 7 +-
javascript/packages/fury/lib/platformBuffer.ts | 86 +-
javascript/packages/fury/lib/reader/index.ts | 21 +-
javascript/packages/fury/lib/type.ts | 5 +-
javascript/packages/fury/lib/util.ts | 5 +-
javascript/packages/fury/lib/writer/index.ts | 18 +-
javascript/packages/hps/binding.gyp | 6 +-
javascript/packages/hps/index.ts | 27 +-
javascript/packages/hps/package.json | 10 +-
.../packages/hps/{index.ts => scripts/build.js} | 29 +-
javascript/packages/hps/scripts/preinstall.js | 56 ++
javascript/packages/hps/src/fastcall.cc | 146 +++-
javascript/packages/hps/src/v8-fast-api-calls.h | 957 ---------------------
javascript/test/hps.test.ts | 33 +-
javascript/test/platformBuffer.test.ts | 30 +-
25 files changed, 652 insertions(+), 1214 deletions(-)
diff --git a/docs/guide/java_serialization_guide.md
b/docs/guide/java_serialization_guide.md
index 9a792d7b..c4ee8f1d 100644
--- a/docs/guide/java_serialization_guide.md
+++ b/docs/guide/java_serialization_guide.md
@@ -102,7 +102,7 @@ public class Example {
| `compressLong` | Enables or disables long compression
for smaller size.
[...]
| `compressString` | Enables or disables string compression
for smaller size.
[...]
| `classLoader` | The classloader should not be updated;
Fury caches class metadata. Use `LoaderBinding` or `ThreadSafeFury` for
classloader updates.
[...]
-| `compatibleMode` | Type forward/backward compatibility
config. Also Related to `checkClassVersion` config. `SCHEMA_CONSISTENT`: Class
schema must be consistent between serialization peer and deserialization peer.
`COMPATIBLE`: Class schema can be different between serialization peer and
deserialization peer. They can add/delete fields independently. [See
more](#class-inconsistency-and-class-version-check).
[...]
+| `compatibleMode` | Type forward/backward compatibility
config. Also Related to `checkClassVersion` config. `SCHEMA_CONSISTENT`: Class
schema must be consistent between serialization peer and deserialization peer.
`COMPATIBLE`: Class schema can be different between serialization peer and
deserialization peer. They can add/delete fields independently. [See
more](#class-inconsistency-and-class-version-check).
[...]
| `checkClassVersion` | Determines whether to check the
consistency of the class schema. If enabled, Fury checks, writes, and checks
consistency using the `classVersionHash`. It will be automatically disabled
when `CompatibleMode#COMPATIBLE` is enabled. Disabling is not recommended
unless you can ensure the class won't evolve.
[...]
| `checkJdkClassSerializable` | Enables or disables checking of
`Serializable` interface for classes under `java.*`. If a class under `java.*`
is not `Serializable`, Fury will throw an `UnsupportedOperationException`.
[...]
| `registerGuavaTypes` | Whether to pre-register Guava types
such as `RegularImmutableMap`/`RegularImmutableList`. These types are not
public API, but seem pretty stable.
[...]
@@ -125,7 +125,7 @@ public class Example {
Single thread fury:
```java
-Fury fury=Fury.builder()
+Fury fury = Fury.builder()
.withLanguage(Language.JAVA)
// enable reference tracking for shared/circular reference.
// Disable it will have better performance if no duplicate reference.
@@ -137,14 +137,14 @@ Fury fury=Fury.builder()
// enable async multi-threaded compilation.
.withAsyncCompilation(true)
.build();
- byte[]bytes=fury.serialize(object);
- System.out.println(fury.deserialize(bytes));
+byte[] bytes = fury.serialize(object);
+System.out.println(fury.deserialize(bytes));
```
Thread-safe fury:
```java
-ThreadSafeFury fury=Fury.builder()
+ThreadSafeFury fury = Fury.builder()
.withLanguage(Language.JAVA)
// enable reference tracking for shared/circular reference.
// Disable it will have better performance if no duplicate reference.
@@ -160,10 +160,45 @@ ThreadSafeFury fury=Fury.builder()
// enable async multi-threaded compilation.
.withAsyncCompilation(true)
.buildThreadSafeFury();
- byte[]bytes=fury.serialize(object);
- System.out.println(fury.deserialize(bytes));
+byte[] bytes = fury.serialize(object);
+System.out.println(fury.deserialize(bytes));
```
+### Handling Class Schema Evolution in Serialization
+
+In many systems, the schema of a class used for serialization may change over
time. For instance, fields within a class
+may be added or removed. When serialization and deserialization processes use
different versions of jars, the schema of
+the class being deserialized may differ from the one used during serialization.
+
+By default, Fury serializes objects using the
`CompatibleMode.SCHEMA_CONSISTENT` mode. This mode assumes that the
+deserialization process uses the same class schema as the serialization
process, minimizing payload overhead.
+However, if there is a schema inconsistency, deserialization will fail.
+
+If the schema is expected to change, to make deserialization succeed, i.e.
schema forward/backward compatibility.
+Users must configure Fury to use `CompatibleMode.COMPATIBLE`. This can be done
using the
+`FuryBuilder#withCompatibleMode(CompatibleMode.COMPATIBLE)` method.
+In this compatible mode, deserialization can handle schema changes such as
missing or extra fields, allowing it to
+succeed even when the serialization and deserialization processes have
different class schemas.
+
+Here is an example of creating Fury to support schema evolution:
+
+```java
+Fury fury = Fury.builder()
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .build();
+
+byte[] bytes = fury.serialize(object);
+System.out.println(fury.deserialize(bytes));
+```
+
+This compatible mode involves serializing class metadata into the serialized
output. Despite Fury's use of
+sophisticated compression techniques to minimize overhead, there is still some
additional space cost associated with
+class metadata.
+
+To further reduce metadata costs, Fury introduces a class metadata sharing
mechanism, which allows the metadata to be
+sent to the deserialization process only once. For more details, please refer
to the [Meta Sharing](#MetaSharing)
+section.
+
### Smaller size
`FuryBuilder#withIntCompressed`/`FuryBuilder#withLongCompressed` can be used
to compress int/long for smaller size.
@@ -184,9 +219,9 @@ For long compression, fury support two encoding:
- Otherwise write as 9 bytes: `| 0b1 | little-endian 8bytes long |`
- Fury PVL(Progressive Variable-length Long) Encoding:
- First bit in every byte indicate whether has next byte. if first bit is
set, then next byte will be read util
- first bit of next byte is unset.
+ first bit of next byte is unset.
- Negative number will be converted to positive number by `(v << 1) ^ (v >>
63)` to reduce cost of small negative
- numbers.
+ numbers.
If a number are `long` type, it can't be represented by smaller bytes mostly,
the compression won't get good enough
result,
@@ -199,22 +234,18 @@ space savings.
Deep copy example:
```java
-Fury fury=Fury.builder()
- ...
- .withRefCopy(true).build();
- SomeClass a=xxx;
- SomeClass copied=fury.copy(a)
+Fury fury = Fury.builder().withRefCopy(true).build();
+SomeClass a = xxx;
+SomeClass copied = fury.copy(a);
```
Make fury deep copy ignore circular and shared reference, this deep copy mode
will ignore circular and shared reference.
Same reference of an object graph will be copied into different objects in one
`Fury#copy`.
```java
-Fury fury=Fury.builder()
- ...
- .withRefCopy(false).build();
- SomeClass a=xxx;
- SomeClass copied=fury.copy(a)
+Fury fury = Fury.builder().withRefCopy(false).build();
+SomeClass a = xxx;
+SomeClass copied = fury.copy(a);
```
### Implement a customized serializer
@@ -257,8 +288,8 @@ class FooSerializer extends Serializer<Foo> {
Register serializer:
```java
-Fury fury=getFury();
- fury.registerSerializer(Foo.class,new FooSerializer(fury));
+Fury fury = getFury();
+fury.registerSerializer(Foo.class, new FooSerializer(fury));
```
### Security & Class Registration
@@ -279,9 +310,9 @@ Note that class registration order is important,
serialization and deserializati
should have same registration order.
```java
-Fury fury=xxx;
- fury.register(SomeClass.class);
- fury.register(SomeClass1.class,200);
+Fury fury = xxx;
+fury.register(SomeClass.class);
+fury.register(SomeClass1.class,200);
```
If you invoke `FuryBuilder#requireClassRegistration(false)` to disable class
registration check,
@@ -290,19 +321,20 @@ allowed
for serialization. For example, you can allow classes started with
`org.example.*` by:
```java
-Fury fury=xxx;
-
fury.getClassResolver().setClassChecker((classResolver,className)->className.startsWith("org.example."));
+Fury fury = xxx;
+fury.getClassResolver().setClassChecker(
+ (classResolver, className) -> className.startsWith("org.example."));
```
```java
-AllowListChecker checker=new
AllowListChecker(AllowListChecker.CheckLevel.STRICT);
- ThreadSafeFury fury=new ThreadLocalFury(classLoader->{
- Fury
f=Fury.builder().requireClassRegistration(true).withClassLoader(classLoader).build();
+AllowListChecker checker = new
AllowListChecker(AllowListChecker.CheckLevel.STRICT);
+ThreadSafeFury fury = new ThreadLocalFury(classLoader -> {
+ Fury f =
Fury.builder().requireClassRegistration(true).withClassLoader(classLoader).build();
f.getClassResolver().setClassChecker(checker);
checker.addListener(f.getClassResolver());
return f;
- });
- checker.allowClass("org.example.*");
+});
+checker.allowClass("org.example.*");
```
Fury also provided a `org.apache.fury.resolver.AllowListChecker` which is
allowed/disallowed list based checker to
@@ -360,30 +392,30 @@ forward/backward compatibility automatically.
// // share meta across serialization.
// .withMetaContextShare(true)
// Not thread-safe fury.
-MetaContext context=xxx;
- fury.getSerializationContext().setMetaContext(context);
- byte[]bytes=fury.serialize(o);
+MetaContext context = xxx;
+fury.getSerializationContext().setMetaContext(context);
+byte[] bytes = fury.serialize(o);
// Not thread-safe fury.
- MetaContext context=xxx;
- fury.getSerializationContext().setMetaContext(context);
- fury.deserialize(bytes)
+MetaContext context = xxx;
+fury.getSerializationContext().setMetaContext(context);
+fury.deserialize(bytes);
// Thread-safe fury
- fury.setClassLoader(beanA.getClass().getClassLoader());
- byte[]serialized=fury.execute(
- f->{
- f.getSerializationContext().setMetaContext(context);
- return f.serialize(beanA);
+fury.setClassLoader(beanA.getClass().getClassLoader());
+byte[] serialized = fury.execute(
+ f -> {
+ f.getSerializationContext().setMetaContext(context);
+ return f.serialize(beanA);
}
- );
+);
// thread-safe fury
- fury.setClassLoader(beanA.getClass().getClassLoader());
- Object newObj=fury.execute(
- f->{
- f.getSerializationContext().setMetaContext(context);
- return f.deserialize(serialized);
+fury.setClassLoader(beanA.getClass().getClassLoader());
+Object newObj = fury.execute(
+ f -> {
+ f.getSerializationContext().setMetaContext(context);
+ return f.deserialize(serialized);
}
- );
+);
```
### Deserialize non-existent classes
@@ -404,10 +436,10 @@ Fury support mapping object from one type to another type.
> Notes:
>
> 1. This mapping will execute a deep copy, all mapped fields are serialized
> into binary and
-deserialized from that binary to map into another type.
+ deserialized from that binary to map into another type.
> 2. All struct types must be registered with same ID, otherwise Fury can not
> mapping to correct struct type.
-> Be careful when you use `Fury#register(Class)`, because fury will allocate
an auto-grown ID which might be
-> inconsistent if you register classes with different order between Fury
instance.
+ > Be careful when you use `Fury#register(Class)`, because fury will
allocate an auto-grown ID which might be
+ > inconsistent if you register classes with different order between Fury
instance.
```java
public class StructMappingExample {
@@ -460,12 +492,12 @@ the binary are generated by jdk serialization, you use
following pattern to make
then upgrade serialization to fury in an async rolling-up way:
```java
-if(JavaSerializer.serializedByJDK(bytes)){
+if (JavaSerializer.serializedByJDK(bytes)) {
ObjectInputStream objectInputStream=xxx;
return objectInputStream.readObject();
- }else{
+} else {
return fury.deserialize(bytes);
- }
+}
```
### Upgrade fury
@@ -482,18 +514,18 @@ serialized data
using code like following to keep binary compatibility:
```java
-MemoryBuffer buffer=xxx;
- buffer.writeVarInt32(2);
- fury.serialize(buffer,obj);
+MemoryBuffer buffer = xxx;
+buffer.writeVarInt32(2);
+fury.serialize(buffer, obj);
```
Then for deserialization, you need:
```java
-MemoryBuffer buffer=xxx;
- int furyVersion=buffer.readVarInt32()
- Fury fury=getFury(furyVersion);
- fury.deserialize(buffer);
+MemoryBuffer buffer = xxx;
+int furyVersion = buffer.readVarInt32();
+Fury fury = getFury(furyVersion);
+fury.deserialize(buffer);
```
`getFury` is a method to load corresponding fury, you can shade and relocate
different version of fury to different
@@ -520,9 +552,38 @@ consistent between serialization and deserialization.
### Deserialize POJO into another type
-Fury allows you to serialize one POJO and deserialize it into a different
POJO. To achieve this, configure Fury with
+Fury allows you to serialize one POJO and deserialize it into a different
POJO. The different POJO means the schema inconsistency. Users must to
configure Fury with
`CompatibleMode` set to `org.apache.fury.config.CompatibleMode.COMPATIBLE`.
+```java
+public class DeserializeIntoType {
+ static class Struct1 {
+ int f1;
+ String f2;
+
+ public Struct1(int f1, String f2) {
+ this.f1 = f1;
+ this.f2 = f2;
+ }
+ }
+
+ static class Struct2 {
+ int f1;
+ String f2;
+ double f3;
+ }
+
+ static ThreadSafeFury fury = Fury.builder()
+ .withCompatibleMode(CompatibleMode.COMPATIBLE).buildThreadSafeFury();
+
+ public static void main(String[] args) {
+ Struct1 struct1 = new Struct1(10, "abc");
+ byte[] data = fury.serializeJavaObject(struct1);
+ Struct2 struct2 = (Struct2) fury.deserializeJavaObject(bytes,
Struct2.class);
+ }
+}
+```
+
### Use wrong API for deserialization
If you serialize an object by invoking `Fury#serialize`, you should invoke
`Fury#deserialize` for deserialization
diff --git a/java/benchmark/pom.xml b/java/benchmark/pom.xml
index 8d4e1a80..6e9eb5bf 100644
--- a/java/benchmark/pom.xml
+++ b/java/benchmark/pom.xml
@@ -139,16 +139,6 @@
<artifactId>smoothie-map</artifactId>
<version>2.0.2</version>
</dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>${protobuf.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.flatbuffers</groupId>
- <artifactId>flatbuffers-java</artifactId>
- <version>2.0.3</version>
- </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java
b/java/fury-core/src/main/java/org/apache/fury/Fury.java
index 939340fe..cfc01656 100644
--- a/java/fury-core/src/main/java/org/apache/fury/Fury.java
+++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java
@@ -208,6 +208,11 @@ public final class Fury implements BaseFury {
return classResolver.getSerializerFactory();
}
+ public <T> Serializer<T> getSerializer(Class<T> cls) {
+ Preconditions.checkNotNull(cls);
+ return classResolver.getSerializer(cls);
+ }
+
@Override
public MemoryBuffer serialize(Object obj, long address, int size) {
MemoryBuffer buffer = MemoryUtils.buffer(address, size);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
index 840f3285..b6cf5157 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
@@ -1410,11 +1410,10 @@ public class ClassResolver {
return classInfo;
}
-
public ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer, Class<?>
targetClass) {
assert metaContextShareEnabled;
ClassInfo classInfo =
- readClassInfoWithMetaShare(buffer,
fury.getSerializationContext().getMetaContext());
+ readClassInfoWithMetaShare(buffer,
fury.getSerializationContext().getMetaContext());
Class<?> readClass = classInfo.getCls();
// replace target class if needed
if (targetClass != readClass) {
@@ -1423,8 +1422,8 @@ public class ClassResolver {
if (newClassInfo == null) {
// similar to create serializer for `NonexistentMetaShared`
newClassInfo =
- getMetaSharedClassInfo(
- classInfo.classDef.replaceRootClassTo(this, targetClass),
targetClass);
+ getMetaSharedClassInfo(
+ classInfo.classDef.replaceRootClassTo(this, targetClass),
targetClass);
extRegistry.transformedClassInfo.put(key, newClassInfo);
}
return newClassInfo;
diff --git a/java/fury-extensions/pom.xml b/java/fury-extensions/pom.xml
new file mode 100644
index 00000000..6f6acdb7
--- /dev/null
+++ b/java/fury-extensions/pom.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <groupId>org.apache.fury</groupId>
+ <artifactId>fury-parent</artifactId>
+ <version>0.9.0</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>fury-extensions</artifactId>
+
+ <description>
+ Apache Fury™ is a blazingly fast multi-language serialization framework
powered by jit and zero-copy.
+
+ Apache Fury (incubating) is an effort undergoing incubation at the Apache
+ Software Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+ Incubation is required of all newly accepted projects until a further
review
+ indicates that the infrastructure, communications, and decision making
process
+ have stabilized in a manner consistent with other successful ASF projects.
+
+ While incubation status is not necessarily a reflection of the completeness
+ or stability of the code, it does indicate that the project has yet to be
+ fully endorsed by the ASF.
+ </description>
+
+ <properties>
+ <zstd.version>1.5.6-9</zstd.version>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ <fury.java.rootdir>${basedir}/..</fury.java.rootdir>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ <version>${zstd.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.fury</groupId>
+ <artifactId>fury-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.fury</groupId>
+ <artifactId>fury-test-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifestEntries>
+
<Automatic-Module-Name>org.apache.fury.extension</Automatic-Module-Name>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git
a/java/fury-extensions/src/main/java/org/apache/fury/meta/ZstdMetaCompressor.java
b/java/fury-extensions/src/main/java/org/apache/fury/meta/ZstdMetaCompressor.java
new file mode 100644
index 00000000..d61f6052
--- /dev/null
+++
b/java/fury-extensions/src/main/java/org/apache/fury/meta/ZstdMetaCompressor.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fury.meta;
+
+import com.github.luben.zstd.Zstd;
+import com.github.luben.zstd.ZstdException;
+import java.util.Arrays;
+
+public class ZstdMetaCompressor implements MetaCompressor {
+
+ @Override
+ public byte[] compress(byte[] data, int offset, int size) {
+ long maxCompressedSize = Zstd.compressBound(size);
+ if (maxCompressedSize > Integer.MAX_VALUE) {
+ throw new ZstdException(Zstd.errGeneric(), "Max output size is greater
than MAX_INT");
+ }
+ byte[] compressedData = new byte[(int) maxCompressedSize];
+ long originalSize =
+ Zstd.compressByteArray(
+ compressedData,
+ 0,
+ (int) maxCompressedSize,
+ data,
+ offset,
+ size,
+ Zstd.defaultCompressionLevel());
+
+ return Arrays.copyOf(compressedData, (int) originalSize);
+ }
+
+ @Override
+ public byte[] decompress(byte[] data, int offset, int size) {
+ int decompressedSize = (int) Zstd.getFrameContentSize(data, offset, size,
false);
+ byte[] decompressedBytes = new byte[decompressedSize];
+ long originalSize =
+ Zstd.decompressByteArray(decompressedBytes, 0, decompressedSize, data,
offset, size);
+ return Arrays.copyOf(decompressedBytes, (int) originalSize);
+ }
+
+ @Override
+ public int hashCode() {
+ return ZstdMetaCompressor.class.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ return o != null && getClass() == o.getClass();
+ }
+}
diff --git
a/java/fury-extensions/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java
b/java/fury-extensions/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java
new file mode 100644
index 00000000..d581e941
--- /dev/null
+++
b/java/fury-extensions/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fury.meta;
+
+import static org.apache.fury.meta.ClassDefEncoder.buildFieldsInfo;
+import static org.apache.fury.meta.ClassDefEncoder.getClassFields;
+
+import java.util.List;
+import org.apache.fury.Fury;
+import org.apache.fury.memory.MemoryBuffer;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class ClassDefEncoderTest {
+
+ static class TestFieldsOrderClass1 {
+ private int intField2;
+ private boolean booleanField;
+ private Object objField;
+ private long longField;
+ }
+
+ @Test
+ public void testBasicClassDef_zstdMetaCompressor() throws Exception {
+ Fury fury =
+ Fury.builder().withMetaShare(true).withMetaCompressor(new
ZstdMetaCompressor()).build();
+ Class<TestFieldsOrderClass1> type = TestFieldsOrderClass1.class;
+ List<ClassDef.FieldInfo> fieldsInfo =
buildFieldsInfo(fury.getClassResolver(), type);
+ MemoryBuffer buffer =
+ ClassDefEncoder.encodeClassDef(
+ fury.getClassResolver(), type, getClassFields(type, fieldsInfo),
true);
+ ClassDef classDef = ClassDef.readClassDef(fury.getClassResolver(), buffer);
+ Assert.assertEquals(classDef.getClassName(), type.getName());
+ Assert.assertEquals(classDef.getFieldsInfo().size(),
type.getDeclaredFields().length);
+ Assert.assertEquals(classDef.getFieldsInfo(), fieldsInfo);
+ }
+}
diff --git a/java/pom.xml b/java/pom.xml
index ad42a734..0314fe88 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -72,6 +72,7 @@
<modules>
<module>fury-format</module>
<module>fury-core</module>
+ <module>fury-extensions</module>
<module>fury-test-core</module>
<module>fury-testsuite</module>
</modules>
diff --git a/javascript/.gitignore b/javascript/.gitignore
index 8216c380..40b7ff8f 100644
--- a/javascript/.gitignore
+++ b/javascript/.gitignore
@@ -1,2 +1,3 @@
package-lock.json
coverage
+packages/hps/includes
\ No newline at end of file
diff --git a/javascript/benchmark/index.js b/javascript/benchmark/index.js
index 52d0b267..9de0cb0a 100644
--- a/javascript/benchmark/index.js
+++ b/javascript/benchmark/index.js
@@ -19,7 +19,7 @@
const Fury = require("@furyjs/fury");
const utils = require("@furyjs/fury/dist/lib/util");
-const hps = require('@furyjs/hps');
+const hps = require('@furyjs/hps').default;
const fury = new Fury.default({ hps, refTracking: false, useSliceString: true
});
const Benchmark = require("benchmark");
const protobuf = require("protobufjs");
diff --git a/javascript/jest.config.js b/javascript/jest.config.js
index ce814269..363bdd2b 100644
--- a/javascript/jest.config.js
+++ b/javascript/jest.config.js
@@ -18,11 +18,9 @@
*/
/** @type {import('ts-jest').JestConfigWithTsJest} */
-const semver = require("semver");
-const hpsEnable = semver.gt(process.versions.node, '20.0.0') &&
process.platform !== 'win32';
module.exports = {
- collectCoverage: hpsEnable,
+ collectCoverage: true,
preset: 'ts-jest',
testEnvironment: 'node',
collectCoverageFrom: [
@@ -31,9 +29,6 @@ module.exports = {
"!**/build/**",
"!packages/fury/lib/murmurHash3.ts"
],
- "testPathIgnorePatterns" : [
- hpsEnable ? null : "(.*)/hps.test.ts$",
- ].filter(Boolean),
transform: {
'\\.ts$': ['ts-jest', {
tsconfig: {
diff --git a/javascript/packages/fury/lib/platformBuffer.ts
b/javascript/packages/fury/lib/platformBuffer.ts
index e3a0978d..2cecadac 100644
--- a/javascript/packages/fury/lib/platformBuffer.ts
+++ b/javascript/packages/fury/lib/platformBuffer.ts
@@ -19,10 +19,11 @@
import { hasBuffer } from "./util";
-let utf8Encoder: TextEncoder | null;
-let textDecoder: TextDecoder | null;
+const utf8Encoder = new TextEncoder();
+const utf8Decoder = new TextDecoder("utf-8");
+const utf16LEDecoder = new TextDecoder("utf-16le");
-export type SupportedEncodings = "latin1" | "utf8";
+export type SupportedEncodings = "latin1" | "utf8" | "utf16le";
export interface PlatformBuffer extends Uint8Array {
toString(encoding?: SupportedEncodings, start?: number, end?: number):
string;
@@ -32,59 +33,55 @@ export interface PlatformBuffer extends Uint8Array {
export class BrowserBuffer extends Uint8Array implements PlatformBuffer {
write(string: string, offset: number, encoding: SupportedEncodings =
"utf8"): void {
- if (encoding === "latin1") {
- return this.latin1Write(string, offset);
+ switch (encoding) {
+ case "utf8":
+ return this.utf8Write(string, offset);
+ case "utf16le":
+ return this.ucs2Write(string, offset);
+ case "latin1":
+ return this.latin1Write(string, offset);
+ default:
+ break;
+ }
+ }
+
+ private ucs2Write(string: string, offset: number): void {
+ for (let i = 0; i < string.length; i++) {
+ const codePoint = string.charCodeAt(i);
+ this[offset++] = codePoint & 0xFF;
+ this[offset++] = (codePoint >> 8) & 0xFF;
}
- return this.utf8Write(string, offset);
}
toString(encoding: SupportedEncodings = "utf8", start = 0, end =
this.length): string {
- if (encoding === "latin1") {
- return this.latin1Slice(start, end);
+ switch (encoding) {
+ case "utf8":
+ return this.utf8Slice(start, end);
+ case "utf16le":
+ return this.utf16LESlice(start, end);
+ case "latin1":
+ return this.latin1Slice(start, end);
+ default:
+ return "";
}
- return this.utf8Slice(start, end);
}
static alloc(size: number) {
return new BrowserBuffer(new Uint8Array(size));
}
- latin1Write(string: string, offset: number) {
+ private latin1Write(string: string, offset: number) {
let index = 0;
for (; index < string.length; index++) {
this[offset++] = string.charCodeAt(index);
}
}
- utf8Write(string: string, offset: number) {
- let c1: number;
- let c2: number;
- for (let i = 0; i < string.length; ++i) {
- c1 = string.charCodeAt(i);
- if (c1 < 128) {
- this[offset++] = c1;
- } else if (c1 < 2048) {
- this[offset++] = (c1 >> 6) | 192;
- this[offset++] = (c1 & 63) | 128;
- } else if (
- (c1 & 0xfc00) === 0xd800
- && ((c2 = string.charCodeAt(i + 1)) & 0xfc00) === 0xdc00
- ) {
- c1 = 0x10000 + ((c1 & 0x03ff) << 10) + (c2 & 0x03ff);
- ++i;
- this[offset++] = (c1 >> 18) | 240;
- this[offset++] = ((c1 >> 12) & 63) | 128;
- this[offset++] = ((c1 >> 6) & 63) | 128;
- this[offset++] = (c1 & 63) | 128;
- } else {
- this[offset++] = (c1 >> 12) | 224;
- this[offset++] = ((c1 >> 6) & 63) | 128;
- this[offset++] = (c1 & 63) | 128;
- }
- }
+ private utf8Write(string: string, offset: number) {
+ utf8Encoder.encodeInto(string, this.subarray(offset));
}
- latin1Slice(start: number, end: number) {
+ private latin1Slice(start: number, end: number) {
if (end - start < 1) {
return "";
}
@@ -95,16 +92,18 @@ export class BrowserBuffer extends Uint8Array implements
PlatformBuffer {
return str;
}
- utf8Slice(start: number, end: number) {
+ private utf16LESlice(start: number, end: number) {
if (end - start < 1) {
return "";
}
+ return utf16LEDecoder.decode(this.subarray(start, end));
+ }
- if (!textDecoder) {
- textDecoder = new TextDecoder("utf-8");
+ private utf8Slice(start: number, end: number) {
+ if (end - start < 1) {
+ return "";
}
-
- return textDecoder.decode(this.subarray(start, end));
+ return utf8Decoder.decode(this.subarray(start, end));
}
copy(target: Uint8Array, targetStart?: number, sourceStart?: number,
sourceEnd?: number) {
@@ -150,8 +149,5 @@ export const fromString
= hasBuffer
? (str: string) => Buffer.from(str) as unknown as PlatformBuffer
: (str: string) => {
- if (!utf8Encoder) {
- utf8Encoder = new TextEncoder();
- }
return new BrowserBuffer(utf8Encoder.encode(str));
};
diff --git a/javascript/packages/fury/lib/reader/index.ts
b/javascript/packages/fury/lib/reader/index.ts
index 5731127a..46fe1aa7 100644
--- a/javascript/packages/fury/lib/reader/index.ts
+++ b/javascript/packages/fury/lib/reader/index.ts
@@ -17,7 +17,7 @@
* under the License.
*/
-import { Config, LATIN1 } from "../type";
+import { Config, LATIN1, UTF16, UTF8 } from "../type";
import { isNodeEnv } from "../util";
import { PlatformBuffer, alloc, fromUint8Array } from "../platformBuffer";
import { readLatin1String } from "./string";
@@ -124,10 +124,23 @@ export class BinaryReader {
return result;
}
+ stringUtf16LE(len: number) {
+ const result = this.platformBuffer.toString("utf16le", this.cursor,
this.cursor + len);
+ this.cursor += len;
+ return result;
+ }
+
stringOfVarUInt32() {
- const isLatin1 = this.uint8() === LATIN1;
- const len = this.varUInt32();
- return isLatin1 ? this.stringLatin1(len) : this.stringUtf8(len);
+ switch (this.uint8()) {
+ case LATIN1:
+ return this.stringLatin1(this.varUInt32());
+ case UTF8:
+ return this.stringUtf8(this.varUInt32());
+ case UTF16:
+ return this.stringUtf16LE(this.varUInt32());
+ default:
+ break;
+ }
}
stringLatin1(len: number) {
diff --git a/javascript/packages/fury/lib/type.ts
b/javascript/packages/fury/lib/type.ts
index 23956da8..6c76e9c0 100644
--- a/javascript/packages/fury/lib/type.ts
+++ b/javascript/packages/fury/lib/type.ts
@@ -94,10 +94,9 @@ export const HalfMinInt32 = MinInt32 / 2;
export const LATIN1 = 0;
export const UTF8 = 1;
-
+export const UTF16 = 2;
export interface Hps {
- isLatin1: (str: string) => boolean;
- stringCopy: (str: string, dist: Uint8Array, offset: number) => void;
+ serializeString: (str: string, dist: Uint8Array, offset: number) => number;
}
export enum Mode {
diff --git a/javascript/packages/fury/lib/util.ts
b/javascript/packages/fury/lib/util.ts
index 39042704..cba73927 100644
--- a/javascript/packages/fury/lib/util.ts
+++ b/javascript/packages/fury/lib/util.ts
@@ -17,8 +17,7 @@
* under the License.
*/
-import { ObjectTypeDescription, Type, TypeDescription } from "./description";
-import { InternalSerializerType } from "./type";
+import { Type, TypeDescription } from "./description";
export const isNodeEnv: boolean
= typeof process !== "undefined"
@@ -107,7 +106,7 @@ export const data2Description = (
.map(([key, value]) => {
return [key, data2Description(value, `${tag}.${key}`)];
})
- .filter(([_, v]) => Boolean(v)),
+ .filter(([, v]) => Boolean(v)),
),
);
}
diff --git a/javascript/packages/fury/lib/writer/index.ts
b/javascript/packages/fury/lib/writer/index.ts
index 69ef3090..d8984ac3 100644
--- a/javascript/packages/fury/lib/writer/index.ts
+++ b/javascript/packages/fury/lib/writer/index.ts
@@ -188,22 +188,8 @@ export class BinaryWriter {
}
stringOfVarUInt32Fast(v: string) {
- const { isLatin1: detectIsLatin1, stringCopy } = this.config.hps!;
- const isLatin1 = detectIsLatin1(v);
- const len = isLatin1 ? v.length : strByteLength(v);
- this.dataView.setUint8(this.cursor++, isLatin1 ? LATIN1 : UTF8);
- this.varUInt32(len);
- this.reserve(len);
- if (isLatin1) {
- stringCopy(v, this.platformBuffer, this.cursor);
- } else {
- if (len < 40) {
- this.fastWriteStringUtf8(v, this.platformBuffer, this.cursor);
- } else {
- this.platformBuffer.write(v, this.cursor, "utf8");
- }
- }
- this.cursor += len;
+ const { serializeString } = this.config.hps!;
+ this.cursor = serializeString(v, this.platformBuffer, this.cursor);
}
stringOfVarUInt32Slow(v: string) {
diff --git a/javascript/packages/hps/binding.gyp
b/javascript/packages/hps/binding.gyp
index 73b11b41..a9ba8c71 100644
--- a/javascript/packages/hps/binding.gyp
+++ b/javascript/packages/hps/binding.gyp
@@ -20,11 +20,11 @@
{
"target_name": "hps",
"sources": [
- "src/fastcall.cc",
- "src/v8-fast-api-calls.h"
+ "src/fastcall.cc"
],
"include_dirs" : [
- "<!(node -e \"require('nan')\")"
+ "<!(node -e \"require('nan')\")",
+ "includes"
],
"cflags": ["-g"]
}
diff --git a/javascript/packages/hps/index.ts b/javascript/packages/hps/index.ts
index f991eb4b..b67177bd 100644
--- a/javascript/packages/hps/index.ts
+++ b/javascript/packages/hps/index.ts
@@ -17,16 +17,27 @@
* under the License.
*/
-const hps: Hps = require("bindings")("hps.node");
-
interface Hps {
- isLatin1: (str: string) => boolean;
- stringCopy: (str: string, dist: Uint8Array, offset: number) => void;
+ serializeString: (dist: Uint8Array, str: string, offset: number, maxLength:
number) => number;
}
-const { isLatin1, stringCopy } = hps;
+const build = () => {
+ try {
+ const hps: Hps = require("bindings")("hps.node");
+ const { serializeString: _serializeString } = hps;
-export {
- isLatin1,
- stringCopy,
+ return {
+ serializeString: (v: string, dist: Uint8Array, offset: number) => {
+ if (typeof v !== "string") {
+ throw new Error(`isLatin1 requires string but got ${typeof v}`);
+ }
+ // todo boundary check
+ return _serializeString(dist, v, offset, 0);
+ },
+ };
+ } catch (error) {
+ return null;
+ }
};
+
+export default build();
diff --git a/javascript/packages/hps/package.json
b/javascript/packages/hps/package.json
index b235a318..0dca7570 100644
--- a/javascript/packages/hps/package.json
+++ b/javascript/packages/hps/package.json
@@ -6,19 +6,21 @@
"files": [
"dist",
"src",
+ "scripts/preinstall.js",
"binding.gyp"
],
"gypfile": false,
"scripts": {
- "postinstall": "node -e \"if (process.version.match(/v(\\d+)/)[1] >= 20 &&
process.platform !== 'win32') { require('child_process').execSync('npx node-gyp
rebuild') } \"",
- "build": "node -e \"if (process.version.match(/v(\\d+)/)[1] >= 20 &&
process.platform !== 'win32') { require('child_process').execSync('npx node-gyp
rebuild && tsc') } \"",
+ "preinstall": "node ./scripts/preinstall.js",
+ "build": "node ./scripts/build.js",
"prepublishOnly": "npm run build"
},
"license": "Apache-2.0",
"dependencies": {
"bindings": "~1.2.1",
- "nan": "^2.17.0",
- "node-gyp": "^9.4.0"
+ "fs-extra": "^11.3.0",
+ "nan": "^2.22.0",
+ "node-gyp": "^11.0.0"
},
"engines": {
"node": "^20.0.0"
diff --git a/javascript/packages/hps/index.ts
b/javascript/packages/hps/scripts/build.js
similarity index 56%
copy from javascript/packages/hps/index.ts
copy to javascript/packages/hps/scripts/build.js
index f991eb4b..09673636 100644
--- a/javascript/packages/hps/index.ts
+++ b/javascript/packages/hps/scripts/build.js
@@ -17,16 +17,25 @@
* under the License.
*/
-const hps: Hps = require("bindings")("hps.node");
+const { spawn } = require("node:child_process");
+const semver = require("semver");
+const { engines } = require("../package.json");
+const versionValid = semver.satisfies(process.version, engines.node);
-interface Hps {
- isLatin1: (str: string) => boolean;
- stringCopy: (str: string, dist: Uint8Array, offset: number) => void;
+function watchError(child) {
+ child.on("error", (error) => {
+ console.error(error);
+ process.exit(1);
+ });
+ child.on("exit", (code, signal) => {
+ if (code !== 0) {
+ process.exit(code);
+ }
+ });
}
-const { isLatin1, stringCopy } = hps;
-
-export {
- isLatin1,
- stringCopy,
-};
+if (versionValid) {
+ const gyp = spawn("npx", ["node-gyp", "rebuild"], { stdio: 'inherit', shell:
true });
+ watchError(gyp);
+}
+watchError(spawn("npx", ["tsc"], { stdio: 'inherit', shell: true }));
diff --git a/javascript/packages/hps/scripts/preinstall.js
b/javascript/packages/hps/scripts/preinstall.js
new file mode 100644
index 00000000..9a1ad291
--- /dev/null
+++ b/javascript/packages/hps/scripts/preinstall.js
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const { spawn } = require("node:child_process");
+const path = require("node:path");
+const version = process.version;
+const fs = require('fs-extra');
+const semver = require("semver");
+const { engines } = require("../package.json");
+const versionValid = semver.satisfies(process.version, engines.node);
+
+async function downloadDeps(urls) {
+ await Promise.all(urls.map(({url, dist}) => {
+ new Promise((resolve, reject) => {
+ fs.ensureDirSync(path.dirname(dist));
+ const process = spawn('curl', [url, '-o', dist]);
+ process.on("close", () => {
+ console.log("finish")
+ resolve();
+ });
+ process.on("error", () => {
+ console.error(`download from ${url} failed`);
+ reject();
+ });
+ })
+ }))
+}
+
+async function main() {
+ await downloadDeps([
+ {
+ url:
`https://raw.githubusercontent.com/nodejs/node/refs/tags/${version}/deps/v8/include/v8-fast-api-calls.h`,
+ dist: path.join(__dirname, "..", "includes/v8-fast-api-calls.h")
+ }
+ ]);
+}
+
+if (versionValid) {
+ main();
+}
diff --git a/javascript/packages/hps/src/fastcall.cc
b/javascript/packages/hps/src/fastcall.cc
index 38f1cda6..2edd27ad 100644
--- a/javascript/packages/hps/src/fastcall.cc
+++ b/javascript/packages/hps/src/fastcall.cc
@@ -19,46 +19,124 @@
#include "v8-fast-api-calls.h"
#include <nan.h>
+using v8::Local;
+using v8::String;
+using v8::Value;
-void IsLatin1Slow(const v8::FunctionCallbackInfo<v8::Value> &info) {
- v8::Local<v8::String> input = info[0].As<v8::String>();
- info.GetReturnValue().Set(Nan::New(input->IsOneByte()));
+template <typename T> constexpr T RoundUp(T a, T b) {
+ return a % b != 0 ? a + b - (a % b) : a;
}
-bool IsLatin1Fast(v8::Local<v8::Value> receiver,
- const v8::FastOneByteString &source) {
- return true;
+template <typename T, typename U> constexpr T *AlignUp(T *ptr, U alignment) {
+ return reinterpret_cast<T *>(
+ RoundUp(reinterpret_cast<uintptr_t>(ptr), alignment));
}
-static v8::CFunction fast_is_latin1(v8::CFunction::Make(IsLatin1Fast));
+uint32_t writeVarUint32(uint8_t *dst, uint32_t offset, int32_t value) {
+ if (value >> 7 == 0) {
+ dst[offset] = (uint8_t)value;
+ return 1;
+ }
+ if (value >> 14 == 0) {
+ dst[offset++] = (uint8_t)((value & 0x7F) | 0x80);
+ dst[offset++] = (uint8_t)(value >> 7);
+ return 2;
+ }
+ if (value >> 21 == 0) {
+ dst[offset++] = (uint8_t)((value & 0x7F) | 0x80);
+ dst[offset++] = (uint8_t)(value >> 7 | 0x80);
+ dst[offset++] = (uint8_t)(value >> 14);
+ return 3;
+ }
+ if (value >> 28 == 0) {
+ dst[offset++] = (uint8_t)((value & 0x7F) | 0x80);
+ dst[offset++] = (uint8_t)(value >> 7 | 0x80);
+ dst[offset++] = (uint8_t)(value >> 14 | 0x80);
+ dst[offset++] = (uint8_t)(value >> 21);
+ return 4;
+ }
+ dst[offset++] = (uint8_t)((value & 0x7F) | 0x80);
+ dst[offset++] = (uint8_t)(value >> 7 | 0x80);
+ dst[offset++] = (uint8_t)(value >> 14 | 0x80);
+ dst[offset++] = (uint8_t)(value >> 21 | 0x80);
+ dst[offset++] = (uint8_t)(value >> 28);
+ return 5;
+}
-void StringCopySlow(const v8::FunctionCallbackInfo<v8::Value> &info) {
- v8::Isolate *isolate = info.GetIsolate();
- v8::Local<v8::String> source = info[0].As<v8::String>();
- char *type_array =
- (char
*)info[1].As<v8::Uint8Array>()->Buffer()->GetBackingStore()->Data();
- int32_t offset =
- info[2]->Int32Value(info.GetIsolate()->GetCurrentContext()).FromJust();
+enum Encoding { LATIN1, UTF8, UTF16 };
- v8::String::Utf8Value source_value(isolate, source);
+uint32_t writeUCS2(v8::Isolate *isolate, uint8_t *buf, Local<String> str,
+ int flags) {
+ uint16_t *const dst = reinterpret_cast<uint16_t *>(buf);
- for (size_t i = 0; i < source_value.length(); i++) {
- *(type_array + (offset++)) = *((*source_value) + i);
+ uint16_t *const aligned_dst = AlignUp(dst, sizeof(*dst));
+ size_t nchars;
+ if (aligned_dst == dst) {
+ nchars = str->Write(isolate, dst, 0, -1, flags);
+ return nchars * sizeof(*dst);
}
- info.GetReturnValue().Set(Nan::New(100));
+
+ nchars = str->Write(isolate, aligned_dst, 0, str->Length() - 1, flags);
+
+ memmove(dst, aligned_dst, nchars * sizeof(*dst));
+
+ uint16_t last;
+ str->Write(isolate, &last, nchars, 1, flags);
+ memcpy(buf + nchars * sizeof(*dst), &last, sizeof(last));
+ nchars++;
+
+ return nchars * sizeof(*dst);
+}
+
+static void serializeString(const v8::FunctionCallbackInfo<v8::Value> &args) {
+ auto isolate = args.GetIsolate();
+ auto context = isolate->GetCurrentContext();
+ v8::Local<v8::Uint8Array> dst = args[0].As<v8::Uint8Array>();
+ v8::Local<v8::String> str = args[1].As<v8::String>();
+ uint32_t offset = args[2].As<v8::Number>()->Uint32Value(context).ToChecked();
+
+ bool is_one_byte = str->IsOneByte();
+ uint8_t *dst_data =
+ reinterpret_cast<uint8_t *>(dst->Buffer()->GetBackingStore()->Data());
+
+ if (is_one_byte && str->IsExternalOneByte()) {
+ dst_data[offset++] = Encoding::LATIN1; // encoding
+ offset += writeVarUint32(dst_data, offset, str->Length()); // length
+ const auto src = str->GetExternalOneByteStringResource()->data();
+ memcpy(dst_data + offset, src, str->Length());
+ offset += str->Length();
+ } else {
+ v8::HandleScope scope(isolate);
+ int flags = String::HINT_MANY_WRITES_EXPECTED |
+ String::NO_NULL_TERMINATION | String::REPLACE_INVALID_UTF8;
+ if (is_one_byte) {
+ dst_data[offset++] = Encoding::LATIN1; // encoding
+ offset += writeVarUint32(dst_data, offset, str->Length()); // length
+ offset += str->WriteOneByte(isolate, dst_data + offset, 0, str->Length(),
+ flags);
+ } else {
+ dst_data[offset++] = Encoding::UTF16; //
encoding
+ offset += writeVarUint32(dst_data, offset, str->Length() * 2); // length
+ offset += writeUCS2(isolate, dst_data + offset, str, flags);
+ }
+ }
+
+ args.GetReturnValue().Set(offset);
}
-void StringCopyFast(v8::Local<v8::Value> receiver,
- const v8::FastOneByteString &source,
- const v8::FastApiTypedArray<u_int8_t> &ab,
- u_int32_t offset) {
- uint8_t *ptr;
- ab.getStorageIfAligned(&ptr);
- int32_t i = 0;
- memcpy(ptr + offset, source.data, source.length);
+static uint32_t serializeStringFast(Local<Value> receiver,
+ const v8::FastApiTypedArray<uint8_t> &dst,
+ const v8::FastOneByteString &src,
+ uint32_t offset, uint32_t max_length) {
+ uint8_t *dst_data;
+ dst.getStorageIfAligned(&dst_data);
+ dst_data[offset++] = Encoding::LATIN1; // encoding
+ offset += writeVarUint32(dst_data, src.length, offset); // length
+ memcpy(dst_data + offset, src.data, src.length);
+ return offset + src.length;
}
-static v8::CFunction string_copy_fast(v8::CFunction::Make(StringCopyFast));
+v8::CFunction fast_serialize_string(v8::CFunction::Make(serializeStringFast));
v8::Local<v8::FunctionTemplate> FastFunction(v8::Isolate *isolate,
v8::FunctionCallback callback,
@@ -72,14 +150,12 @@ v8::Local<v8::FunctionTemplate> FastFunction(v8::Isolate
*isolate,
void Init(v8::Local<v8::Object> exports) {
v8::Local<v8::Context> context = exports->GetCreationContextChecked();
v8::Isolate *isolate = context->GetIsolate();
- exports->Set(context, Nan::New("isLatin1").ToLocalChecked(),
- FastFunction(isolate, IsLatin1Slow, &fast_is_latin1)
- ->GetFunction(context)
- .ToLocalChecked());
- exports->Set(context, Nan::New("stringCopy").ToLocalChecked(),
- FastFunction(isolate, StringCopySlow, &string_copy_fast)
- ->GetFunction(context)
- .ToLocalChecked());
+ exports
+ ->Set(context, Nan::New("serializeString").ToLocalChecked(),
+ FastFunction(isolate, serializeString, &fast_serialize_string)
+ ->GetFunction(context)
+ .ToLocalChecked())
+ .Check();
}
NODE_MODULE(fury_util, Init)
\ No newline at end of file
diff --git a/javascript/packages/hps/src/v8-fast-api-calls.h
b/javascript/packages/hps/src/v8-fast-api-calls.h
deleted file mode 100644
index 0fe7cd24..00000000
--- a/javascript/packages/hps/src/v8-fast-api-calls.h
+++ /dev/null
@@ -1,957 +0,0 @@
-// Copyright 2020 the V8 project authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-/**
- * This file provides additional API on top of the default one for making
- * API calls, which come from embedder C++ functions. The functions are being
- * called directly from optimized code, doing all the necessary typechecks
- * in the compiler itself, instead of on the embedder side. Hence the "fast"
- * in the name. Example usage might look like:
- *
- * \code
- * void FastMethod(int param, bool another_param);
- *
- * v8::FunctionTemplate::New(isolate, SlowCallback, data,
- * signature, length, constructor_behavior
- * side_effect_type,
- * &v8::CFunction::Make(FastMethod));
- * \endcode
- *
- * By design, fast calls are limited by the following requirements, which
- * the embedder should enforce themselves:
- * - they should not allocate on the JS heap;
- * - they should not trigger JS execution.
- * To enforce them, the embedder could use the existing
- * v8::Isolate::DisallowJavascriptExecutionScope and a utility similar to
- * Blink's NoAllocationScope:
- *
https://source.chromium.org/chromium/chromium/src/+/master:third_party/blink/renderer/platform/heap/thread_state_scopes.h;l=16
- *
- * Due to these limitations, it's not directly possible to report errors by
- * throwing a JS exception or to otherwise do an allocation. There is an
- * alternative way of creating fast calls that supports falling back to the
- * slow call and then performing the necessary allocation. When one creates
- * the fast method by using CFunction::MakeWithFallbackSupport instead of
- * CFunction::Make, the fast callback gets as last parameter an output
variable,
- * through which it can request falling back to the slow call. So one might
- * declare their method like:
- *
- * \code
- * void FastMethodWithFallback(int param, FastApiCallbackOptions& options);
- * \endcode
- *
- * If the callback wants to signal an error condition or to perform an
- * allocation, it must set options.fallback to true and do an early return from
- * the fast method. Then V8 checks the value of options.fallback and if it's
- * true, falls back to executing the SlowCallback, which is capable of
reporting
- * the error (either by throwing a JS exception or logging to the console) or
- * doing the allocation. It's the embedder's responsibility to ensure that the
- * fast callback is idempotent up to the point where error and fallback
- * conditions are checked, because otherwise executing the slow callback might
- * produce visible side-effects twice.
- *
- * An example for custom embedder type support might employ a way to wrap/
- * unwrap various C++ types in JSObject instances, e.g:
- *
- * \code
- *
- * // Helper method with a check for field count.
- * template <typename T, int offset>
- * inline T* GetInternalField(v8::Local<v8::Object> wrapper) {
- * assert(offset < wrapper->InternalFieldCount());
- * return reinterpret_cast<T*>(
- * wrapper->GetAlignedPointerFromInternalField(offset));
- * }
- *
- * class CustomEmbedderType {
- * public:
- * // Returns the raw C object from a wrapper JS object.
- * static CustomEmbedderType* Unwrap(v8::Local<v8::Object> wrapper) {
- * return GetInternalField<CustomEmbedderType,
- * kV8EmbedderWrapperObjectIndex>(wrapper);
- * }
- * static void FastMethod(v8::Local<v8::Object> receiver_obj, int param) {
- * CustomEmbedderType* receiver = static_cast<CustomEmbedderType*>(
- * receiver_obj->GetAlignedPointerFromInternalField(
- * kV8EmbedderWrapperObjectIndex));
- *
- * // Type checks are already done by the optimized code.
- * // Then call some performance-critical method like:
- * // receiver->Method(param);
- * }
- *
- * static void SlowMethod(
- * const v8::FunctionCallbackInfo<v8::Value>& info) {
- * v8::Local<v8::Object> instance =
- * v8::Local<v8::Object>::Cast(info.Holder());
- * CustomEmbedderType* receiver = Unwrap(instance);
- * // TODO: Do type checks and extract {param}.
- * receiver->Method(param);
- * }
- * };
- *
- * // TODO(mslekova): Clean-up these constants
- * // The constants kV8EmbedderWrapperTypeIndex and
- * // kV8EmbedderWrapperObjectIndex describe the offsets for the type info
- * // struct and the native object, when expressed as internal field indices
- * // within a JSObject. The existance of this helper function assumes that
- * // all embedder objects have their JSObject-side type info at the same
- * // offset, but this is not a limitation of the API itself. For a detailed
- * // use case, see the third example.
- * static constexpr int kV8EmbedderWrapperTypeIndex = 0;
- * static constexpr int kV8EmbedderWrapperObjectIndex = 1;
- *
- * // The following setup function can be templatized based on
- * // the {embedder_object} argument.
- * void SetupCustomEmbedderObject(v8::Isolate* isolate,
- * v8::Local<v8::Context> context,
- * CustomEmbedderType* embedder_object) {
- * isolate->set_embedder_wrapper_type_index(
- * kV8EmbedderWrapperTypeIndex);
- * isolate->set_embedder_wrapper_object_index(
- * kV8EmbedderWrapperObjectIndex);
- *
- * v8::CFunction c_func =
- * MakeV8CFunction(CustomEmbedderType::FastMethod);
- *
- * Local<v8::FunctionTemplate> method_template =
- * v8::FunctionTemplate::New(
- * isolate, CustomEmbedderType::SlowMethod, v8::Local<v8::Value>(),
- * v8::Local<v8::Signature>(), 1, v8::ConstructorBehavior::kAllow,
- * v8::SideEffectType::kHasSideEffect, &c_func);
- *
- * v8::Local<v8::ObjectTemplate> object_template =
- * v8::ObjectTemplate::New(isolate);
- * object_template->SetInternalFieldCount(
- * kV8EmbedderWrapperObjectIndex + 1);
- * object_template->Set(isolate, "method", method_template);
- *
- * // Instantiate the wrapper JS object.
- * v8::Local<v8::Object> object =
- * object_template->NewInstance(context).ToLocalChecked();
- * object->SetAlignedPointerInInternalField(
- * kV8EmbedderWrapperObjectIndex,
- * reinterpret_cast<void*>(embedder_object));
- *
- * // TODO: Expose {object} where it's necessary.
- * }
- * \endcode
- *
- * For instance if {object} is exposed via a global "obj" variable,
- * one could write in JS:
- * function hot_func() {
- * obj.method(42);
- * }
- * and once {hot_func} gets optimized, CustomEmbedderType::FastMethod
- * will be called instead of the slow version, with the following arguments:
- * receiver := the {embedder_object} from above
- * param := 42
- *
- * Currently supported return types:
- * - void
- * - bool
- * - int32_t
- * - uint32_t
- * - float32_t
- * - float64_t
- * Currently supported argument types:
- * - pointer to an embedder type
- * - JavaScript array of primitive types
- * - bool
- * - int32_t
- * - uint32_t
- * - int64_t
- * - uint64_t
- * - float32_t
- * - float64_t
- *
- * The 64-bit integer types currently have the IDL (unsigned) long long
- * semantics: https://heycam.github.io/webidl/#abstract-opdef-converttoint
- * In the future we'll extend the API to also provide conversions from/to
- * BigInt to preserve full precision.
- * The floating point types currently have the IDL (unrestricted) semantics,
- * which is the only one used by WebGL. We plan to add support also for
- * restricted floats/doubles, similarly to the BigInt conversion policies.
- * We also differ from the specific NaN bit pattern that WebIDL prescribes
- * (https://heycam.github.io/webidl/#es-unrestricted-float) in that Blink
- * passes NaN values as-is, i.e. doesn't normalize them.
- *
- * To be supported types:
- * - TypedArrays and ArrayBuffers
- * - arrays of embedder types
- *
- *
- * The API offers a limited support for function overloads:
- *
- * \code
- * void FastMethod_2Args(int param, bool another_param);
- * void FastMethod_3Args(int param, bool another_param, int third_param);
- *
- * v8::CFunction fast_method_2args_c_func =
- * MakeV8CFunction(FastMethod_2Args);
- * v8::CFunction fast_method_3args_c_func =
- * MakeV8CFunction(FastMethod_3Args);
- * const v8::CFunction fast_method_overloads[] = {fast_method_2args_c_func,
- * fast_method_3args_c_func};
- * Local<v8::FunctionTemplate> method_template =
- * v8::FunctionTemplate::NewWithCFunctionOverloads(
- * isolate, SlowCallback, data, signature, length,
- * constructor_behavior, side_effect_type,
- * {fast_method_overloads, 2});
- * \endcode
- *
- * In this example a single FunctionTemplate is associated to multiple C++
- * functions. The overload resolution is currently only based on the number of
- * arguments passed in a call. For example, if this method_template is
- * registered with a wrapper JS object as described above, a call with two
- * arguments:
- * obj.method(42, true);
- * will result in a fast call to FastMethod_2Args, while a call with three or
- * more arguments:
- * obj.method(42, true, 11);
- * will result in a fast call to FastMethod_3Args. Instead a call with less
than
- * two arguments, like:
- * obj.method(42);
- * would not result in a fast call but would fall back to executing the
- * associated SlowCallback.
- */
-
-#ifndef INCLUDE_V8_FAST_API_CALLS_H_
-#define INCLUDE_V8_FAST_API_CALLS_H_
-
-#include <stddef.h>
-#include <stdint.h>
-
-#include <tuple>
-#include <type_traits>
-
-#include "v8-internal.h" // NOLINT(build/include_directory)
-#include "v8-local-handle.h" // NOLINT(build/include_directory)
-#include "v8-typed-array.h" // NOLINT(build/include_directory)
-#include "v8-value.h" // NOLINT(build/include_directory)
-#include "v8config.h" // NOLINT(build/include_directory)
-
-namespace v8 {
-
-class Isolate;
-
-class CTypeInfo {
- public:
- enum class Type : uint8_t {
- kVoid,
- kBool,
- kUint8,
- kInt32,
- kUint32,
- kInt64,
- kUint64,
- kFloat32,
- kFloat64,
- kPointer,
- kV8Value,
- kSeqOneByteString,
- kApiObject, // This will be deprecated once all users have
- // migrated from v8::ApiObject to v8::Local<v8::Value>.
- kAny, // This is added to enable untyped representation of fast
- // call arguments for test purposes. It can represent any of
- // the other types stored in the same memory as a union (see
- // the AnyCType struct declared below). This allows for
- // uniform passing of arguments w.r.t. their location
- // (in a register or on the stack), independent of their
- // actual type. It's currently used by the arm64 simulator
- // and can be added to the other simulators as well when fast
- // calls having both GP and FP params need to be supported.
- };
-
- // kCallbackOptionsType is not part of the Type enum
- // because it is only used internally. Use value 255 that is larger
- // than any valid Type enum.
- static constexpr Type kCallbackOptionsType = Type(255);
-
- enum class SequenceType : uint8_t {
- kScalar,
- kIsSequence, // sequence<T>
- kIsTypedArray, // TypedArray of T or any ArrayBufferView if T
- // is void
- kIsArrayBuffer // ArrayBuffer
- };
-
- enum class Flags : uint8_t {
- kNone = 0,
- kAllowSharedBit = 1 << 0, // Must be an ArrayBuffer or TypedArray
- kEnforceRangeBit = 1 << 1, // T must be integral
- kClampBit = 1 << 2, // T must be integral
- kIsRestrictedBit = 1 << 3, // T must be float or double
- };
-
- explicit constexpr CTypeInfo(
- Type type, SequenceType sequence_type = SequenceType::kScalar,
- Flags flags = Flags::kNone)
- : type_(type), sequence_type_(sequence_type), flags_(flags) {}
-
- typedef uint32_t Identifier;
- explicit constexpr CTypeInfo(Identifier identifier)
- : CTypeInfo(static_cast<Type>(identifier >> 16),
- static_cast<SequenceType>((identifier >> 8) & 255),
- static_cast<Flags>(identifier & 255)) {}
- constexpr Identifier GetId() const {
- return static_cast<uint8_t>(type_) << 16 |
- static_cast<uint8_t>(sequence_type_) << 8 |
- static_cast<uint8_t>(flags_);
- }
-
- constexpr Type GetType() const { return type_; }
- constexpr SequenceType GetSequenceType() const { return sequence_type_; }
- constexpr Flags GetFlags() const { return flags_; }
-
- static constexpr bool IsIntegralType(Type type) {
- return type == Type::kUint8 || type == Type::kInt32 ||
- type == Type::kUint32 || type == Type::kInt64 ||
- type == Type::kUint64;
- }
-
- static constexpr bool IsFloatingPointType(Type type) {
- return type == Type::kFloat32 || type == Type::kFloat64;
- }
-
- static constexpr bool IsPrimitive(Type type) {
- return IsIntegralType(type) || IsFloatingPointType(type) ||
- type == Type::kBool;
- }
-
- private:
- Type type_;
- SequenceType sequence_type_;
- Flags flags_;
-};
-
-struct FastApiTypedArrayBase {
- public:
- // Returns the length in number of elements.
- size_t V8_EXPORT length() const { return length_; }
- // Checks whether the given index is within the bounds of the collection.
- void V8_EXPORT ValidateIndex(size_t index) const;
-
- protected:
- size_t length_ = 0;
-};
-
-template <typename T>
-struct FastApiTypedArray : public FastApiTypedArrayBase {
- public:
- V8_INLINE T get(size_t index) const {
-#ifdef DEBUG
- ValidateIndex(index);
-#endif // DEBUG
- T tmp;
- memcpy(&tmp, reinterpret_cast<T*>(data_) + index, sizeof(T));
- return tmp;
- }
-
- bool getStorageIfAligned(T** elements) const {
- if (reinterpret_cast<uintptr_t>(data_) % alignof(T) != 0) {
- return false;
- }
- *elements = reinterpret_cast<T*>(data_);
- return true;
- }
-
- private:
- // This pointer should include the typed array offset applied.
- // It's not guaranteed that it's aligned to sizeof(T), it's only
- // guaranteed that it's 4-byte aligned, so for 8-byte types we need to
- // provide a special implementation for reading from it, which hides
- // the possibly unaligned read in the `get` method.
- void* data_;
-};
-
-// Any TypedArray. It uses kTypedArrayBit with base type void
-// Overloaded args of ArrayBufferView and TypedArray are not supported
-// (for now) because the generic “any” ArrayBufferView doesn’t have its
-// own instance type. It could be supported if we specify that
-// TypedArray<T> always has precedence over the generic ArrayBufferView,
-// but this complicates overload resolution.
-struct FastApiArrayBufferView {
- void* data;
- size_t byte_length;
-};
-
-struct FastApiArrayBuffer {
- void* data;
- size_t byte_length;
-};
-
-struct FastOneByteString {
- const char* data;
- uint32_t length;
-};
-
-class V8_EXPORT CFunctionInfo {
- public:
- // Construct a struct to hold a CFunction's type information.
- // |return_info| describes the function's return type.
- // |arg_info| is an array of |arg_count| CTypeInfos describing the
- // arguments. Only the last argument may be of the special type
- // CTypeInfo::kCallbackOptionsType.
- CFunctionInfo(const CTypeInfo& return_info, unsigned int arg_count,
- const CTypeInfo* arg_info);
-
- const CTypeInfo& ReturnInfo() const { return return_info_; }
-
- // The argument count, not including the v8::FastApiCallbackOptions
- // if present.
- unsigned int ArgumentCount() const {
- return HasOptions() ? arg_count_ - 1 : arg_count_;
- }
-
- // |index| must be less than ArgumentCount().
- // Note: if the last argument passed on construction of CFunctionInfo
- // has type CTypeInfo::kCallbackOptionsType, it is not included in
- // ArgumentCount().
- const CTypeInfo& ArgumentInfo(unsigned int index) const;
-
- bool HasOptions() const {
- // The options arg is always the last one.
- return arg_count_ > 0 && arg_info_[arg_count_ - 1].GetType() ==
- CTypeInfo::kCallbackOptionsType;
- }
-
- private:
- const CTypeInfo return_info_;
- const unsigned int arg_count_;
- const CTypeInfo* arg_info_;
-};
-
-struct FastApiCallbackOptions;
-
-// Provided for testing.
-struct AnyCType {
- AnyCType() : int64_value(0) {}
-
- union {
- bool bool_value;
- int32_t int32_value;
- uint32_t uint32_value;
- int64_t int64_value;
- uint64_t uint64_value;
- float float_value;
- double double_value;
- void* pointer_value;
- Local<Object> object_value;
- Local<Array> sequence_value;
- const FastApiTypedArray<uint8_t>* uint8_ta_value;
- const FastApiTypedArray<int32_t>* int32_ta_value;
- const FastApiTypedArray<uint32_t>* uint32_ta_value;
- const FastApiTypedArray<int64_t>* int64_ta_value;
- const FastApiTypedArray<uint64_t>* uint64_ta_value;
- const FastApiTypedArray<float>* float_ta_value;
- const FastApiTypedArray<double>* double_ta_value;
- const FastOneByteString* string_value;
- FastApiCallbackOptions* options_value;
- };
-};
-
-static_assert(
- sizeof(AnyCType) == 8,
- "The AnyCType struct should have size == 64 bits, as this is assumed "
- "by EffectControlLinearizer.");
-
-class V8_EXPORT CFunction {
- public:
- constexpr CFunction() : address_(nullptr), type_info_(nullptr) {}
-
- const CTypeInfo& ReturnInfo() const { return type_info_->ReturnInfo(); }
-
- const CTypeInfo& ArgumentInfo(unsigned int index) const {
- return type_info_->ArgumentInfo(index);
- }
-
- unsigned int ArgumentCount() const { return type_info_->ArgumentCount(); }
-
- const void* GetAddress() const { return address_; }
- const CFunctionInfo* GetTypeInfo() const { return type_info_; }
-
- enum class OverloadResolution { kImpossible, kAtRuntime, kAtCompileTime };
-
- // Returns whether an overload between this and the given CFunction can
- // be resolved at runtime by the RTTI available for the arguments or at
- // compile time for functions with different number of arguments.
- OverloadResolution GetOverloadResolution(const CFunction* other) {
- // Runtime overload resolution can only deal with functions with the
- // same number of arguments. Functions with different arity are handled
- // by compile time overload resolution though.
- if (ArgumentCount() != other->ArgumentCount()) {
- return OverloadResolution::kAtCompileTime;
- }
-
- // The functions can only differ by a single argument position.
- int diff_index = -1;
- for (unsigned int i = 0; i < ArgumentCount(); ++i) {
- if (ArgumentInfo(i).GetSequenceType() !=
- other->ArgumentInfo(i).GetSequenceType()) {
- if (diff_index >= 0) {
- return OverloadResolution::kImpossible;
- }
- diff_index = i;
-
- // We only support overload resolution between sequence types.
- if (ArgumentInfo(i).GetSequenceType() ==
- CTypeInfo::SequenceType::kScalar ||
- other->ArgumentInfo(i).GetSequenceType() ==
- CTypeInfo::SequenceType::kScalar) {
- return OverloadResolution::kImpossible;
- }
- }
- }
-
- return OverloadResolution::kAtRuntime;
- }
-
- template <typename F>
- static CFunction Make(F* func) {
- return ArgUnwrap<F*>::Make(func);
- }
-
- // Provided for testing purposes.
- template <typename R, typename... Args, typename R_Patch,
- typename... Args_Patch>
- static CFunction Make(R (*func)(Args...),
- R_Patch (*patching_func)(Args_Patch...)) {
- CFunction c_func = ArgUnwrap<R (*)(Args...)>::Make(func);
- static_assert(
- sizeof...(Args_Patch) == sizeof...(Args),
- "The patching function must have the same number of arguments.");
- c_func.address_ = reinterpret_cast<void*>(patching_func);
- return c_func;
- }
-
- CFunction(const void* address, const CFunctionInfo* type_info);
-
- private:
- const void* address_;
- const CFunctionInfo* type_info_;
-
- template <typename F>
- class ArgUnwrap {
- static_assert(sizeof(F) != sizeof(F),
- "CFunction must be created from a function pointer.");
- };
-
- template <typename R, typename... Args>
- class ArgUnwrap<R (*)(Args...)> {
- public:
- static CFunction Make(R (*func)(Args...));
- };
-};
-
-/**
- * A struct which may be passed to a fast call callback, like so:
- * \code
- * void FastMethodWithOptions(int param, FastApiCallbackOptions& options);
- * \endcode
- */
-struct FastApiCallbackOptions {
- /**
- * Creates a new instance of FastApiCallbackOptions for testing purpose. The
- * returned instance may be filled with mock data.
- */
- static FastApiCallbackOptions CreateForTesting(Isolate* isolate) {
- return {false, {0}, nullptr};
- }
-
- /**
- * If the callback wants to signal an error condition or to perform an
- * allocation, it must set options.fallback to true and do an early return
- * from the fast method. Then V8 checks the value of options.fallback and if
- * it's true, falls back to executing the SlowCallback, which is capable of
- * reporting the error (either by throwing a JS exception or logging to the
- * console) or doing the allocation. It's the embedder's responsibility to
- * ensure that the fast callback is idempotent up to the point where error
and
- * fallback conditions are checked, because otherwise executing the slow
- * callback might produce visible side-effects twice.
- */
- bool fallback;
-
- /**
- * The `data` passed to the FunctionTemplate constructor, or `undefined`.
- * `data_ptr` allows for default constructing FastApiCallbackOptions.
- */
- union {
- uintptr_t data_ptr;
- v8::Local<v8::Value> data;
- };
-
- /**
- * When called from WebAssembly, a view of the calling module's memory.
- */
- FastApiTypedArray<uint8_t>* const wasm_memory;
-};
-
-namespace internal {
-
-// Helper to count the number of occurances of `T` in `List`
-template <typename T, typename... List>
-struct count : std::integral_constant<int, 0> {};
-template <typename T, typename... Args>
-struct count<T, T, Args...>
- : std::integral_constant<std::size_t, 1 + count<T, Args...>::value> {};
-template <typename T, typename U, typename... Args>
-struct count<T, U, Args...> : count<T, Args...> {};
-
-template <typename RetBuilder, typename... ArgBuilders>
-class CFunctionInfoImpl : public CFunctionInfo {
- static constexpr int kOptionsArgCount =
- count<FastApiCallbackOptions&, ArgBuilders...>();
- static constexpr int kReceiverCount = 1;
-
- static_assert(kOptionsArgCount == 0 || kOptionsArgCount == 1,
- "Only one options parameter is supported.");
-
- static_assert(sizeof...(ArgBuilders) >= kOptionsArgCount + kReceiverCount,
- "The receiver or the options argument is missing.");
-
- public:
- constexpr CFunctionInfoImpl()
- : CFunctionInfo(RetBuilder::Build(), sizeof...(ArgBuilders),
- arg_info_storage_),
- arg_info_storage_{ArgBuilders::Build()...} {
- constexpr CTypeInfo::Type kReturnType = RetBuilder::Build().GetType();
- static_assert(kReturnType == CTypeInfo::Type::kVoid ||
- kReturnType == CTypeInfo::Type::kBool ||
- kReturnType == CTypeInfo::Type::kInt32 ||
- kReturnType == CTypeInfo::Type::kUint32 ||
- kReturnType == CTypeInfo::Type::kFloat32 ||
- kReturnType == CTypeInfo::Type::kFloat64 ||
- kReturnType == CTypeInfo::Type::kPointer ||
- kReturnType == CTypeInfo::Type::kAny,
- "64-bit int, string and api object values are not currently "
- "supported return types.");
- }
-
- private:
- const CTypeInfo arg_info_storage_[sizeof...(ArgBuilders)];
-};
-
-template <typename T>
-struct TypeInfoHelper {
- static_assert(sizeof(T) != sizeof(T), "This type is not supported");
-};
-
-#define SPECIALIZE_GET_TYPE_INFO_HELPER_FOR(T, Enum) \
- template <> \
- struct TypeInfoHelper<T> { \
- static constexpr CTypeInfo::Flags Flags() { \
- return CTypeInfo::Flags::kNone; \
- } \
- \
- static constexpr CTypeInfo::Type Type() { return CTypeInfo::Type::Enum; } \
- static constexpr CTypeInfo::SequenceType SequenceType() { \
- return CTypeInfo::SequenceType::kScalar; \
- } \
- };
-
-template <CTypeInfo::Type type>
-struct CTypeInfoTraits {};
-
-#define DEFINE_TYPE_INFO_TRAITS(CType, Enum) \
- template <> \
- struct CTypeInfoTraits<CTypeInfo::Type::Enum> { \
- using ctype = CType; \
- };
-
-#define PRIMITIVE_C_TYPES(V) \
- V(bool, kBool) \
- V(uint8_t, kUint8) \
- V(int32_t, kInt32) \
- V(uint32_t, kUint32) \
- V(int64_t, kInt64) \
- V(uint64_t, kUint64) \
- V(float, kFloat32) \
- V(double, kFloat64) \
- V(void*, kPointer)
-
-// Same as above, but includes deprecated types for compatibility.
-#define ALL_C_TYPES(V) \
- PRIMITIVE_C_TYPES(V) \
- V(void, kVoid) \
- V(v8::Local<v8::Value>, kV8Value) \
- V(v8::Local<v8::Object>, kV8Value) \
- V(AnyCType, kAny)
-
-// ApiObject was a temporary solution to wrap the pointer to the v8::Value.
-// Please use v8::Local<v8::Value> in new code for the arguments and
-// v8::Local<v8::Object> for the receiver, as ApiObject will be deprecated.
-
-ALL_C_TYPES(SPECIALIZE_GET_TYPE_INFO_HELPER_FOR)
-PRIMITIVE_C_TYPES(DEFINE_TYPE_INFO_TRAITS)
-
-#undef PRIMITIVE_C_TYPES
-#undef ALL_C_TYPES
-
-#define SPECIALIZE_GET_TYPE_INFO_HELPER_FOR_TA(T, Enum) \
- template <> \
- struct TypeInfoHelper<const FastApiTypedArray<T>&> { \
- static constexpr CTypeInfo::Flags Flags() { \
- return CTypeInfo::Flags::kNone; \
- } \
- \
- static constexpr CTypeInfo::Type Type() { return CTypeInfo::Type::Enum; } \
- static constexpr CTypeInfo::SequenceType SequenceType() { \
- return CTypeInfo::SequenceType::kIsTypedArray; \
- } \
- };
-
-#define TYPED_ARRAY_C_TYPES(V) \
- V(uint8_t, kUint8) \
- V(int32_t, kInt32) \
- V(uint32_t, kUint32) \
- V(int64_t, kInt64) \
- V(uint64_t, kUint64) \
- V(float, kFloat32) \
- V(double, kFloat64)
-
-TYPED_ARRAY_C_TYPES(SPECIALIZE_GET_TYPE_INFO_HELPER_FOR_TA)
-
-#undef TYPED_ARRAY_C_TYPES
-
-template <>
-struct TypeInfoHelper<v8::Local<v8::Array>> {
- static constexpr CTypeInfo::Flags Flags() { return CTypeInfo::Flags::kNone; }
-
- static constexpr CTypeInfo::Type Type() { return CTypeInfo::Type::kVoid; }
- static constexpr CTypeInfo::SequenceType SequenceType() {
- return CTypeInfo::SequenceType::kIsSequence;
- }
-};
-
-template <>
-struct TypeInfoHelper<v8::Local<v8::Uint32Array>> {
- static constexpr CTypeInfo::Flags Flags() { return CTypeInfo::Flags::kNone; }
-
- static constexpr CTypeInfo::Type Type() { return CTypeInfo::Type::kUint32; }
- static constexpr CTypeInfo::SequenceType SequenceType() {
- return CTypeInfo::SequenceType::kIsTypedArray;
- }
-};
-
-template <>
-struct TypeInfoHelper<FastApiCallbackOptions&> {
- static constexpr CTypeInfo::Flags Flags() { return CTypeInfo::Flags::kNone; }
-
- static constexpr CTypeInfo::Type Type() {
- return CTypeInfo::kCallbackOptionsType;
- }
- static constexpr CTypeInfo::SequenceType SequenceType() {
- return CTypeInfo::SequenceType::kScalar;
- }
-};
-
-template <>
-struct TypeInfoHelper<const FastOneByteString&> {
- static constexpr CTypeInfo::Flags Flags() { return CTypeInfo::Flags::kNone; }
-
- static constexpr CTypeInfo::Type Type() {
- return CTypeInfo::Type::kSeqOneByteString;
- }
- static constexpr CTypeInfo::SequenceType SequenceType() {
- return CTypeInfo::SequenceType::kScalar;
- }
-};
-
-#define STATIC_ASSERT_IMPLIES(COND, ASSERTION, MSG) \
- static_assert(((COND) == 0) || (ASSERTION), MSG)
-
-} // namespace internal
-
-template <typename T, CTypeInfo::Flags... Flags>
-class V8_EXPORT CTypeInfoBuilder {
- public:
- using BaseType = T;
-
- static constexpr CTypeInfo Build() {
- constexpr CTypeInfo::Flags kFlags =
- MergeFlags(internal::TypeInfoHelper<T>::Flags(), Flags...);
- constexpr CTypeInfo::Type kType = internal::TypeInfoHelper<T>::Type();
- constexpr CTypeInfo::SequenceType kSequenceType =
- internal::TypeInfoHelper<T>::SequenceType();
-
- STATIC_ASSERT_IMPLIES(
- uint8_t(kFlags) & uint8_t(CTypeInfo::Flags::kAllowSharedBit),
- (kSequenceType == CTypeInfo::SequenceType::kIsTypedArray ||
- kSequenceType == CTypeInfo::SequenceType::kIsArrayBuffer),
- "kAllowSharedBit is only allowed for TypedArrays and ArrayBuffers.");
- STATIC_ASSERT_IMPLIES(
- uint8_t(kFlags) & uint8_t(CTypeInfo::Flags::kEnforceRangeBit),
- CTypeInfo::IsIntegralType(kType),
- "kEnforceRangeBit is only allowed for integral types.");
- STATIC_ASSERT_IMPLIES(
- uint8_t(kFlags) & uint8_t(CTypeInfo::Flags::kClampBit),
- CTypeInfo::IsIntegralType(kType),
- "kClampBit is only allowed for integral types.");
- STATIC_ASSERT_IMPLIES(
- uint8_t(kFlags) & uint8_t(CTypeInfo::Flags::kIsRestrictedBit),
- CTypeInfo::IsFloatingPointType(kType),
- "kIsRestrictedBit is only allowed for floating point types.");
- STATIC_ASSERT_IMPLIES(kSequenceType ==
CTypeInfo::SequenceType::kIsSequence,
- kType == CTypeInfo::Type::kVoid,
- "Sequences are only supported from void type.");
- STATIC_ASSERT_IMPLIES(
- kSequenceType == CTypeInfo::SequenceType::kIsTypedArray,
- CTypeInfo::IsPrimitive(kType) || kType == CTypeInfo::Type::kVoid,
- "TypedArrays are only supported from primitive types or void.");
-
- // Return the same type with the merged flags.
- return CTypeInfo(internal::TypeInfoHelper<T>::Type(),
- internal::TypeInfoHelper<T>::SequenceType(), kFlags);
- }
-
- private:
- template <typename... Rest>
- static constexpr CTypeInfo::Flags MergeFlags(CTypeInfo::Flags flags,
- Rest... rest) {
- return CTypeInfo::Flags(uint8_t(flags) | uint8_t(MergeFlags(rest...)));
- }
- static constexpr CTypeInfo::Flags MergeFlags() { return CTypeInfo::Flags(0);
}
-};
-
-namespace internal {
-template <typename RetBuilder, typename... ArgBuilders>
-class CFunctionBuilderWithFunction {
- public:
- explicit constexpr CFunctionBuilderWithFunction(const void* fn) : fn_(fn) {}
-
- template <CTypeInfo::Flags... Flags>
- constexpr auto Ret() {
- return CFunctionBuilderWithFunction<
- CTypeInfoBuilder<typename RetBuilder::BaseType, Flags...>,
- ArgBuilders...>(fn_);
- }
-
- template <unsigned int N, CTypeInfo::Flags... Flags>
- constexpr auto Arg() {
- // Return a copy of the builder with the Nth arg builder merged with
- // template parameter pack Flags.
- return ArgImpl<N, Flags...>(
- std::make_index_sequence<sizeof...(ArgBuilders)>());
- }
-
- // Provided for testing purposes.
- template <typename Ret, typename... Args>
- auto Patch(Ret (*patching_func)(Args...)) {
- static_assert(
- sizeof...(Args) == sizeof...(ArgBuilders),
- "The patching function must have the same number of arguments.");
- fn_ = reinterpret_cast<void*>(patching_func);
- return *this;
- }
-
- auto Build() {
- static CFunctionInfoImpl<RetBuilder, ArgBuilders...> instance;
- return CFunction(fn_, &instance);
- }
-
- private:
- template <bool Merge, unsigned int N, CTypeInfo::Flags... Flags>
- struct GetArgBuilder;
-
- // Returns the same ArgBuilder as the one at index N, including its flags.
- // Flags in the template parameter pack are ignored.
- template <unsigned int N, CTypeInfo::Flags... Flags>
- struct GetArgBuilder<false, N, Flags...> {
- using type =
- typename std::tuple_element<N, std::tuple<ArgBuilders...>>::type;
- };
-
- // Returns an ArgBuilder with the same base type as the one at index N,
- // but merges the flags with the flags in the template parameter pack.
- template <unsigned int N, CTypeInfo::Flags... Flags>
- struct GetArgBuilder<true, N, Flags...> {
- using type = CTypeInfoBuilder<
- typename std::tuple_element<N,
-
std::tuple<ArgBuilders...>>::type::BaseType,
- std::tuple_element<N, std::tuple<ArgBuilders...>>::type::Build()
- .GetFlags(),
- Flags...>;
- };
-
- // Return a copy of the CFunctionBuilder, but merges the Flags on
- // ArgBuilder index N with the new Flags passed in the template parameter
- // pack.
- template <unsigned int N, CTypeInfo::Flags... Flags, size_t... I>
- constexpr auto ArgImpl(std::index_sequence<I...>) {
- return CFunctionBuilderWithFunction<
- RetBuilder, typename GetArgBuilder<N == I, I, Flags...>::type...>(fn_);
- }
-
- const void* fn_;
-};
-
-class CFunctionBuilder {
- public:
- constexpr CFunctionBuilder() {}
-
- template <typename R, typename... Args>
- constexpr auto Fn(R (*fn)(Args...)) {
- return CFunctionBuilderWithFunction<CTypeInfoBuilder<R>,
- CTypeInfoBuilder<Args>...>(
- reinterpret_cast<const void*>(fn));
- }
-};
-
-} // namespace internal
-
-// static
-template <typename R, typename... Args>
-CFunction CFunction::ArgUnwrap<R (*)(Args...)>::Make(R (*func)(Args...)) {
- return internal::CFunctionBuilder().Fn(func).Build();
-}
-
-using CFunctionBuilder = internal::CFunctionBuilder;
-
-static constexpr CTypeInfo kTypeInfoInt32 = CTypeInfo(CTypeInfo::Type::kInt32);
-static constexpr CTypeInfo kTypeInfoFloat64 =
- CTypeInfo(CTypeInfo::Type::kFloat64);
-
-/**
- * Copies the contents of this JavaScript array to a C++ buffer with
- * a given max_length. A CTypeInfo is passed as an argument,
- * instructing different rules for conversion (e.g. restricted float/double).
- * The element type T of the destination array must match the C type
- * corresponding to the CTypeInfo (specified by CTypeInfoTraits).
- * If the array length is larger than max_length or the array is of
- * unsupported type, the operation will fail, returning false. Generally, an
- * array which contains objects, undefined, null or anything not convertible
- * to the requested destination type, is considered unsupported. The operation
- * returns true on success. `type_info` will be used for conversions.
- */
-template <CTypeInfo::Identifier type_info_id, typename T>
-bool V8_EXPORT V8_WARN_UNUSED_RESULT TryToCopyAndConvertArrayToCppBuffer(
- Local<Array> src, T* dst, uint32_t max_length);
-
-template <>
-bool V8_EXPORT V8_WARN_UNUSED_RESULT
-TryToCopyAndConvertArrayToCppBuffer<CTypeInfoBuilder<int32_t>::Build().GetId(),
- int32_t>(Local<Array> src, int32_t* dst,
- uint32_t max_length);
-
-template <>
-bool V8_EXPORT V8_WARN_UNUSED_RESULT
-TryToCopyAndConvertArrayToCppBuffer<CTypeInfoBuilder<uint32_t>::Build().GetId(),
- uint32_t>(Local<Array> src, uint32_t* dst,
- uint32_t max_length);
-
-template <>
-bool V8_EXPORT V8_WARN_UNUSED_RESULT
-TryToCopyAndConvertArrayToCppBuffer<CTypeInfoBuilder<float>::Build().GetId(),
- float>(Local<Array> src, float* dst,
- uint32_t max_length);
-
-template <>
-bool V8_EXPORT V8_WARN_UNUSED_RESULT
-TryToCopyAndConvertArrayToCppBuffer<CTypeInfoBuilder<double>::Build().GetId(),
- double>(Local<Array> src, double* dst,
- uint32_t max_length);
-
-} // namespace v8
-
-#endif // INCLUDE_V8_FAST_API_CALLS_H_
diff --git a/javascript/test/hps.test.ts b/javascript/test/hps.test.ts
index 8d8bc1bc..40be8d4e 100644
--- a/javascript/test/hps.test.ts
+++ b/javascript/test/hps.test.ts
@@ -17,28 +17,27 @@
* under the License.
*/
-import { isLatin1, stringCopy } from '../packages/hps/index';
+import { BinaryReader } from '@furyjs/fury/dist/lib/reader';
+import hps from '../packages/hps/index';
import { describe, expect, test } from '@jest/globals';
-describe('hps', () => {
- test('should isLatin1 work', () => {
- for (let index = 0; index < 10000; index++) {
- var is = isLatin1("hello");
- expect(is).toBe(true)
-
- var is = isLatin1("😁");
- expect(is).toBe(false)
- }
- });
+const skipableDescribe = (hps ? describe : describe.skip);
- test('should stringCopy work', () => {
+skipableDescribe('hps', () => {
+ test.only('should isLatin1 work', () => {
+ const { serializeString } = hps!;
for (let index = 0; index < 10000; index++) {
- const dist = new Uint8Array(5);
- stringCopy("hello", dist, 0);
- expect([...dist]).toEqual([104, 101, 108, 108, 111])
+ const bf = Buffer.alloc(100);
+ serializeString("hello", bf, 0);
+ var reader = new BinaryReader({});
+ reader.reset(bf);
+ expect(reader.stringOfVarUInt32()).toBe("hello")
+
+ serializeString("😁", bf, 0);
+ var reader = new BinaryReader({});
+ reader.reset(bf);
+ expect(reader.stringOfVarUInt32()).toBe("😁")
}
});
});
-
-
diff --git a/javascript/test/platformBuffer.test.ts
b/javascript/test/platformBuffer.test.ts
index cc633fc4..e0f6362e 100644
--- a/javascript/test/platformBuffer.test.ts
+++ b/javascript/test/platformBuffer.test.ts
@@ -40,37 +40,20 @@ describe('platformBuffer', () => {
test('should latin1Write work', () => {
const bb = BrowserBuffer.alloc(100);
- bb.latin1Write("hello, world", 0);
+ bb.write("hello, world", 0, "latin1");
- const str = bb.latin1Slice(0, 12);
+ const str = bb.toString("latin1", 0, 12);
expect(str).toBe("hello, world");
-
- const str2 = bb.toString('latin1', 0, 12);
- expect(str2).toBe("hello, world");
- });
-
- test('should write latin1 work', () => {
- const bb = BrowserBuffer.alloc(100);
- bb.write("hello, world", 0, 'latin1');
-
- const str = bb.latin1Slice(0, 12);
- expect(str).toBe("hello, world");
-
- const str2 = bb.toString('latin1', 0, 12);
- expect(str2).toBe("hello, world");
});
test('should utf8Write work', () => {
const rawStr = "我是Fury, 你好！😁א";
const bb = BrowserBuffer.alloc(100);
- bb.utf8Write(rawStr, 0);
+ bb.write(rawStr, 0);
- const str = bb.utf8Slice(0, 27);
+ const str = bb.toString("utf8", 0, 27);
expect(str).toBe(rawStr);
-
- const str2 = bb.toString('utf8', 0, 27);
- expect(str2).toBe(rawStr);
});
test('should utf8 work', () => {
@@ -78,9 +61,6 @@ describe('platformBuffer', () => {
const bb = BrowserBuffer.alloc(100);
bb.write(rawStr, 0, 'utf8');
- const str = bb.utf8Slice(0, 27);
- expect(str).toBe(rawStr);
-
const str2 = bb.toString('utf8', 0, 27);
expect(str2).toBe(rawStr);
});
@@ -93,7 +73,7 @@ describe('platformBuffer', () => {
test('should copy work', () => {
const bb = BrowserBuffer.alloc(100);
- bb.latin1Write("hello", 0);
+ bb.write("hello", 0, 'latin1');
const target = new Uint8Array(5);
bb.copy(target, 0, 0, 5);
expect([...target]).toEqual([ 104, 101, 108, 108, 111 ])
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
