(fury) branch main updated: feat: use TextDecoder to decode buffer (#1699)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fury.git
The following commit(s) were added to refs/heads/main by this push:
new d25ccbb1 feat: use TextDecoder to decode buffer (#1699)
d25ccbb1 is described below
commit d25ccbb1803f137cc5e9bf15fa060ae67c5be7ad
Author: 野声
AuthorDate: Wed Jul 3 17:23:28 2024 +0800
feat: use TextDecoder to decode buffer (#1699)
## What does this PR do?
Use the browser builtin decode utils: TextDecoder
https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder
This can improve a lot of performance. see the diff of the benchmark.
and the original implementation will cause many `minor gc` in
javascript, the longer the string, the longer the GC time.
## Related issues
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
---
javascript/packages/fury/lib/platformBuffer.ts | 25 -
platform-buffer.jpg| Bin 0 -> 37116 bytes
2 files changed, 8 insertions(+), 17 deletions(-)
diff --git a/javascript/packages/fury/lib/platformBuffer.ts
b/javascript/packages/fury/lib/platformBuffer.ts
index b28df55b..e3a0978d 100644
--- a/javascript/packages/fury/lib/platformBuffer.ts
+++ b/javascript/packages/fury/lib/platformBuffer.ts
@@ -19,6 +19,9 @@
import { hasBuffer } from "./util";
+let utf8Encoder: TextEncoder | null;
+let textDecoder: TextDecoder | null;
+
export type SupportedEncodings = "latin1" | "utf8";
export interface PlatformBuffer extends Uint8Array {
@@ -96,22 +99,12 @@ export class BrowserBuffer extends Uint8Array implements
PlatformBuffer {
if (end - start < 1) {
return "";
}
-let str = "";
-for (let i = start; i < end;) {
- const t = this[i++];
- if (t <= 0x7F) {
-str += String.fromCharCode(t);
- } else if (t >= 0xC0 && t < 0xE0) {
-str += String.fromCharCode((t & 0x1F) << 6 | this[i++] & 0x3F);
- } else if (t >= 0xE0 && t < 0xF0) {
-str += String.fromCharCode((t & 0xF) << 12 | (this[i++] & 0x3F) << 6 |
this[i++] & 0x3F);
- } else if (t >= 0xF0) {
-const t2 = ((t & 7) << 18 | (this[i++] & 0x3F) << 12 | (this[i++] &
0x3F) << 6 | this[i++] & 0x3F) - 0x1;
-str += String.fromCharCode(0xD800 + (t2 >> 10));
-str += String.fromCharCode(0xDC00 + (t2 & 0x3FF));
- }
+
+if (!textDecoder) {
+ textDecoder = new TextDecoder("utf-8");
}
-return str;
+
+return textDecoder.decode(this.subarray(start, end));
}
copy(target: Uint8Array, targetStart?: number, sourceStart?: number,
sourceEnd?: number) {
@@ -153,8 +146,6 @@ export const alloc = (hasBuffer ? Buffer.allocUnsafe :
BrowserBuffer.alloc) as u
export const strByteLength = hasBuffer ? Buffer.byteLength :
BrowserBuffer.byteLength;
-let utf8Encoder: TextEncoder | null;
-
export const fromString
= hasBuffer
? (str: string) => Buffer.from(str) as unknown as PlatformBuffer
diff --git a/platform-buffer.jpg b/platform-buffer.jpg
new file mode 100644
index ..0b69b424
Binary files /dev/null and b/platform-buffer.jpg differ
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch rust-setup deleted (was 110179ad)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a change to branch rust-setup in repository https://gitbox.apache.org/repos/asf/incubator-fury.git was 110179ad Add rust toolchain The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository. - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: chore(rust): Setup rustfmt and toolchain for rust (#1677)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 257564f4 chore(rust): Setup rustfmt and toolchain for rust (#1677)
257564f4 is described below
commit 257564f4d051595d657104f77b96f6d8203a7683
Author: Xuanwo
AuthorDate: Thu Jun 6 20:21:25 2024 +0800
chore(rust): Setup rustfmt and toolchain for rust (#1677)
---
ci/run_ci.py | 2 --
rust/rust-toolchain.toml | 20
rust/rustfmt.toml| 19 +++
3 files changed, 39 insertions(+), 2 deletions(-)
diff --git a/ci/run_ci.py b/ci/run_ci.py
index df2c0d37..05c9fc1e 100644
--- a/ci/run_ci.py
+++ b/ci/run_ci.py
@@ -86,8 +86,6 @@ def _run_cpp():
def _run_rust():
-_exec_cmd("rustup component add clippy-preview")
-_exec_cmd("rustup component add rustfmt")
logging.info("Executing fury rust tests")
_cd_project_subdir("rust")
diff --git a/rust/rust-toolchain.toml b/rust/rust-toolchain.toml
new file mode 100644
index ..870d7eb7
--- /dev/null
+++ b/rust/rust-toolchain.toml
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[toolchain]
+channel = "stable"
+components = ["rustfmt", "clippy"]
diff --git a/rust/rustfmt.toml b/rust/rustfmt.toml
new file mode 100644
index ..39be343c
--- /dev/null
+++ b/rust/rustfmt.toml
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+edition = "2021"
+reorder_imports = true
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury-site) branch main updated: update download page for 0.5.1 (#124)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git The following commit(s) were added to refs/heads/main by this push: new 9707c9b update download page for 0.5.1 (#124) 9707c9b is described below commit 9707c9b15daf58073d8792241d3e5ac15e0a6a08 Author: Shawn Yang AuthorDate: Tue May 28 18:45:30 2024 +0800 update download page for 0.5.1 (#124) --- src/pages/download.md | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pages/download.md b/src/pages/download.md index 3e740d5..b6cb98f 100644 --- a/src/pages/download.md +++ b/src/pages/download.md @@ -11,11 +11,11 @@ For binary install, please see Fury [install](/docs/start/install/) document. ## The latest release -The latest source release is 0.5.0: +The latest source release is 0.5.1: | Version | Date | Source | Release Notes | |-|||---| -| 0.5.0 |2024-05-03 | [apache-fury-0.5.0-incubating-src.tar.gz](https://www.apache.org/dyn/closer.lua/incubator/fury/0.5.0/apache-fury-0.5.0-incubating-src.tar.gz?action=download) ([asc](https://downloads.apache.org/incubator/fury/0.5.0/apache-fury-0.5.0-incubating-src.tar.gz.asc), [sha512](https://downloads.apache.org/incubator/fury/0.5.0/apache-fury-0.5.0-incubating-src.tar.gz.sha512)) | [release notes](https://github.com/apache/incubator-fury/releases/tag/v0.5.0) | +| 0.5.1 |2024-05-28 | [apache-fury-0.5.1-incubating-src.tar.gz](https://www.apache.org/dyn/closer.lua/incubator/fury/0.5.1/apache-fury-0.5.1-incubating-src.tar.gz?action=download) ([asc](https://downloads.apache.org/incubator/fury/0.5.1/apache-fury-0.5.1-incubating-src.tar.gz.asc), [sha512](https://downloads.apache.org/incubator/fury/0.5.1/apache-fury-0.5.1-incubating-src.tar.gz.sha512)) | [release notes](https://github.com/apache/incubator-fury/releases/tag/v0.5.1) | ## All archived releases @@ -34,13 +34,13 @@ These files are named after the files they relate to but have `.sha512/.asc` ext To verify the SHA digests, you need the `.tgz` and its associated `.tgz.sha512` file. An example command: ```bash -sha512sum --check apache-fury-incubating-0.5.0-src.tar.gz +sha512sum --check apache-fury-incubating-0.5.1-src.tar.gz ``` It should output something like: ```bash -apache-fury-incubating-0.5.0-src.tar.gz: OK +apache-fury-incubating-0.5.1-src.tar.gz: OK ``` ### Verifying Signatures @@ -56,7 +56,7 @@ gpg --import KEYS Then you can verify signature: ```bash -gpg --verify apache-fury-incubating-0.5.0-src.tar.gz.asc apache-fury-incubating-0.5.0-src.tar.gz +gpg --verify apache-fury-incubating-0.5.1-src.tar.gz.asc apache-fury-incubating-0.5.1-src.tar.gz ``` If something like the following appears, it means the signature is correct: - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: feat(spec): remove list/map header from type meta spec (#1590)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury.git The following commit(s) were added to refs/heads/main by this push: new 3b61eca8 feat(spec): remove list/map header from type meta spec (#1590) 3b61eca8 is described below commit 3b61eca87c7fb7b6332f05ef5a5b6ee1927e2118 Author: Shawn Yang AuthorDate: Sun Apr 28 16:38:14 2024 +0800 feat(spec): remove list/map header from type meta spec (#1590) ## What does this PR do? This PR removes list/map header from type meta spec. Such header may be computed ahead sometimes. But it may need to compute based the data at runtime. If we write header into type meta, we may still need to compute and write header when serialization. This will introduce extra space cost. Instead, we can write all such header at runtime, and when creating/generating serializer, we can compute part of such header ahead, and let remaing parts of header computed at serialization. When deserialization, the deserializer can generate different deserializer based on read header, which will be more efficient. ## Related issues #1413 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- docs/specification/java_serialization_spec.md | 26 ++ docs/specification/xlang_serialization_spec.md | 22 +- 2 files changed, 11 insertions(+), 37 deletions(-) diff --git a/docs/specification/java_serialization_spec.md b/docs/specification/java_serialization_spec.md index 242a1416..38dd5326 100644 --- a/docs/specification/java_serialization_spec.md +++ b/docs/specification/java_serialization_spec.md @@ -178,25 +178,8 @@ Meta header is a 64 bits number value encoded in little endian order. meta for such types is written separately instead of inlining here is to reduce meta space cost if object of this type is serialized in current object graph multiple times, and the field value may be null too. -- Collection Type Info: collection type will have an extra byte for elements info. - Users can use annotation to provide those info. -- elements type same -- elements tracking ref -- elements nullability -- elements declared type -- Map Type Info: map type will have an extra byte for kv items info. - Users can use annotation to provide those info. -- keys type same -- keys tracking ref -- keys nullability -- keys declared type -- values type same -- values tracking ref -- values nullability -- values declared type - Field name: If type id is set, type id will be used instead. Otherwise meta string encoding length and data will - be - written instead. + be written instead. Field order are left as implementation details, which is not exposed to specification, the deserialization need to resort fields based on Fury field comparator. In this way, fury can compute statistics for field names or types and @@ -394,6 +377,9 @@ which will be encoded by elements header, each use one bit: By default, all bits are unset, which means all elements won't track ref, all elements are same type, not null and the actual element is the declared type in the custom class field. +The implementation can generate different deserialization code based read header, and look up the generated code from a +linear map/list. + Elements data Based on the elements header, the serialization of elements data may skip `ref flag`/`null flag`/`element class info`. @@ -469,8 +455,8 @@ format will be: |KV header | N*2 objects | ``` -`KV header` will be a header marked by `MapFieldInfo` in java. For languages such as golang, this can be computed in -advance for non-interface types in most times. +`KV header` will be a header marked by `MapFieldInfo` in java. The implementation can generate different deserialization +code based read header, and look up the generated code from a linear map/list. ### Enum diff --git a/docs/specification/xlang_serialization_spec.md b/docs/specification/xlang_serialization_spec.md index bbfdc76e..0583e7e0 100644 --- a/docs/specification/xlang_serialization_spec.md +++ b/docs/specification/xlang_serialization_spec.md @@ -303,22 +303,6 @@ Meta header is a 64 bits number value encoded in little endian order. - Otherwise it will be encoded as `OBJECT_ID` if it isn't `final` and `FINAL_OBJECT_ID` if it's `final`. The meta for such types is written separately instead of inlining here is to reduce meta space cost if object of this type
(incubator-fury) branch main updated: feat(java): enable deserializeUnexistedClass by default (#1575)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new ffc500a2 feat(java): enable deserializeUnexistedClass by default
(#1575)
ffc500a2 is described below
commit ffc500a2fe8ea5af9f0b5aa7c49e7a3709c8e5f8
Author: Shawn Yang
AuthorDate: Fri Apr 26 07:57:21 2024 +0800
feat(java): enable deserializeUnexistedClass by default (#1575)
---
docs/guide/java_serialization_guide.md | 211 -
.../java/org/apache/fury/config/FuryBuilder.java | 9 +-
2 files changed, 133 insertions(+), 87 deletions(-)
diff --git a/docs/guide/java_serialization_guide.md
b/docs/guide/java_serialization_guide.md
index 8f076765..a86c39e4 100644
--- a/docs/guide/java_serialization_guide.md
+++ b/docs/guide/java_serialization_guide.md
@@ -96,24 +96,24 @@ public class Example {
## FuryBuilder options
-| Option Name | Description
[...]
-|-|-
[...]
-| `timeRefIgnored`| Whether to ignore reference tracking
of all time types registered in `TimeSerializers` and subclasses of those types
when ref tracking is enabled. If ignored, ref tracking of every time type can
be enabled by invoking `Fury#registerSerializer(Class, Serializer)`. For
example, `fury.registerSerializer(Date.class, new DateSerializer(fury, true))`.
Note that enabling ref tracking should happen before serializer codegen of any
types which contain time [...]
-| `compressInt` | Enables or disables int compression
for smaller size.
[...]
-| `compressLong` | Enables or disables long compression
for smaller size.
[...]
-| `compressString`| Enables or disables string compression
for smaller size.
[...]
-| `classLoader` | The classloader should not be updated;
Fury caches class metadata. Use `LoaderBinding` or `ThreadSafeFury` for
classloader updates.
[...]
-| `compatibleMode`| Type forward/backward compatibility
config. Also Related to `checkClassVersion` config. `SCHEMA_CONSISTENT`: Class
schema must be consistent between serialization peer and deserialization peer.
`COMPATIBLE`: Class schema can be different between serialization peer and
deserialization peer. They can add/delete fields independently
(incubator-fury) branch main updated: chore(doc): fixed a typo (#1572)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury.git The following commit(s) were added to refs/heads/main by this push: new dd9f9128 chore(doc): fixed a typo (#1572) dd9f9128 is described below commit dd9f9128d24b418f6a420880c7780ce83d6448a2 Author: huisman6 AuthorDate: Thu Apr 25 10:58:39 2024 +0800 chore(doc): fixed a typo (#1572) Update java_serialization_guide.md ## What does this PR do? ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- docs/guide/java_serialization_guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/java_serialization_guide.md b/docs/guide/java_serialization_guide.md index 673e6a8c..8f076765 100644 --- a/docs/guide/java_serialization_guide.md +++ b/docs/guide/java_serialization_guide.md @@ -341,9 +341,9 @@ Object newObj = fury.execute( ); ``` -### Deserialize un-exited classes. +### Deserialize non-existent classes -Fury support deserializing unexisted classes, this feature can be enabled +Fury support deserializing non-existent classes, this feature can be enabled by `FuryBuilder#deserializeUnexistedClass(true)`. When enabled, and metadata sharing enabled, Fury will store the deserialized data of this type in a lazy subclass of Map. By using the lazy map implemented by Fury, the rebalance cost of filling map during deserialization can be avoided, which further improves performance. If this data is sent to - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: fix(javascript): fix macos javascript ci failure (#1567)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new d3a78768 fix(javascript): fix macos javascript ci failure (#1567)
d3a78768 is described below
commit d3a787687b18c73e23fefaa198abb7504cba1e24
Author: Shawn Yang
AuthorDate: Wed Apr 24 19:44:34 2024 +0800
fix(javascript): fix macos javascript ci failure (#1567)
---
.github/workflows/ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c3528629..c22abdea 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -136,7 +136,7 @@ jobs:
strategy:
matrix:
node-version: [14.x, 16.x, 18.x, 20.x]
-os: [ubuntu-latest, macos-latest, windows-2022]
+os: [ubuntu-latest, macos-13, windows-2022]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: feat(javascript): Implement Xlang map (#1549)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 53ee111c feat(javascript): Implement Xlang map (#1549)
53ee111c is described below
commit 53ee111c312debb362a61fc5045db709d876e9ce
Author: weipeng
AuthorDate: Mon Apr 22 23:44:59 2024 +0800
feat(javascript): Implement Xlang map (#1549)
## What does this PR do?
1. Implement the map protocol.
2. Cache the primitive type serializer, guess serializer by data type
every time is unnecessary.
In this PR , the code generator is simple call the AnySerializer which
is much more slower then jit code, I will implement the code generator
next PR.
## Does this PR introduce any user-facing change?
No user-facing change.
## Benchmark
Before:
| (index)| Values |
|-||
| serialize | 365|
| deserialize | 193|
After:
| (index)| Values |
|-||
| serialize | 519|
| deserialize | 238|
-
Co-authored-by: wangweipeng
---
javascript/benchmark/map.js| 60 ++
javascript/packages/fury/lib/classResolver.ts | 50 +++--
javascript/packages/fury/lib/gen/collection.ts | 13 +-
javascript/packages/fury/lib/gen/map.ts| 276 -
javascript/packages/fury/lib/meta.ts | 17 ++
javascript/packages/fury/lib/writer/index.ts | 8 +
6 files changed, 345 insertions(+), 79 deletions(-)
diff --git a/javascript/benchmark/map.js b/javascript/benchmark/map.js
new file mode 100644
index ..8e4df29f
--- /dev/null
+++ b/javascript/benchmark/map.js
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const Fury = require("@furyjs/fury");
+const hps = require('@furyjs/hps');
+const fury = new Fury.default({ hps, refTracking: false, useSliceString: true
});
+const Benchmark = require("benchmark");
+const Type = Fury.Type;
+
+
+
+const { serialize, deserialize, serializeVolatile } =
fury.registerSerializer(Type.map(Type.any(), Type.any()));
+const sample = new Map([["foo", "ba1"],["foo1", "ba1"],["foo2",
"ba1"],["foo3", "ba1"],["foo4", "ba1"],["foo5", "ba1"]]);
+const furyAb = serialize(sample);
+
+
+async function start() {
+
+ let result = {
+serialize: 0,
+deserialize: 0
+ }
+
+ {
+var suite = new Benchmark.Suite();
+suite
+ .add("serialize", function () {
+serializeVolatile(sample).dispose()
+ })
+ .add("deserialize", function () {
+deserialize(furyAb)
+ })
+ .on("complete", function (e) {
+e.currentTarget.forEach(({ name, hz }) => {
+ result[name] = Math.ceil(hz / 1);
+});
+ })
+ .run({ async: false });
+ }
+
+ console.table(result);
+
+}
+start();
diff --git a/javascript/packages/fury/lib/classResolver.ts
b/javascript/packages/fury/lib/classResolver.ts
index c2d0f4c4..27dbbc64 100644
--- a/javascript/packages/fury/lib/classResolver.ts
+++ b/javascript/packages/fury/lib/classResolver.ts
@@ -72,6 +72,7 @@ const uninitSerialize = {
fixedSize: 0,
type: InternalSerializerType.ANY,
needToWriteRef: false,
+typeId: null,
},
};
@@ -114,8 +115,26 @@ export default class SerializerResolver {
this.registerSerializer(fury, Type.float16Array());
this.registerSerializer(fury, Type.float32Array());
this.registerSerializer(fury, Type.float64Array());
+
+this.numberSerializer =
this.getSerializerById(SerializerResolver.getTypeIdByInternalSerializerType(InternalSerializerType.FLOAT64));
+this.int64Serializer =
this.getSerializerById(SerializerResolver.getTypeIdByInternalSerializerType(InternalSerializerType.INT64));
+this.boolSerializer =
this.getSerializerById(SerializerResolver.getTypeI
(incubator-fury) branch main updated: perf(java): fix fury input stream read slow (#1551)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new f246aafc perf(java): fix fury input stream read slow (#1551)
f246aafc is described below
commit f246aafc920455f6e65dd898b0ab31c0be3f6cc1
Author: Shawn Yang
AuthorDate: Sat Apr 20 14:40:36 2024 +0800
perf(java): fix fury input stream read slow (#1551)
---
.../java/org/apache/fury/io/FuryInputStream.java | 35 +-
1 file changed, 21 insertions(+), 14 deletions(-)
diff --git
a/java/fury-core/src/main/java/org/apache/fury/io/FuryInputStream.java
b/java/fury-core/src/main/java/org/apache/fury/io/FuryInputStream.java
index 4b81de9b..87cf82f5 100644
--- a/java/fury-core/src/main/java/org/apache/fury/io/FuryInputStream.java
+++ b/java/fury-core/src/main/java/org/apache/fury/io/FuryInputStream.java
@@ -53,24 +53,15 @@ public class FuryInputStream extends InputStream implements
FuryStreamReader {
MemoryBuffer buffer = this.buffer;
byte[] heapMemory = buffer.getHeapMemory();
int offset = buffer.size();
-int targetSize = offset + minFillSize;
-if (targetSize > heapMemory.length) {
- int newSize;
- if (targetSize < BUFFER_GROW_STEP_THRESHOLD) {
-newSize = targetSize << 2;
- } else {
-newSize = (int) (targetSize * 1.5);
- }
- byte[] newBuffer = new byte[newSize];
- System.arraycopy(heapMemory, 0, newBuffer, 0, buffer.size());
- buffer.initHeapBuffer(newBuffer, 0, buffer.size());
- heapMemory = newBuffer;
+if (offset + minFillSize > heapMemory.length) {
+ heapMemory = growBuffer(minFillSize, buffer);
}
try {
int read;
- read = stream.read(heapMemory, offset, Math.min(stream.available(),
heapMemory.length));
+ int len = heapMemory.length - offset;
+ read = stream.read(heapMemory, offset, len);
while (read < minFillSize) {
-int newRead = stream.read(heapMemory, offset + read, minFillSize -
read);
+int newRead = stream.read(heapMemory, offset + read, len - read);
if (newRead < 0) {
throw new IndexOutOfBoundsException("No enough data in the stream "
+ stream);
}
@@ -83,6 +74,22 @@ public class FuryInputStream extends InputStream implements
FuryStreamReader {
}
}
+ private static byte[] growBuffer(int minFillSize, MemoryBuffer buffer) {
+int newSize;
+int targetSize = buffer.size() + minFillSize;
+if (targetSize < BUFFER_GROW_STEP_THRESHOLD) {
+ newSize = targetSize << 2;
+} else {
+ newSize = (int) (targetSize * 1.5);
+}
+byte[] newBuffer = new byte[newSize];
+byte[] heapMemory = buffer.getHeapMemory();
+System.arraycopy(heapMemory, 0, newBuffer, 0, buffer.size());
+buffer.initHeapBuffer(newBuffer, 0, buffer.size());
+heapMemory = newBuffer;
+return heapMemory;
+ }
+
@Override
public void readTo(byte[] dst, int dstIndex, int len) {
MemoryBuffer buf = buffer;
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: chore(doc): rename guide document name (#1538)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 0d074216 chore(doc): rename guide document name (#1538)
0d074216 is described below
commit 0d074216964e2bf1918d438524a139b0fd8aa1f2
Author: Shawn Yang
AuthorDate: Fri Apr 19 12:25:51 2024 +0800
chore(doc): rename guide document name (#1538)
---
README.md | 2 +-
docs/README.md| 4 ++--
.../guide/{java_object_graph_guide.md => java_serialization_guide.md} | 2 +-
.../{xlang_object_graph_guide.md => xlang_serialization_guide.md} | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/README.md b/README.md
index 24543cbd..6b8fe19a 100644
--- a/README.md
+++ b/README.md
@@ -144,7 +144,7 @@ go get github.com/apache/incubator-fury/go/fury
```
## Quickstart
-Here we give a quick start about how to use Fury, see [user
guide](https://github.com/apache/incubator-fury/blob/main/docs/README.md) for
more details about
[java](https://github.com/apache/incubator-fury/blob/main/docs/guide/java_object_graph_guide.md),
[cross
language](https://github.com/apache/incubator-fury/blob/main/docs/guide/xlang_object_graph_guide.md),
and [row
format](https://github.com/apache/incubator-fury/blob/main/docs/guide/row_format_guide.md).
+Here we give a quick start about how to use Fury, see [user
guide](docs/README.md) for more details about
[java](docs/guide/java_serialization_guide.md), [cross
language](docs/guide/xlang_serialization_guide.md), and [row
format](docs/guide/row_format_guide.md).
### Fury java object graph serialization
If you don't have cross-language requirements, using this mode will
diff --git a/docs/README.md b/docs/README.md
index bb09506d..6de510a0 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,6 +1,6 @@
# User Guide
-- For Cross Language Object Graph Guide, see [xlang serialization
guide](guide/xlang_object_graph_guide.md) doc.
-- For Java Object Graph Guide, see [java serialization
guide](guide/java_object_graph_guide.md) doc.
+- For Cross Language Object Graph Guide, see [xlang serialization
guide](guide/xlang_serialization_guide.md) doc.
+- For Java Object Graph Guide, see [java serialization
guide](guide/java_serialization_guide.md) doc.
- For Row Format Guide, see [row format guide](guide/row_format_guide.md) doc.
- For Scala Guide, see [scala guide](guide/scala_guide.md) doc.
- For using Fury with GraalVM native image, see [graalvm native image
guide](guide/graalvm_guide.md) doc.
diff --git a/docs/guide/java_object_graph_guide.md
b/docs/guide/java_serialization_guide.md
similarity index 99%
rename from docs/guide/java_object_graph_guide.md
rename to docs/guide/java_serialization_guide.md
index 997b7a12..673e6a8c 100644
--- a/docs/guide/java_object_graph_guide.md
+++ b/docs/guide/java_serialization_guide.md
@@ -1,5 +1,5 @@
---
-title: Java Object Graph Guide
+title: Java Serialization Guide
sidebar_position: 0
id: java_object_graph_guide
---
diff --git a/docs/guide/xlang_object_graph_guide.md
b/docs/guide/xlang_serialization_guide.md
similarity index 99%
rename from docs/guide/xlang_object_graph_guide.md
rename to docs/guide/xlang_serialization_guide.md
index 62ae1880..02738281 100644
--- a/docs/guide/xlang_object_graph_guide.md
+++ b/docs/guide/xlang_serialization_guide.md
@@ -1,5 +1,5 @@
---
-title: Xlang Object Graph Guide
+title: Xlang Serialization Guide
sidebar_position: 2
id: xlang_object_graph_guide
---
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: fix(java): skip weakmap test for openj9 (#1536)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 5e3ebc0d fix(java): skip weakmap test for openj9 (#1536)
5e3ebc0d is described below
commit 5e3ebc0d49229df9c3c749d03f217b13a5b516d8
Author: Shawn Yang
AuthorDate: Thu Apr 18 18:36:03 2024 +0800
fix(java): skip weakmap test for openj9 (#1536)
---
.../test/java/org/apache/fury/collection/MultiKeyWeakMapTest.java| 5 +
1 file changed, 5 insertions(+)
diff --git
a/java/fury-core/src/test/java/org/apache/fury/collection/MultiKeyWeakMapTest.java
b/java/fury-core/src/test/java/org/apache/fury/collection/MultiKeyWeakMapTest.java
index 35f0be2c..adde8d06 100644
---
a/java/fury-core/src/test/java/org/apache/fury/collection/MultiKeyWeakMapTest.java
+++
b/java/fury-core/src/test/java/org/apache/fury/collection/MultiKeyWeakMapTest.java
@@ -24,6 +24,8 @@ import java.lang.reflect.Field;
import java.util.Set;
import org.apache.fury.logging.Logger;
import org.apache.fury.logging.LoggerFactory;
+import org.apache.fury.util.unsafe._JDKAccess;
+import org.testng.SkipException;
import org.testng.annotations.Test;
public class MultiKeyWeakMapTest {
@@ -32,6 +34,9 @@ public class MultiKeyWeakMapTest {
@SuppressWarnings("unchecked")
@Test(timeOut = 6)
public void testMap() throws Exception {
+if (_JDKAccess.IS_OPEN_J9) {
+ throw new SkipException("OpenJ9 unsupported");
+}
MultiKeyWeakMap map = new MultiKeyWeakMap<>();
Field referencesField =
MultiKeyWeakMap.class.getDeclaredField("REFERENCES");
referencesField.setAccessible(true);
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: feat(spec): add doc for why map use chunk by chunk spec (#1508)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury.git The following commit(s) were added to refs/heads/main by this push: new 971951a4 feat(spec): add doc for why map use chunk by chunk spec (#1508) 971951a4 is described below commit 971951a473afd1ae44d295cf1d08b0d2d625de57 Author: Shawn Yang AuthorDate: Mon Apr 15 19:26:34 2024 +0800 feat(spec): add doc for why map use chunk by chunk spec (#1508) --- docs/specification/xlang_serialization_spec.md | 17 - 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/specification/xlang_serialization_spec.md b/docs/specification/xlang_serialization_spec.md index 15094c52..4641d2ba 100644 --- a/docs/specification/xlang_serialization_spec.md +++ b/docs/specification/xlang_serialization_spec.md @@ -559,7 +559,7 @@ Map iteration is too expensive, Fury won't compute the header like for list sinc Users can use `MapFieldInfo` annotation to provide the header in advance. Otherwise Fury will use first key-value pair to predict header optimistically, and update the chunk header if the prediction failed at some pair. -Fury will serialize the map chunk by chunk, every chunk has 127 pairs at most. +Fury will serialize the map chunk by chunk, every chunk has 255 pairs at most. ``` |1 byte | 1 byte | variable bytes | @@ -592,6 +592,21 @@ format will be: `KV header` will be a header marked by `MapFieldInfo` in java. For languages such as golang, this can be computed in advance for non-interface types most times. + Why serialize chunk by chunk? + +When fury will use first key-value pair to predict header optimistically, it can't know how many pairs have same +meta(tracking kef ref, key has null and so on). If we don't write chunk by chunk with max chunk size, we must write at +least `X` bytes to take up a place for later to update the number which has same elements, `X` is the num_bytes for +encoding varint encoding of map size. + +And most map size are smaller than 255, if all pairs have same data, the chunk will be 1. This is common in golang/rust, +which object are not reference by default. + +Also, if only one or two keys have different meta, we can make it into a different chunk, so that most pairs can share +meta. + +The implementation can accumulate read count with map size to decide whether to read more chunks. + ### enum Enums are serialized as an unsigned var int. If the order of enum values change, the deserialized enum value may not be - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: fix(java): fix create NilLogger (#1506)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 7433ac87 fix(java): fix create NilLogger (#1506)
7433ac87 is described below
commit 7433ac87dbed29871a853c3708674b0286f92a12
Author: Shawn Yang
AuthorDate: Sun Apr 14 20:37:44 2024 +0800
fix(java): fix create NilLogger (#1506)
## What does this PR do?
THis PR fix fix NilLogger creation introduced in #1503
## Related issues
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
---
java/fury-core/src/main/java/org/apache/fury/logging/LoggerFactory.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/java/fury-core/src/main/java/org/apache/fury/logging/LoggerFactory.java
b/java/fury-core/src/main/java/org/apache/fury/logging/LoggerFactory.java
index c5f290f4..c5cd46da 100644
--- a/java/fury-core/src/main/java/org/apache/fury/logging/LoggerFactory.java
+++ b/java/fury-core/src/main/java/org/apache/fury/logging/LoggerFactory.java
@@ -71,7 +71,7 @@ public class LoggerFactory {
*/
public static Logger getLogger(Class clazz) {
if (disableLogging) {
- return new FuryLogger(clazz, logLevel);
+ return new NilLogger();
} else {
if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE || !useSlf4jLogger) {
return new FuryLogger(clazz, logLevel);
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: chore(javaScript): enhancement code readability (#1505)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new e8d59d66 chore(javaScript): enhancement code readability (#1505)
e8d59d66 is described below
commit e8d59d66eb76fde48dc5b30611e87c6e5226fad4
Author: weipeng
AuthorDate: Sun Apr 14 19:30:44 2024 +0800
chore(javaScript): enhancement code readability (#1505)
## What does this PR do?
**Reduce the use of closures to enhance code readability.**
We used closures extensively in the past, which could easily trigger
Inline-Cache in `V8`, but this practice was detrimental to code
readability. Now that the infrastructure is stable, we can replace them
with classes to enhance readability without affecting performance.
## Does this PR introduce any user-facing change?
There are no user-facing change.
## Benchmark
| (index) | serialize | deserialize |
|--||--|
| fury | 151| 120 |
| protobuf | 29 | 39 |
| json | 28 | 35 |
-
Co-authored-by: wangweipeng
---
javascript/packages/fury/lib/classResolver.ts | 7 +-
javascript/packages/fury/lib/fury.ts | 14 +-
javascript/packages/fury/lib/gen/serializer.ts| 5 +-
javascript/packages/fury/lib/reader/index.ts | 298 +-
javascript/packages/fury/lib/referenceResolver.ts | 62 ++--
javascript/packages/fury/lib/type.ts | 5 -
javascript/packages/fury/lib/writer/index.ts | 354 ++
javascript/test/io.test.ts| 84 ++---
javascript/test/reader.test.ts| 4 +-
javascript/test/referenceResolve.test.ts | 10 +-
javascript/test/util.js | 6 +-
javascript/test/writer.test.ts| 4 +-
12 files changed, 400 insertions(+), 453 deletions(-)
diff --git a/javascript/packages/fury/lib/classResolver.ts
b/javascript/packages/fury/lib/classResolver.ts
index 2156640b..c2d0f4c4 100644
--- a/javascript/packages/fury/lib/classResolver.ts
+++ b/javascript/packages/fury/lib/classResolver.ts
@@ -17,13 +17,14 @@
* under the License.
*/
-import { InternalSerializerType, Serializer, BinaryReader, BinaryWriter as
TBinaryWriter } from "./type";
+import { InternalSerializerType, Serializer } from "./type";
import { fromString } from "./platformBuffer";
import { x64hash128 } from "./murmurHash3";
import { BinaryWriter } from "./writer";
import { generateSerializer } from "./gen";
import { Type, TypeDescription } from "./description";
import Fury from "./fury";
+import { BinaryReader } from "./reader";
const USESTRINGVALUE = 0;
const USESTRINGID = 1;
@@ -157,7 +158,7 @@ export default class SerializerResolver {
static tagBuffer(tag: string) {
const tagBuffer = fromString(tag);
const bufferLen = tagBuffer.byteLength;
-const writer = BinaryWriter({});
+const writer = new BinaryWriter({});
let tagHash = x64hash128(tagBuffer, 47).getBigUint64(0);
if (tagHash === 0n) {
@@ -177,7 +178,7 @@ export default class SerializerResolver {
const fullBuffer = SerializerResolver.tagBuffer(tag);
return {
- write: (binaryWriter: TBinaryWriter) => {
+ write: (binaryWriter: BinaryWriter) => {
const tagIndex = this.writeStringIndex[idx];
if (tagIndex > -1) {
// equivalent of: `uint8(USESTRINGID); int16(tagIndex)`
diff --git a/javascript/packages/fury/lib/fury.ts
b/javascript/packages/fury/lib/fury.ts
index aa53e13a..2889ecc3 100644
--- a/javascript/packages/fury/lib/fury.ts
+++ b/javascript/packages/fury/lib/fury.ts
@@ -21,16 +21,16 @@ import ClassResolver from "./classResolver";
import { BinaryWriter } from "./writer";
import { BinaryReader } from "./reader";
import { ReferenceResolver } from "./referenceResolver";
-import { ConfigFlags, Serializer, Config, Language, BinaryReader as
BinaryReaderType, BinaryWriter as BinaryWriterType } from "./type";
+import { ConfigFlags, Serializer, Config, Language } from "./type";
import { OwnershipError } from "./error";
import { InputType, ResultType, TypeDescription } from "./description";
import { generateSerializer, AnySerializer } from "./gen";
export default class {
- binaryReader: BinaryReaderType;
- binaryWriter: BinaryWriterType;
+ binaryReader: BinaryReader;
+ binaryWriter: BinaryWriter;
classResolver = new ClassResolver();
- referenceResolver: ReturnType;
+ referenceResolver: ReferenceResolver;
(incubator-fury) branch main updated: fix(java): make slf4j optional (#1494)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury.git The following commit(s) were added to refs/heads/main by this push: new 06b8e2b7 fix(java): make slf4j optional (#1494) 06b8e2b7 is described below commit 06b8e2b70c4e74d4147058330e630e4a04845d07 Author: Shawn Yang AuthorDate: Sat Apr 13 13:56:27 2024 +0800 fix(java): make slf4j optional (#1494) ## What does this PR do? make slf4j optional ## Related issues #1485 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- java/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/pom.xml b/java/pom.xml index bda70865..883a050e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -92,6 +92,7 @@ org.slf4j slf4j-api 2.0.12 +true com.google.guava - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: chore(java): rename remaing get/put int/float (#1500)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 125e43bb chore(java): rename remaing get/put int/float (#1500)
125e43bb is described below
commit 125e43bb0d820e6f2cf08fafd7f6890ab4fc
Author: Shawn Yang
AuthorDate: Sat Apr 13 12:28:05 2024 +0800
chore(java): rename remaing get/put int/float (#1500)
## What does this PR do?
rename remaing get/put int/float
## Related issues
#1480
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
---
.../apache/fury/benchmark/CompressStringSuite.java | 6 +-
.../src/main/java/org/apache/fury/Fury.java| 12 +-
.../apache/fury/builder/ObjectCodecBuilder.java| 12 +-
.../java/org/apache/fury/io/FuryInputStream.java | 2 +-
.../main/java/org/apache/fury/memory/BitUtils.java | 26 ++--
.../java/org/apache/fury/memory/MemoryBuffer.java | 139 +++--
.../apache/fury/resolver/EnumStringResolver.java | 10 +-
.../apache/fury/serializer/BufferSerializers.java | 2 +-
.../fury/serializer/PrimitiveSerializers.java | 2 +-
.../apache/fury/serializer/StringSerializer.java | 29 ++---
.../java/org/apache/fury/memory/BitUtilsTest.java | 2 +-
.../org/apache/fury/memory/MemoryBufferTest.java | 68 +-
.../java/org/apache/fury/format/row/Getters.java | 10 +-
.../java/org/apache/fury/format/row/Setters.java | 10 +-
.../org/apache/fury/format/row/ValueVisitor.java | 10 +-
.../apache/fury/format/row/binary/BinaryArray.java | 2 +-
.../apache/fury/format/row/binary/BinaryMap.java | 6 +-
.../apache/fury/format/row/binary/BinaryRow.java | 4 +-
.../apache/fury/format/row/binary/BinaryUtils.java | 10 +-
.../apache/fury/format/row/binary/UnsafeTrait.java | 62 -
.../row/binary/writer/BinaryArrayWriter.java | 16 +--
.../format/row/binary/writer/BinaryRowWriter.java | 22 ++--
.../format/row/binary/writer/BinaryWriter.java | 18 +--
.../org/apache/fury/format/type/DataTypes.java | 2 +-
.../apache/fury/format/vectorized/ArrowWriter.java | 14 +--
.../fury/format/row/binary/BinaryArrayTest.java| 4 +-
.../fury/format/row/binary/BinaryMapTest.java | 2 +-
.../fury/format/row/binary/BinaryRowTest.java | 24 ++--
28 files changed, 267 insertions(+), 259 deletions(-)
diff --git
a/java/benchmark/src/main/java/org/apache/fury/benchmark/CompressStringSuite.java
b/java/benchmark/src/main/java/org/apache/fury/benchmark/CompressStringSuite.java
index 3bc042e9..dc66db92 100644
---
a/java/benchmark/src/main/java/org/apache/fury/benchmark/CompressStringSuite.java
+++
b/java/benchmark/src/main/java/org/apache/fury/benchmark/CompressStringSuite.java
@@ -53,7 +53,7 @@ public class CompressStringSuite {
MemoryBuffer directBuffer = CompressStringSuite.directBuffer;
char[] latinStrChars = CompressStringSuite.latinStrChars;
for (int i = 0; i < latinStrChars.length; i++) {
- directBuffer.put(i, (byte) (latinStrChars[i]));
+ directBuffer.putByte(i, (byte) (latinStrChars[i]));
}
return directBuffer;
}
@@ -81,8 +81,8 @@ public class CompressStringSuite {
for (int i = 0; i < utf16StrChars.length; i++) {
int index = i << 1;
char c = utf16StrChars[i];
- directBuffer.put(index++, (byte) (c));
- directBuffer.put(index, (byte) (c >> 8));
+ directBuffer.putByte(index++, (byte) (c));
+ directBuffer.putByte(index, (byte) (c >> 8));
}
return directBuffer;
}
diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java
b/java/fury-core/src/main/java/org/apache/fury/Fury.java
index d9f875a4..90b31913 100644
--- a/java/fury-core/src/main/java/org/apache/fury/Fury.java
+++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java
@@ -228,7 +228,7 @@ public final class Fury implements BaseFury {
byte bitmap = 0;
if (obj == null) {
bitmap |= isNilFlag;
- buffer.put(maskIndex, bitmap);
+ buffer.putByte(maskIndex, bitmap);
return buffer;
}
// set endian.
@@ -244,7 +244,7 @@ public final class Fury implements BaseFury {
if (bufferCallback != null) {
bitmap |= isOutOfBandFlag;
}
-buffer.put(maskIndex, bitmap);
+buffer.putByte(maskIndex, bitmap);
try {
jitContext.lock();
if (depth != 0) {
@@ -311,7 +311,7 @@ public final class Fury implements BaseFury {
int startOffset = buffer.writerIndex();
buffer.writeInt32(-1); // preserve 4-byte for nativeObjects start
offsets.
writeRef(buffer, obj);
- b
(incubator-fury) branch main updated: feat(JavaScript): implement xlang protocol (#1487)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 417f064b feat(JavaScript): implement xlang protocol (#1487)
417f064b is described below
commit 417f064b705a55fd9fe0270b464863d54199c327
Author: weipeng
AuthorDate: Thu Apr 11 22:32:03 2024 +0800
feat(JavaScript): implement xlang protocol (#1487)
## What does this PR do?
1. Remove unsigned types which are removed in xlang protocol.
2. Rename `long int double` to `int64 int32 float64` to ensure
consistency with other languages
3. Implements the float16 writer and float16 reader.
## Related issues
## Does this PR introduce any user-facing change?
The JavaScript implementation has not been released yet, so there are no
user-facing changes.
## Benchmark
There are no fluctuations in the benchmark.
| (index) | serialize | deserialize |
|--|---|-|
| fury | 116 | 118 |
| protobuf | 25| 33 |
| json | 20| 23 |
-
Co-authored-by: wangweipeng
---
javascript/packages/fury/lib/classResolver.ts | 120 +++
javascript/packages/fury/lib/description.ts| 132 ++---
javascript/packages/fury/lib/gen/any.ts| 3 +-
javascript/packages/fury/lib/gen/builder.ts| 28 +++--
javascript/packages/fury/lib/gen/collection.ts | 4 +-
javascript/packages/fury/lib/gen/datetime.ts | 4 +-
javascript/packages/fury/lib/gen/index.ts | 7 +-
javascript/packages/fury/lib/gen/number.ts | 57 +
javascript/packages/fury/lib/gen/object.ts | 9 +-
javascript/packages/fury/lib/gen/oneof.ts | 115 --
javascript/packages/fury/lib/gen/serializer.ts | 5 +-
javascript/packages/fury/lib/gen/set.ts| 2 +-
javascript/packages/fury/lib/gen/typedArray.ts | 17 +--
javascript/packages/fury/lib/meta.ts | 64 +-
javascript/packages/fury/lib/reader/index.ts | 41 ++-
javascript/packages/fury/lib/referenceResolver.ts | 3 +-
javascript/packages/fury/lib/type.ts | 66 ++-
.../fury/lib/{writer.ts => writer/index.ts}| 24 ++--
javascript/packages/fury/lib/writer/number.ts | 47
javascript/test/array.test.ts | 73 ++--
javascript/test/binary.test.ts | 2 +-
javascript/test/datetime.test.ts | 2 +-
javascript/test/fixtures/tuple.ts | 2 +-
javascript/test/fury.test.ts | 2 +-
javascript/test/io.test.ts | 18 +--
javascript/test/number.test.ts | 86 ++
javascript/test/object.test.ts | 24 ++--
javascript/test/oneof.test.ts | 90 --
javascript/test/protocol/struct.test.ts| 4 +-
javascript/test/referenceResolve.test.ts | 5 +-
30 files changed, 447 insertions(+), 609 deletions(-)
diff --git a/javascript/packages/fury/lib/classResolver.ts
b/javascript/packages/fury/lib/classResolver.ts
index 62cc1b0b..2156640b 100644
--- a/javascript/packages/fury/lib/classResolver.ts
+++ b/javascript/packages/fury/lib/classResolver.ts
@@ -84,7 +84,7 @@ export default class SerializerResolver {
private writeStringIndex: number[] = [];
private registerSerializer(fury: Fury, description: TypeDescription) {
-return fury.classResolver.registerSerializerById(description.type,
generateSerializer(fury, description));
+return
fury.classResolver.registerSerializerById(SerializerResolver.getTypeIdByInternalSerializerType(description.type),
generateSerializer(fury, description));
}
private initInternalSerializer(fury: Fury) {
@@ -92,27 +92,27 @@ export default class SerializerResolver {
this.registerSerializer(fury, Type.array(Type.any()));
this.registerSerializer(fury, Type.map(Type.any(), Type.any()));
this.registerSerializer(fury, Type.bool());
-this.registerSerializer(fury, Type.uint8());
this.registerSerializer(fury, Type.int8());
-this.registerSerializer(fury, Type.uint16());
this.registerSerializer(fury, Type.int16());
-this.registerSerializer(fury, Type.uint32());
this.registerSerializer(fury, Type.int32());
-this.registerSerializer(fury, Type.uint64());
+this.registerSerializer(fury, Type.varInt32());
this.registerSerializer(fury, Type.int64());
-this.registerSerializer(fury, Type.float());
-this.registerSerializer(fury, Type.double());
+this.registerSerializer(fury, Type.sliInt64());
+this.registerSerializer(fury, Type.floa
(incubator-fury) branch main updated: chore: add pull request template (#1477)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury.git The following commit(s) were added to refs/heads/main by this push: new 6ee0a2b9 chore: add pull request template (#1477) 6ee0a2b9 is described below commit 6ee0a2b9aa74859d6d44aeadf4bfcf15a888239b Author: LiangliangSui <116876207+liangliang...@users.noreply.github.com> AuthorDate: Tue Apr 9 21:33:24 2024 +0800 chore: add pull request template (#1477) --- .github/pull_request_template.md | 43 1 file changed, 43 insertions(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index ..6e5d4d60 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,43 @@ + + +## What does this PR do? + + + + +## Related issues + + + + +## Does this PR introduce any user-facing change? + + + +- [ ] Does this PR introduce any public API change? +- [ ] Does this PR introduce any binary protocol compatibility change? + + +## Benchmark + + - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: feat(spec): add xlang type mapping (#1468)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury.git The following commit(s) were added to refs/heads/main by this push: new 95979623 feat(spec): add xlang type mapping (#1468) 95979623 is described below commit 959796233fdd25a3bf47bc2dd75c6d60e5d7d08e Author: Shawn Yang AuthorDate: Sat Apr 6 20:30:50 2024 +0800 feat(spec): add xlang type mapping (#1468) --- docs/guide/DEVELOPMENT.md | 2 +- docs/guide/xlang_type_mapping.md | 73 +- docs/specification/xlang_serialization_spec.md | 2 - 3 files changed, 73 insertions(+), 4 deletions(-) diff --git a/docs/guide/DEVELOPMENT.md b/docs/guide/DEVELOPMENT.md index 5d71166a..bfab5417 100644 --- a/docs/guide/DEVELOPMENT.md +++ b/docs/guide/DEVELOPMENT.md @@ -1,6 +1,6 @@ --- title: Development -sidebar_position: 6 +sidebar_position: 7 id: development --- diff --git a/docs/guide/xlang_type_mapping.md b/docs/guide/xlang_type_mapping.md index f529d4ef..bec22957 100644 --- a/docs/guide/xlang_type_mapping.md +++ b/docs/guide/xlang_type_mapping.md @@ -4,4 +4,75 @@ sidebar_position: 3 id: xlang_type_mapping --- -Coming soon. +Note: + +- For type definition, see [Type Systems in Spec](../specification/xlang_serialization_spec.md#type-systems) +- `int16_t[x]/vector<~>` indicates `int16_t[x]/vector` +- The cross-language serialization is not stable, do not use it in your production environment. + +# Type Mapping + +| Type | Type ID | Java| Python | Javascript | C++| Golang | Rust | +||-|-|--|-||--|--| +| bool | 1 | bool/Boolean| bool | Boolean | bool | bool | bool | +| int8 | 2 | byte/Byte | int/pyfury.Int8 | Type.int8() | int8_t | int8 | i8 | +| int16 | 3 | short/Short | int/pyfury.Int16 | Type.int16()| int16_t| int16| i6 | +| int32 | 4 | int/Integer | int/pyfury.Int32 | Type.int32()| int32_t| int32| i32 | +| var_int32 | 5 | int/Integer | int/pyfury.VarInt32 | Type.varint32() | fury::varint32_t | fury.varint32| fury::varint32 | +| int64 | 6 | long/Long | int/pyfury.Int64 | Type.int64()| int64_t| int64| i64 | +| var_int64 | 7 | long/Long | int/pyfury.VarInt64 | Type.varint64() | fury::varint64_t | fury.varint64| fury::varint64 | +| sli_int64 | 8 | long/Long | int/pyfury.SliInt64 | Type.sliint64() | fury::sliint64_t | fury.sliint64| fury::sliint64 | +| float16| 9 | float/Float | float/pyfury.Float16 | Type.float16() | fury::float16_t| fury.float16 | fury::f16 | +| float32| 10 | float/Float | float/pyfury.Float32 | Type.float32() | float | float32 | f32 | +| float64| 11 | double/Double | float/pyfury.Float64 | Type.float64() | double | float64 | f64 | +| string | 12 | String | str | String | string | string | String/str | +| enum | 13 | Enum subclasses | enum subclasses | / | enum | /| enum | +| list | 14 | List/Collection | list/tuple | array | vector | slice| Vec | +| set| 15 | Set | set | / | set| fury.Set | Set | +| map| 16 | Map | dict | Map | unordered_map | map | HashMap | +| duration | 17 | Duration| timedelta| Number | duration | Duration | Duration | +| timestamp | 18 | Instant | datetime | Number | std::chrono::nanoseconds | Time | DateTime | +| decimal| 19 | BigDecimal | Decimal |
(incubator-fury) branch main updated: feat(java): refactor readVarUint32 algorithm (#1462)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 55d625a1 feat(java): refactor readVarUint32 algorithm (#1462)
55d625a1 is described below
commit 55d625a1035f9612a7d7996eb01d1c51a0e24ef0
Author: Shawn Yang
AuthorDate: Fri Apr 5 08:40:56 2024 +0800
feat(java): refactor readVarUint32 algorithm (#1462)
---
.../java/org/apache/fury/memory/MemoryBuffer.java | 40 --
1 file changed, 21 insertions(+), 19 deletions(-)
diff --git
a/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
b/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
index 45db5ddc..7e8ffae3 100644
--- a/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
+++ b/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
@@ -1287,26 +1287,28 @@ public final class MemoryBuffer {
if (size - readIdx < 5) {
return readPositiveVarIntSlow();
}
-// varint are written using little endian byte order, so read by little
endian byte order.
+// | 1bit + 7bits | 1bit + 7bits | 1bit + 7bits | 1bit + 7bits |
int fourByteValue = unsafeGetInt(readIdx);
-int b = fourByteValue & 0xFF;
-readIdx++; // read one byte
-int result = b & 0x7F;
-if ((b & 0x80) != 0) {
- readIdx++; // read one byte
- b = (fourByteValue >>> 8) & 0xFF;
- result |= (b & 0x7F) << 7;
- if ((b & 0x80) != 0) {
-readIdx++; // read one byte
-b = (fourByteValue >>> 16) & 0xFF;
-result |= (b & 0x7F) << 14;
-if ((b & 0x80) != 0) {
- readIdx++; // read one byte
- b = (fourByteValue >>> 24) & 0xFF;
- result |= (b & 0x7F) << 21;
- if ((b & 0x80) != 0) {
-b = unsafeGet(readIdx++); // read one byte
-result |= (b & 0x7F) << 28;
+readIdx++;
+int result = fourByteValue & 0x7F;
+// Duplicate and manual inline for performance.
+// noinspection Duplicates
+if ((fourByteValue & 0x80) != 0) {
+ readIdx++;
+ // 0x3f80: 0b111 << 7
+ result |= (fourByteValue >>> 1) & 0x3f80;
+ // 0x8000: 0b1 << 15
+ if ((fourByteValue & 0x8000) != 0) {
+readIdx++;
+// 0x1fc000: 0b111 << 14
+result |= (fourByteValue >>> 2) & 0x1fc000;
+// 0x80: 0b1 << 23
+if ((fourByteValue & 0x80) != 0) {
+ readIdx++;
+ // 0xfe0: 0b111 << 21
+ result |= (fourByteValue >>> 3) & 0xfe0;
+ if ((fourByteValue & 0x8000) != 0) {
+result |= (UNSAFE.getByte(heapMemory, address + readIdx++) & 0x7F)
<< 28;
}
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: fix(java): refine writeVarUint64 readbility (#1464)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 2ebdad05 fix(java): refine writeVarUint64 readbility (#1464)
2ebdad05 is described below
commit 2ebdad055b159c06908835c093374e9b723f8bfb
Author: Shawn Yang
AuthorDate: Fri Apr 5 08:41:44 2024 +0800
fix(java): refine writeVarUint64 readbility (#1464)
---
.../java/org/apache/fury/memory/MemoryBuffer.java | 23 --
1 file changed, 8 insertions(+), 15 deletions(-)
diff --git
a/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
b/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
index 7e8ffae3..6c933318 100644
--- a/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
+++ b/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
@@ -1620,64 +1620,57 @@ public final class MemoryBuffer {
this.writerIndex = writerIndex + 1;
return 1;
}
-varInt |= 0x80;
-varInt |= ((value & 0x7F) << 8);
+varInt |= (int) ((value & 0x7F) << 8) | 0x80;
value >>>= 7;
if (value == 0) {
unsafePutInt(writerIndex, varInt);
this.writerIndex = writerIndex + 2;
return 2;
}
-varInt |= (0x80 << 8);
-varInt |= ((value & 0x7F) << 16);
+varInt |= (int) ((value & 0x7F) << 16) | 0x8000;
value >>>= 7;
if (value == 0) {
unsafePutInt(writerIndex, varInt);
this.writerIndex = writerIndex + 3;
return 3;
}
-varInt |= (0x80 << 16);
-varInt |= ((value & 0x7F) << 24);
+varInt |= (int) ((value & 0x7F) << 24) | 0x80;
value >>>= 7;
if (value == 0) {
unsafePutInt(writerIndex, varInt);
this.writerIndex = writerIndex + 4;
return 4;
}
-varInt |= (0x80L << 24);
long varLong = (varInt & 0xL);
-varLong |= ((value & 0x7F) << 32);
+varLong |= ((value & 0x7F) << 32) | 0x8000L;
value >>>= 7;
if (value == 0) {
unsafePutLong(writerIndex, varLong);
this.writerIndex = writerIndex + 5;
return 5;
}
-varLong |= (0x80L << 32);
-varLong |= ((value & 0x7F) << 40);
+varLong |= ((value & 0x7F) << 40) | 0x80L;
value >>>= 7;
if (value == 0) {
unsafePutLong(writerIndex, varLong);
this.writerIndex = writerIndex + 6;
return 6;
}
-varLong |= (0x80L << 40);
-varLong |= ((value & 0x7F) << 48);
+varLong |= ((value & 0x7F) << 48) | 0x8000L;
value >>>= 7;
if (value == 0) {
unsafePutLong(writerIndex, varLong);
this.writerIndex = writerIndex + 7;
return 7;
}
-varLong |= (0x80L << 48);
-varLong |= ((value & 0x7F) << 56);
+varLong |= ((value & 0x7F) << 56) | 0x80L;
value >>>= 7;
if (value == 0) {
unsafePutLong(writerIndex, varLong);
this.writerIndex = writerIndex + 8;
return 8;
}
-varLong |= (0x80L << 56);
+varLong |= 0x8000L;
unsafePutLong(writerIndex, varLong);
UNSAFE.putByte(heapMemory, address + writerIndex + 8, (byte) (value &
0xFF));
this.writerIndex = writerIndex + 9;
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: feat(java): rewrite readVarUint64 algorithm (#1463)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new beb508fd feat(java): rewrite readVarUint64 algorithm (#1463)
beb508fd is described below
commit beb508fd6da09ca4d52cf3dccd6675ae14d37490
Author: Shawn Yang
AuthorDate: Fri Apr 5 08:40:17 2024 +0800
feat(java): rewrite readVarUint64 algorithm (#1463)
---
.../java/org/apache/fury/memory/MemoryBuffer.java | 67 +++---
1 file changed, 33 insertions(+), 34 deletions(-)
diff --git
a/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
b/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
index 748aea14..45db5ddc 100644
--- a/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
+++ b/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
@@ -1695,40 +1695,39 @@ public final class MemoryBuffer {
return readPositiveVarLongSlow();
}
// varint are written using little endian byte order, so read by little
endian byte order.
-long eightByteValue = unsafeGetLong(readIdx);
-long b = eightByteValue & 0xFF;
-readIdx++; // read one byte
-long result = b & 0x7F;
-if ((b & 0x80) != 0) {
- readIdx++; // read one byte
- b = (eightByteValue >>> 8) & 0xFF;
- result |= (b & 0x7F) << 7;
- if ((b & 0x80) != 0) {
-readIdx++; // read one byte
-b = (eightByteValue >>> 16) & 0xFF;
-result |= (b & 0x7F) << 14;
-if ((b & 0x80) != 0) {
- readIdx++; // read one byte
- b = (eightByteValue >>> 24) & 0xFF;
- result |= (b & 0x7F) << 21;
- if ((b & 0x80) != 0) {
-readIdx++; // read one byte
-b = (eightByteValue >>> 32) & 0xFF;
-result |= (b & 0x7F) << 28;
-if ((b & 0x80) != 0) {
- readIdx++; // read one byte
- b = (eightByteValue >>> 40) & 0xFF;
- result |= (b & 0x7F) << 35;
- if ((b & 0x80) != 0) {
-readIdx++; // read one byte
-b = (eightByteValue >>> 48) & 0xFF;
-result |= (b & 0x7F) << 42;
-if ((b & 0x80) != 0) {
- readIdx++; // read one byte
- b = (eightByteValue >>> 56) & 0xFF;
- result |= (b & 0x7F) << 49;
- if ((b & 0x80) != 0) {
-b = unsafeGet(readIdx++); // read one byte
+long value = unsafeGetLong(readIdx);
+// Duplicate and manual inline for performance.
+// noinspection Duplicates
+readIdx++;
+long result = value & 0x7F;
+if ((value & 0x80) != 0) {
+ readIdx++;
+ // 0x3f80: 0b111 << 7
+ result |= (value >>> 1) & 0x3f80;
+ // 0x8000: 0b1 << 15
+ if ((value & 0x8000) != 0) {
+readIdx++;
+// 0x1fc000: 0b111 << 14
+result |= (value >>> 2) & 0x1fc000;
+// 0x80: 0b1 << 23
+if ((value & 0x80) != 0) {
+ readIdx++;
+ // 0xfe0: 0b111 << 21
+ result |= (value >>> 3) & 0xfe0;
+ if ((value & 0x8000L) != 0) {
+readIdx++;
+result |= (value >>> 4) & 0x7f000L;
+if ((value & 0x80L) != 0) {
+ readIdx++;
+ result |= (value >>> 5) & 0x3f8L;
+ if ((value & 0x8000L) != 0) {
+readIdx++;
+result |= (value >>> 6) & 0x1fc00L;
+if ((value & 0x80L) != 0) {
+ readIdx++;
+ result |= (value >>> 7) & 0xfeL;
+ if ((value & 0x8000L) != 0) {
+long b = UNSAFE.getByte(heapMemory, address + readIdx++);
result |= b << 56;
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury-site) branch main updated: add fury serialization specification tab (#108)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git
The following commit(s) were added to refs/heads/main by this push:
new d6f914d add fury serialization specification tab (#108)
d6f914d is described below
commit d6f914d6b2da8473949c4b42ff1f8a303f968117
Author: Shawn Yang
AuthorDate: Wed Apr 3 08:08:34 2024 +0800
add fury serialization specification tab (#108)
---
docusaurus.config.ts | 6 ++
sidebars.ts | 1 +
2 files changed, 7 insertions(+)
diff --git a/docusaurus.config.ts b/docusaurus.config.ts
index 72f9f42..a51d9e6 100644
--- a/docusaurus.config.ts
+++ b/docusaurus.config.ts
@@ -78,6 +78,12 @@ const config: Config = {
position: 'right',
label: 'Guide',
},
+{
+ type: 'docSidebar',
+ sidebarId: 'specificationSidebar',
+ position: 'right',
+ label: 'Specification',
+},
{to: '/blog', label: 'Blog', position: 'right'},
{
type: 'dropdown',
diff --git a/sidebars.ts b/sidebars.ts
index b4b97e3..d539b14 100644
--- a/sidebars.ts
+++ b/sidebars.ts
@@ -15,6 +15,7 @@ const sidebars: SidebarsConfig = {
// tutorialSidebar: [{type: 'autogenerated', dirName: '.'}],
startSidebar: [{type: 'autogenerated', dirName: 'start'}],
introductionSidebar: [{type: 'autogenerated', dirName: 'introduction'}],
+ specificationSidebar: [{type: 'autogenerated', dirName: 'specification'}],
guideSidebar: [{type: 'autogenerated', dirName: 'guide'}],
// But you can create a sidebar manually
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: chore: update rules (#1448)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 9341868c chore: update rules (#1448)
9341868c is described below
commit 9341868c6ae093a07c738d13b6e2bacfd84dedfb
Author: Shawn Yang
AuthorDate: Mon Apr 1 13:11:54 2024 +0800
chore: update rules (#1448)
---
.asf.yaml | 9 +
1 file changed, 1 insertion(+), 8 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
index b35272ad..2a03475b 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -47,13 +47,6 @@ github:
merge: false
rebase: false
protected_branches:
-main:
- required_status_checks:
-strict: false
- required_pull_request_reviews:
-dismiss_stale_reviews: false
-required_approving_review_count: 1
-required_linear_history: true
releases-0.4.1: {}
releases-0.4.0: {}
releases-0.3.1: {}
@@ -61,7 +54,7 @@ github:
releases-0.2.1: {}
releases-0.2.0: {}
releases-0.1.2: {}
-
+
notifications:
commits: commits@fury.apache.org
issues: commits@fury.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: feat(java): avoid recompilation when gc happens for memory pressure (#1411)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 0437556e feat(java): avoid recompilation when gc happens for memory
pressure (#1411)
0437556e is described below
commit 0437556ef4f3445869923f6b90806f565fda0e33
Author: Shawn Yang
AuthorDate: Sun Mar 17 18:47:34 2024 +0800
feat(java): avoid recompilation when gc happens for memory pressure (#1411)
---
.../java/org/apache/fury/builder/CodecUtils.java | 22 --
.../org/apache/fury/codegen/CodeGenerator.java | 16
.../org/apache/fury/resolver/ClassResolver.java| 16
.../main/java/org/apache/fury/util/DelayedRef.java | 48 ++
.../org/apache/fury/codegen/CodeGeneratorTest.java | 6 +--
5 files changed, 94 insertions(+), 14 deletions(-)
diff --git
a/java/fury-core/src/main/java/org/apache/fury/builder/CodecUtils.java
b/java/fury-core/src/main/java/org/apache/fury/builder/CodecUtils.java
index e5cd5a54..d13be51e 100644
--- a/java/fury-core/src/main/java/org/apache/fury/builder/CodecUtils.java
+++ b/java/fury-core/src/main/java/org/apache/fury/builder/CodecUtils.java
@@ -24,6 +24,7 @@ import java.util.Collections;
import org.apache.fury.Fury;
import org.apache.fury.codegen.CodeGenerator;
import org.apache.fury.codegen.CompileUnit;
+import org.apache.fury.resolver.ClassResolver;
import org.apache.fury.resolver.FieldResolver;
import org.apache.fury.serializer.Serializer;
import org.apache.fury.type.ClassDef;
@@ -81,14 +82,29 @@ public class CodecUtils {
if (beanClassClassLoader == null) {
beanClassClassLoader = fury.getClass().getClassLoader();
}
+ClassResolver classResolver = fury.getClassResolver();
try {
// generated code imported fury classes.
beanClassClassLoader.loadClass(Fury.class.getName());
- codeGenerator =
CodeGenerator.getSharedCodeGenerator(beanClassClassLoader);
+ codeGenerator = classResolver.getCodeGenerator(beanClassClassLoader);
+ if (codeGenerator == null) {
+codeGenerator =
CodeGenerator.getSharedCodeGenerator(beanClassClassLoader);
+// Hold strong reference of {@link CodeGenerator}, so the referent of
`DelayedRef`
+// won't be null.
+classResolver.setCodeGenerator(beanClassClassLoader, codeGenerator);
+ }
} catch (ClassNotFoundException e) {
codeGenerator =
- CodeGenerator.getSharedCodeGenerator(
- beanClassClassLoader, fury.getClass().getClassLoader());
+ classResolver.getCodeGenerator(beanClassClassLoader,
fury.getClass().getClassLoader());
+ ClassLoader[] loaders = {beanClassClassLoader,
fury.getClass().getClassLoader()};
+ if (codeGenerator == null) {
+codeGenerator =
+CodeGenerator.getSharedCodeGenerator(
+beanClassClassLoader, fury.getClass().getClassLoader());
+// Hold strong reference of {@link CodeGenerator}, so the referent of
`DelayedRef`
+// won't be null.
+classResolver.setCodeGenerator(loaders, codeGenerator);
+ }
}
ClassLoader classLoader =
codeGenerator.compile(
diff --git
a/java/fury-core/src/main/java/org/apache/fury/codegen/CodeGenerator.java
b/java/fury-core/src/main/java/org/apache/fury/codegen/CodeGenerator.java
index c6be7358..da3bc7e0 100644
--- a/java/fury-core/src/main/java/org/apache/fury/codegen/CodeGenerator.java
+++ b/java/fury-core/src/main/java/org/apache/fury/codegen/CodeGenerator.java
@@ -23,7 +23,6 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import java.lang.ref.SoftReference;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -41,6 +40,7 @@ import org.apache.fury.collection.Collections;
import org.apache.fury.collection.MultiKeyWeakMap;
import org.apache.fury.util.ClassLoaderUtils;
import org.apache.fury.util.ClassLoaderUtils.ByteArrayClassLoader;
+import org.apache.fury.util.DelayedRef;
import org.apache.fury.util.GraalvmSupport;
import org.apache.fury.util.LoggerFactory;
import org.apache.fury.util.Preconditions;
@@ -71,9 +71,9 @@ public class CodeGenerator {
// FIXME The classloaders will only be reclaimed when the generated class
are not be referenced.
// FIXME CodeGenerator may reference to classloader, thus cause circular
reference, neither can
// be gc.
- private static final WeakHashMap>
sharedCodeGenerator =
+ private static final WeakHashMap>
sharedCodeGenerator =
new WeakHashMap<>();
- private static final MultiKeyWeakMap>
sharedCodeGenerator2 =
+ private static final MultiKeyWeakMap>
sharedCodeGener
(incubator-fury) branch main updated: fix(java): fix immutable collection ref tracking (#1403)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 5aba0c87 fix(java): fix immutable collection ref tracking (#1403)
5aba0c87 is described below
commit 5aba0c8780fb7fdff96a63ec60cde631eb107cb7
Author: Shawn Yang
AuthorDate: Mon Mar 11 22:16:05 2024 +0800
fix(java): fix immutable collection ref tracking (#1403)
fix immutable collection ref tracking in #1401
---
.../ImmutableCollectionSerializersTest.java| 32 +
.../collection/GuavaCollectionSerializers.java | 29 ++--
.../collection/ImmutableCollectionSerializers.java | 3 -
.../test/java/org/apache/fury/FuryTestBase.java| 22 ++
.../collection/GuavaCollectionSerializersTest.java | 82 +-
5 files changed, 108 insertions(+), 60 deletions(-)
diff --git
a/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/ImmutableCollectionSerializersTest.java
b/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/ImmutableCollectionSerializersTest.java
index d0e1a108..e31bafc0 100644
---
a/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/ImmutableCollectionSerializersTest.java
+++
b/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/ImmutableCollectionSerializersTest.java
@@ -24,9 +24,14 @@ import static
org.apache.fury.integration_tests.TestUtils.serDeCheck;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import lombok.AllArgsConstructor;
+import lombok.Data;
import org.apache.fury.Fury;
+import org.apache.fury.ThreadSafeFury;
+import org.apache.fury.config.Language;
import org.apache.fury.test.bean.CollectionFields;
import org.apache.fury.test.bean.MapFields;
+import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@@ -88,4 +93,31 @@ public class ImmutableCollectionSerializersTest {
collectionFields.map2 = Map.of("1", "2", "3", "4");
serDeCheck(fury, collectionFields);
}
+
+ @Data
+ @AllArgsConstructor
+ public static class Pojo {
+List> data;
+ }
+
+ @DataProvider
+ public static Object[][] refTrackingAndCodegen() {
+return new Object[][] {{false, false}, {true, false}, {false, true},
{true, true}};
+ }
+
+ @Test(dataProvider = "refTrackingAndCodegen")
+ void testNestedRefTracking(boolean trackingRef, boolean codegen) {
+Pojo pojo = new Pojo(List.of(List.of(1, 2), List.of(2, 2)));
+ThreadSafeFury fury =
+Fury.builder()
+.withLanguage(Language.JAVA)
+.requireClassRegistration(false)
+.withCodegen(codegen)
+.withRefTracking(trackingRef)
+.buildThreadSafeFury();
+
+byte[] bytes = fury.serialize(pojo);
+Pojo deserializedPojo = (Pojo) fury.deserialize(bytes);
+Assert.assertEquals(deserializedPojo, pojo);
+ }
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java
index f6d0e889..532837ae 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java
@@ -56,9 +56,7 @@ public class GuavaCollectionSerializers {
int size = buffer.readPositiveVarInt();
List list = new ArrayList<>();
xreadElements(fury, buffer, list, size);
- T immutableList = xnewInstance(list);
- fury.getRefResolver().reference(immutableList);
- return immutableList;
+ return xnewInstance(list);
}
protected abstract T xnewInstance(Collection collection);
@@ -82,7 +80,6 @@ public class GuavaCollectionSerializers {
public T onCollectionRead(Collection collection) {
Object[] elements = ((CollectionContainer) collection).elements;
ImmutableList list = ImmutableList.copyOf(elements);
- fury.getRefResolver().reference(list);
return (T) list;
}
@@ -133,9 +130,7 @@ public class GuavaCollectionSerializers {
@Override
public T onCollectionRead(Collection collection) {
Object[] elements = ((CollectionContainer) collection).elements;
- T t = (T) function.apply(elements);
- fury.getRefResolver().reference(t);
- return t;
+ return (T) function.apply(elements);
}
@Override
@@ -166,9 +161,7 @@ public class GuavaCollectionSerializers {
@Override
public T onCollectionRead(Collection collection) {
Object[] elements = ((CollectionContainer) collection).elements;
- T t = (T) ImmutableSet
(incubator-fury) branch main updated: feat(JavaScript): create zero-copy buffer when convert (#1386)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 25d45146 feat(JavaScript): create zero-copy buffer when convert (#1386)
25d45146 is described below
commit 25d451461ed904b8b0110cb8f408e404499f10d0
Author: 野声
AuthorDate: Tue Feb 27 18:21:33 2024 +0800
feat(JavaScript): create zero-copy buffer when convert (#1386)
---
javascript/packages/fury/lib/fury.ts | 3 ++-
javascript/packages/fury/lib/platformBuffer.ts | 4 +++-
javascript/packages/fury/lib/reader/index.ts | 2 +-
javascript/packages/fury/package.json | 2 +-
4 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/javascript/packages/fury/lib/fury.ts
b/javascript/packages/fury/lib/fury.ts
index 1ea7d26c..aa53e13a 100644
--- a/javascript/packages/fury/lib/fury.ts
+++ b/javascript/packages/fury/lib/fury.ts
@@ -78,11 +78,12 @@ export default class {
if (!isCrossLanguage) {
throw new Error("support crosslanguage mode only");
}
-this.binaryReader.uint8(); // skip language
const isOutOfBandEnabled = (bitmap & ConfigFlags.isOutOfBandFlag) ===
ConfigFlags.isOutOfBandFlag;
if (isOutOfBandEnabled) {
throw new Error("outofband mode is not supported now");
}
+
+this.binaryReader.uint8(); // skip language
this.binaryReader.int32(); // native object offset. should skip.
javascript support cross mode only
this.binaryReader.int32(); // native object size. should skip.
return serializer.read();
diff --git a/javascript/packages/fury/lib/platformBuffer.ts
b/javascript/packages/fury/lib/platformBuffer.ts
index fb95f456..b28df55b 100644
--- a/javascript/packages/fury/lib/platformBuffer.ts
+++ b/javascript/packages/fury/lib/platformBuffer.ts
@@ -140,7 +140,9 @@ export class BrowserBuffer extends Uint8Array implements
PlatformBuffer {
export const fromUint8Array = hasBuffer
? (ab: Buffer | Uint8Array) => {
if (!Buffer.isBuffer(ab)) {
-return (Buffer.from(ab) as unknown as PlatformBuffer);
+//
https://nodejs.org/docs/latest/api/buffer.html#static-method-bufferfromarraybuffer-byteoffset-length
+// Create a zero-copy Buffer wrapper around the ArrayBuffer pointed to
by the Uint8Array
+return (Buffer.from(ab.buffer, ab.byteOffset, ab.byteLength) as
unknown as PlatformBuffer);
} else {
return ab as unknown as PlatformBuffer;
}
diff --git a/javascript/packages/fury/lib/reader/index.ts
b/javascript/packages/fury/lib/reader/index.ts
index ef308e79..e0df6000 100644
--- a/javascript/packages/fury/lib/reader/index.ts
+++ b/javascript/packages/fury/lib/reader/index.ts
@@ -35,7 +35,7 @@ export const BinaryReader = (config: Config) => {
function reset(ab: Uint8Array) {
buffer = fromUint8Array(ab);
byteLength = buffer.byteLength;
-dataView = new DataView(buffer.buffer, buffer.byteOffset);
+dataView = new DataView(buffer.buffer, buffer.byteOffset, byteLength);
if (sliceStringEnable) {
bigString = buffer.toString("latin1", 0, byteLength);
}
diff --git a/javascript/packages/fury/package.json
b/javascript/packages/fury/package.json
index 92af2bc3..c0b79053 100644
--- a/javascript/packages/fury/package.json
+++ b/javascript/packages/fury/package.json
@@ -1,6 +1,6 @@
{
"name": "@furyjs/fury",
- "version": "0.5.8-beta",
+ "version": "0.5.9-beta",
"description": "A blazing fast multi-language serialization framework
powered by jit and zero-copy",
"main": "dist/index.js",
"scripts": {
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury-site) branch main updated: Fix incorrent usage (#103)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git
The following commit(s) were added to refs/heads/main by this push:
new 5e0754a Fix incorrent usage (#103)
5e0754a is described below
commit 5e0754a5cb2559668eaf3cc5937bf16f3d90db37
Author: weipeng
AuthorDate: Mon Feb 26 21:52:05 2024 +0800
Fix incorrent usage (#103)
There are some incorrent usage in guide, should fix it
---
docs/start/install.md | 1 -
docs/start/usage.md | 7 +++
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/docs/start/install.md b/docs/start/install.md
index 69d3875..f9cc5d7 100644
--- a/docs/start/install.md
+++ b/docs/start/install.md
@@ -84,7 +84,6 @@ npm install @furyjs/fury
[dependencies]
fury = { git= "https://github.com/apache/incubator-fury.git;, branch = "main" }
-fury_derive = { git= "https://github.com/apache/incubator-fury.git;, branch =
"main" }
lazy_static = { version = "1.4.0" }
```
diff --git a/docs/start/usage.md b/docs/start/usage.md
index 14523af..b9d5999 100644
--- a/docs/start/usage.md
+++ b/docs/start/usage.md
@@ -148,17 +148,16 @@ console.log(result);
### Rust
```rust
-use fury::{from_buffer, to_buffer};
-use fury_derive::{Deserialize, FuryMeta, Serialize};
+use fury::{from_buffer, to_buffer, Fury};
-#[derive(FuryMeta, Deserialize, Serialize, Debug, PartialEq)]
+#[derive(Fury, Debug, PartialEq)]
#[tag("example.foo")]
struct Animal {
name: String,
category: String,
}
-#[derive(FuryMeta, Deserialize, Serialize, Debug, PartialEq)]
+#[derive(Fury, Debug, PartialEq)]
#[tag("example.bar")]
struct Person {
name: String,
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: chore: remove redundant statement (#1376)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new ea1c471c chore: remove redundant statement (#1376)
ea1c471c is described below
commit ea1c471c778606198c19c6497db0d42a5c65a94d
Author: 野声
AuthorDate: Mon Feb 19 17:28:07 2024 +0800
chore: remove redundant statement (#1376)
---
javascript/packages/fury/lib/gen/any.ts | 4 ++--
javascript/packages/fury/package.json | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/javascript/packages/fury/lib/gen/any.ts
b/javascript/packages/fury/lib/gen/any.ts
index e0335f9b..bb6c8d54 100644
--- a/javascript/packages/fury/lib/gen/any.ts
+++ b/javascript/packages/fury/lib/gen/any.ts
@@ -81,8 +81,8 @@ export class AnySerializer {
if (!serializer) {
throw new Error(`Failed to detect the Fury serializer from JavaScript
type: ${typeof v}`);
}
-this.fury.binaryWriter.reserve(serializer?.meta.fixedSize);
-serializer?.write(v);
+this.fury.binaryWriter.reserve(serializer.meta.fixedSize);
+serializer.write(v);
}
}
diff --git a/javascript/packages/fury/package.json
b/javascript/packages/fury/package.json
index 5a5c6249..92af2bc3 100644
--- a/javascript/packages/fury/package.json
+++ b/javascript/packages/fury/package.json
@@ -1,6 +1,6 @@
{
"name": "@furyjs/fury",
- "version": "0.5.7-beta",
+ "version": "0.5.8-beta",
"description": "A blazing fast multi-language serialization framework
powered by jit and zero-copy",
"main": "dist/index.js",
"scripts": {
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: chore: update benchmark (#1375)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 4998e567 chore: update benchmark (#1375)
4998e567 is described below
commit 4998e5677b752d59a182f7cde0d8399762f75720
Author: 野声
AuthorDate: Mon Feb 19 17:25:54 2024 +0800
chore: update benchmark (#1375)
---
javascript/benchmark/platform-buffer-draw.py | 60 ---
javascript/benchmark/platform-buffer.jpg | Bin 55653 -> 36432 bytes
javascript/benchmark/platform-buffer.js | 40 ++
3 files changed, 31 insertions(+), 69 deletions(-)
diff --git a/javascript/benchmark/platform-buffer-draw.py
b/javascript/benchmark/platform-buffer-draw.py
index 9ff73d5b..5c9a803e 100644
--- a/javascript/benchmark/platform-buffer-draw.py
+++ b/javascript/benchmark/platform-buffer-draw.py
@@ -19,31 +19,31 @@ import matplotlib.pyplot as plt
import sys
[
-_,
browser_utf8_write,
browser_write,
-browser_write_1,
native_write,
browser_to_string,
native_to_string,
-] = sys.argv[0:7]
+] = sys.argv[1:6]
-# 创建图形和子图
-fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(15, 5), sharey=True)
+browser_utf8_write = int(browser_utf8_write)
+browser_write = int(browser_write)
+native_write = int(native_write)
+browser_to_string = int(browser_to_string)
+native_to_string = int(native_to_string)
+
+fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(10, 5), sharey=True)
-# 绘制第一部分比较：browser utf8Write 和 browser write
axs[0].bar(
-["browser utf8Write", "browser write"],
-[browser_utf8_write, browser_write],
-color=["b", "g"],
+["browser utf8Write", "browser write", "native write"],
+[browser_utf8_write, browser_write, native_write],
+color=["b", "g", "r"],
)
-axs[0].set_title("Browser UTF8 Write vs Browser Write")
-axs[0].set_xlabel("Operation Type")
-axs[0].set_xticklabels(["browser utf8Write", "browser write"])
-axs[0].set_ylabel("Tps")
+axs[0].set_title("Write Comparison")
+axs[0].set_xticklabels(["browser utf8Write", "browser write", "native write"])
+axs[0].set_ylabel("TPS")
-# 在柱形图上添加数值标签
for p in axs[0].patches:
axs[0].annotate(
format(p.get_height(), ".0f"),
@@ -54,38 +54,16 @@ for p in axs[0].patches:
textcoords="offset points",
)
-# 绘制第二部分比较：browser write 和 native write
axs[1].bar(
-["browser write", "native write"], [browser_write_1, native_write],
color=["g", "r"]
-)
-axs[1].set_title("Browser Write vs Native Write")
-axs[1].set_xlabel("Operation Type")
-axs[1].set_xticklabels(["browser write", "native write"])
-
-# 在柱形图上添加数值标签
-for p in axs[1].patches:
-axs[1].annotate(
-format(p.get_height(), ".0f"),
-(p.get_x() + p.get_width() / 2.0, p.get_height()),
-ha="center",
-va="center",
-xytext=(0, 9),
-textcoords="offset points",
-)
-
-# 绘制第三部分比较：browser toString 和 native toString
-axs[2].bar(
["browser toString", "native toString"],
[browser_to_string, native_to_string],
color=["b", "r"],
)
-axs[2].set_title("Browser ToString vs Native ToString")
-axs[2].set_xlabel("Operation Type")
-axs[2].set_xticklabels(["browser toString", "native toString"])
+axs[1].set_title("toString Comparison")
+axs[1].set_xticklabels(["browser toString", "native toString"])
-# 在柱形图上添加数值标签
-for p in axs[2].patches:
-axs[2].annotate(
+for p in axs[1].patches:
+axs[1].annotate(
format(p.get_height(), ".0f"),
(p.get_x() + p.get_width() / 2.0, p.get_height()),
ha="center",
@@ -94,9 +72,7 @@ for p in axs[2].patches:
textcoords="offset points",
)
-# 调整布局以避免重叠
plt.tight_layout()
-# 显示图形
plt.show()
fig.savefig("./platform-buffer.jpg")
diff --git a/javascript/benchmark/platform-buffer.jpg
b/javascript/benchmark/platform-buffer.jpg
index c6ed82fa..d13eb58c 100644
Binary files a/javascript/benchmark/platform-buffer.jpg and
b/javascript/benchmark/platform-buffer.jpg differ
diff --git a/javascript/benchmark/platform-buffer.js
b/javascript/benchmark/platform-buffer.js
index cacd4bd1..bf56235a 100644
--- a/javascript/benchmark/platform-buffer.js
+++ b/javascript/benchmark/platform-buffer.js
@@ -29,14 +29,14 @@ const jsonString = JSON.stringify({
async function start() {
const result = {
-browserCompare: {},
-browserVsNativeWrit
(incubator-fury) branch main updated: feat(JavaScript): Make PlatformBuffer available if has Buffer polyfill (#1373)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new c2238b7f feat(JavaScript): Make PlatformBuffer available if has Buffer
polyfill (#1373)
c2238b7f is described below
commit c2238b7f75c53c61d2bb866ec9676aeb0f2d1057
Author: 野声
AuthorDate: Mon Feb 19 14:39:41 2024 +0800
feat(JavaScript): Make PlatformBuffer available if has Buffer polyfill
(#1373)
If user has a polyfill(process/buffer) in browser, furyjs cannot work.
---
javascript/benchmark/platform-buffer-draw.py | 102 ++
javascript/benchmark/platform-buffer.jpg | Bin 0 -> 55653 bytes
javascript/benchmark/platform-buffer.js| 113 +
javascript/packages/fury/lib/platformBuffer.ts | 35 +---
javascript/packages/fury/lib/reader/index.ts | 6 +-
javascript/packages/fury/lib/reader/string.ts | 2 +-
javascript/packages/fury/lib/util.ts | 2 +
javascript/packages/fury/lib/writer.ts | 6 +-
javascript/packages/fury/package.json | 4 +-
javascript/test/platformBuffer.test.ts | 51 +--
javascript/test/reader.test.ts | 2 +-
11 files changed, 297 insertions(+), 26 deletions(-)
diff --git a/javascript/benchmark/platform-buffer-draw.py
b/javascript/benchmark/platform-buffer-draw.py
new file mode 100644
index ..9ff73d5b
--- /dev/null
+++ b/javascript/benchmark/platform-buffer-draw.py
@@ -0,0 +1,102 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import matplotlib.pyplot as plt
+import sys
+
+[
+_,
+browser_utf8_write,
+browser_write,
+browser_write_1,
+native_write,
+browser_to_string,
+native_to_string,
+] = sys.argv[0:7]
+
+# 创建图形和子图
+fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(15, 5), sharey=True)
+
+# 绘制第一部分比较：browser utf8Write 和 browser write
+axs[0].bar(
+["browser utf8Write", "browser write"],
+[browser_utf8_write, browser_write],
+color=["b", "g"],
+)
+axs[0].set_title("Browser UTF8 Write vs Browser Write")
+axs[0].set_xlabel("Operation Type")
+axs[0].set_xticklabels(["browser utf8Write", "browser write"])
+axs[0].set_ylabel("Tps")
+
+
+# 在柱形图上添加数值标签
+for p in axs[0].patches:
+axs[0].annotate(
+format(p.get_height(), ".0f"),
+(p.get_x() + p.get_width() / 2.0, p.get_height()),
+ha="center",
+va="center",
+xytext=(0, 9),
+textcoords="offset points",
+)
+
+# 绘制第二部分比较：browser write 和 native write
+axs[1].bar(
+["browser write", "native write"], [browser_write_1, native_write],
color=["g", "r"]
+)
+axs[1].set_title("Browser Write vs Native Write")
+axs[1].set_xlabel("Operation Type")
+axs[1].set_xticklabels(["browser write", "native write"])
+
+# 在柱形图上添加数值标签
+for p in axs[1].patches:
+axs[1].annotate(
+format(p.get_height(), ".0f"),
+(p.get_x() + p.get_width() / 2.0, p.get_height()),
+ha="center",
+va="center",
+xytext=(0, 9),
+textcoords="offset points",
+)
+
+# 绘制第三部分比较：browser toString 和 native toString
+axs[2].bar(
+["browser toString", "native toString"],
+[browser_to_string, native_to_string],
+color=["b", "r"],
+)
+axs[2].set_title("Browser ToString vs Native ToString")
+axs[2].set_xlabel("Operation Type")
+axs[2].set_xticklabels(["browser toString", "native toString"])
+
+# 在柱形图上添加数值标签
+for p in axs[2].patches:
+axs[2].annotate(
+format(p.get_height(), ".0f"),
+(p.get_x() + p.get_width() / 2.0, p.get_height()),
+ha="center",
+va="center",
+xytext=(0, 9),
+textcoords="offset points",
+)
+
+# 调整布局以避免重叠
+plt.tight_layout()
+
+#
(incubator-fury) branch main updated: docs(JavaScript): Add build and test documentation for JavaScript (#1360)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury.git The following commit(s) were added to refs/heads/main by this push: new ba374001 docs(JavaScript): Add build and test documentation for JavaScript (#1360) ba374001 is described below commit ba374001dc9b6905171dbea7ca0dd64cbf09316d Author: LiangliangSui <116876207+liangliang...@users.noreply.github.com> AuthorDate: Tue Jan 30 11:28:48 2024 +0800 docs(JavaScript): Add build and test documentation for JavaScript (#1360) --- CONTRIBUTING.md | 22 +++--- docs/guide/DEVELOPMENT.md | 18 ++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1f8c33e3..22115e0d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # How to contribute to Fury -## Finding good first issues +## Finding good first issues See [Good First Issues](https://github.com/apache/incubator-fury/contribute). @@ -20,7 +20,9 @@ For example, here are good PR titles: For more details, please check [pr-lint.yml](./.github/workflows/pr-lint.yml). -## 離 Testing +## Testing + +For environmental requirements, please check [DEVELOPMENT.md](./docs/guide/DEVELOPMENT.md). ### Python @@ -57,7 +59,14 @@ cd rust cargo test ``` -## Code Style +### JavaScript + +```bash +cd javascript +npm run test +``` + +## Code Style Run all checks: `bash ci/format.sh --all`. @@ -108,6 +117,13 @@ cd rust cargo fmt ``` +### JavaScript + +```bash +cd javascript +npm run lint +``` + ## Debug ### Java diff --git a/docs/guide/DEVELOPMENT.md b/docs/guide/DEVELOPMENT.md index bdc5d591..5d71166a 100644 --- a/docs/guide/DEVELOPMENT.md +++ b/docs/guide/DEVELOPMENT.md @@ -82,3 +82,21 @@ cargo test ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` + +### Build Fury JavaScript + +```bash +cd javascript +npm install + +# run build +npm run build +# run test +npm run test +``` + + Environment Requirements + +- node 14+ +- npm 8+ + - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: feat(JavaScript): Support oneof (#1348)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 2f2e60f8 feat(JavaScript): Support oneof (#1348)
2f2e60f8 is described below
commit 2f2e60f8d1ee4f60cd93c26d88bdd6d90d5dc12e
Author: weipeng
AuthorDate: Fri Jan 26 17:23:48 2024 +0800
feat(JavaScript): Support oneof (#1348)
### 1. Fixed a performance bug that caused writeInt32 to slow down
Before:
```JavaScript
// reader/index.ts
function writeVarInt32() {
buffer.byteLength
}
```
After:
```JavaScript
// reader/index.ts
const byteLength = buffer.byteLength;
function writeVarInt32() {
byteLength
}
```
The byteLength property in a Buffer is slow to access. It appears that
when we access byteLength, the V8 engine processes it using a hash
lookup. so we store it in closure.
### 2. Support Oneof
Sometimes, the data we want to serialize does not have a confirmed type;
instead, it could be one of several confirmed types. If we use an object
to handle this situation, the size of the resulting binary will be too
large, as it will contain much unused information.
usage:
```JavaScript
const oneOfThree = Type.oneof({
option1: Type.string(),
option2: Type.object("foo", {
a: Type.int32()
}),
option3: Type.int32(),
});
const fury = new Fury({ refTracking: true });
const { serialize, deserialize } =
fury.registerSerializer(oneOfThree);
const obj = {
option1: "hello"
}
const input = serialize(obj);
const result = deserialize(
input
);
expect(result).toEqual(obj.option1)
```
---
javascript/package.json | 3 +-
javascript/packages/fury/lib/description.ts | 205 --
javascript/packages/fury/lib/fury.ts | 8 +-
javascript/packages/fury/lib/gen/any.ts | 43 -
javascript/packages/fury/lib/gen/builder.ts | 7 +
javascript/packages/fury/lib/gen/index.ts | 11 +-
javascript/packages/fury/lib/gen/object.ts| 2 +-
javascript/packages/fury/lib/gen/oneof.ts | 115
javascript/packages/fury/lib/gen/serializer.ts| 17 +-
javascript/packages/fury/lib/meta.ts | 7 +-
javascript/packages/fury/lib/platformBuffer.ts| 10 +-
javascript/packages/fury/lib/reader/index.ts | 8 +-
javascript/packages/fury/lib/referenceResolver.ts | 2 +-
javascript/packages/fury/lib/type.ts | 25 +--
javascript/packages/hps/index.ts | 4 +-
javascript/test/any.test.ts | 10 +-
javascript/test/oneof.test.ts | 90 ++
17 files changed, 467 insertions(+), 100 deletions(-)
diff --git a/javascript/package.json b/javascript/package.json
index a0345c20..653adfdd 100644
--- a/javascript/package.json
+++ b/javascript/package.json
@@ -1,7 +1,8 @@
{
"scripts": {
"test": "npm run build && jest",
-"build": "npm run build -w packages/fury -w packages/hps",
+"clear": "rm -rf ./packages/fury/dist && rm -rf ./packages/hps/dist",
+"build": "npm run clear && npm run build -w packages/fury -w packages/hps",
"lint": "eslint .",
"lint-fix": "eslint . --fix"
},
diff --git a/javascript/packages/fury/lib/description.ts
b/javascript/packages/fury/lib/description.ts
index 42855e4f..d332e740 100644
--- a/javascript/packages/fury/lib/description.ts
+++ b/javascript/packages/fury/lib/description.ts
@@ -20,112 +20,136 @@
import { InternalSerializerType } from "./type";
export interface TypeDescription {
- type: InternalSerializerType
- label?: string
+ type: InternalSerializerType;
+ label?: string;
}
export interface ObjectTypeDescription extends TypeDescription {
options: {
-props: { [key: string]: TypeDescription }
-tag: string
- }
+props: { [key: string]: TypeDescription };
+tag: string;
+ };
}
export interface EnumTypeDescription extends TypeDescription {
options: {
-inner: { [key: string]: any }
- }
+inner: { [key: string]: any };
+ };
+}
+
+export interface OneofTypeDescription extends TypeDescription {
+ options: {
+inner: { [key: string]: TypeDescription };
+ };
}
export interface ArrayTypeDescription extends TypeDescription {
options: {
-inner: TypeDescription
- }
+inner: TypeDescription;
+ };
}
export interface TupleTypeDescription extends TypeDescri
(incubator-fury) branch main updated: ci(JavaScript): JavaScript ci is ported from run_ci.sh to run_ci.py (#1358)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new cb49d116 ci(JavaScript): JavaScript ci is ported from run_ci.sh to
run_ci.py (#1358)
cb49d116 is described below
commit cb49d116cac25457c1e607f9d56d2eae97ee0cfa
Author: LiangliangSui <116876207+liangliang...@users.noreply.github.com>
AuthorDate: Fri Jan 26 14:32:07 2024 +0800
ci(JavaScript): JavaScript ci is ported from run_ci.sh to run_ci.py (#1358)
1. JavaScript CI implemented using python.
2. Enable windows CI and macos CI for JavaScript.
Signed-off-by: LiangliangSui
---
.github/workflows/ci.yml | 14 --
ci/run_ci.py | 26 --
2 files changed, 36 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c95a07b6..4dc5c5d6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -107,10 +107,11 @@ jobs:
javascript:
name: JavaScript CI
-runs-on: ubuntu-latest
strategy:
matrix:
node-version: [14.x, 16.x, 18.x, 20.x]
+os: [ubuntu-latest, macos-latest, windows-2022]
+runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Use Node.js ${{ matrix.node-version }}
@@ -119,8 +120,17 @@ jobs:
node-version: ${{ matrix.node-version }}
- name: Upgrade npm
run: npm install -g npm@8
+ # node-gyp needs to use python and relies on the distutils module.
+ # The distutils module has been removed starting from python 3.12
+ # (see https://docs.python.org/3.10/library/distutils.html). Some
+ # OS (such as macos -latest) uses python3.12 by default, so python 3.8
+ # is used here to avoid this problem.
+ - name: Set up Python3.8
+uses: actions/setup-python@v2
+with:
+ python-version: 3.8
- name: Run CI with NodeJS
-run: ./ci/run_ci.sh javascript
+run: python ./ci/run_ci.py javascript
rust:
name: Rust CI
diff --git a/ci/run_ci.py b/ci/run_ci.py
index 205e5c10..fe7a220b 100644
--- a/ci/run_ci.py
+++ b/ci/run_ci.py
@@ -29,6 +29,9 @@ BAZEL_VERSION = "6.3.2"
PYARROW_VERSION = "14.0.0"
+PROJECT_ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"../")
+
+
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
@@ -71,6 +74,10 @@ def _get_bazel_download_url():
)
+def _cd_project_subdir(subdir):
+os.chdir(os.path.join(PROJECT_ROOT_DIR, subdir))
+
+
def _run_cpp():
_install_cpp_deps()
# run test
@@ -84,8 +91,7 @@ def _run_rust():
_exec_cmd("rustup component add clippy-preview")
_exec_cmd("rustup component add rustfmt")
logging.info("Executing fury rust tests")
-cur_script_abs_path = os.path.split(os.path.realpath(__file__))[0]
-os.chdir(os.path.join(cur_script_abs_path, "../rust"))
+_cd_project_subdir("rust")
cmds = (
"cargo doc --no-deps --document-private-items --all-features --open",
@@ -102,6 +108,14 @@ def _run_rust():
logging.info("Executing fury rust tests succeeds")
+def _run_js():
+logging.info("Executing fury javascript tests.")
+_cd_project_subdir("javascript")
+_exec_cmd("npm install")
+_exec_cmd("npm run test")
+logging.info("Executing fury javascript tests succeeds.")
+
+
def _install_cpp_deps():
_exec_cmd(f"pip install pyarrow=={PYARROW_VERSION}")
_exec_cmd("pip install psutil")
@@ -156,6 +170,14 @@ def _parse_args():
)
rust_parser.set_defaults(func=_run_rust)
+js_parser = subparsers.add_parser(
+"javascript",
+description="Run Javascript CI",
+help="Run Javascript CI",
+formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+)
+js_parser.set_defaults(func=_run_js)
+
args = parser.parse_args()
arg_dict = dict(vars(args))
del arg_dict["func"]
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury-site) branch main updated: update: go get not need https:// (#102)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git The following commit(s) were added to refs/heads/main by this push: new 0fb989f update: go get not need https:// (#102) 0fb989f is described below commit 0fb989f4a97b1731b041daf9354ec3222910771f Author: loopFY <34152277+767829...@users.noreply.github.com> AuthorDate: Fri Jan 26 09:59:30 2024 +0800 update: go get not need https:// (#102) $ go get https://github.com/apache/incubator-fury/go/fury go: malformed module path "https:/github.com/apache/incubator-fury/go/fury": invalid char ':' Signed-off-by: Fang Yuan --- docs/start/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/start/install.md b/docs/start/install.md index e428a45..69d3875 100644 --- a/docs/start/install.md +++ b/docs/start/install.md @@ -68,7 +68,7 @@ pip install pyfury ### Golang ```bash -go get https://github.com/apache/incubator-fury/go/fury +go get github.com/apache/incubator-fury/go/fury ``` ### JavaScript - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: chore(JavaScript): Fix using npm run build error (#1357)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new c25695c5 chore(JavaScript): Fix using npm run build error (#1357)
c25695c5 is described below
commit c25695c520256aac7adfb5a3ece8f7fabb190acc
Author: LiangliangSui <116876207+liangliang...@users.noreply.github.com>
AuthorDate: Thu Jan 25 21:15:59 2024 +0800
chore(JavaScript): Fix using npm run build error (#1357)
---
javascript/packages/hps/package.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/javascript/packages/hps/package.json
b/javascript/packages/hps/package.json
index fdce6f02..704a71b5 100644
--- a/javascript/packages/hps/package.json
+++ b/javascript/packages/hps/package.json
@@ -11,7 +11,7 @@
"gypfile": false,
"scripts": {
"postinstall": "node -e \"if (process.version.match(/v(\\d+)/)[1] >= 20 &&
process.platform !== 'win32') { require('child_process').execSync('npx node-gyp
rebuild') } \"",
-"build": "npx node-gyp rebuild && tsc",
+"build": "node -e \"if (process.version.match(/v(\\d+)/)[1] >= 20 &&
process.platform !== 'win32') { require('child_process').execSync('npx node-gyp
rebuild && tsc') } \"",
"prepublishOnly": "npm run build"
},
"license": "Apache",
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: chore(JavaScript): fix test ci on early Node (#1356)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 01ad634f chore(JavaScript): fix test ci on early Node (#1356)
01ad634f is described below
commit 01ad634f5bea9f088af8e32575748ef73a8de3e9
Author: weipeng
AuthorDate: Thu Jan 25 18:12:41 2024 +0800
chore(JavaScript): fix test ci on early Node (#1356)
1. Npm 6 doesn't support workspace, upgrade it before run test
2. There are Some node-gyp compile problems on windows, ignore it
temporary
#1350
---
.github/workflows/ci.yml | 2 ++
ci/run_ci.sh | 3 +--
javascript/jest.config.js| 2 +-
javascript/packages/hps/package.json | 3 ++-
4 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index df5ffb8a..c95a07b6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -117,6 +117,8 @@ jobs:
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
+ - name: Upgrade npm
+run: npm install -g npm@8
- name: Run CI with NodeJS
run: ./ci/run_ci.sh javascript
diff --git a/ci/run_ci.sh b/ci/run_ci.sh
index 8e20..61caaab3 100755
--- a/ci/run_ci.sh
+++ b/ci/run_ci.sh
@@ -201,8 +201,7 @@ case $1 in
testcode=$?
if [[ $testcode -ne 0 ]]; then
echo "Executing fury javascript tests failed"
-# TODO(bigtech) enable js ci
-# exit $testcode
+exit $testcode
fi
echo "Executing fury javascript tests succeeds"
;;
diff --git a/javascript/jest.config.js b/javascript/jest.config.js
index 4b9f5e81..ce814269 100644
--- a/javascript/jest.config.js
+++ b/javascript/jest.config.js
@@ -19,7 +19,7 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
const semver = require("semver");
-const hpsEnable = semver.gt(process.versions.node, '20.0.0')
+const hpsEnable = semver.gt(process.versions.node, '20.0.0') &&
process.platform !== 'win32';
module.exports = {
collectCoverage: hpsEnable,
diff --git a/javascript/packages/hps/package.json
b/javascript/packages/hps/package.json
index 4febf187..fdce6f02 100644
--- a/javascript/packages/hps/package.json
+++ b/javascript/packages/hps/package.json
@@ -8,8 +8,9 @@
"src",
"binding.gyp"
],
+ "gypfile": false,
"scripts": {
-"postinstall": "npx node-gyp rebuild",
+"postinstall": "node -e \"if (process.version.match(/v(\\d+)/)[1] >= 20 &&
process.platform !== 'win32') { require('child_process').execSync('npx node-gyp
rebuild') } \"",
"build": "npx node-gyp rebuild && tsc",
"prepublishOnly": "npm run build"
},
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: chore(JavaScript): Fix JavaScript test CI (#1355)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 5e897321 chore(JavaScript): Fix JavaScript test CI (#1355)
5e897321 is described below
commit 5e897321683c4f38b42bb6b635c77bf1473657ba
Author: weipeng
AuthorDate: Wed Jan 24 12:12:49 2024 +0800
chore(JavaScript): Fix JavaScript test CI (#1355)
1. JavaScript codebase contains a c++ module which requires the nodejs
version greater than 20, we should ignore it when the version is not
satisfied to prevent it from breaking test ci
2. Remove coverage threshold temporarily before the xlang protocol
release
3. Add 14.x and 16.x to the docker envs
#1350
---
.github/workflows/ci.yml | 2 +-
javascript/jest.config.js | 25 ++
javascript/package.json | 6 ++-
javascript/packages/fury/package.json | 6 +--
javascript/test/reader.test.ts| 51
javascript/test/string.test.ts| 91 ---
javascript/test/writer.test.ts| 4 +-
7 files changed, 87 insertions(+), 98 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 973358f4..df5ffb8a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -110,7 +110,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
-node-version: [14.x, 20.x]
+node-version: [14.x, 16.x, 18.x, 20.x]
steps:
- uses: actions/checkout@v3
- name: Use Node.js ${{ matrix.node-version }}
diff --git a/javascript/jest.config.js b/javascript/jest.config.js
index 7059d3bb..4b9f5e81 100644
--- a/javascript/jest.config.js
+++ b/javascript/jest.config.js
@@ -18,8 +18,11 @@
*/
/** @type {import('ts-jest').JestConfigWithTsJest} */
+const semver = require("semver");
+const hpsEnable = semver.gt(process.versions.node, '20.0.0')
+
module.exports = {
- collectCoverage: true,
+ collectCoverage: hpsEnable,
preset: 'ts-jest',
testEnvironment: 'node',
collectCoverageFrom: [
@@ -28,6 +31,9 @@ module.exports = {
"!**/build/**",
"!packages/fury/lib/murmurHash3.ts"
],
+ "testPathIgnorePatterns" : [
+hpsEnable ? null : "(.*)/hps.test.ts$",
+ ].filter(Boolean),
transform: {
'\\.ts$': ['ts-jest', {
tsconfig: {
@@ -38,12 +44,13 @@ module.exports = {
}
}],
},
- coverageThreshold: {
-global: {
- branches: 91,
- functions: 99,
- lines: 98,
- statements: 98
-}
- }
+ // todo: JavaScript codebase is iterating rapidly, remove this restriction
temporary
+ // coverageThreshold: {
+ // global: {
+ // branches: 91,
+ // functions: 99,
+ // lines: 98,
+ // statements: 98
+ // }
+ // }
};
\ No newline at end of file
diff --git a/javascript/package.json b/javascript/package.json
index e8c6b3df..a0345c20 100644
--- a/javascript/package.json
+++ b/javascript/package.json
@@ -14,6 +14,10 @@
"@stylistic/eslint-plugin": "^1.5.1",
"@types/js-beautify": "^1.14.3",
"eslint": "^8.55.0",
-"js-beautify": "^1.14.11"
+"js-beautify": "^1.14.11",
+"jest": "^29.5.0",
+"jest-junit": "^16.0.0",
+"ts-jest": "^29.0.2",
+"typescript": "^4.8.4"
}
}
diff --git a/javascript/packages/fury/package.json
b/javascript/packages/fury/package.json
index 98758518..7283cd19 100644
--- a/javascript/packages/fury/package.json
+++ b/javascript/packages/fury/package.json
@@ -17,11 +17,7 @@
"@typescript-eslint/parser": "^5.40.0",
"benchmark": "^2.1.4",
"eslint": "^8.25.0",
-"jest": "^29.5.0",
-"jest-junit": "^16.0.0",
-"protobufjs": "^7.2.4",
-"ts-jest": "^29.0.2",
-"typescript": "^4.8.4"
+"protobufjs": "^7.2.4"
},
"dependencies": {
"node-gyp": "^9.4.0",
diff --git a/javascript/test/reader.test.ts b/javascript/test/reader.test.ts
index 0e0e6dc6..a707d393 100644
--- a/javascript/test/reader.test.ts
+++ b/javascript/test/reader.test.ts
@@ -17,42 +17,33 @@
* under the License.
*/
-import { alloc } from '@furyjs/fury/lib/platformBuffer';
-import { BinaryReader } from '@furyjs/fury/lib/reader';
-import { Config } from '@furyjs/fury/lib/type';
-import { BinaryWriter } from '@furyjs/fury/lib/writer';
+import { alloc } from '../packages/fury/lib/platformBuffer';
+import { BinaryReader } from '../packag
(incubator-fury) branch main updated: feat(JavaScript): Implement the collection protocol (#1337)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 96ace333 feat(JavaScript): Implement the collection protocol (#1337)
96ace333 is described below
commit 96ace3331f34f8a2bb64436632d8371766a665c2
Author: weipeng
AuthorDate: Tue Jan 16 16:20:17 2024 +0800
feat(JavaScript): Implement the collection protocol (#1337)
Now we have introduced a new binary layout for Collection, which is
utilized by both Array and Set. The Collection is defined in the
[SPEC](https://github.com/apache/incubator-fury/blob/3da32797beb25fd39b06c42f8c8b9e67eec5184d/docs/protocols/java_object_graph_spec.md#unsigned-int).
This PR implements the Collection in JavaScript.
1. Array and Set will extend CollectionSerializerGenerator to generate
code.
2. The Xlang Collection is not stable yet; it may undergo changes in the
future, particularly regarding polymorphism, although the changes are
expected to be minimal.
3. We have refined the reference checks, which were previously
disorganized.
4. The refactor of collection types has resulted in performance
improvements. In our sample benchmark, we observed an increase in
performance of nearly 20%.
#1330
---
javascript/packages/fury/index.ts | 41 +-
javascript/packages/fury/lib/any.ts| 150 -
javascript/packages/fury/lib/classResolver.ts | 48 +-
javascript/packages/fury/lib/fury.ts | 115 ++--
javascript/packages/fury/lib/gen/any.ts| 96 +++-
javascript/packages/fury/lib/gen/array.ts | 48 +-
javascript/packages/fury/lib/gen/binary.ts | 6 +-
javascript/packages/fury/lib/gen/builder.ts| 32 +-
javascript/packages/fury/lib/gen/collection.ts | 322 +++
javascript/packages/fury/lib/gen/index.ts | 12 +-
javascript/packages/fury/lib/gen/map.ts| 6 +-
javascript/packages/fury/lib/gen/object.ts | 22 +-
javascript/packages/fury/lib/gen/router.ts | 10 +
javascript/packages/fury/lib/gen/serializer.ts | 173 --
javascript/packages/fury/lib/gen/set.ts| 48 +-
javascript/packages/fury/lib/gen/tuple.ts | 6 +-
javascript/packages/fury/lib/gen/typedArray.ts | 11 +-
javascript/packages/fury/lib/meta.ts | 68 ++-
javascript/packages/fury/lib/referenceResolver.ts | 12 +-
javascript/packages/fury/lib/type.ts | 6 +-
javascript/test/__snapshots__/codeGen.test.ts.snap | 606 -
javascript/test/any.test.ts| 6 +-
javascript/test/array.test.ts | 7 +-
javascript/test/codeGen.test.ts| 41 --
javascript/test/object.test.ts | 2 +-
25 files changed, 779 insertions(+), 1115 deletions(-)
diff --git a/javascript/packages/fury/index.ts
b/javascript/packages/fury/index.ts
index 5334f007..165b7f38 100644
--- a/javascript/packages/fury/index.ts
+++ b/javascript/packages/fury/index.ts
@@ -17,18 +17,14 @@
* under the License.
*/
-import {
- generateSerializer,
-} from "./lib/gen";
import {
ObjectTypeDescription,
TypeDescription,
ArrayTypeDescription,
Type,
- ToRecordType,
} from "./lib/description";
-import { Serializer, Fury, InternalSerializerType, Config } from "./lib/type";
-import FuryInternal from "./lib/fury";
+import { Serializer, InternalSerializerType } from "./lib/type";
+import Fury from "./lib/fury";
export {
Serializer,
@@ -39,35 +35,4 @@ export {
Type,
};
-export default class {
- constructor(private config?: Config) { }
- private fury: Fury = FuryInternal(this.config || {});
-
- registerSerializer(description: T) {
-const serializer = generateSerializer(this.fury, description);
-return {
- serializer,
- serialize: (data: ToRecordType) => {
-return this.fury.serialize(data, serializer);
- },
- serializeVolatile: (data: ToRecordType) => {
-return this.fury.serializeVolatile(data, serializer);
- },
- deserialize: (bytes: Uint8Array) => {
-return this.fury.deserialize(bytes, serializer) as ToRecordType;
- },
-};
- }
-
- serializeVolatile(v: any, serialize?: Serializer) {
-return this.fury.serializeVolatile(v, serialize);
- }
-
- serialize(v: any, serialize?: Serializer) {
-return this.fury.serialize(v, serialize);
- }
-
- deserialize(bytes: Uint8Array) {
-return this.fury.deserialize(bytes);
- }
-}
+export default Fury;
diff --git a/javascript/packages/fury/lib/any.ts
b/javascript/packages/fury/lib/any.ts
deleted file mode 100644
index 7fc0d6f1..
--- a/javascript/packages/fury/lib/any.ts
++
(incubator-fury) branch main updated: chore(JavaScript): Drop optional chaining expression (#1338)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 08899bbc chore(JavaScript): Drop optional chaining expression (#1338)
08899bbc is described below
commit 08899bbcb9f268801c52b09d4a130c4f7e117324
Author: 野声
AuthorDate: Fri Jan 12 16:33:13 2024 +0800
chore(JavaScript): Drop optional chaining expression (#1338)
some old compiler did not recognize the `?.` expression(sadly).
---
javascript/packages/fury/lib/gen/index.ts | 9 ++---
javascript/packages/fury/lib/writer.ts| 2 +-
javascript/packages/fury/package.json | 2 +-
3 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/javascript/packages/fury/lib/gen/index.ts
b/javascript/packages/fury/lib/gen/index.ts
index 1a347bc1..4a9d1e3a 100644
--- a/javascript/packages/fury/lib/gen/index.ts
+++ b/javascript/packages/fury/lib/gen/index.ts
@@ -45,10 +45,13 @@ export const generate = (fury: Fury, description:
TypeDescription) => {
const generator = new InnerGeneratorClass(description, new
CodecBuilder(scope, fury), scope);
const funcString = generator.toSerializer();
- const afterCodeGenerated = fury.config?.hooks?.afterCodeGenerated;
- if (typeof afterCodeGenerated === "function") {
-return new Function(afterCodeGenerated(funcString));
+ if (fury.config && fury.config.hooks) {
+const afterCodeGenerated = fury.config.hooks.afterCodeGenerated;
+if (typeof afterCodeGenerated === "function") {
+ return new Function(afterCodeGenerated(funcString));
+}
}
+
return new Function(funcString);
};
diff --git a/javascript/packages/fury/lib/writer.ts
b/javascript/packages/fury/lib/writer.ts
index 31cfddf0..c88f53bc 100644
--- a/javascript/packages/fury/lib/writer.ts
+++ b/javascript/packages/fury/lib/writer.ts
@@ -333,7 +333,7 @@ export const BinaryWriter = (config: Config) => {
varUInt32,
varUInt64,
varInt64,
-stringOfVarUInt32: config?.hps
+stringOfVarUInt32: config && config.hps
? stringOfVarUInt32Fast()
: stringOfVarUInt32Slow,
bufferWithoutMemCheck,
diff --git a/javascript/packages/fury/package.json
b/javascript/packages/fury/package.json
index 6cfa0fb4..98758518 100644
--- a/javascript/packages/fury/package.json
+++ b/javascript/packages/fury/package.json
@@ -1,6 +1,6 @@
{
"name": "@furyjs/fury",
- "version": "0.5.5-beta",
+ "version": "0.5.6-beta",
"description": "A blazing fast multi-language serialization framework
powered by jit and zero-copy",
"main": "dist/index.js",
"scripts": {
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury-site) branch main updated: Update footer css (#101)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git
The following commit(s) were added to refs/heads/main by this push:
new d8b789e Update footer css (#101)
d8b789e is described below
commit d8b789ee99baaf93f07ebd72e8f397110ef6a7d9
Author: LiangliangSui <116876207+liangliang...@users.noreply.github.com>
AuthorDate: Wed Jan 10 12:29:27 2024 +0800
Update footer css (#101)
---
docusaurus.config.ts | 2 +-
src/css/custom.css | 8
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/docusaurus.config.ts b/docusaurus.config.ts
index ce571f9..1ea9471 100644
--- a/docusaurus.config.ts
+++ b/docusaurus.config.ts
@@ -176,7 +176,7 @@ const config: Config = {
},
],
logo: {
-width: 320,
+width: 200,
src: "/img/apache-incubator.svg",
href: "https://incubator.apache.org/;,
alt: "Apache Incubator logo"
diff --git a/src/css/custom.css b/src/css/custom.css
index bae8e3a..c6d2b24 100644
--- a/src/css/custom.css
+++ b/src/css/custom.css
@@ -50,3 +50,11 @@
background: url("data:image/svg+xml,%3Csvg viewBox='0 0 24 24'
xmlns='http://www.w3.org/2000/svg'%3E%3Cpath fill='white' d='M12 .297c-6.63
0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577
0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633
17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236
1.07 1.835 2.809 1.305
3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93
0-1.31.465-2.38 1.235-3.22-.135- [...]
no-repeat;
}
+
+.footer__copyright {
+ font-size: .7em;
+}
+
+.footer__links {
+ text-align: center;
+}
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: feat(JavaScript): enhance performance 64bits number (#1320)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new a80dce67 feat(JavaScript): enhance performance 64bits number (#1320)
a80dce67 is described below
commit a80dce67f5e00c3825bb788b87fc558671346110
Author: weipeng
AuthorDate: Tue Jan 9 10:33:04 2024 +0800
feat(JavaScript): enhance performance 64bits number (#1320)
1. Creating BigInts is too performance-intensive; we'll use uint32
instead.
2. Reduce memory reads as much as possiable; Replace calle uint8 four
times by uint32 once.
---
.../fury/lib/{reader.ts => reader/index.ts}| 216 -
.../packages/fury/lib/{ => reader}/string.ts | 71 +--
javascript/packages/fury/lib/writer.ts | 29 ++-
javascript/packages/fury/package.json | 2 +-
javascript/test/util.js| 6 +
5 files changed, 209 insertions(+), 115 deletions(-)
diff --git a/javascript/packages/fury/lib/reader.ts
b/javascript/packages/fury/lib/reader/index.ts
similarity index 50%
rename from javascript/packages/fury/lib/reader.ts
rename to javascript/packages/fury/lib/reader/index.ts
index 46b4e5fe..e32b02f7 100644
--- a/javascript/packages/fury/lib/reader.ts
+++ b/javascript/packages/fury/lib/reader/index.ts
@@ -17,10 +17,10 @@
* under the License.
*/
-import { Config, LATIN1 } from "./type";
-import { isNodeEnv } from "./util";
-import { PlatformBuffer, alloc, fromUint8Array } from "./platformBuffer";
-import { read1, read10, read11, read12, read13, read14, read15, read2, read3,
read4, read5, read6, read7, read8, read9 } from "./string";
+import { Config, LATIN1 } from "../type";
+import { isNodeEnv } from "../util";
+import { PlatformBuffer, alloc, fromUint8Array } from "../platformBuffer";
+import { readLatin1String } from "./string";
export const BinaryReader = (config: Config) => {
const sliceStringEnable = isNodeEnv && config.useSliceString;
@@ -135,42 +135,7 @@ export const BinaryReader = (config: Config) => {
function stringLatin1Slow(len: number) {
const rawCursor = cursor;
cursor += len;
-switch (len) {
- case 0:
-return "";
- case 1:
-return read1(buffer, rawCursor);
- case 2:
-return read2(buffer, rawCursor);
- case 3:
-return read3(buffer, rawCursor);
- case 4:
-return read4(buffer, rawCursor);
- case 5:
-return read5(buffer, rawCursor);
- case 6:
-return read6(buffer, rawCursor);
- case 7:
-return read7(buffer, rawCursor);
- case 8:
-return read8(buffer, rawCursor);
- case 9:
-return read9(buffer, rawCursor);
- case 10:
-return read10(buffer, rawCursor);
- case 11:
-return read11(buffer, rawCursor);
- case 12:
-return read12(buffer, rawCursor);
- case 13:
-return read13(buffer, rawCursor);
- case 14:
-return read14(buffer, rawCursor);
- case 15:
-return read15(buffer, rawCursor);
- default:
-return buffer.latin1Slice(rawCursor, cursor);
-}
+return readLatin1String(buffer, len, rawCursor);
}
function binary(len: number) {
@@ -186,29 +151,45 @@ export const BinaryReader = (config: Config) => {
return result;
}
- function zigZag(v: number) {
-return (v >> 1) ^ -(v & 1);
- }
-
- function zigZagBigInt(v: bigint) {
-return (v >> 1n) ^ -(v & 1n);
- }
-
function varUInt32() {
-let byte_ = uint8();
-let result = byte_ & 0x7f;
-if ((byte_ & 0x80) != 0) {
- byte_ = uint8();
- result |= (byte_ & 0x7f) << 7;
- if ((byte_ & 0x80) != 0) {
-byte_ = uint8();
-result |= (byte_ & 0x7f) << 14;
-if ((byte_ & 0x80) != 0) {
- byte_ = uint8();
- result |= (byte_ & 0x7f) << 21;
- if ((byte_ & 0x80) != 0) {
-byte_ = uint8();
-result |= (byte_) << 28;
+// Reduce memory reads as much as possible. Reading a uint32 at once is
far faster than reading four uint8s separately.
+if (buffer.byteLength - cursor >= 5) {
+ const u32 = dataView.getUint32(cursor++, true);
+ let result = u32 & 0x7f;
+ if ((u32 & 0x80) != 0) {
+cursor++;
+const b2 = u32 >> 8;
+result |= (b2 & 0x7f) << 7;
+if ((b2 & 0x80) != 0) {
+ cursor++;
+ const b3 = u32 >> 16;
+ result |= (b3 & 0x7f) << 14;
+ if ((b3 & 0x80) != 0) {
+cursor++;
+const b4 = u32 >> 24;
+result |=
(incubator-fury) branch main updated: perf(java): merge perftests into benchmark (#1318)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new bcd36656 perf(java): merge perftests into benchmark (#1318)
bcd36656 is described below
commit bcd366562f7ecf96970ebef05a1877dd48805d76
Author: Shawn Yang
AuthorDate: Sun Jan 7 19:08:25 2024 +0800
perf(java): merge perftests into benchmark (#1318)
---
.gitattributes | 1 -
ci/deploy.sh | 4 +-
ci/format.sh | 4 +-
ci/run_ci.sh | 3 +
integration_tests/README.md| 3 +-
integration_tests/perftests/README.md | 24 --
integration_tests/perftests/pom.xml| 251 -
.../UserTypeDeserializeSuite.java | 67 --
.../integration_tests/UserTypeSerializeSuite.java | 83 ---
java/fury-benchmark/README.md | 13 +-
java/fury-benchmark/pom.xml| 78 +++
.../fury/benchmark/UserTypeDeserializeSuite.java | 23 ++
.../fury/benchmark/UserTypeSerializeSuite.java | 25 ++
.../org/apache/fury/benchmark}/state/Example.java | 2 +-
.../fury/benchmark}/state/FlatBuffersState.java| 21 +-
.../fury/benchmark}/state/ProtoBuffersState.java | 9 +-
.../org/apache/fury/benchmark}/state/bench.fbs | 0
.../fury/benchmark}/state/generated/FBSBar.java| 24 +-
.../fury/benchmark}/state/generated/FBSFoo.java| 4 +-
.../fury/benchmark}/state/generated/FBSImage.java | 8 +-
.../fury/benchmark}/state/generated/FBSMedia.java | 4 +-
.../state/generated/FBSMediaContent.java | 16 +-
.../fury/benchmark}/state/generated/FBSPlayer.java | 2 +-
.../fury/benchmark}/state/generated/FBSSample.java | 4 +-
.../fury/benchmark}/state/generated/FBSSize.java | 2 +-
.../fury-benchmark}/src/main/proto/bench.proto | 0
.../benchmark}/state/FlatBuffersStateTest.java | 11 +-
.../benchmark}/state/ProtoBuffersStateTest.java| 11 +-
licenserc.toml | 2 +-
29 files changed, 211 insertions(+), 488 deletions(-)
diff --git a/.gitattributes b/.gitattributes
index a2e46878..4c3924f5 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1 @@
java/fury-benchmark export-ignore
-integration_tests/perftests export-ignore
\ No newline at end of file
diff --git a/ci/deploy.sh b/ci/deploy.sh
index bb9d4be8..e5d1ad40 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -89,8 +89,6 @@ bump_java_version() {
echo "Set fury integration_tests version to $version"
cd "$ROOT/integration_tests/jdk_compatibility_tests"
mvn versions:set -DnewVersion="$version"
- cd "$ROOT/integration_tests/perftests"
- mvn versions:set -DnewVersion="$version"
cd "$ROOT/integration_tests/latest_jdk_tests"
mvn versions:set -DnewVersion="$version"
}
@@ -176,4 +174,4 @@ python) # Deploy wheel to pypi
echo "Execute command $*"
"$@"
;;
-esac
\ No newline at end of file
+esac
diff --git a/ci/format.sh b/ci/format.sh
index 433dc47b..54750890 100755
--- a/ci/format.sh
+++ b/ci/format.sh
@@ -13,7 +13,7 @@ BLACK_VERSION_REQUIRED="22.1.0"
SHELLCHECK_VERSION_REQUIRED="0.7.1"
install_nodejs() {
- #intall nodejs
+ #intall nodejs
filename="node-v16.17.1-linux-x64"
pkg="$filename.tar.gz"
NODE_URL="https://nodejs.org/dist/v16.17.1/$pkg;
@@ -188,7 +188,7 @@ format_java() {
cd "$ROOT/java/fury-benchmark"
mvn -T10 --no-transfer-progress spotless:apply
cd "$ROOT/integration_tests"
- dirs=("graalvm_tests" "jdk_compatibility_tests" "latest_jdk_tests"
"perftests")
+ dirs=("graalvm_tests" "jdk_compatibility_tests" "latest_jdk_tests")
for d in "${dirs[@]}" ; do
pushd "$d"
mvn -T10 --no-transfer-progress spotless:apply
diff --git a/ci/run_ci.sh b/ci/run_ci.sh
index 9d7fc9f4..52b8a87a 100755
--- a/ci/run_ci.sh
+++ b/ci/run_ci.sh
@@ -109,6 +109,9 @@ graalvm_test() {
integration_tests() {
cd "$ROOT"/java
mvn -T10 -B --no-transfer-progress clean install -DskipTests
+ echo "benchmark tests"
+ cd "$ROOT"/java/fury-benchmark
+ mvn -T10 -B --no-transfer-progress clean test -Pjmh
echo "Start latest jdk tests"
cd "$ROOT"/integration_tests/latest_jdk_tests
echo "latest_jdk_tests: JDK 21"
diff --git a/integration_tests/README.md b/integration_tests/README.md
index 2dc5eaad..f806f7a4 100644
--- a/integration_tests/README.md
+++ b/integra
(incubator-fury) branch main updated: [Java] remove vague desc in drop-in replacement jdk serialization libs (#1291)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury.git The following commit(s) were added to refs/heads/main by this push: new e67e2537 [Java] remove vague desc in drop-in replacement jdk serialization libs (#1291) e67e2537 is described below commit e67e25377d5ce12639c407bbd93cee877b4aa514 Author: Shawn Yang AuthorDate: Wed Jan 3 18:42:29 2024 +0800 [Java] remove vague desc in drop-in replacement jdk serialization libs (#1291) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b4d3fc2f..8bc117af 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ https://fury.apache.org In addition to cross-language serialization, Fury also features at: -- Drop-in replace Java serialization frameworks such as JDK/Kryo/Hessian without modifying any code, but 100x faster. - It can greatly improve the efficiency of high-performance RPC calls, data transfer, and object persistence. +- Drop-in replace Java serialization frameworks such as JDK/Kryo/Hessian, but 100x faster at most, which can greatly improve + the efficiency of high-performance RPC calls, data transfer, and object persistence. - **100% compatible** with JDK serialization API with much faster implementation: supporting JDK `writeObject/readObject/writeReplace/readResolve/readObjectNoData/Externalizable` API. - Supports **Java 8~21**, Java 17+ `record` is supported too. - Support [AOT compilation serialization](docs/guide/graalvm_guide.md) for **GraalVM native image**, and no reflection/serialization json config are needed. - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: [JavaScript] Implement the standard protocol (#1286)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 20f18493 [JavaScript] Implement the standard protocol (#1286)
20f18493 is described below
commit 20f1849353ace58804e7c09ef916d26c4b167dd9
Author: weipeng
AuthorDate: Tue Jan 2 15:57:30 2024 +0800
[JavaScript] Implement the standard protocol (#1286)
1. Make the tag string lazy to avoid creating strings.
2. Add the serializeVolatile API to take ownership of the bufferWriter
and avoid buffer copying, which is 30% faster than the serialize
operation. There is a usage limitation; the bufferWriter remains
inaccessible until the dispose method from the result of
serializeVolatile has been called.
4. Remove the useLatin1 configuration, which is a standard in the Fury
protocol.
Before:
After:
---
javascript/benchmark/index.js | 2 +-
javascript/benchmark/sample.jpg| Bin 0 -> 27259 bytes
javascript/packages/fury/index.ts | 7 +++
javascript/packages/fury/lib/classResolver.ts | 50 +---
javascript/packages/fury/lib/error.ts | 28 +
javascript/packages/fury/lib/fury.ts | 24 +++-
.../packages/fury/lib/internalSerializer/number.ts | 8 +--
javascript/packages/fury/lib/reader.ts | 9 ++-
javascript/packages/fury/lib/type.ts | 1 -
javascript/packages/fury/lib/writer.ts | 26 ++--
javascript/test/array.test.ts | 2 +-
javascript/test/io.test.ts | 20 ++-
javascript/test/protocol/struct.test.ts| 52
javascript/test/reader.test.ts | 5 --
javascript/test/string.test.ts | 2 -
javascript/test/writer.test.ts | 66 +
16 files changed, 201 insertions(+), 101 deletions(-)
diff --git a/javascript/benchmark/index.js b/javascript/benchmark/index.js
index eb74e305..5dd96f0f 100644
--- a/javascript/benchmark/index.js
+++ b/javascript/benchmark/index.js
@@ -20,7 +20,7 @@
const Fury = require("@furyjs/fury");
const utils = require("../test/util");
const hps = require('@furyjs/hps');
-const fury = new Fury.default({ hps, refTracking: false, useLatin1: true,
useSliceString: true });
+const fury = new Fury.default({ hps, refTracking: false, useSliceString: true
});
const Benchmark = require("benchmark");
const protobuf = require("protobufjs");
const path = require('path');
diff --git a/javascript/benchmark/sample.jpg b/javascript/benchmark/sample.jpg
new file mode 100644
index ..5c98c2d2
Binary files /dev/null and b/javascript/benchmark/sample.jpg differ
diff --git a/javascript/packages/fury/index.ts
b/javascript/packages/fury/index.ts
index fd5d1fa4..b305c457 100644
--- a/javascript/packages/fury/index.ts
+++ b/javascript/packages/fury/index.ts
@@ -57,12 +57,19 @@ export default class {
serialize: (data: ToRecordType) => {
return this.fury.serialize(data, serializer);
},
+ serializeVolatile: (data: ToRecordType) => {
+return this.fury.serializeVolatile(data, serializer);
+ },
deserialize: (bytes: Uint8Array) => {
return this.fury.deserialize(bytes, serializer) as ToRecordType;
},
};
}
+ serializeVolatile(v: any, serialize?: Serializer) {
+return this.fury.serializeVolatile(v, serialize);
+ }
+
serialize(v: any, serialize?: Serializer) {
return this.fury.serialize(v, serialize);
}
diff --git a/javascript/packages/fury/lib/classResolver.ts
b/javascript/packages/fury/lib/classResolver.ts
index e2553ad8..d344e511 100644
--- a/javascript/packages/fury/lib/classResolver.ts
+++ b/javascript/packages/fury/lib/classResolver.ts
@@ -34,12 +34,39 @@ import { BinaryWriter } from "./writer";
const USESTRINGVALUE = 0;
const USESTRINGID = 1;
+class Lazystring {
+ private string: string | null = null;
+ private start: number | null = null;
+ private len: number | null = null;
+
+ static fromPair(start: number, len: number) {
+const result = new Lazystring();
+result.start = start;
+result.len = len;
+return result;
+ }
+
+ static fromString(str: string) {
+const result = new Lazystring();
+result.string = str;
+return result;
+ }
+
+ toString(binaryReader: BinaryReader) {
+if (this.string == null) {
+ const str = binaryReader.stringUtf8At(this.start!,
(incubator-fury) branch main updated: Update twitter id in README.md to ApacheFury (#1270)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury.git The following commit(s) were added to refs/heads/main by this push: new 26f1fc39 Update twitter id in README.md to ApacheFury (#1270) 26f1fc39 is described below commit 26f1fc398249d6c7f663e4ba92a83e086cdf9677 Author: Shawn Yang AuthorDate: Thu Dec 28 11:11:59 2023 +0800 Update twitter id in README.md to ApacheFury (#1270) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32c07c8d..b4d3fc2f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [](https://github.com/apache/incubator-fury/actions/workflows/ci.yml) [](https://join.slack.com/t/fury-project/shared_invite/zt-1u8soj4qc-ieYEu7ciHOqA2mo47llS8A) -[](https://twitter.com/fury_community) +[](https://twitter.com/ApacheFury) [](https://search.maven.org/#search|gav|1|g:"org.furyio"%20AND%20a:"fury-core") - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch chaokunyang-patch-1 deleted (was c0c43f5e)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a change to branch chaokunyang-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-fury.git was c0c43f5e Update twitter id in README.md to ApacheFury The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository. - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury-site) branch main updated: Update twitter id in README.md to ApacheFury (#99)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git
The following commit(s) were added to refs/heads/main by this push:
new 5f95b72 Update twitter id in README.md to ApacheFury (#99)
5f95b72 is described below
commit 5f95b7271af3f76a2d0afe2622d4fcb16f41b049
Author: Shawn Yang
AuthorDate: Thu Dec 28 11:11:49 2023 +0800
Update twitter id in README.md to ApacheFury (#99)
---
docusaurus.config.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docusaurus.config.ts b/docusaurus.config.ts
index b1a41c2..ce571f9 100644
--- a/docusaurus.config.ts
+++ b/docusaurus.config.ts
@@ -140,7 +140,7 @@ const config: Config = {
},
{
label: 'Twitter',
- href: 'https://twitter.com/fury_community',
+ href: 'https://twitter.com/ApacheFury',
},
],
},
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury-site) branch update_twitter_id_to_apache_fury deleted (was 8f23bfa)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a change to branch update_twitter_id_to_apache_fury in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git was 8f23bfa Update twitter id in README.md to ApacheFury The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository. - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: [Java][Doc] Switch Option and Description column for better readability (#1267)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 36c6fe35 [Java][Doc] Switch Option and Description column for better
readability (#1267)
36c6fe35 is described below
commit 36c6fe35e7fea7723ca16a75d09a22558f07577c
Author: Shawn Yang
AuthorDate: Thu Dec 28 08:46:09 2023 +0800
[Java][Doc] Switch Option and Description column for better readability
(#1267)
---
docs/guide/java_object_graph_guide.md | 42 +--
1 file changed, 21 insertions(+), 21 deletions(-)
diff --git a/docs/guide/java_object_graph_guide.md
b/docs/guide/java_object_graph_guide.md
index fbcf918b..9ab05b17 100644
--- a/docs/guide/java_object_graph_guide.md
+++ b/docs/guide/java_object_graph_guide.md
@@ -94,28 +94,28 @@ public class Example {
}
```
-## Advanced Usage
+## FuryBuilder options
+
+| Option Name | Description
[...]
+|-|-
[...]
+| `timeRefIgnored`| Whether to ignore reference tracking
of all time types registered in `TimeSerializers` and subclasses of those types
when ref tracking is enabled. If ignored, ref tracking of every time type can
be enabled by invoking `Fury#registerSerializer(Class, Serializer)`. For
example, `fury.registerSerializer(Date.class, new DateSerializer(fury, true))`.
Note that enabling ref tracking should happen before serializer codegen of any
types which contain time [...]
+| `compressInt` | Enables or disables int compression
for smaller size.
[...]
+| `compressLong` | Enables or disables long compression
for smaller size.
[...]
+| `compressString`| Enables or disables string compression
for smaller size.
[...]
+| `classLoader` | The classloader should not be updated;
Fury caches class metadata. Use `LoaderBinding` or `ThreadSafeFury` for
classloader updates.
[...]
+| `compatibleMode`| Type forward/backward compatibility
config. Also Related to `checkClassVersion` config. `SCHEMA_CONSISTENT`: Class
schema must be consistent between serialization peer and deserialization peer.
`COMPATIBLE`: Class schema can be different between serialization peer and
deserialization peer. They can add/delete fields independently
(incubator-fury-site) branch update_apache_license deleted (was 17033d3)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a change to branch update_apache_license in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git was 17033d3 update apache license The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository. - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury-site) branch main updated: update apache license (#97)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git
The following commit(s) were added to refs/heads/main by this push:
new 603330d update apache license (#97)
603330d is described below
commit 603330daa61fa69322070eebfd6a1eb1af311de2
Author: Shawn Yang
AuthorDate: Thu Dec 28 08:44:56 2023 +0800
update apache license (#97)
---
LICENSE | 6 +++---
NOTICE | 5 +
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/LICENSE b/LICENSE
index c8db697..f49a4e1 100644
--- a/LICENSE
+++ b/LICENSE
@@ -178,7 +178,7 @@
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "{}"
+ boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
@@ -186,7 +186,7 @@
same "printed page" as the copyright notice for easier
identification within third-party archives.
- Copyright 2023 FURY authors
+ Copyright [] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -198,4 +198,4 @@
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
- limitations under the License.
+ limitations under the License.
\ No newline at end of file
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 000..ea67774
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,5 @@
+Apache Fury (Incubating)
+Copyright 2023 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury-site) branch update_fury_java_install deleted (was 8c1a5fc)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a change to branch update_fury_java_install in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git was 8c1a5fc update fury java install The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository. - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury-site) branch main updated: update fury java install (#98)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git The following commit(s) were added to refs/heads/main by this push: new 2c3df2a update fury java install (#98) 2c3df2a is described below commit 2c3df2a7329a3feda4e8f77bf7b8c0ca2cd3fc17 Author: Shawn Yang AuthorDate: Thu Dec 28 08:44:19 2023 +0800 update fury java install (#98) --- docs/start/install.md | 18 ++ 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/start/install.md b/docs/start/install.md index f755c58..e428a45 100644 --- a/docs/start/install.md +++ b/docs/start/install.md @@ -11,8 +11,8 @@ Nightly snapshot: ```xml -sonatype -https://s01.oss.sonatype.org/content/repositories/snapshots +apache +https://repository.apache.org/snapshots/ false @@ -22,13 +22,13 @@ Nightly snapshot: - org.furyio + org.apache.fury fury-core 0.5.0-SNAPSHOT @@ -40,18 +40,20 @@ Release version: org.furyio fury-core - 0.4.0 + 0.4.1 ``` +Maven groupId will be changed to `org.apache.fury` when next version is released. + ### Scala ```sbt -libraryDependencies += "org.furyio" % "fury-core" % "0.4.0" +libraryDependencies += "org.furyio" % "fury-core" % "0.4.1" ``` ### Python @@ -60,7 +62,7 @@ libraryDependencies += "org.furyio" % "fury-core" % "0.4.0" # Python wheel will be released in the future. # Currently you need to specify `--pre` to install # the unstable version. -pip install pyfury --pre +pip install pyfury ``` ### Golang - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: [JavaScript] Improve tag write performance (#1241)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new dd2f24aa [JavaScript] Improve tag write performance (#1241)
dd2f24aa is described below
commit dd2f24aa865e751560cdd4876aa2b1d485723a49
Author: 野声
AuthorDate: Wed Dec 27 21:38:56 2023 +0800
[JavaScript] Improve tag write performance (#1241)
---
javascript/packages/fury/lib/classResolver.ts | 52 ++
javascript/packages/fury/lib/codeGen.ts| 60 +--
javascript/packages/fury/lib/platformBuffer.ts | 12 +++
javascript/packages/fury/lib/util.ts | 2 -
javascript/test/__snapshots__/codeGen.test.ts.snap | 110 ++---
5 files changed, 128 insertions(+), 108 deletions(-)
diff --git a/javascript/packages/fury/lib/classResolver.ts
b/javascript/packages/fury/lib/classResolver.ts
index f54bf070..e2553ad8 100644
--- a/javascript/packages/fury/lib/classResolver.ts
+++ b/javascript/packages/fury/lib/classResolver.ts
@@ -25,11 +25,12 @@ import mapSerializer from "./internalSerializer/map";
import setSerializer from "./internalSerializer/set";
import boolSerializer from "./internalSerializer/bool";
import { uInt16Serializer, int16Serializer, int32Serializer, uInt32Serializer,
uInt64Serializer, floatSerializer, doubleSerializer, uInt8Serializer,
int64Serializer, int8Serializer } from "./internalSerializer/number";
-import { InternalSerializerType, Serializer, Fury, BinaryReader, BinaryWriter
} from "./type";
+import { InternalSerializerType, Serializer, Fury, BinaryReader, BinaryWriter
as TBinaryWriter } from "./type";
import anySerializer from "./internalSerializer/any";
-import { PlatformBuffer, fromUint8Array } from "./platformBuffer";
+import { fromString } from "./platformBuffer";
import { x64hash128 } from "./murmurHash3";
-import { BinaryWriter as BufferWriter } from "./writer";
+import { BinaryWriter } from "./writer";
+
const USESTRINGVALUE = 0;
const USESTRINGID = 1;
@@ -39,7 +40,8 @@ export default class SerializerResolver {
};
private readStringPool: string[] = [];
- private writeStringIndex: string[] = [];
+ private writeStringCount = 0;
+ private writeStringIndex: number[] = [];
private initInternalSerializer(fury: Fury) {
const _anySerializer = anySerializer(fury);
@@ -77,7 +79,7 @@ export default class SerializerResolver {
reset() {
this.readStringPool = [];
-this.writeStringIndex = [];
+this.writeStringIndex.fill(-1);
}
getSerializerById(id: InternalSerializerType) {
@@ -97,30 +99,40 @@ export default class SerializerResolver {
return this.customSerializer[tag];
}
- tagToBuffer(tag: string) {
-const tagBuffer = fromUint8Array(new TextEncoder().encode(tag));
+ createTagWriter(tag: string) {
+this.writeStringIndex.push(-1);
+const idx = this.writeStringIndex.length - 1;
+const tagBuffer = fromString(tag);
+const bufferLen = tagBuffer.byteLength;
+
+const writer = BinaryWriter({});
+
let tagHash = x64hash128(tagBuffer, 47).getBigUint64(0);
if (tagHash === BigInt(0)) {
tagHash = BigInt(1);
}
-const bufferLen = tagBuffer.byteLength;
-const writer = BufferWriter({});
+
writer.uint8(USESTRINGVALUE);
writer.uint64(tagHash);
writer.int16(bufferLen);
writer.bufferWithoutMemCheck(tagBuffer, bufferLen);
-return writer.dump();
- }
- writeTag(binaryWriter: BinaryWriter, tag: string, bf: PlatformBuffer,
byteLength: number) {
-const index = this.writeStringIndex.indexOf(tag);
-if (index > -1) {
- binaryWriter.uint8(USESTRINGID);
- binaryWriter.int16(index);
- return;
-}
-this.writeStringIndex.push(tag);
-binaryWriter.bufferWithoutMemCheck(bf, byteLength);
+const fullBuffer = writer.dump();
+
+return {
+ write: (binaryWriter: TBinaryWriter) => {
+const tagIndex = this.writeStringIndex[idx];
+if (tagIndex > -1) {
+ // equivalent of: `uint8(USESTRINGID); int16(tagIndex)`
+ binaryWriter.int24((tagIndex << 8) | USESTRINGID);
+ return;
+}
+
+this.writeStringIndex[idx] = this.writeStringCount++;
+binaryWriter.buffer(fullBuffer);
+ },
+ bufferLen,
+};
}
detectTag(binaryReader: BinaryReader) {
diff --git a/javascript/packages/fury/lib/codeGen.ts
b/javascript/packages/fury/lib/codeGen.ts
index 8ff1ae43..da3ab1e6 100644
--- a/javascript/packages/fury/lib/codeGen.ts
+++ b/javascript/packages/fury/lib/codeGen.ts
@@ -24,6 +24,7 @@ import setSerializer from "./internalSerializer/set";
import { arraySerializer } from "./internalSerializer/array";
import
(incubator-fury-site) branch main updated: Add more links and refine words in footer (#95)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury-site.git
The following commit(s) were added to refs/heads/main by this push:
new 155fc9a Add more links and refine words in footer (#95)
155fc9a is described below
commit 155fc9a7b9fe8ee7275307fa583e3b3575cf3eea
Author: Twice
AuthorDate: Wed Dec 27 22:28:42 2023 +0900
Add more links and refine words in footer (#95)
---
docusaurus.config.ts | 53 ++--
src/pages/index.tsx | 4 ++--
2 files changed, 41 insertions(+), 16 deletions(-)
diff --git a/docusaurus.config.ts b/docusaurus.config.ts
index c2632af..c97954d 100644
--- a/docusaurus.config.ts
+++ b/docusaurus.config.ts
@@ -13,11 +13,6 @@ const config: Config = {
// For GitHub pages deployment, it is often '//'
baseUrl: '/',
- // GitHub pages deployment config.
- // If you aren't using GitHub pages, you don't need these.
- organizationName: 'Apache', // Usually your GitHub org/user name.
- projectName: 'fury', // Usually your repo name.
-
onBrokenLinks: 'throw',
onBrokenMarkdownLinks: 'warn',
@@ -95,6 +90,14 @@ const config: Config = {
{
title: 'Community',
items: [
+{
+ label: 'Mailing list',
+ href: 'https://lists.apache.org/list.html?d...@fury.apache.org',
+},
+{
+ label: 'Slack',
+ href:
'https://join.slack.com/t/fury-project/shared_invite/zt-1u8soj4qc-ieYEu7ciHOqA2mo47llS8A',
+},
{
label: 'Twitter',
href: 'https://twitter.com/fury_community',
@@ -102,27 +105,49 @@ const config: Config = {
],
},
{
- title: 'Help',
+ title: 'Docs',
items: [
{
- label: 'Blog',
- to: '/blog',
+ label: 'Install',
+ to: '/docs/start/install',
+},
+{
+ label: 'Usage',
+ to: '/docs/start/usage',
+},
+{
+ label: 'Benchmark',
+ to: '/docs/introduction/benchmark',
},
+ ],
+},
+{
+ title: 'Repositories',
+ items: [
{
- label: 'GitHub',
+ label: 'Fury',
href: 'https://github.com/apache/incubator-fury',
},
+{
+ label: 'Website',
+ href: 'https://github.com/apache/incubator-fury-site',
+},
],
},
],
logo: {
-width: 256,
+width: 320,
src: "/img/apache-incubator.svg",
+href: "https://incubator.apache.org/;,
+alt: "Apache Incubator logo"
},
- copyright: `Apache Fury is an effort undergoing incubation at The Apache
Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is
required of all newly accepted projects until a further review indicates that
the infrastructure, communications, and decision making process have stabilized
in a manner consistent with other successful ASF projects. While incubation
status is not necessarily a reflection of the completeness or stability of the
code, it does indicate t [...]
-
- Copyright © ${new Date().getFullYear()} The Apache Software Foundation,
Licensed under the Apache License, Version 2.0.
- Apache, the names of Apache projects, and the feather logo are either
registered trademarks or trademarks of the Apache Software Foundation in the
United States and/or other countries.`,
+ copyright: `
+ Apache Fury is an effort undergoing incubation at The Apache
Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is
required of all newly accepted projects until a further review indicates that
the infrastructure, communications, and decision making process have stabilized
in a manner consistent with other successful ASF projects. While incubation
status is not necessarily a reflection of the completeness or stability of the
code, it does indicate that the [...]
+
+Copyright © ${new Date().getFullYear()} The Apache Software
Foundation, Licensed under the Apache License, Version 2.0.
+Apache, the names of Apache projects, and the feather logo are either
registered trademarks or trademarks of the Apache Software Foundation in the
United States and/or other countries.
+
+ `,
},
prism: {
theme: prismThemes.github,
diff --git a/src/pages/index.tsx b/src/pages/index.tsx
index 921e43c..a83f60b 100644
--- a/src/pages/index.tsx
+++ b/src/pages/index.tsx
@@ -45,8 +45,8 @@ export default function Home(): JSX.Element {
return (
<>
+title={`${siteConfig.title}`}
+de
(incubator-fury) branch main updated: [DOC] doc sync stay consistent with the site (#1266)
This is an automated email from the ASF dual-hosted git repository. wangweipeng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-fury.git The following commit(s) were added to refs/heads/main by this push: new ae1fe200 [DOC] doc sync stay consistent with the site (#1266) ae1fe200 is described below commit ae1fe200f7eacb00b78fc15f0510694ce0ce7ad8 Author: weipeng AuthorDate: Wed Dec 27 16:45:11 2023 +0800 [DOC] doc sync stay consistent with the site (#1266) The structure and frontmatter of the site has some changes, so it is necessary to ensure that the results synced over remain consistent Co-authored-by: theweipeng --- .github/sync.yml | 2 +- docs/guide/DEVELOPMENT.md | 3 ++- docs/guide/graalvm_guide.md| 5 +++-- docs/guide/java_object_graph_guide.md | 3 ++- docs/guide/row_format_guide.md | 3 ++- docs/guide/scala_guide.md | 3 ++- docs/guide/xlang_object_graph_guide.md | 3 ++- 7 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/sync.yml b/.github/sync.yml index b3339b06..bed00023 100644 --- a/.github/sync.yml +++ b/.github/sync.yml @@ -19,4 +19,4 @@ apache/incubator-fury-site@main: - source: docs/guide/ dest: docs/guide/ - source: docs/benchmarks/ -dest: public/benchmarks/ +dest: static/img/benchmarks/ diff --git a/docs/guide/DEVELOPMENT.md b/docs/guide/DEVELOPMENT.md index 55bee4bf..836f5db8 100644 --- a/docs/guide/DEVELOPMENT.md +++ b/docs/guide/DEVELOPMENT.md @@ -1,6 +1,7 @@ # How to build to Fury diff --git a/docs/guide/graalvm_guide.md b/docs/guide/graalvm_guide.md index 7f4848d0..8327b257 100644 --- a/docs/guide/graalvm_guide.md +++ b/docs/guide/graalvm_guide.md @@ -1,6 +1,7 @@ # GraalVM Native Image @@ -130,7 +131,7 @@ When Fury compression is enabled: - Struct: Fury is `24x speed, 31% size` compared to JDK. - Pojo: Fury is `12x speed, 48% size` compared to JDK. -See [[Benchmark.java](../../integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/Benchmark.java)] for benchmark code. +See [[Benchmark.java](https://github.com/apache/incubator-fury/blob/main/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/Benchmark.java)] for benchmark code. ### Struct Benchmark Class Fields diff --git a/docs/guide/java_object_graph_guide.md b/docs/guide/java_object_graph_guide.md index bde9f467..fbcf918b 100644 --- a/docs/guide/java_object_graph_guide.md +++ b/docs/guide/java_object_graph_guide.md @@ -1,6 +1,7 @@ # Java object graph serialization diff --git a/docs/guide/row_format_guide.md b/docs/guide/row_format_guide.md index 4b9ec323..6e231ff7 100644 --- a/docs/guide/row_format_guide.md +++ b/docs/guide/row_format_guide.md @@ -1,6 +1,7 @@ ## Row format protocol diff --git a/docs/guide/scala_guide.md b/docs/guide/scala_guide.md index ae1aa2f4..e08709ff 100644 --- a/docs/guide/scala_guide.md +++ b/docs/guide/scala_guide.md @@ -1,6 +1,7 @@ # Scala serialization diff --git a/docs/guide/xlang_object_graph_guide.md b/docs/guide/xlang_object_graph_guide.md index c2dd9945..bd2789e8 100644 --- a/docs/guide/xlang_object_graph_guide.md +++ b/docs/guide/xlang_object_graph_guide.md @@ -1,6 +1,7 @@ ## Cross-language object graph serialization - To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
(incubator-fury) branch main updated: chore: reflect PPMC members set up (#1262)
This is an automated email from the ASF dual-hosted git repository.
wangweipeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 890f4314 chore: reflect PPMC members set up (#1262)
890f4314 is described below
commit 890f4314f34106b1dae792cf2a0adbc228b00a97
Author: tison
AuthorDate: Wed Dec 27 08:47:10 2023 +0800
chore: reflect PPMC members set up (#1262)
cc @chaokunyang @wangweipeng2 waiting for your approval to check the
permission.
---
.asf.yaml | 5 -
1 file changed, 5 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
index 11b64c0c..e9940948 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -61,11 +61,6 @@ github:
releases-0.2.1: {}
releases-0.2.0: {}
releases-0.1.2: {}
-
- # TODO: remove them after their apache ids are set up
- collaborators:
-- chaokunyang
-- wangweipeng2
notifications:
commits: commits@fury.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
