This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new fc5725a2a feat(binding/nodejs): add TimeoutLayer, LoggingLayer and
ThrottleLayer in nodejs binding (#6772)
fc5725a2a is described below
commit fc5725a2a007fbf8e322d05596e72f5a787c7ce4
Author: Kilerd Chan <[email protected]>
AuthorDate: Fri Nov 14 18:03:56 2025 +0800
feat(binding/nodejs): add TimeoutLayer, LoggingLayer and ThrottleLayer in
nodejs binding (#6772)
* feat: add TimeoutLayer, LoggingLayer and ThrottleLayer in nodejs binding
* refactor: remove redundant param check in binding
---
bindings/nodejs/Cargo.toml | 1 +
bindings/nodejs/README.md | 155 ++++++++++++++++++++
bindings/nodejs/generated.d.ts | 136 ++++++++++++++++++
bindings/nodejs/generated.js | 3 +
bindings/nodejs/src/layer.rs | 207 +++++++++++++++++++++++++++
bindings/nodejs/tests/suites/layer.suite.mjs | 59 +++++++-
6 files changed, 560 insertions(+), 1 deletion(-)
diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml
index bdce8f7a2..f35f9f0f0 100644
--- a/bindings/nodejs/Cargo.toml
+++ b/bindings/nodejs/Cargo.toml
@@ -161,6 +161,7 @@ napi-derive = "3.2.2"
# this crate won't be published, we always use the local version
opendal = { version = ">=0", path = "../../core", features = [
"blocking",
+ "layers-throttle",
] }
tokio = "1"
diff --git a/bindings/nodejs/README.md b/bindings/nodejs/README.md
index 2f2545c8c..fcbd5c51e 100644
--- a/bindings/nodejs/README.md
+++ b/bindings/nodejs/README.md
@@ -65,6 +65,161 @@ async function main() {
main();
```
+## Layers
+
+OpenDAL provides layers to add additional functionality to operators. You can
chain multiple layers together to build powerful data access pipelines.
+
+### Available Layers
+
+- **RetryLayer** - Retry failed operations automatically
+- **ConcurrentLimitLayer** - Limit concurrent requests
+- **TimeoutLayer** - Add timeout for operations
+- **LoggingLayer** - Log all operations
+- **ThrottleLayer** - Limit bandwidth usage
+
+### TimeoutLayer
+
+Prevents operations from hanging indefinitely:
+
+```javascript
+import { Operator, TimeoutLayer } from "opendal";
+
+const op = new Operator("fs", { root: "/tmp" });
+
+const timeout = new TimeoutLayer();
+timeout.timeout = 10000; // 10 seconds for non-IO operations (ms)
+timeout.ioTimeout = 3000; // 3 seconds for IO operations (ms)
+op.layer(timeout.build());
+```
+
+**Default values:** timeout: 60s, ioTimeout: 10s
+
+### LoggingLayer
+
+Add structured logging for debugging and monitoring:
+
+```javascript
+import { Operator, LoggingLayer } from "opendal";
+
+const op = new Operator("fs", { root: "/tmp" });
+
+const logging = new LoggingLayer();
+op.layer(logging.build());
+
+// All operations will be logged
+await op.write("test.txt", "Hello World");
+```
+
+Enable logging output:
+
+```bash
+# Show debug logs
+RUST_LOG=debug node app.js
+
+# Show only OpenDAL logs
+RUST_LOG=opendal::services=debug node app.js
+```
+
+### ThrottleLayer
+
+Control bandwidth usage with rate limiting:
+
+```javascript
+import { Operator, ThrottleLayer } from "opendal";
+
+const op = new Operator("s3", {
+ bucket: "my-bucket",
+ region: "us-east-1",
+});
+
+// Limit to 10 KiB/s with 10 MiB burst
+const throttle = new ThrottleLayer(10 * 1024, 10 * 1024 * 1024);
+op.layer(throttle.build());
+```
+
+**Parameters:**
+
+- `bandwidth`: Maximum bytes per second
+- `burst`: Maximum burst size (must be larger than any operation size)
+
+### RetryLayer
+
+Automatically retry temporary failed operations:
+
+```javascript
+import { Operator, RetryLayer } from "opendal";
+
+const op = new Operator("s3", {
+ bucket: "my-bucket",
+ region: "us-east-1",
+});
+
+const retry = new RetryLayer();
+retry.maxTimes = 3;
+retry.jitter = true;
+op.layer(retry.build());
+```
+
+### ConcurrentLimitLayer
+
+Limit concurrent requests to storage services:
+
+```javascript
+import { Operator, ConcurrentLimitLayer } from "opendal";
+
+const op = new Operator("s3", {
+ bucket: "my-bucket",
+ region: "us-east-1",
+});
+
+// Allow max 1024 concurrent operations
+const limit = new ConcurrentLimitLayer(1024);
+limit.httpPermits = 512; // Limit HTTP requests separately
+op.layer(limit.build());
+```
+
+### Combining Multiple Layers
+
+Stack multiple layers for comprehensive control:
+
+```javascript
+import {
+ Operator,
+ LoggingLayer,
+ TimeoutLayer,
+ RetryLayer,
+ ThrottleLayer,
+} from "opendal";
+
+const op = new Operator("s3", {
+ bucket: "my-bucket",
+ region: "us-east-1",
+});
+
+// Layer 1: Logging for observability
+const logging = new LoggingLayer();
+op.layer(logging.build());
+
+// Layer 2: Timeout protection
+const timeout = new TimeoutLayer();
+timeout.timeout = 30000;
+timeout.ioTimeout = 10000;
+op.layer(timeout.build());
+
+// Layer 3: Retry on failures
+const retry = new RetryLayer();
+retry.maxTimes = 3;
+retry.jitter = true;
+op.layer(retry.build());
+
+// Layer 4: Bandwidth throttling
+const throttle = new ThrottleLayer(100 * 1024, 10 * 1024 * 1024);
+op.layer(throttle.build());
+
+// Now the operator has full production-ready protection
+await op.write("data.json", JSON.stringify(data));
+```
+
## Usage with Next.js
Config automatically be bundled by
[Next.js](https://nextjs.org/docs/app/api-reference/config/next-config-js/serverExternalPackages).
diff --git a/bindings/nodejs/generated.d.ts b/bindings/nodejs/generated.d.ts
index 3473095c4..68ba74179 100644
--- a/bindings/nodejs/generated.d.ts
+++ b/bindings/nodejs/generated.d.ts
@@ -293,6 +293,44 @@ export declare class Lister {
next(): Promise<Entry | null>
}
+/**
+ * Logging layer
+ *
+ * Add log for every operation.
+ *
+ * # Logging
+ *
+ * - OpenDAL will log in structural way.
+ * - Every operation will start with a `started` log entry.
+ * - Every operation will finish with the following status:
+ * - `succeeded`: the operation is successful, but might have more to take.
+ * - `finished`: the whole operation is finished.
+ * - `failed`: the operation returns an unexpected error.
+ * - The default log level while expected error happened is `Warn`.
+ * - The default log level while unexpected failure happened is `Error`.
+ *
+ * # Examples
+ *
+ * ```javascript
+ * const op = new Operator("fs", { root: "/tmp" })
+ *
+ * const logging = new LoggingLayer();
+ * op.layer(logging.build());
+ * ```
+ *
+ * # Output
+ *
+ * To enable logging output, set the `RUST_LOG` environment variable:
+ *
+ * ```shell
+ * RUST_LOG=debug node app.js
+ * ```
+ */
+export declare class LoggingLayer {
+ constructor()
+ build(): ExternalObject<Layer>
+}
+
/** Metadata carries all metadata associated with a path. */
export declare class Metadata {
/** Returns true if the <op.stat> object describes a file system directory.
*/
@@ -876,6 +914,104 @@ export declare class RetryLayer {
build(): ExternalObject<Layer>
}
+/**
+ * Throttle layer
+ *
+ * Add a bandwidth rate limiter to the underlying services.
+ *
+ * # Throttle
+ *
+ * There are several algorithms when it come to rate limiting techniques.
+ * This throttle layer uses Generic Cell Rate Algorithm (GCRA) provided by
Governor.
+ * By setting the `bandwidth` and `burst`, we can control the byte flow rate
of underlying services.
+ *
+ * # Note
+ *
+ * When setting the ThrottleLayer, always consider the largest possible
operation size as the burst size,
+ * as **the burst size should be larger than any possible byte length to allow
it to pass through**.
+ *
+ * # Examples
+ *
+ * This example limits bandwidth to 10 KiB/s and burst size to 10 MiB.
+ *
+ * ```javascript
+ * const op = new Operator("fs", { root: "/tmp" })
+ *
+ * const throttle = new ThrottleLayer(10 * 1024, 10000 * 1024);
+ * op.layer(throttle.build());
+ * ```
+ */
+export declare class ThrottleLayer {
+ /**
+ * Create a new `ThrottleLayer` with given bandwidth and burst.
+ *
+ * # Arguments
+ *
+ * - `bandwidth`: the maximum number of bytes allowed to pass through per
second.
+ * - `burst`: the maximum number of bytes allowed to pass through at once.
+ *
+ * # Notes
+ *
+ * Validation (bandwidth and burst must be greater than 0) is handled by the
Rust core layer.
+ */
+ constructor(bandwidth: number, burst: number)
+ build(): ExternalObject<Layer>
+}
+
+/**
+ * Timeout layer
+ *
+ * Add timeout for every operation to avoid slow or unexpected hang operations.
+ *
+ * # Notes
+ *
+ * `TimeoutLayer` treats all operations in two kinds:
+ *
+ * - Non IO Operation like `stat`, `delete` they operate on a single file. We
control
+ * them by setting `timeout`.
+ * - IO Operation like `read`, `Reader::read` and `Writer::write`, they
operate on data directly, we
+ * control them by setting `io_timeout`.
+ *
+ * # Default
+ *
+ * - timeout: 60 seconds
+ * - io_timeout: 10 seconds
+ *
+ * # Examples
+ *
+ * ```javascript
+ * const op = new Operator("fs", { root: "/tmp" })
+ *
+ * const timeout = new TimeoutLayer();
+ * timeout.timeout = 10000; // 10 seconds for non-IO ops (in milliseconds)
+ * timeout.ioTimeout = 3000; // 3 seconds for IO ops (in milliseconds)
+ *
+ * op.layer(timeout.build());
+ * ```
+ */
+export declare class TimeoutLayer {
+ constructor()
+ /**
+ * Set timeout for non-IO operations (stat, delete, etc.)
+ *
+ * # Notes
+ *
+ * - The unit is millisecond.
+ * - Default is 60000ms (60 seconds).
+ */
+ set timeout(v: number)
+ /**
+ * Set timeout for IO operations (read, write, etc.)
+ *
+ * # Notes
+ *
+ * - The unit is millisecond.
+ * - Default is 10000ms (10 seconds).
+ */
+ set ioTimeout(v: number)
+ build(): ExternalObject<Layer>
+}
+
/**
* Writer is designed to write data into a given path in an asynchronous
* manner.
diff --git a/bindings/nodejs/generated.js b/bindings/nodejs/generated.js
index 5f1b7e986..68f81d9ff 100644
--- a/bindings/nodejs/generated.js
+++ b/bindings/nodejs/generated.js
@@ -535,9 +535,12 @@ module.exports.ConcurrentLimitLayer =
nativeBinding.ConcurrentLimitLayer
module.exports.Entry = nativeBinding.Entry
module.exports.Layer = nativeBinding.Layer
module.exports.Lister = nativeBinding.Lister
+module.exports.LoggingLayer = nativeBinding.LoggingLayer
module.exports.Metadata = nativeBinding.Metadata
module.exports.Operator = nativeBinding.Operator
module.exports.Reader = nativeBinding.Reader
module.exports.RetryLayer = nativeBinding.RetryLayer
+module.exports.ThrottleLayer = nativeBinding.ThrottleLayer
+module.exports.TimeoutLayer = nativeBinding.TimeoutLayer
module.exports.Writer = nativeBinding.Writer
module.exports.EntryMode = nativeBinding.EntryMode
diff --git a/bindings/nodejs/src/layer.rs b/bindings/nodejs/src/layer.rs
index 80b2c65d7..ff6e9dd2b 100644
--- a/bindings/nodejs/src/layer.rs
+++ b/bindings/nodejs/src/layer.rs
@@ -233,3 +233,210 @@ impl ConcurrentLimitLayer {
External::new(Layer { inner: Box::new(l) })
}
}
+
+impl NodeLayer for opendal::layers::TimeoutLayer {
+ fn layer(&self, op: opendal::Operator) -> opendal::Operator {
+ op.layer(self.clone())
+ }
+}
+
+/// Timeout layer
+///
+/// Add timeout for every operation to avoid slow or unexpected hang
operations.
+///
+/// # Notes
+///
+/// `TimeoutLayer` treats all operations in two kinds:
+///
+/// - Non IO Operation like `stat`, `delete` they operate on a single file. We
control
+/// them by setting `timeout`.
+/// - IO Operation like `read`, `Reader::read` and `Writer::write`, they
operate on data directly, we
+/// control them by setting `io_timeout`.
+///
+/// # Default
+///
+/// - timeout: 60 seconds
+/// - io_timeout: 10 seconds
+///
+/// # Examples
+///
+/// ```javascript
+/// const op = new Operator("fs", { root: "/tmp" })
+///
+/// const timeout = new TimeoutLayer();
+/// timeout.timeout = 10000; // 10 seconds for non-IO ops (in
milliseconds)
+/// timeout.ioTimeout = 3000; // 3 seconds for IO ops (in milliseconds)
+///
+/// op.layer(timeout.build());
+/// ```
+#[derive(Default)]
+#[napi]
+pub struct TimeoutLayer {
+ timeout: Option<f64>,
+ io_timeout: Option<f64>,
+}
+
+#[napi]
+impl TimeoutLayer {
+ #[napi(constructor)]
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Set timeout for non-IO operations (stat, delete, etc.)
+ ///
+ /// # Notes
+ ///
+ /// - The unit is millisecond.
+ /// - Default is 60000ms (60 seconds).
+ #[napi(setter)]
+ pub fn timeout(&mut self, v: f64) {
+ self.timeout = Some(v);
+ }
+
+ /// Set timeout for IO operations (read, write, etc.)
+ ///
+ /// # Notes
+ ///
+ /// - The unit is millisecond.
+ /// - Default is 10000ms (10 seconds).
+ #[napi(setter)]
+ pub fn io_timeout(&mut self, v: f64) {
+ self.io_timeout = Some(v);
+ }
+
+ #[napi]
+ pub fn build(&self) -> External<Layer> {
+ let mut l = opendal::layers::TimeoutLayer::default();
+
+ if let Some(timeout) = self.timeout {
+ l = l.with_timeout(Duration::from_millis(timeout as u64));
+ }
+ if let Some(io_timeout) = self.io_timeout {
+ l = l.with_io_timeout(Duration::from_millis(io_timeout as u64));
+ }
+
+ External::new(Layer { inner: Box::new(l) })
+ }
+}
+
+impl NodeLayer for opendal::layers::LoggingLayer {
+ fn layer(&self, op: opendal::Operator) -> opendal::Operator {
+ op.layer(opendal::layers::LoggingLayer::default())
+ }
+}
+
+/// Logging layer
+///
+/// Add log for every operation.
+///
+/// # Logging
+///
+/// - OpenDAL will log in structural way.
+/// - Every operation will start with a `started` log entry.
+/// - Every operation will finish with the following status:
+/// - `succeeded`: the operation is successful, but might have more to take.
+/// - `finished`: the whole operation is finished.
+/// - `failed`: the operation returns an unexpected error.
+/// - The default log level while expected error happened is `Warn`.
+/// - The default log level while unexpected failure happened is `Error`.
+///
+/// # Examples
+///
+/// ```javascript
+/// const op = new Operator("fs", { root: "/tmp" })
+///
+/// const logging = new LoggingLayer();
+/// op.layer(logging.build());
+/// ```
+///
+/// # Output
+///
+/// To enable logging output, set the `RUST_LOG` environment variable:
+///
+/// ```shell
+/// RUST_LOG=debug node app.js
+/// ```
+#[napi]
+pub struct LoggingLayer;
+
+impl Default for LoggingLayer {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+#[napi]
+impl LoggingLayer {
+ #[napi(constructor)]
+ pub fn new() -> Self {
+ Self
+ }
+
+ #[napi]
+ pub fn build(&self) -> External<Layer> {
+ let l = opendal::layers::LoggingLayer::default();
+ External::new(Layer { inner: Box::new(l) })
+ }
+}
+
+impl NodeLayer for opendal::layers::ThrottleLayer {
+ fn layer(&self, op: opendal::Operator) -> opendal::Operator {
+ op.layer(self.clone())
+ }
+}
+
+/// Throttle layer
+///
+/// Add a bandwidth rate limiter to the underlying services.
+///
+/// # Throttle
+///
+/// There are several algorithms when it come to rate limiting techniques.
+/// This throttle layer uses Generic Cell Rate Algorithm (GCRA) provided by
Governor.
+/// By setting the `bandwidth` and `burst`, we can control the byte flow rate
of underlying services.
+///
+/// # Note
+///
+/// When setting the ThrottleLayer, always consider the largest possible
operation size as the burst size,
+/// as **the burst size should be larger than any possible byte length to
allow it to pass through**.
+///
+/// # Examples
+///
+/// This example limits bandwidth to 10 KiB/s and burst size to 10 MiB.
+///
+/// ```javascript
+/// const op = new Operator("fs", { root: "/tmp" })
+///
+/// const throttle = new ThrottleLayer(10 * 1024, 10000 * 1024);
+/// op.layer(throttle.build());
+/// ```
+#[napi]
+pub struct ThrottleLayer {
+ bandwidth: u32,
+ burst: u32,
+}
+
+#[napi]
+impl ThrottleLayer {
+ /// Create a new `ThrottleLayer` with given bandwidth and burst.
+ ///
+ /// # Arguments
+ ///
+ /// - `bandwidth`: the maximum number of bytes allowed to pass through per
second.
+ /// - `burst`: the maximum number of bytes allowed to pass through at once.
+ ///
+ /// # Notes
+ ///
+ /// Validation (bandwidth and burst must be greater than 0) is handled by
the Rust core layer.
+ #[napi(constructor)]
+ pub fn new(bandwidth: u32, burst: u32) -> Self {
+ Self { bandwidth, burst }
+ }
+
+ #[napi]
+ pub fn build(&self) -> External<Layer> {
+ let l = opendal::layers::ThrottleLayer::new(self.bandwidth,
self.burst);
+ External::new(Layer { inner: Box::new(l) })
+ }
+}
diff --git a/bindings/nodejs/tests/suites/layer.suite.mjs
b/bindings/nodejs/tests/suites/layer.suite.mjs
index dc9f0307c..9eb6c68b9 100644
--- a/bindings/nodejs/tests/suites/layer.suite.mjs
+++ b/bindings/nodejs/tests/suites/layer.suite.mjs
@@ -19,7 +19,7 @@
import { test, assert } from 'vitest'
-import { RetryLayer, ConcurrentLimitLayer } from '../../index.mjs'
+import { RetryLayer, ConcurrentLimitLayer, TimeoutLayer, LoggingLayer,
ThrottleLayer } from '../../index.mjs'
/**
* @param {import("../../index").Operator} op
@@ -52,4 +52,61 @@ export function run(op) {
assert.ok(layerOp)
assert.ok(layerOp.capability())
})
+
+ test('test operator with timeout layer', () => {
+ const timeoutLayer = new TimeoutLayer()
+ timeoutLayer.timeout = 10000
+ timeoutLayer.ioTimeout = 5000
+
+ const layerOp = op.layer(timeoutLayer.build())
+
+ assert.ok(layerOp)
+ assert.ok(layerOp.capability())
+ })
+
+ test('test operator with timeout layer using default values', () => {
+ const timeoutLayer = new TimeoutLayer()
+
+ const layerOp = op.layer(timeoutLayer.build())
+
+ assert.ok(layerOp)
+ assert.ok(layerOp.capability())
+ })
+
+ test('test operator with logging layer', () => {
+ const loggingLayer = new LoggingLayer()
+
+ const layerOp = op.layer(loggingLayer.build())
+
+ assert.ok(layerOp)
+ assert.ok(layerOp.capability())
+ })
+
+ test('test operator with throttle layer', () => {
+ const throttleLayer = new ThrottleLayer(10 * 1024, 1024 * 1024)
+
+ const layerOp = op.layer(throttleLayer.build())
+
+ assert.ok(layerOp)
+ assert.ok(layerOp.capability())
+ })
+
+ test('test operator with multiple layers', () => {
+ const loggingLayer = new LoggingLayer()
+ const timeoutLayer = new TimeoutLayer()
+ timeoutLayer.timeout = 30000
+ timeoutLayer.ioTimeout = 10000
+ const retryLayer = new RetryLayer()
+ retryLayer.maxTimes = 3
+ const throttleLayer = new ThrottleLayer(100 * 1024, 10 * 1024 * 1024)
+
+ const layerOp = op
+ .layer(loggingLayer.build())
+ .layer(timeoutLayer.build())
+ .layer(retryLayer.build())
+ .layer(throttleLayer.build())
+
+ assert.ok(layerOp)
+ assert.ok(layerOp.capability())
+ })
}
