This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new b8455654a fix(binding/nodejs): add missing lister methods (#6769)
b8455654a is described below
commit b8455654a45746cd3491656a3c9375d26b0f78c0
Author: Kilerd Chan <[email protected]>
AuthorDate: Mon Nov 10 23:27:41 2025 +0800
fix(binding/nodejs): add missing lister methods (#6769)
fix(nodejs): add missing lister methods
---
bindings/nodejs/generated.d.ts | 62 +++++++
bindings/nodejs/src/lib.rs | 87 +++++++++
bindings/nodejs/tests/suites/asyncLister.suite.mjs | 203 +++++++++++++++++++++
bindings/nodejs/tests/suites/index.mjs | 4 +
bindings/nodejs/tests/suites/syncLister.suite.mjs | 203 +++++++++++++++++++++
5 files changed, 559 insertions(+)
diff --git a/bindings/nodejs/generated.d.ts b/bindings/nodejs/generated.d.ts
index c2219d529..3473095c4 100644
--- a/bindings/nodejs/generated.d.ts
+++ b/bindings/nodejs/generated.d.ts
@@ -676,6 +676,68 @@ export declare class Operator {
* ```
*/
listSync(path: string, options?: ListOptions | undefined | null):
Array<Entry>
+ /**
+ * Create a lister to list entries at given path.
+ *
+ * This function returns a Lister that can be used to iterate over entries
+ * in a streaming manner, which is more memory-efficient for large
directories.
+ *
+ * An error will be returned if given path doesn't end with `/`.
+ *
+ * ### Example
+ *
+ * ```javascript
+ * const lister = await op.lister("path/to/dir/");
+ * let entry;
+ * while ((entry = await lister.next()) !== null) {
+ * console.log(entry.path());
+ * }
+ * ```
+ *
+ * #### List recursively
+ *
+ * With `recursive` option, you can list recursively.
+ *
+ * ```javascript
+ * const lister = await op.lister("path/to/dir/", { recursive: true });
+ * let entry;
+ * while ((entry = await lister.next()) !== null) {
+ * console.log(entry.path());
+ * }
+ * ```
+ */
+ lister(path: string, options?: ListOptions | undefined | null):
Promise<Lister>
+ /**
+ * Create a lister to list entries at given path synchronously.
+ *
+ * This function returns a BlockingLister that can be used to iterate over
entries
+ * in a streaming manner, which is more memory-efficient for large
directories.
+ *
+ * An error will be returned if given path doesn't end with `/`.
+ *
+ * ### Example
+ *
+ * ```javascript
+ * const lister = op.listerSync("path/to/dir/");
+ * let entry;
+ * while ((entry = lister.next()) !== null) {
+ * console.log(entry.path());
+ * }
+ * ```
+ *
+ * #### List recursively
+ *
+ * With `recursive` option, you can list recursively.
+ *
+ * ```javascript
+ * const lister = op.listerSync("path/to/dir/", { recursive: true });
+ * let entry;
+ * while ((entry = lister.next()) !== null) {
+ * console.log(entry.path());
+ * }
+ * ```
+ */
+ listerSync(path: string, options?: ListOptions | undefined | null):
BlockingLister
/**
* Get a presigned request for read.
*
diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs
index 932539ea6..de7ca38bc 100644
--- a/bindings/nodejs/src/lib.rs
+++ b/bindings/nodejs/src/lib.rs
@@ -638,6 +638,93 @@ impl Operator {
Ok(l.into_iter().map(Entry).collect())
}
+ /// Create a lister to list entries at given path.
+ ///
+ /// This function returns a Lister that can be used to iterate over entries
+ /// in a streaming manner, which is more memory-efficient for large
directories.
+ ///
+ /// An error will be returned if given path doesn't end with `/`.
+ ///
+ /// ### Example
+ ///
+ /// ```javascript
+ /// const lister = await op.lister("path/to/dir/");
+ /// let entry;
+ /// while ((entry = await lister.next()) !== null) {
+ /// console.log(entry.path());
+ /// }
+ /// ```
+ ///
+ /// #### List recursively
+ ///
+ /// With `recursive` option, you can list recursively.
+ ///
+ /// ```javascript
+ /// const lister = await op.lister("path/to/dir/", { recursive: true });
+ /// let entry;
+ /// while ((entry = await lister.next()) !== null) {
+ /// console.log(entry.path());
+ /// }
+ /// ```
+ #[napi]
+ pub async fn lister(
+ &self,
+ path: String,
+ options: Option<options::ListOptions>,
+ ) -> Result<Lister> {
+ let options = options.map_or(ListOptions::default(),
ListOptions::from);
+ let l = self
+ .async_op
+ .lister_options(&path, options)
+ .await
+ .map_err(format_napi_error)?;
+
+ Ok(Lister(l))
+ }
+
+ /// Create a lister to list entries at given path synchronously.
+ ///
+ /// This function returns a BlockingLister that can be used to iterate
over entries
+ /// in a streaming manner, which is more memory-efficient for large
directories.
+ ///
+ /// An error will be returned if given path doesn't end with `/`.
+ ///
+ /// ### Example
+ ///
+ /// ```javascript
+ /// const lister = op.listerSync("path/to/dir/");
+ /// let entry;
+ /// while ((entry = lister.next()) !== null) {
+ /// console.log(entry.path());
+ /// }
+ /// ```
+ ///
+ /// #### List recursively
+ ///
+ /// With `recursive` option, you can list recursively.
+ ///
+ /// ```javascript
+ /// const lister = op.listerSync("path/to/dir/", { recursive: true });
+ /// let entry;
+ /// while ((entry = lister.next()) !== null) {
+ /// console.log(entry.path());
+ /// }
+ /// ```
+ #[napi]
+ pub fn lister_sync(
+ &self,
+ path: String,
+ options: Option<options::ListOptions>,
+ ) -> Result<BlockingLister> {
+ let options = options.map_or(ListOptions::default(),
ListOptions::from);
+ let l = self
+ .blocking_op
+ .lister_options(&path, options)
+ .map_err(format_napi_error)?;
+
+ Ok(BlockingLister(l))
+ }
+
/// Get a presigned request for read.
///
/// Unit of `expires` is seconds.
diff --git a/bindings/nodejs/tests/suites/asyncLister.suite.mjs
b/bindings/nodejs/tests/suites/asyncLister.suite.mjs
new file mode 100644
index 000000000..1b5ab1129
--- /dev/null
+++ b/bindings/nodejs/tests/suites/asyncLister.suite.mjs
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { randomUUID } from 'node:crypto'
+import path from 'node:path'
+import { test, describe, expect } from 'vitest'
+
+/**
+ * @param {import("../../index").Operator} op
+ */
+export function run(op) {
+ const capability = op.capability()
+
+ describe.runIf(capability.write && capability.read &&
capability.list)('async lister tests', () => {
+ test('test basic lister', async () => {
+ const dirname = `random_dir_${randomUUID()}/`
+ await op.createDir(dirname)
+
+ const expected = ['file1', 'file2', 'file3']
+ for (const entry of expected) {
+ await op.write(path.join(dirname, entry), 'test_content')
+ }
+
+ const lister = await op.lister(dirname)
+ const actual = []
+ let entry
+ while ((entry = await lister.next()) !== null) {
+ if (entry.path() !== dirname) {
+ actual.push(entry.path().slice(dirname.length))
+ }
+ }
+
+ expect(actual.sort()).toEqual(expected.sort())
+
+ await op.removeAll(dirname)
+ })
+
+ test('test lister returns same results as list', async () => {
+ const dirname = `random_dir_${randomUUID()}/`
+ await op.createDir(dirname)
+
+ const expected = Array.from({ length: 5 }, (_, i) =>
`file_${i}_${randomUUID()}`)
+ for (const entry of expected) {
+ await op.write(path.join(dirname, entry), 'content')
+ }
+
+ // Get results using list()
+ const listResults = await op.list(dirname)
+ const listPaths = listResults
+ .filter((item) => item.path() !== dirname)
+ .map((item) => item.path())
+ .sort()
+
+ // Get results using lister()
+ const lister = await op.lister(dirname)
+ const listerPaths = []
+ let entry
+ while ((entry = await lister.next()) !== null) {
+ if (entry.path() !== dirname) {
+ listerPaths.push(entry.path())
+ }
+ }
+ listerPaths.sort()
+
+ expect(listerPaths).toEqual(listPaths)
+
+ await op.removeAll(dirname)
+ })
+
+ test.runIf(capability.listWithRecursive)('lister with recursive', async ()
=> {
+ const dirname = `random_dir_${randomUUID()}/`
+ await op.createDir(dirname)
+
+ const expected = ['x/', 'x/y', 'x/x/', 'x/x/y', 'x/x/x/', 'x/x/x/y',
'x/x/x/x/']
+ for (const entry of expected) {
+ if (entry.endsWith('/')) {
+ await op.createDir(path.join(dirname, entry))
+ } else {
+ await op.write(path.join(dirname, entry), 'test_scan')
+ }
+ }
+
+ const lister = await op.lister(dirname, { recursive: true })
+ const actual = []
+ let entry
+ while ((entry = await lister.next()) !== null) {
+ if (entry.metadata().isFile()) {
+ actual.push(entry.path().slice(dirname.length))
+ }
+ }
+
+ expect(actual.length).toEqual(3)
+ expect(actual.sort()).toEqual(['x/x/x/y', 'x/x/y', 'x/y'])
+
+ await op.removeAll(dirname)
+ })
+
+ test.runIf(capability.listWithStartAfter)('lister with start after', async
() => {
+ const dirname = `random_dir_${randomUUID()}/`
+ await op.createDir(dirname)
+
+ const given = Array.from({ length: 6 }, (_, i) => path.join(dirname,
`file_${i}_${randomUUID()}`))
+ for (const entry of given) {
+ await op.write(entry, 'content')
+ }
+
+ const lister = await op.lister(dirname, { startAfter: given[2] })
+ const actual = []
+ let entry
+ while ((entry = await lister.next()) !== null) {
+ if (entry.path() !== dirname) {
+ actual.push(entry.path())
+ }
+ }
+
+ const expected = given.slice(3)
+ expect(actual).toEqual(expected)
+
+ await op.removeAll(dirname)
+ })
+
+ test.runIf(capability.listWithLimit)('lister with limit', async () => {
+ const dirname = `random_dir_${randomUUID()}/`
+ await op.createDir(dirname)
+
+ const given = Array.from({ length: 10 }, (_, i) => path.join(dirname,
`file_${i}_${randomUUID()}`))
+ for (const entry of given) {
+ await op.write(entry, 'data')
+ }
+
+ const lister = await op.lister(dirname, { limit: 5 })
+ const actual = []
+ let entry
+ while ((entry = await lister.next()) !== null) {
+ actual.push(entry.path())
+ }
+
+ // With limit, we should get the paginated results
+ expect(actual.length).toBeGreaterThan(0)
+
+ await op.removeAll(dirname)
+ })
+
+ test('test empty directory lister', async () => {
+ const dirname = `empty_dir_${randomUUID()}/`
+ await op.createDir(dirname)
+
+ const lister = await op.lister(dirname)
+ let entry
+ let count = 0
+ while ((entry = await lister.next()) !== null) {
+ if (entry.path() !== dirname) {
+ count++
+ }
+ }
+
+ expect(count).toEqual(0)
+
+ await op.removeAll(dirname)
+ })
+
+ test('test lister metadata', async () => {
+ const dirname = `random_dir_${randomUUID()}/`
+ await op.createDir(dirname)
+
+ const filename = `file_${randomUUID()}`
+ const content = 'test content for metadata'
+ await op.write(path.join(dirname, filename), content)
+
+ const lister = await op.lister(dirname)
+ let entry
+ let found = false
+ while ((entry = await lister.next()) !== null) {
+ if (entry.path() === path.join(dirname, filename)) {
+ const meta = entry.metadata()
+ expect(meta.isFile()).toBe(true)
+ expect(meta.isDirectory()).toBe(false)
+ found = true
+ }
+ }
+
+ expect(found).toBe(true)
+
+ await op.removeAll(dirname)
+ })
+ })
+}
diff --git a/bindings/nodejs/tests/suites/index.mjs
b/bindings/nodejs/tests/suites/index.mjs
index 791b15c48..36d7de093 100644
--- a/bindings/nodejs/tests/suites/index.mjs
+++ b/bindings/nodejs/tests/suites/index.mjs
@@ -31,6 +31,8 @@ import { run as AsyncReadOptionsTestRun } from
'./asyncReadOptions.suite.mjs'
import { run as SyncReadOptionsTestRun } from './syncReadOptions.suite.mjs'
import { run as AsyncListOptionsTestRun } from './asyncListOptions.suite.mjs'
import { run as SyncListOptionsTestRun } from './syncListOptions.suite.mjs'
+import { run as AsyncListerTestRun } from './asyncLister.suite.mjs'
+import { run as SyncListerTestRun } from './syncLister.suite.mjs'
import { run as AsyncDeleteOptionsTestRun } from
'./asyncDeleteOptions.suite.mjs'
import { run as SyncDeleteOptionsTestRun } from './syncDeleteOptions.suite.mjs'
import { run as AsyncWriteOptionsTestRun } from './asyncWriteOptions.suite.mjs'
@@ -71,6 +73,8 @@ export function runner(testName, scheme) {
SyncReadOptionsTestRun(operator)
AsyncListOptionsTestRun(operator)
SyncListOptionsTestRun(operator)
+ AsyncListerTestRun(operator)
+ SyncListerTestRun(operator)
AsyncDeleteOptionsTestRun(operator)
SyncDeleteOptionsTestRun(operator)
AsyncWriteOptionsTestRun(operator)
diff --git a/bindings/nodejs/tests/suites/syncLister.suite.mjs
b/bindings/nodejs/tests/suites/syncLister.suite.mjs
new file mode 100644
index 000000000..44ee9e6aa
--- /dev/null
+++ b/bindings/nodejs/tests/suites/syncLister.suite.mjs
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { randomUUID } from 'node:crypto'
+import path from 'node:path'
+import { test, describe, expect } from 'vitest'
+
+/**
+ * @param {import("../../index").Operator} op
+ */
+export function run(op) {
+ const capability = op.capability()
+
+ describe.runIf(capability.write && capability.read && capability.list)('sync
lister tests', () => {
+ test('test basic lister', () => {
+ const dirname = `random_dir_${randomUUID()}/`
+ op.createDirSync(dirname)
+
+ const expected = ['file1', 'file2', 'file3']
+ for (const entry of expected) {
+ op.writeSync(path.join(dirname, entry), 'test_content')
+ }
+
+ const lister = op.listerSync(dirname)
+ const actual = []
+ let entry
+ while ((entry = lister.next()) !== null) {
+ if (entry.path() !== dirname) {
+ actual.push(entry.path().slice(dirname.length))
+ }
+ }
+
+ expect(actual.sort()).toEqual(expected.sort())
+
+ op.removeAllSync(dirname)
+ })
+
+ test('test lister returns same results as list', () => {
+ const dirname = `random_dir_${randomUUID()}/`
+ op.createDirSync(dirname)
+
+ const expected = Array.from({ length: 5 }, (_, i) =>
`file_${i}_${randomUUID()}`)
+ for (const entry of expected) {
+ op.writeSync(path.join(dirname, entry), 'content')
+ }
+
+ // Get results using listSync()
+ const listResults = op.listSync(dirname)
+ const listPaths = listResults
+ .filter((item) => item.path() !== dirname)
+ .map((item) => item.path())
+ .sort()
+
+ // Get results using listerSync()
+ const lister = op.listerSync(dirname)
+ const listerPaths = []
+ let entry
+ while ((entry = lister.next()) !== null) {
+ if (entry.path() !== dirname) {
+ listerPaths.push(entry.path())
+ }
+ }
+ listerPaths.sort()
+
+ expect(listerPaths).toEqual(listPaths)
+
+ op.removeAllSync(dirname)
+ })
+
+ test.runIf(capability.listWithRecursive)('lister with recursive', () => {
+ const dirname = `random_dir_${randomUUID()}/`
+ op.createDirSync(dirname)
+
+ const expected = ['x/', 'x/y', 'x/x/', 'x/x/y', 'x/x/x/', 'x/x/x/y',
'x/x/x/x/']
+ for (const entry of expected) {
+ if (entry.endsWith('/')) {
+ op.createDirSync(path.join(dirname, entry))
+ } else {
+ op.writeSync(path.join(dirname, entry), 'test_scan')
+ }
+ }
+
+ const lister = op.listerSync(dirname, { recursive: true })
+ const actual = []
+ let entry
+ while ((entry = lister.next()) !== null) {
+ if (entry.metadata().isFile()) {
+ actual.push(entry.path().slice(dirname.length))
+ }
+ }
+
+ expect(actual.length).toEqual(3)
+ expect(actual.sort()).toEqual(['x/x/x/y', 'x/x/y', 'x/y'])
+
+ op.removeAllSync(dirname)
+ })
+
+ test.runIf(capability.listWithStartAfter)('lister with start after', () =>
{
+ const dirname = `random_dir_${randomUUID()}/`
+ op.createDirSync(dirname)
+
+ const given = Array.from({ length: 6 }, (_, i) => path.join(dirname,
`file_${i}_${randomUUID()}`))
+ for (const entry of given) {
+ op.writeSync(entry, 'content')
+ }
+
+ const lister = op.listerSync(dirname, { startAfter: given[2] })
+ const actual = []
+ let entry
+ while ((entry = lister.next()) !== null) {
+ if (entry.path() !== dirname) {
+ actual.push(entry.path())
+ }
+ }
+
+ const expected = given.slice(3)
+ expect(actual).toEqual(expected)
+
+ op.removeAllSync(dirname)
+ })
+
+ test.runIf(capability.listWithLimit)('lister with limit', () => {
+ const dirname = `random_dir_${randomUUID()}/`
+ op.createDirSync(dirname)
+
+ const given = Array.from({ length: 10 }, (_, i) => path.join(dirname,
`file_${i}_${randomUUID()}`))
+ for (const entry of given) {
+ op.writeSync(entry, 'data')
+ }
+
+ const lister = op.listerSync(dirname, { limit: 5 })
+ const actual = []
+ let entry
+ while ((entry = lister.next()) !== null) {
+ actual.push(entry.path())
+ }
+
+ // With limit, we should get the paginated results
+ expect(actual.length).toBeGreaterThan(0)
+
+ op.removeAllSync(dirname)
+ })
+
+ test('test empty directory lister', () => {
+ const dirname = `empty_dir_${randomUUID()}/`
+ op.createDirSync(dirname)
+
+ const lister = op.listerSync(dirname)
+ let entry
+ let count = 0
+ while ((entry = lister.next()) !== null) {
+ if (entry.path() !== dirname) {
+ count++
+ }
+ }
+
+ expect(count).toEqual(0)
+
+ op.removeAllSync(dirname)
+ })
+
+ test('test lister metadata', () => {
+ const dirname = `random_dir_${randomUUID()}/`
+ op.createDirSync(dirname)
+
+ const filename = `file_${randomUUID()}`
+ const content = 'test content for metadata'
+ op.writeSync(path.join(dirname, filename), content)
+
+ const lister = op.listerSync(dirname)
+ let entry
+ let found = false
+ while ((entry = lister.next()) !== null) {
+ if (entry.path() === path.join(dirname, filename)) {
+ const meta = entry.metadata()
+ expect(meta.isFile()).toBe(true)
+ expect(meta.isDirectory()).toBe(false)
+ found = true
+ }
+ }
+
+ expect(found).toBe(true)
+
+ op.removeAllSync(dirname)
+ })
+ })
+}
