This is an automated email from the ASF dual-hosted git repository.
biyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d60110eb51 [core] Add lance table type for rest catalog (#5977)
d60110eb51 is described below
commit d60110eb51b54f0a574f3275a5e47900e44ee87c
Author: YeJunHao <[email protected]>
AuthorDate: Wed Jul 30 11:40:19 2025 +0800
[core] Add lance table type for rest catalog (#5977)
---
.../shortcodes/generated/core_configuration.html | 2 +-
.../src/main/java/org/apache/paimon/TableType.java | 3 +-
.../org/apache/paimon/catalog/CatalogUtils.java | 19 +++
.../org/apache/paimon/table/lance/LanceTable.java | 88 ++++++++++++++
.../apache/paimon/table/lance/LanceTableImpl.java | 130 +++++++++++++++++++++
.../org/apache/paimon/rest/RESTCatalogTest.java | 30 +++++
6 files changed, 270 insertions(+), 2 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index 77237c198e..311914ebb0 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -1198,7 +1198,7 @@ If the data size allocated for the sorting task is
uneven,which may lead to perf
<td><h5>type</h5></td>
<td style="word-wrap: break-word;">table</td>
<td><p>Enum</p></td>
- <td>Type of the table.<br /><br />Possible values:<ul><li>"table":
Normal Paimon table.</li><li>"format-table": A file format table refers to a
directory that contains multiple files of the same
format.</li><li>"materialized-table": A materialized table combines normal
Paimon table and materialized SQL.</li><li>"object-table": An object table
combines normal Paimon table and object location.</li></ul></td>
+ <td>Type of the table.<br /><br />Possible values:<ul><li>"table":
Normal Paimon table.</li><li>"format-table": A file format table refers to a
directory that contains multiple files of the same
format.</li><li>"materialized-table": A materialized table combines normal
Paimon table and materialized SQL.</li><li>"object-table": An object table
combines normal Paimon table and object location.</li><li>"lance-table": A
lance table, see 'https://lancedb.github.io/lance/'.</li></ul></td>
</tr>
<tr>
<td><h5>upsert-key</h5></td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/TableType.java
b/paimon-api/src/main/java/org/apache/paimon/TableType.java
index 5f3f1f0d1f..215070cd02 100644
--- a/paimon-api/src/main/java/org/apache/paimon/TableType.java
+++ b/paimon-api/src/main/java/org/apache/paimon/TableType.java
@@ -33,7 +33,8 @@ public enum TableType implements DescribedEnum {
"materialized-table",
"A materialized table combines normal Paimon table and
materialized SQL."),
OBJECT_TABLE(
- "object-table", "An object table combines normal Paimon table and
object location.");
+ "object-table", "An object table combines normal Paimon table and
object location."),
+ LANCE_TABLE("lance-table", "A lance table, see
'https://lancedb.github.io/lance/'.");
private final String value;
private final String description;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java
b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java
index 766b708663..2b79263438 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java
@@ -33,6 +33,7 @@ import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.FileStoreTableFactory;
import org.apache.paimon.table.FormatTable;
import org.apache.paimon.table.Table;
+import org.apache.paimon.table.lance.LanceTable;
import org.apache.paimon.table.object.ObjectTable;
import org.apache.paimon.table.system.AllTableOptionsTable;
import org.apache.paimon.table.system.CatalogOptionsTable;
@@ -211,6 +212,10 @@ public class CatalogUtils {
return toObjectTable(identifier, schema, dataFileIO);
}
+ if (options.type() == TableType.LANCE_TABLE) {
+ return toLanceTable(identifier, schema, dataFileIO);
+ }
+
CatalogEnvironment catalogEnv =
new CatalogEnvironment(
identifier,
@@ -305,4 +310,18 @@ public class CatalogUtils {
.comment(schema.comment())
.build();
}
+
+ private static LanceTable toLanceTable(
+ Identifier identifier, TableSchema schema, Function<Path, FileIO>
fileIO) {
+ Map<String, String> options = schema.options();
+ String location = options.get(CoreOptions.PATH.key());
+ return LanceTable.builder()
+ .fileIO(fileIO.apply(new Path(location)))
+ .identifier(identifier)
+ .location(location)
+ .rowType(schema.logicalRowType())
+ .options(options)
+ .comment(schema.comment())
+ .build();
+ }
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java
new file mode 100644
index 0000000000..93146a41a8
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.lance;
+
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.types.RowType;
+
+import java.util.Map;
+
+/**
+ * A lance table format table, paimon does not support read and write
operation on this table yet.
+ */
+public interface LanceTable extends Table {
+
+ /** Object location in file system. */
+ String location();
+
+ @Override
+ LanceTable copy(Map<String, String> dynamicOptions);
+
+ /** Create a new builder for {@link LanceTable}. */
+ static Builder builder() {
+ return new Builder();
+ }
+
+ /** Builder for {@link LanceTable}. */
+ class Builder {
+
+ private Identifier identifier;
+ private FileIO fileIO;
+ private RowType rowType;
+ private String location;
+ private Map<String, String> options;
+ private String comment;
+
+ public Builder identifier(Identifier identifier) {
+ this.identifier = identifier;
+ return this;
+ }
+
+ public Builder fileIO(FileIO fileIO) {
+ this.fileIO = fileIO;
+ return this;
+ }
+
+ public Builder rowType(RowType rowType) {
+ this.rowType = rowType;
+ return this;
+ }
+
+ public Builder location(String location) {
+ this.location = location;
+ return this;
+ }
+
+ public Builder options(Map<String, String> options) {
+ this.options = options;
+ return this;
+ }
+
+ public Builder comment(String comment) {
+ this.comment = comment;
+ return this;
+ }
+
+ public LanceTable build() {
+ return new LanceTableImpl(identifier, fileIO, rowType, location,
options, comment);
+ }
+ }
+}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTableImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTableImpl.java
new file mode 100644
index 0000000000..2669ae699a
--- /dev/null
+++
b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTableImpl.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.lance;
+
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.stats.Statistics;
+import org.apache.paimon.table.ReadonlyTable;
+import org.apache.paimon.table.source.InnerTableRead;
+import org.apache.paimon.table.source.InnerTableScan;
+import org.apache.paimon.types.RowType;
+
+import javax.annotation.Nullable;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/** An implementation for {@link LanceTable}. */
+public class LanceTableImpl implements ReadonlyTable, LanceTable {
+
+ private final Identifier identifier;
+ private final FileIO fileIO;
+ private final RowType rowType;
+ private final String location;
+ private final Map<String, String> options;
+ @Nullable private final String comment;
+
+ public LanceTableImpl(
+ Identifier identifier,
+ FileIO fileIO,
+ RowType rowType,
+ String location,
+ Map<String, String> options,
+ @Nullable String comment) {
+ this.identifier = identifier;
+ this.fileIO = fileIO;
+ this.rowType = rowType;
+ this.location = location;
+ this.options = options;
+ this.comment = comment;
+ }
+
+ @Override
+ public String name() {
+ return identifier.getTableName();
+ }
+
+ @Override
+ public String fullName() {
+ return identifier.getFullName();
+ }
+
+ @Override
+ public RowType rowType() {
+ return rowType;
+ }
+
+ @Override
+ public List<String> partitionKeys() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<String> primaryKeys() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Map<String, String> options() {
+ return options;
+ }
+
+ @Override
+ public Optional<String> comment() {
+ return Optional.ofNullable(comment);
+ }
+
+ @Override
+ public Optional<Statistics> statistics() {
+ return ReadonlyTable.super.statistics();
+ }
+
+ @Override
+ public FileIO fileIO() {
+ return fileIO;
+ }
+
+ @Override
+ public String location() {
+ return location;
+ }
+
+ @Override
+ public LanceTable copy(Map<String, String> dynamicOptions) {
+ Map<String, String> newOptions = new HashMap<>(options);
+ newOptions.putAll(dynamicOptions);
+ return new LanceTableImpl(identifier, fileIO, rowType, location,
newOptions, comment);
+ }
+
+ @Override
+ public InnerTableScan newScan() {
+ throw new UnsupportedOperationException(
+ "LanceTable does not support InnerTableScan. Use newRead()
instead.");
+ }
+
+ @Override
+ public InnerTableRead newRead() {
+ throw new UnsupportedOperationException(
+ "LanceTable does not support InnerTableRead. Use newScan()
instead.");
+ }
+}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
index 8755cb74fe..b1b52520ce 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
@@ -2039,6 +2039,36 @@ public abstract class RESTCatalogTest extends
CatalogTestBase {
assertThat(files).containsExactlyInAnyOrder("my_file1", "my_file2",
"dir1/my_file3");
}
+ @Test
+ public void testCreateLanceTable() throws Exception {
+ Catalog catalog = newRestCatalogWithDataToken();
+ catalog.createDatabase("test_db", false);
+ List<String> tables = catalog.listTables("test_db");
+ assertThat(tables).isEmpty();
+
+ Map<String, String> options = new HashMap<>();
+ options.put("type", "lance-table");
+ options.put("a", "b");
+ Schema schema =
+ new Schema(
+ Lists.newArrayList(
+ new DataField(0, "pk", DataTypes.INT()),
+ new DataField(1, "col1", DataTypes.STRING()),
+ new DataField(2, "col2", DataTypes.STRING())),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ options,
+ "");
+ catalog.createTable(Identifier.create("test_db", "table1"), schema,
false);
+
+ tables = catalog.listTables("test_db");
+ Table table = catalog.getTable(Identifier.create("test_db", "table1"));
+ assertThat(table.options()).containsEntry("a", "b");
+ assertThat(table.options().containsKey("path")).isTrue();
+ assertThat(table.fileIO()).isInstanceOf(RESTTokenFileIO.class);
+ assertThat(tables).containsExactlyInAnyOrder("table1");
+ }
+
private TestPagedResponse generateTestPagedResponse(
Map<String, String> queryParams,
List<Integer> testData,
