This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new b8a087a400 [#9792] improvement(lance): Add dataset version handling in
table operations (#9793)
b8a087a400 is described below
commit b8a087a400d44c387128dd4531b408124dd06279
Author: Mini Yu <[email protected]>
AuthorDate: Wed Jan 28 03:05:54 2026 +0800
[#9792] improvement(lance): Add dataset version handling in table
operations (#9793)
### What changes were proposed in this pull request?
This pull request introduces support for tracking and returning the
version of a Lance table throughout its creation and description APIs.
The changes ensure that the table version is stored as a property,
propagated through responses, and verified in integration tests.
### Why are the changes needed?
Make APIs in the Lance REST server according to the docs.
Fix: #9792
### Does this PR introduce _any_ user-facing change?
N/A.
### How was this patch tested?
ITs
---
.../catalog/lakehouse/lance/LanceTableOperations.java | 18 +++++++++++++++++-
.../ops/gravitino/GravitinoLanceTableOperations.java | 11 +++++++++--
.../gravitino/lance/common/utils/LanceConstants.java | 1 +
.../lance/integration/test/LanceRESTServiceIT.java | 2 ++
4 files changed, 29 insertions(+), 3 deletions(-)
diff --git
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
index e5f0519a20..d702843819 100644
---
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
+++
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
@@ -22,6 +22,7 @@ import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_CREAT
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_REGISTER;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import com.lancedb.lance.Dataset;
import com.lancedb.lance.WriteParams;
import com.lancedb.lance.index.DistanceType;
@@ -260,6 +261,8 @@ public class LanceTableOperations extends
ManagedTableOperations {
throws NoSuchSchemaException, TableAlreadyExistsException {
if (register) {
+ // Currently, register operation does not read the schema from the
underlying Lance dataset.
+ // So we can't get the version of the dataset here.
return super.createTable(
ident, columns, comment, properties, partitions, distribution,
sortOrders, indexes);
}
@@ -285,8 +288,21 @@ public class LanceTableOperations extends
ManagedTableOperations {
new
WriteParams.Builder().withStorageOptions(storageProps).build())) {
// Only create the table metadata in Gravitino after the Lance dataset
is successfully
// created.
+ long datasetVersion = ignored.version();
+ Map<String, String> updatedProperties =
+ ImmutableMap.<String, String>builder()
+ .putAll(properties)
+ .put(LanceConstants.LANCE_TABLE_VERSION,
String.valueOf(datasetVersion))
+ .build();
return super.createTable(
- ident, columns, comment, properties, partitions, distribution,
sortOrders, indexes);
+ ident,
+ columns,
+ comment,
+ updatedProperties,
+ partitions,
+ distribution,
+ sortOrders,
+ indexes);
} catch (NoSuchSchemaException e) {
throw e;
} catch (TableAlreadyExistsException e) {
diff --git
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
index f94ffdcdbd..66dae0e017 100644
---
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
+++
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
@@ -24,6 +24,7 @@ import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_CREAT
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_LOCATION;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_CREATE_EMPTY;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_FORMAT;
+import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_VERSION;
import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET;
import com.google.common.base.Preconditions;
@@ -92,7 +93,10 @@ public class GravitinoLanceTableOperations implements
LanceTableOperations {
response.setProperties(table.properties());
response.setLocation(table.properties().get(LANCE_LOCATION));
response.setSchema(toJsonArrowSchema(table.columns()));
- response.setVersion(null);
+ response.setVersion(
+ Optional.ofNullable(table.properties().get(LANCE_TABLE_VERSION))
+ .map(Long::valueOf)
+ .orElse(null));
response.setStorageOptions(LancePropertiesUtils.getLanceStorageOptions(table.properties()));
return response;
}
@@ -147,7 +151,10 @@ public class GravitinoLanceTableOperations implements
LanceTableOperations {
// Extract storage options from table properties. All storage options
stores in table
// properties.
response.setStorageOptions(LancePropertiesUtils.getLanceStorageOptions(t.properties()));
- response.setVersion(null);
+ response.setVersion(
+ Optional.ofNullable(t.properties().get(LANCE_TABLE_VERSION))
+ .map(Long::valueOf)
+ .orElse(null));
response.setLocation(t.properties().get(LANCE_LOCATION));
response.setProperties(t.properties());
return response;
diff --git
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
index 3dd5e6fd60..991c6b8f26 100644
---
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
+++
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
@@ -37,6 +37,7 @@ public class LanceConstants {
public static final String LANCE_TABLE_REGISTER = "lance.register";
+ public static final String LANCE_TABLE_VERSION = "lance.version";
// Mark whether it is to create an empty Lance table(no data files)
public static final String LANCE_TABLE_CREATE_EMPTY = "lance.create-empty";
diff --git
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
index 2060d93dfd..51ffb0cfb8 100644
---
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
+++
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
@@ -500,12 +500,14 @@ public class LanceRESTServiceIT extends BaseIT {
Assertions.assertEquals("v1", response.getProperties().get("key1"));
Assertions.assertEquals("value_a", response.getStorageOptions().get("a"));
Assertions.assertEquals("value_b", response.getStorageOptions().get("b"));
+ Assertions.assertNotNull(response.getVersion());
DescribeTableRequest describeTableRequest = new DescribeTableRequest();
describeTableRequest.setId(ids);
DescribeTableResponse loadTable = ns.describeTable(describeTableRequest);
Assertions.assertNotNull(loadTable);
Assertions.assertEquals(location, loadTable.getLocation());
+ Assertions.assertNotNull(loadTable.getVersion());
List<JsonArrowField> jsonArrowFields = loadTable.getSchema().getFields();
for (int i = 0; i < jsonArrowFields.size(); i++) {
