This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 72c25d58c [core] Add Table.uuid method (#4213)
72c25d58c is described below
commit 72c25d58c575d4966fd2cb12fcc883071e393890
Author: Jingsong Lee <[email protected]>
AuthorDate: Wed Nov 13 11:27:07 2024 +0800
[core] Add Table.uuid method (#4213)
---
.../org/apache/paimon/catalog/AbstractCatalog.java | 32 ++++++++++++++++++++--
.../org/apache/paimon/schema/SchemaManager.java | 19 +++++++++++++
.../paimon/table/AbstractFileStoreTable.java | 9 ++++++
.../apache/paimon/table/CatalogEnvironment.java | 10 ++++++-
.../paimon/table/DelegatedFileStoreTable.java | 5 ++++
.../main/java/org/apache/paimon/table/Table.java | 9 ++++++
.../org/apache/paimon/catalog/CatalogTestBase.java | 12 ++++++++
.../java/org/apache/paimon/hive/HiveCatalog.java | 19 +++++++++++--
8 files changed, 108 insertions(+), 7 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
index a1cf941cd..93018f12c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
@@ -424,16 +424,17 @@ public abstract class AbstractCatalog implements Catalog {
protected Table getDataOrFormatTable(Identifier identifier) throws
TableNotExistException {
Preconditions.checkArgument(identifier.getSystemTableName() == null);
- TableSchema tableSchema = getDataTableSchema(identifier);
+ TableMeta tableMeta = getDataTableMeta(identifier);
return FileStoreTableFactory.create(
fileIO,
getTableLocation(identifier),
- tableSchema,
+ tableMeta.schema,
new CatalogEnvironment(
identifier,
+ tableMeta.uuid,
Lock.factory(
lockFactory().orElse(null),
lockContext().orElse(null), identifier),
- metastoreClientFactory(identifier,
tableSchema).orElse(null),
+ metastoreClientFactory(identifier,
tableMeta.schema).orElse(null),
lineageMetaFactory));
}
@@ -475,6 +476,10 @@ public abstract class AbstractCatalog implements Catalog {
}
}
+ protected TableMeta getDataTableMeta(Identifier identifier) throws
TableNotExistException {
+ return new TableMeta(getDataTableSchema(identifier), null);
+ }
+
protected abstract TableSchema getDataTableSchema(Identifier identifier)
throws TableNotExistException;
@@ -627,4 +632,25 @@ public abstract class AbstractCatalog implements Catalog {
}
});
}
+
+ /** Table metadata. */
+ protected static class TableMeta {
+
+ private final TableSchema schema;
+ @Nullable private final String uuid;
+
+ public TableMeta(TableSchema schema, @Nullable String uuid) {
+ this.schema = schema;
+ this.uuid = uuid;
+ }
+
+ public TableSchema schema() {
+ return schema;
+ }
+
+ @Nullable
+ public String uuid() {
+ return uuid;
+ }
+ }
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
index 28cc69cf9..86e365a88 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
@@ -78,6 +78,7 @@ import static
org.apache.paimon.catalog.AbstractCatalog.DB_SUFFIX;
import static org.apache.paimon.catalog.Identifier.UNKNOWN_DATABASE;
import static org.apache.paimon.utils.BranchManager.DEFAULT_MAIN_BRANCH;
import static org.apache.paimon.utils.FileUtils.listVersionedFiles;
+import static org.apache.paimon.utils.Preconditions.checkArgument;
import static org.apache.paimon.utils.Preconditions.checkState;
/** Schema Manager to manage schema versions. */
@@ -123,6 +124,24 @@ public class SchemaManager implements Serializable {
}
}
+ public long earliestCreationTime() {
+ try {
+ long earliest = 0;
+ if (!schemaExists(0)) {
+ Optional<Long> min =
+ listVersionedFiles(fileIO, schemaDirectory(),
SCHEMA_PREFIX)
+ .reduce(Math::min);
+ checkArgument(min.isPresent());
+ earliest = min.get();
+ }
+
+ Path schemaPath = toSchemaPath(earliest);
+ return fileIO.getFileStatus(schemaPath).getModificationTime();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
public List<TableSchema> listAll() {
return
listAllIds().stream().map(this::schema).collect(Collectors.toList());
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
index af0c3d71e..07c0e8864 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
@@ -167,6 +167,15 @@ abstract class AbstractFileStoreTable implements
FileStoreTable {
: identifier;
}
+ @Override
+ public String uuid() {
+ if (catalogEnvironment.uuid() != null) {
+ return catalogEnvironment.uuid();
+ }
+ long earliestCreationTime = schemaManager().earliestCreationTime();
+ return fullName() + "." + earliestCreationTime;
+ }
+
@Override
public Optional<Statistics> statistics() {
Snapshot snapshot = TimeTravelUtil.resolveSnapshot(this);
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java
b/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java
index ebaff1266..9ff5f9b4f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java
@@ -36,23 +36,26 @@ public class CatalogEnvironment implements Serializable {
private static final long serialVersionUID = 1L;
@Nullable private final Identifier identifier;
+ @Nullable private final String uuid;
private final Lock.Factory lockFactory;
@Nullable private final MetastoreClient.Factory metastoreClientFactory;
@Nullable private final LineageMetaFactory lineageMetaFactory;
public CatalogEnvironment(
@Nullable Identifier identifier,
+ @Nullable String uuid,
Lock.Factory lockFactory,
@Nullable MetastoreClient.Factory metastoreClientFactory,
@Nullable LineageMetaFactory lineageMetaFactory) {
this.identifier = identifier;
+ this.uuid = uuid;
this.lockFactory = lockFactory;
this.metastoreClientFactory = metastoreClientFactory;
this.lineageMetaFactory = lineageMetaFactory;
}
public static CatalogEnvironment empty() {
- return new CatalogEnvironment(null, Lock.emptyFactory(), null, null);
+ return new CatalogEnvironment(null, null, Lock.emptyFactory(), null,
null);
}
@Nullable
@@ -60,6 +63,11 @@ public class CatalogEnvironment implements Serializable {
return identifier;
}
+ @Nullable
+ public String uuid() {
+ return uuid;
+ }
+
public Lock.Factory lockFactory() {
return lockFactory;
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java
index f6f3930ba..2b369e500 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java
@@ -72,6 +72,11 @@ public abstract class DelegatedFileStoreTable implements
FileStoreTable {
return wrapped.fullName();
}
+ @Override
+ public String uuid() {
+ return wrapped.uuid();
+ }
+
@Override
public SnapshotReader newSnapshotReader() {
return wrapped.newSnapshotReader();
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/Table.java
b/paimon-core/src/main/java/org/apache/paimon/table/Table.java
index db6848f5f..7ed7ba48a 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/Table.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/Table.java
@@ -52,10 +52,19 @@ public interface Table extends Serializable {
/** A name to identify this table. */
String name();
+ /** Full name of the table, default is database.tableName. */
default String fullName() {
return name();
}
+ /**
+ * UUID of the table, metastore can provide the true UUID of this table,
default is the full
+ * name.
+ */
+ default String uuid() {
+ return fullName();
+ }
+
/** Returns the row type of this table. */
RowType rowType();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
index f130920a7..98a9b92c5 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
@@ -948,4 +948,16 @@ public abstract class CatalogTestBase {
.isInstanceOf(Catalog.TableNotExistException.class);
assertThat(catalog.getTable(newIdentifier)).isInstanceOf(FormatTable.class);
}
+
+ @Test
+ public void testTableUUID() throws Exception {
+ catalog.createDatabase("test_db", false);
+ Identifier identifier = Identifier.create("test_db", "test_table");
+ catalog.createTable(identifier, DEFAULT_TABLE_SCHEMA, false);
+ Table table = catalog.getTable(identifier);
+ String uuid = table.uuid();
+ assertThat(uuid).startsWith(identifier.getFullName() + ".");
+ assertThat(Long.parseLong(uuid.substring((identifier.getFullName() +
".").length())))
+ .isGreaterThan(0);
+ }
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index 2bf16c0f4..8b8b62934 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -411,6 +411,18 @@ public class HiveCatalog extends AbstractCatalog {
}
}
+ @Override
+ protected TableMeta getDataTableMeta(Identifier identifier) throws
TableNotExistException {
+ return getDataTableMeta(identifier, getHmsTable(identifier));
+ }
+
+ private TableMeta getDataTableMeta(Identifier identifier, Table table)
+ throws TableNotExistException {
+ return new TableMeta(
+ getDataTableSchema(identifier, table),
+ identifier.getFullName() + "." + table.getCreateTime());
+ }
+
@Override
public TableSchema getDataTableSchema(Identifier identifier) throws
TableNotExistException {
Table table = getHmsTable(identifier);
@@ -567,18 +579,19 @@ public class HiveCatalog extends AbstractCatalog {
Preconditions.checkArgument(identifier.getSystemTableName() == null);
Table table = getHmsTable(identifier);
try {
- TableSchema tableSchema = getDataTableSchema(identifier, table);
+ TableMeta tableMeta = getDataTableMeta(identifier, table);
return FileStoreTableFactory.create(
fileIO,
getTableLocation(identifier, table),
- tableSchema,
+ tableMeta.schema(),
new CatalogEnvironment(
identifier,
+ tableMeta.uuid(),
Lock.factory(
lockFactory().orElse(null),
lockContext().orElse(null),
identifier),
- metastoreClientFactory(identifier,
tableSchema).orElse(null),
+ metastoreClientFactory(identifier,
tableMeta.schema()).orElse(null),
lineageMetaFactory));
} catch (TableNotExistException ignore) {
}
