This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d3612e6cd [core] Get table name and full table name from identifier
(#3686)
d3612e6cd is described below
commit d3612e6cd3738eb06a0e0d19e495f457e01b5c83
Author: Zouxxyy <[email protected]>
AuthorDate: Fri Jul 5 21:49:57 2024 +0800
[core] Get table name and full table name from identifier (#3686)
---
.../java/org/apache/paimon/catalog/AbstractCatalog.java | 1 +
.../apache/paimon/privilege/PrivilegedFileStoreTable.java | 10 ++++++++++
.../org/apache/paimon/table/AbstractFileStoreTable.java | 15 +++++++++++++++
.../org/apache/paimon/table/AppendOnlyFileStoreTable.java | 3 +--
.../java/org/apache/paimon/table/CatalogEnvironment.java | 13 +++++++++++++
.../main/java/org/apache/paimon/table/FileStoreTable.java | 11 -----------
.../org/apache/paimon/table/FileStoreTableFactory.java | 15 ++-------------
.../org/apache/paimon/table/PrimaryKeyFileStoreTable.java | 3 +--
.../test/java/org/apache/paimon/TestAppendFileStore.java | 3 +--
.../src/test/java/org/apache/paimon/TestFileStore.java | 3 +--
.../org/apache/paimon/table/sink/TableCommitTest.java | 5 ++---
.../ContinuousAppendAndCompactFollowUpScannerTest.java | 7 +------
.../paimon/table/source/snapshot/ScannerTestBase.java | 7 +------
.../paimon/table/source/snapshot/SnapshotReaderTest.java | 7 +------
.../java/org/apache/paimon/flink/sink/FlinkSinkTest.java | 3 +--
.../paimon/spark/sql/DDLWithHiveCatalogTestBase.scala | 6 ++++++
16 files changed, 57 insertions(+), 55 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
index 3001faf3b..0c6d19e28 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
@@ -350,6 +350,7 @@ public abstract class AbstractCatalog implements Catalog {
getDataTableLocation(identifier),
tableSchema,
new CatalogEnvironment(
+ identifier,
Lock.factory(
lockFactory().orElse(null),
lockContext().orElse(null), identifier),
metastoreClientFactory(identifier).orElse(null),
diff --git
a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java
b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java
index d33e73b8f..8d77ad612 100644
---
a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java
@@ -61,6 +61,16 @@ public class PrivilegedFileStoreTable implements
FileStoreTable {
this.identifier = identifier;
}
+ @Override
+ public String name() {
+ return wrapped.name();
+ }
+
+ @Override
+ public String fullName() {
+ return wrapped.fullName();
+ }
+
@Override
public SnapshotReader newSnapshotReader() {
privilegeChecker.assertCanSelect(identifier);
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
index 10bc386ca..d735642a6 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
@@ -20,6 +20,7 @@ package org.apache.paimon.table;
import org.apache.paimon.CoreOptions;
import org.apache.paimon.Snapshot;
+import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.consumer.ConsumerManager;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
@@ -103,6 +104,20 @@ abstract class AbstractFileStoreTable implements
FileStoreTable {
this.catalogEnvironment = catalogEnvironment;
}
+ @Override
+ public String name() {
+ Identifier identifier = catalogEnvironment.identifier();
+ return identifier == null ? location().getName() :
identifier.getObjectName();
+ }
+
+ @Override
+ public String fullName() {
+ Identifier identifier = catalogEnvironment.identifier();
+ return identifier == null
+ ? SchemaManager.fromPath(location().toUri().toString(),
true).getFullName()
+ : identifier.getFullName();
+ }
+
@Override
public Optional<Statistics> statistics() {
// todo: support time travel
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java
index f45865d01..0c389e408 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java
@@ -27,7 +27,6 @@ import org.apache.paimon.manifest.ManifestCacheFilter;
import org.apache.paimon.operation.AppendOnlyFileStoreScan;
import org.apache.paimon.operation.AppendOnlyFileStoreWrite;
import org.apache.paimon.operation.FileStoreScan;
-import org.apache.paimon.operation.Lock;
import org.apache.paimon.operation.RawFileSplitRead;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.reader.RecordReader;
@@ -54,7 +53,7 @@ class AppendOnlyFileStoreTable extends AbstractFileStoreTable
{
private transient AppendOnlyFileStore lazyStore;
AppendOnlyFileStoreTable(FileIO fileIO, Path path, TableSchema
tableSchema) {
- this(fileIO, path, tableSchema, new
CatalogEnvironment(Lock.emptyFactory(), null, null));
+ this(fileIO, path, tableSchema, CatalogEnvironment.empty());
}
AppendOnlyFileStoreTable(
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java
b/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java
index 7a3fbc37b..ebaff1266 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java
@@ -18,6 +18,7 @@
package org.apache.paimon.table;
+import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.lineage.LineageMetaFactory;
import org.apache.paimon.metastore.MetastoreClient;
import org.apache.paimon.operation.Lock;
@@ -34,19 +35,31 @@ public class CatalogEnvironment implements Serializable {
private static final long serialVersionUID = 1L;
+ @Nullable private final Identifier identifier;
private final Lock.Factory lockFactory;
@Nullable private final MetastoreClient.Factory metastoreClientFactory;
@Nullable private final LineageMetaFactory lineageMetaFactory;
public CatalogEnvironment(
+ @Nullable Identifier identifier,
Lock.Factory lockFactory,
@Nullable MetastoreClient.Factory metastoreClientFactory,
@Nullable LineageMetaFactory lineageMetaFactory) {
+ this.identifier = identifier;
this.lockFactory = lockFactory;
this.metastoreClientFactory = metastoreClientFactory;
this.lineageMetaFactory = lineageMetaFactory;
}
+ public static CatalogEnvironment empty() {
+ return new CatalogEnvironment(null, Lock.emptyFactory(), null, null);
+ }
+
+ @Nullable
+ public Identifier identifier() {
+ return identifier;
+ }
+
public Lock.Factory lockFactory() {
return lockFactory;
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java
index 7c791bd47..68f053178 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java
@@ -22,7 +22,6 @@ import org.apache.paimon.FileStore;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.manifest.ManifestCacheFilter;
-import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.stats.SimpleStats;
import org.apache.paimon.table.query.LocalTableQuery;
@@ -41,16 +40,6 @@ import java.util.Optional;
*/
public interface FileStoreTable extends DataTable {
- @Override
- default String name() {
- return location().getName();
- }
-
- @Override
- default String fullName() {
- return SchemaManager.fromPath(location().toUri().toString(),
true).getFullName();
- }
-
@Override
default RowType rowType() {
return schema().logicalRowType();
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java
b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java
index e941a89e0..e124fbb27 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java
@@ -22,7 +22,6 @@ import org.apache.paimon.CoreOptions;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
-import org.apache.paimon.operation.Lock;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.schema.TableSchema;
@@ -63,21 +62,11 @@ public class FileStoreTableFactory {
"Schema file not found in
location "
+ tablePath
+ ". Please create
table first."));
- return create(
- fileIO,
- tablePath,
- tableSchema,
- options,
- new CatalogEnvironment(Lock.emptyFactory(), null, null));
+ return create(fileIO, tablePath, tableSchema, options,
CatalogEnvironment.empty());
}
public static FileStoreTable create(FileIO fileIO, Path tablePath,
TableSchema tableSchema) {
- return create(
- fileIO,
- tablePath,
- tableSchema,
- new Options(),
- new CatalogEnvironment(Lock.emptyFactory(), null, null));
+ return create(fileIO, tablePath, tableSchema, new Options(),
CatalogEnvironment.empty());
}
public static FileStoreTable create(
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java
index 8c6925d73..0393b0d6e 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java
@@ -28,7 +28,6 @@ import
org.apache.paimon.mergetree.compact.LookupMergeFunction;
import org.apache.paimon.mergetree.compact.MergeFunctionFactory;
import org.apache.paimon.operation.FileStoreScan;
import org.apache.paimon.operation.KeyValueFileStoreScan;
-import org.apache.paimon.operation.Lock;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.schema.KeyValueFieldsExtractor;
@@ -56,7 +55,7 @@ class PrimaryKeyFileStoreTable extends AbstractFileStoreTable
{
private transient KeyValueFileStore lazyStore;
PrimaryKeyFileStoreTable(FileIO fileIO, Path path, TableSchema
tableSchema) {
- this(fileIO, path, tableSchema, new
CatalogEnvironment(Lock.emptyFactory(), null, null));
+ this(fileIO, path, tableSchema, CatalogEnvironment.empty());
}
PrimaryKeyFileStoreTable(
diff --git
a/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java
b/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java
index 6ca3b3a91..d01210e47 100644
--- a/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java
+++ b/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java
@@ -32,7 +32,6 @@ import org.apache.paimon.io.DataIncrement;
import org.apache.paimon.io.IndexIncrement;
import org.apache.paimon.manifest.ManifestCommittable;
import org.apache.paimon.operation.FileStoreCommitImpl;
-import org.apache.paimon.operation.Lock;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.schema.SchemaUtils;
@@ -80,7 +79,7 @@ public class TestAppendFileStore extends AppendOnlyFileStore {
bucketType,
rowType,
tableName,
- new CatalogEnvironment(Lock.emptyFactory(), null, null));
+ CatalogEnvironment.empty());
this.fileIO = fileIO;
this.commitUser = UUID.randomUUID().toString();
diff --git a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
index 2b0c66380..6a73a120e 100644
--- a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
+++ b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
@@ -39,7 +39,6 @@ import org.apache.paimon.operation.AbstractFileStoreWrite;
import org.apache.paimon.operation.FileStoreCommit;
import org.apache.paimon.operation.FileStoreCommitImpl;
import org.apache.paimon.operation.FileStoreScan;
-import org.apache.paimon.operation.Lock;
import org.apache.paimon.operation.SplitRead;
import org.apache.paimon.options.ExpireConfig;
import org.apache.paimon.options.MemorySize;
@@ -133,7 +132,7 @@ public class TestFileStore extends KeyValueFileStore {
keyValueFieldsExtractor,
mfFactory,
(new Path(root)).getName(),
- new CatalogEnvironment(Lock.emptyFactory(), null, null));
+ CatalogEnvironment.empty());
this.root = root;
this.fileIO = FileIOFinder.find(new Path(root));
this.keySerializer = new InternalRowSerializer(keyType);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
index d9aa14ce6..34abc25b1 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
@@ -24,7 +24,6 @@ import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.io.DataFilePathFactory;
import org.apache.paimon.manifest.ManifestCommittable;
-import org.apache.paimon.operation.Lock;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaManager;
@@ -125,7 +124,7 @@ public class TableCommitTest {
new FailingFileIO(),
new Path(path),
tableSchema,
- new CatalogEnvironment(Lock.emptyFactory(), null,
null));
+ CatalogEnvironment.empty());
String commitUser = UUID.randomUUID().toString();
StreamTableWrite write = table.newWrite(commitUser);
@@ -212,7 +211,7 @@ public class TableCommitTest {
LocalFileIO.create(),
new Path(path),
tableSchema,
- new CatalogEnvironment(Lock.emptyFactory(), null,
null));
+ CatalogEnvironment.empty());
String commitUser = UUID.randomUUID().toString();
StreamTableWrite write = table.newWrite(commitUser);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ContinuousAppendAndCompactFollowUpScannerTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ContinuousAppendAndCompactFollowUpScannerTest.java
index 4d3c3fd90..c32e4fad0 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ContinuousAppendAndCompactFollowUpScannerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ContinuousAppendAndCompactFollowUpScannerTest.java
@@ -21,7 +21,6 @@ package org.apache.paimon.table.source.snapshot;
import org.apache.paimon.Snapshot;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.io.DataFileMetaSerializer;
-import org.apache.paimon.operation.Lock;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaManager;
@@ -145,10 +144,6 @@ public class ContinuousAppendAndCompactFollowUpScannerTest
extends ScannerTestBa
conf.toMap(),
""));
return FileStoreTableFactory.create(
- fileIO,
- tablePath,
- tableSchema,
- conf,
- new CatalogEnvironment(Lock.emptyFactory(), null, null));
+ fileIO, tablePath, tableSchema, conf,
CatalogEnvironment.empty());
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ScannerTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ScannerTestBase.java
index bb237f2d1..defbe24ec 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ScannerTestBase.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ScannerTestBase.java
@@ -27,7 +27,6 @@ import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.FileIOFinder;
import org.apache.paimon.fs.Path;
import org.apache.paimon.mergetree.compact.ConcatRecordReader;
-import org.apache.paimon.operation.Lock;
import org.apache.paimon.options.Options;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.reader.RecordReaderIterator;
@@ -168,11 +167,7 @@ public abstract class ScannerTestBase {
conf.toMap(),
""));
return FileStoreTableFactory.create(
- fileIO,
- tablePath,
- tableSchema,
- conf,
- new CatalogEnvironment(Lock.emptyFactory(), null, null));
+ fileIO, tablePath, tableSchema, conf,
CatalogEnvironment.empty());
}
protected List<Split> toSplits(List<DataSplit> dataSplits) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
index 9cee8a246..02a170c36 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
@@ -27,7 +27,6 @@ import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.FileIOFinder;
import org.apache.paimon.fs.Path;
import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.operation.Lock;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
@@ -369,10 +368,6 @@ public class SnapshotReaderTest {
options.toMap(),
""));
return FileStoreTableFactory.create(
- fileIO,
- tablePath,
- tableSchema,
- options,
- new CatalogEnvironment(Lock.emptyFactory(), null, null));
+ fileIO, tablePath, tableSchema, options,
CatalogEnvironment.empty());
}
}
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java
index 2a1022dce..84246d00a 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java
@@ -24,7 +24,6 @@ import org.apache.paimon.data.InternalRow;
import org.apache.paimon.fs.FileIOFinder;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.operation.KeyValueFileStoreWrite;
-import org.apache.paimon.operation.Lock;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaManager;
@@ -132,6 +131,6 @@ public class FlinkSinkTest {
tablePath,
tableSchema,
options,
- new CatalogEnvironment(Lock.emptyFactory(), null, null));
+ CatalogEnvironment.empty());
}
}
diff --git
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala
index 6769d81c6..a371f3e31 100644
---
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala
+++
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala
@@ -19,6 +19,7 @@
package org.apache.paimon.spark.sql
import org.apache.paimon.spark.PaimonHiveTestBase
+import org.apache.paimon.table.FileStoreTable
import org.apache.spark.sql.{Row, SparkSession}
import org.junit.jupiter.api.Assertions
@@ -47,6 +48,11 @@ abstract class DDLWithHiveCatalogTestBase extends
PaimonHiveTestBase {
Assertions.assertEquals(
getTableLocation("paimon_db.paimon_tbl"),
s"${dBLocation.getCanonicalPath}/paimon_tbl")
+
+ val fileStoreTable = getPaimonScan("SELECT * FROM
paimon_db.paimon_tbl").table
+ .asInstanceOf[FileStoreTable]
+ Assertions.assertEquals("paimon_tbl", fileStoreTable.name())
+ Assertions.assertEquals("paimon_db.paimon_tbl",
fileStoreTable.fullName())
}
}
}
