This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 339aff648a [hive] Use the FileIO obtained from the path in HiveCatalog
(#6992)
339aff648a is described below
commit 339aff648a17c85fbe217c1c6e8cebda5b07c655
Author: Zouxxyy <[email protected]>
AuthorDate: Mon Jan 12 16:46:42 2026 +0800
[hive] Use the FileIO obtained from the path in HiveCatalog (#6992)
---
.../org/apache/paimon/catalog/AbstractCatalog.java | 9 ++---
.../java/org/apache/paimon/hive/HiveCatalog.java | 39 +++++++++++++++++++---
.../apache/paimon/hive/HiveCatalogITCaseBase.java | 6 ++--
3 files changed, 41 insertions(+), 13 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
index 6c98d0fb26..045710356b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
@@ -700,7 +700,7 @@ public abstract class AbstractCatalog implements Catalog {
protected List<String> listDatabasesInFileSystem(Path warehouse) throws
IOException {
List<String> databases = new ArrayList<>();
- for (FileStatus status : fileIO.listDirectories(warehouse)) {
+ for (FileStatus status : fileIO(warehouse).listDirectories(warehouse))
{
Path path = status.getPath();
if (status.isDir() && path.getName().endsWith(DB_SUFFIX)) {
String fileName = path.getName();
@@ -712,7 +712,7 @@ public abstract class AbstractCatalog implements Catalog {
protected List<String> listTablesInFileSystem(Path databasePath) throws
IOException {
List<String> tables = new ArrayList<>();
- for (FileStatus status : fileIO.listDirectories(databasePath)) {
+ for (FileStatus status :
fileIO(databasePath).listDirectories(databasePath)) {
if (status.isDir() && tableExistsInFileSystem(status.getPath(),
DEFAULT_MAIN_BRANCH)) {
tables.add(status.getPath().getName());
}
@@ -721,7 +721,7 @@ public abstract class AbstractCatalog implements Catalog {
}
protected boolean tableExistsInFileSystem(Path tablePath, String
branchName) {
- SchemaManager schemaManager = new SchemaManager(fileIO, tablePath,
branchName);
+ SchemaManager schemaManager = new SchemaManager(fileIO(tablePath),
tablePath, branchName);
// in order to improve the performance, check the schema-0 firstly.
boolean schemaZeroExists = schemaManager.schemaExists(0);
@@ -734,7 +734,8 @@ public abstract class AbstractCatalog implements Catalog {
}
public Optional<TableSchema> tableSchemaInFileSystem(Path tablePath,
String branchName) {
- Optional<TableSchema> schema = new SchemaManager(fileIO, tablePath,
branchName).latest();
+ Optional<TableSchema> schema =
+ new SchemaManager(fileIO(tablePath), tablePath,
branchName).latest();
if (!DEFAULT_MAIN_BRANCH.equals(branchName)) {
schema =
schema.map(
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index f63cd48464..27e8c067dd 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -208,6 +208,15 @@ public class HiveCatalog extends AbstractCatalog {
new SerializableHiveConf(hiveConf), clientClassName,
options));
}
+ @Override
+ protected FileIO fileIO(Path path) {
+ try {
+ return FileIO.get(path, context);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
@Override
public Path getTableLocation(Identifier identifier) {
Table table = null;
@@ -286,7 +295,7 @@ public class HiveCatalog extends AbstractCatalog {
database.getLocationUri() == null
? newDatabasePath(name)
: new Path(database.getLocationUri());
- locationHelper.createPathIfRequired(databasePath, fileIO);
+ locationHelper.createPathIfRequired(databasePath,
fileIO(databasePath));
locationHelper.specifyDatabaseLocation(databasePath, database);
clients.execute(client -> client.createDatabase(database));
} catch (TException | IOException e) {
@@ -628,8 +637,8 @@ public class HiveCatalog extends AbstractCatalog {
protected void dropDatabaseImpl(String name) {
try {
Database database = clients.run(client ->
client.getDatabase(name));
- String location = locationHelper.getDatabaseLocation(database);
- locationHelper.dropPathIfRequired(new Path(location), fileIO);
+ Path location = new
Path(locationHelper.getDatabaseLocation(database));
+ locationHelper.dropPathIfRequired(location, fileIO(location));
clients.execute(client -> client.dropDatabase(name, true, false,
true));
} catch (TException | IOException e) {
throw new RuntimeException("Failed to drop database " + name, e);
@@ -711,6 +720,24 @@ public class HiveCatalog extends AbstractCatalog {
}
}
+ @Override
+ public org.apache.paimon.table.Table getTable(Identifier identifier)
+ throws TableNotExistException {
+ // Hive table's location can be arbitrarily specified (inherited from
the database) and may
+ // use a schema different from that of the default warehouse. Even
though it is not an
+ // external table in this case, the FileIO still needs to be obtained
based on the path.
+ return CatalogUtils.loadTable(
+ this,
+ identifier,
+ this::fileIO,
+ this::fileIO,
+ this::loadTableMetadata,
+ lockFactory().orElse(null),
+ lockContext().orElse(null),
+ context,
+ false);
+ }
+
@Override
protected TableMetadata loadTableMetadata(Identifier identifier) throws
TableNotExistException {
return loadTableMetadata(identifier, getHmsTable(identifier));
@@ -1001,6 +1028,7 @@ public class HiveCatalog extends AbstractCatalog {
// which in Hive metastore.
Path path = getTableLocation(identifier);
try {
+ FileIO fileIO = fileIO(path);
if (fileIO.exists(path)) {
fileIO.deleteDirectoryQuietly(path);
}
@@ -1066,7 +1094,7 @@ public class HiveCatalog extends AbstractCatalog {
} catch (Exception e) {
try {
if (!externalTable) {
- fileIO.deleteDirectoryQuietly(location);
+ fileIO(location).deleteDirectoryQuietly(location);
}
} catch (Exception ee) {
LOG.error("Delete directory[{}] fail for table {}", location,
identifier, ee);
@@ -1128,6 +1156,7 @@ public class HiveCatalog extends AbstractCatalog {
Path fromPath = getTableLocation(fromTable);
Table table = renameHiveTable(fromTable, toTable);
Path toPath = getTableLocation(toTable);
+ FileIO fileIO = fileIO(fromPath);
if (!isExternalTable(table)
&& !fromPath.equals(toPath)
&& !new SchemaManager(fileIO,
fromPath).listAllIds().isEmpty()) {
@@ -1641,7 +1670,7 @@ public class HiveCatalog extends AbstractCatalog {
}
private SchemaManager schemaManager(Identifier identifier, Path location) {
- return new SchemaManager(fileIO, location,
identifier.getBranchNameOrDefault());
+ return new SchemaManager(fileIO(location), location,
identifier.getBranchNameOrDefault());
}
public <T> T runWithLock(Identifier identifier, Callable<T> callable)
throws Exception {
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
index af5ad48a48..cc21d12ca5 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
@@ -51,7 +51,6 @@ import
org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
import org.apache.flink.types.Row;
import org.apache.flink.util.CloseableIterator;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
import org.junit.Rule;
import org.junit.Test;
@@ -202,9 +201,8 @@ public abstract class HiveCatalogITCaseBase {
properties.put("location", dbLocation);
assertThatThrownBy(() -> catalog.createDatabase("location_test_db",
false, properties))
- .hasRootCauseInstanceOf(MetaException.class)
- .hasRootCauseMessage(
- "Got exception: java.io.IOException No FileSystem for
scheme: s3");
+ .hasMessageContaining(
+ "Could not find a file io implementation for scheme
's3' in the classpath.");
}
@Test
