This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 7aff485263436d3d1970ea0d5ad7fca8b9a5cb5e Author: Dapeng Sun(孙大鹏) <[email protected]> AuthorDate: Sat Sep 12 21:39:48 2026 +0800 [core] Improve the error when listing a Format Table custom partition location fails (#9748) --- .../table/format/CatalogSplitEnumerator.java | 44 ++++++++++++++++---- .../format/CatalogManagedPartitionScanTest.java | 48 ++++++++++++++++++++++ 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java index 1425cd538e..a65d224de3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java @@ -116,19 +116,26 @@ final class CatalogSplitEnumerator extends SplitEnumerator { for (Pair<LinkedHashMap<String, String>, Path> partition : partitions) { boolean useCatalogContextFileIO = fileIOResolver.useCatalogContextFileIO(partition.getValue()); - if (useCatalogContextFileIO) { - fileIOResolver.prepare(partition.getValue(), true); - } else if (!tableFileIOPrepared) { - fileIOResolver.prepare(partition.getValue(), false); - tableFileIOPrepared = true; + try { + if (useCatalogContextFileIO) { + fileIOResolver.prepare(partition.getValue(), true); + } else if (!tableFileIOPrepared) { + fileIOResolver.prepare(partition.getValue(), false); + tableFileIOPrepared = true; + } + } catch (IOException e) { + throw new RuntimeException( + listFailureMessage( + partition.getKey(), partition.getValue(), useCatalogContextFileIO), + e); } } Function<Pair<LinkedHashMap<String, String>, Path>, List<Split>> lister = pair -> { BinaryRow partitionRow = toPartitionRow(pair.getKey()); + boolean useCatalogContextFileIO = + fileIOResolver.useCatalogContextFileIO(pair.getValue()); try { - boolean useCatalogContextFileIO = - fileIOResolver.useCatalogContextFileIO(pair.getValue()); return createSplits( fileIOResolver.fileIO(useCatalogContextFileIO), pair.getValue(), @@ -139,7 +146,9 @@ final class CatalogSplitEnumerator extends SplitEnumerator { return Collections.emptyList(); } catch (IOException e) { throw new RuntimeException( - "Failed to list files for partition " + pair.getValue(), e); + listFailureMessage( + pair.getKey(), pair.getValue(), useCatalogContextFileIO), + e); } }; int parallelism = @@ -362,6 +371,25 @@ final class CatalogSplitEnumerator extends SplitEnumerator { spec, table.fullName(), table.partitionKeys())); } + /** Says which credentials listed the partition, so a permission failure is actionable. */ + private String listFailureMessage( + LinkedHashMap<String, String> spec, Path path, boolean useCatalogContextFileIO) { + String message = + String.format( + "Failed to list files for partition '%s' of format table %s at '%s'.", + PartitionPathUtils.generatePartitionName(spec, false), + table.fullName(), + path); + if (!useCatalogContextFileIO) { + return message; + } + return message + + " The partition is registered at a custom location outside the table" + + " directory, so it is listed with the filesystem credentials of the catalog" + + " context (the engine's own configuration, for example fs.oss.* or the Hadoop" + + " configuration), not with the table's data token."; + } + private void warnMissingPartition(LinkedHashMap<String, String> spec, Path path) { LOG.warn( "Partition '{}' of format table {} is registered in the catalog but its directory " diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java index 5b6b2f226c..8e070dd7da 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java @@ -545,6 +545,52 @@ class CatalogManagedPartitionScanTest { assertThat(tableFileIO.listedPaths).isEmpty(); } + @Test + void testCustomLocationListingFailureNamesTheCatalogContextCredentials() throws Exception { + InjectingLocalFileIO clientFileIO = new InjectingLocalFileIO(); + Path externalPath = new Path(tempDir.resolve("external").toUri()); + writeDataFile(clientFileIO, externalPath, "files"); + clientFileIO.failListingContaining("external", new IOException("not found login secrets")); + Catalog catalog = mock(Catalog.class); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn( + new PagedList<>( + Collections.singletonList( + partition("2025", "11", externalPath.toString())), + null)); + Path tablePath = new Path(tempDir.resolve("table").toUri()); + TableRootOnlyLocalFileIO tableFileIO = new TableRootOnlyLocalFileIO(tablePath); + FileIOLoader clientLoader = + new FileIOLoader() { + @Override + public String getScheme() { + return "file"; + } + + @Override + public LocalFileIO load(Path path) { + return clientFileIO; + } + }; + CatalogContext catalogContext = CatalogContext.create(new Options(), clientLoader, null); + FormatTable table = + createTable( + tableFileIO, tablePath, partitionManager(catalog), false, catalogContext); + + // A permission failure on an archive bucket must say that the engine's own credentials + // listed it, or the user keeps looking at the table token. + assertThatThrownBy(() -> new FormatTableScan(table, null, null).plan().splits()) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("partition 'year=2025/month=11'") + .hasMessageContaining(table.fullName()) + .hasMessageContaining("'" + externalPath + "'") + .hasMessageContaining("custom location outside the table directory") + .hasMessageContaining("credentials of the catalog context") + .hasMessageContaining("not with the table's data token") + .hasRootCauseMessage("not found login secrets"); + assertThat(tableFileIO.listedPaths).isEmpty(); + } + @Test void testWhitespacePartitionValueIsVisible() throws Exception { Catalog catalog = mock(Catalog.class); @@ -919,6 +965,8 @@ class CatalogManagedPartitionScanTest { stringPartitionTable(fileIO, tablePath, recordingCatalog(partitions), 4); assertThatThrownBy(() -> new FormatTableScan(table, null, null).plan().splits()) .isInstanceOf(RuntimeException.class) + .hasMessageContaining("partition 'year=2025/month=11'") + .hasMessageNotContaining("custom location") .hasRootCauseInstanceOf(IOException.class) .hasRootCauseMessage("boom"); }
