This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 7904c1d94c [hive] Read tables whose Paimon location is only in table
properties (#10589)
7904c1d94c is described below
commit 7904c1d94ceb9dcb6b29e45dfa2c7fa8a48bf0dd
Author: YangJie <[email protected]>
AuthorDate: Thu Oct 15 20:49:54 2026 -0400
[hive] Read tables whose Paimon location is only in table properties
(#10589)
---
.../apache/paimon/hive/PaimonStorageHandler.java | 10 +++++++
.../paimon/hive/utils/HiveSplitGenerator.java | 35 +++++++++++++++++-----
.../apache/paimon/hive/HiveCatalogITCaseBase.java | 28 +++++++++++++++++
.../apache/paimon/hive/HiveSplitGeneratorTest.java | 18 +++++++++++
4 files changed, 84 insertions(+), 7 deletions(-)
diff --git
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java
index 071434ec11..8036bf4a58 100644
---
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java
+++
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java
@@ -21,10 +21,13 @@ package org.apache.paimon.hive;
import org.apache.paimon.hive.mapred.PaimonInputFormat;
import org.apache.paimon.hive.mapred.PaimonOutputCommitter;
import org.apache.paimon.hive.mapred.PaimonOutputFormat;
+import org.apache.paimon.hive.utils.HiveSplitGenerator;
import org.apache.paimon.utils.JsonSerdeUtil;
+import org.apache.paimon.utils.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler;
@@ -81,6 +84,13 @@ public class PaimonStorageHandler implements
HiveStoragePredicateHandler, HiveSt
Properties properties = tableDesc.getProperties();
String paimonLocation = LocationKeyExtractor.getPaimonLocation(conf,
properties);
map.put(LocationKeyExtractor.INTERNAL_LOCATION, paimonLocation);
+ // always set, so that a value left by another table of the same query
cannot leak in
+ map.put(
+ HiveSplitGenerator.HIVE_PARTITIONED,
+ String.valueOf(
+ !StringUtils.isEmpty(
+ properties.getProperty(
+
hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS))));
String dataFieldJsonStr = getDataFieldsJsonStr(conf, properties);
tableDesc.getProperties().put(PAIMON_TABLE_FIELDS, dataFieldJsonStr);
}
diff --git
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java
index b121c4e4d7..ea3ea52b5f 100644
---
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java
+++
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java
@@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
@@ -63,6 +64,9 @@ public class HiveSplitGenerator {
private static final Logger LOG =
LoggerFactory.getLogger(HiveSplitGenerator.class);
+ /** Whether the Hive table has partition columns, set by {@code
PaimonStorageHandler}. */
+ public static final String HIVE_PARTITIONED =
"paimon_internal_hive_partitioned";
+
public static InputSplit[] generateSplits(
FileStoreTable table, JobConf jobConf, int numSplits) {
List<Predicate> predicates = new ArrayList<>();
@@ -81,13 +85,7 @@ public class HiveSplitGenerator {
List<PaimonInputSplit> splits = new ArrayList<>();
// locations may contain multiple partitions
- for (String location : locations.split(",")) {
- if (!location.startsWith(table.location().toString())) {
- // Hive create dummy file for empty table or partition. If
this location doesn't
- // belong to this table, nothing to do.
- continue;
- }
-
+ for (String location : locationsToPlan(table, jobConf, locations)) {
InnerTableScan scan;
if (tagToPartField != null) {
// the location should be pruned by partition predicate
@@ -130,6 +128,29 @@ public class HiveSplitGenerator {
return splits.toArray(new InputSplit[0]);
}
+ private static List<String> locationsToPlan(
+ FileStoreTable table, JobConf jobConf, String locations) {
+ List<String> belonging = new ArrayList<>();
+ String foreign = null;
+ for (String location : locations.split(",")) {
+ // Hive create dummy file for empty table or partition. If this
location doesn't
+ // belong to this table, nothing to do.
+ if (location.startsWith(table.location().toString())) {
+ belonging.add(location);
+ } else if (foreign == null && !location.isEmpty()) {
+ foreign = location;
+ }
+ }
+ // With LOCATION_IN_PROPERTIES, Hive gives the table a location under
its own warehouse,
+ // which does not start with the paimon location. If the table has no
Hive partitions,
+ // that location stands for the whole table. Hive partitions of such a
table all share
+ // it, so they cannot be told apart and are still skipped.
+ if (belonging.isEmpty() && foreign != null &&
!jobConf.getBoolean(HIVE_PARTITIONED, true)) {
+ return Collections.singletonList(foreign);
+ }
+ return belonging;
+ }
+
private static Optional<Predicate> createPartitionPredicate(
RowType rowType,
List<String> partitionKeys,
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
index 279159eb48..47eb0ca695 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
@@ -1152,6 +1152,34 @@ public abstract class HiveCatalogITCaseBase {
"+I[3004, 3a, 34]");
}
+ @Test
+ @LocationInProperties
+ public void testHiveReadLocationInProperties() throws Exception {
+ tEnv.executeSql("CREATE TABLE t (a INT, b STRING)").await();
+ tEnv.executeSql("INSERT INTO t VALUES (1, 'apple'), (2,
'banana')").await();
+ assertThat(hiveShell.executeQuery("SELECT a, b FROM t"))
+ .containsExactlyInAnyOrder("1\tapple", "2\tbanana");
+ assertThat(hiveShell.executeQuery("SELECT b FROM t WHERE a =
2")).containsExactly("banana");
+ assertThat(hiveShell.executeQuery("SELECT count(*) FROM
t")).containsExactly("2");
+
+ // partitioned in paimon only, so Hive still sees one location for the
whole table
+ tEnv.executeSql("CREATE TABLE pt_t (pt INT, a INT) PARTITIONED BY
(pt)").await();
+ tEnv.executeSql("INSERT INTO pt_t VALUES (1, 10), (2, 20)").await();
+ assertThat(hiveShell.executeQuery("SELECT pt, a FROM pt_t"))
+ .containsExactlyInAnyOrder("1\t10", "2\t20");
+ assertThat(hiveShell.executeQuery("SELECT a FROM pt_t WHERE pt =
2")).containsExactly("20");
+
+ // Hive partitions all share the table location here, so they cannot
be read yet, but
+ // no partition may be read as another one
+ tEnv.executeSql(
+ "CREATE TABLE hive_pt_t (pt INT, a INT) PARTITIONED BY
(pt) "
+ + "WITH ('metastore.partitioned-table' =
'true')")
+ .await();
+ tEnv.executeSql("INSERT INTO hive_pt_t VALUES (1, 10), (2,
20)").await();
+ assertThat(hiveShell.executeQuery("SELECT pt, a FROM hive_pt_t"))
+ .isSubsetOf("1\t10", "2\t20");
+ }
+
private void prepareTestAddPartitionsToMetastore() throws Exception {
tEnv.executeSql(
String.join(
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java
index 250dc00f6a..73ba93682c 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java
@@ -181,6 +181,24 @@ public class HiveSplitGeneratorTest {
assertThat(splits).hasSize(1);
}
+ @Test
+ public void testForeignLocation() throws Exception {
+ FileStoreTable table = createPartitionedTableWithTwoPartitions();
+ JobConf jobConf = new JobConf();
+ jobConf.set(FileInputFormat.INPUT_DIR,
"/hive/warehouse/somewhere-else");
+
+ // With LOCATION_IN_PROPERTIES, Hive gives the table a location under
its own warehouse.
+ // Without Hive partitions it is the location of the whole table.
+ jobConf.setBoolean(HiveSplitGenerator.HIVE_PARTITIONED, false);
+ assertThat(HiveSplitGenerator.generateSplits(table, jobConf,
1)).hasSize(2);
+
+ // All Hive partitions share that location, which names no partition.
Planning it would
+ // read every partition once for each Hive partition. Hive decides
this, not the
+ // metastore.partitioned-table option, which a session can override.
+ jobConf.setBoolean(HiveSplitGenerator.HIVE_PARTITIONED, true);
+ assertThat(HiveSplitGenerator.generateSplits(table, jobConf,
1)).isEmpty();
+ }
+
private InputSplit[] generateSplits(FileStoreTable table, String
partitionDir) {
JobConf jobConf = new JobConf();
jobConf.set(FileInputFormat.INPUT_DIR, tablePath + "/" + partitionDir);
