This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 920f9e535db HIVE-27659: Make partition order configurable if the
metastore does not return all partitions (#4646)
920f9e535db is described below
commit 920f9e535db6270a401db274eef3267d70c1fd2f
Author: Yuming Wang <yumw...@apache.org>
AuthorDate: Wed Sep 6 00:56:59 2023 +0800
HIVE-27659: Make partition order configurable if the metastore does not
return all partitions (#4646)
---
.../apache/hadoop/hive/ql/metadata/TestHive.java | 56 ++++++++++++++++++++++
.../hadoop/hive/metastore/conf/MetastoreConf.java | 5 ++
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 2 +-
3 files changed, 62 insertions(+), 1 deletion(-)
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
index df3ecf6f662..33171ebb0cb 100755
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
@@ -849,6 +849,62 @@ public class TestHive {
}
}
+ @Test
+ public void testGetPartitionsWithMaxLimit() throws Exception {
+ String dbName = Warehouse.DEFAULT_DATABASE_NAME;
+ String tableName = "table_for_get_partitions_with_max_limit";
+
+ try {
+ Map<String, String> part_spec = new HashMap<String, String>();
+
+ Table table = createPartitionedTable(dbName, tableName);
+ part_spec.clear();
+ part_spec.put("ds", "2025-06-30");
+ part_spec.put("hr", "11");
+ hm.createPartition(table, part_spec);
+
+ Thread.sleep(1);
+ part_spec.clear();
+ part_spec.put("ds", "2023-04-15");
+ part_spec.put("hr", "12");
+ hm.createPartition(table, part_spec);
+
+ Thread.sleep(1);
+ part_spec.clear();
+ part_spec.put("ds", "2023-09-01");
+ part_spec.put("hr", "10");
+ hm.createPartition(table, part_spec);
+
+ // Default
+ Assert.assertEquals(
+ ((List<Partition>) hm.getPartitions(table, new HashMap(), (short)
1)).get(0).getTPartition().getValues(),
+ Arrays.asList("2023-04-15", "12"));
+
+ // Sort by "PARTITIONS"."CREATE_TIME" desc
+ hm.setMetaConf(MetastoreConf.ConfVars.PARTITION_ORDER_EXPR.getVarname(),
"\"PARTITIONS\".\"CREATE_TIME\" desc");
+ Assert.assertEquals(
+ ((List<Partition>) hm.getPartitions(table, new HashMap(), (short)
1)).get(0).getTPartition().getValues(),
+ Arrays.asList("2023-09-01", "10"));
+
+ // Sort by "PART_NAME" desc
+ hm.setMetaConf(MetastoreConf.ConfVars.PARTITION_ORDER_EXPR.getVarname(),
"\"PART_NAME\" desc");
+ Assert.assertEquals(
+ ((List<Partition>) hm.getPartitions(table, new HashMap(), (short)
1)).get(0).getTPartition().getValues(),
+ Arrays.asList("2025-06-30", "11"));
+
+ // Test MetaStoreClient
+ Assert.assertEquals(
+ hm.getMSC().listPartitions(table.getDbName(), table.getTableName(),
(short) 1).get(0).getValues(),
+ Arrays.asList("2025-06-30", "11"));
+ } catch (Exception e) {
+ fail("Unexpected exception: " + StringUtils.stringifyException(e));
+ } finally {
+ hm.setMetaConf(MetastoreConf.ConfVars.PARTITION_ORDER_EXPR.getVarname(),
+
MetastoreConf.ConfVars.PARTITION_ORDER_EXPR.getDefaultVal().toString());
+ cleanUpTableQuietly(dbName, tableName);
+ }
+ }
+
private void checkPartitionsConsistency(Table tbl) throws Exception {
Set<Partition> allParts = hm.getAllPartitionsOf(tbl);
List<Partition> allParts2 = hm.getPartitions(tbl);
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 323d953be12..6f597a1739d 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -251,6 +251,7 @@ public class MetastoreConf {
ConfVars.TRY_DIRECT_SQL_DDL,
ConfVars.CLIENT_SOCKET_TIMEOUT,
ConfVars.PARTITION_NAME_WHITELIST_PATTERN,
+ ConfVars.PARTITION_ORDER_EXPR,
ConfVars.CAPABILITY_CHECK,
ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES,
ConfVars.EXPRESSION_PROXY_CLASS
@@ -1266,6 +1267,10 @@ public class MetastoreConf {
PARTITION_NAME_WHITELIST_PATTERN("metastore.partition.name.whitelist.pattern",
"hive.metastore.partition.name.whitelist.pattern", "",
"Partition names will be checked against this regex pattern and
rejected if not matched."),
+ PARTITION_ORDER_EXPR("metastore.partition.order.expr",
+ "hive.metastore.partition.order.expr", "\"PART_NAME\" asc",
+ "The default partition order if the metastore does not return all
partitions. \n" +
+ "It can be sorted based on any column in the PARTITIONS table
(e.g., \"PARTITIONS\".\"CREATE_TIME\" desc, \"PARTITIONS\".\"LAST_ACCESS_TIME\"
desc etc)"),
PART_INHERIT_TBL_PROPS("metastore.partition.inherit.table.properties",
"hive.metastore.partition.inherit.table.properties", "",
"List of comma separated keys occurring in table properties which will
get inherited to newly created partitions. \n" +
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 9d884f2f48a..71de41e2e9e 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -901,7 +901,7 @@ class MetaStoreDirectSql {
final String catNameLcase = normalizeSpace(catName).toLowerCase();
// We have to be mindful of order during filtering if we are not returning
all partitions.
- String orderForFilter = (max != null) ? " order by \"PART_NAME\" asc" : "";
+ String orderForFilter = (max != null) ? " order by " +
MetastoreConf.getVar(conf, ConfVars.PARTITION_ORDER_EXPR) : "";
String queryText =
"select " + PARTITIONS + ".\"PART_ID\" from " + PARTITIONS + ""
