This is an automated email from the ASF dual-hosted git repository. sunchao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 920f9e535db HIVE-27659: Make partition order configurable if the metastore does not return all partitions (#4646) 920f9e535db is described below commit 920f9e535db6270a401db274eef3267d70c1fd2f Author: Yuming Wang <yumw...@apache.org> AuthorDate: Wed Sep 6 00:56:59 2023 +0800 HIVE-27659: Make partition order configurable if the metastore does not return all partitions (#4646) --- .../apache/hadoop/hive/ql/metadata/TestHive.java | 56 ++++++++++++++++++++++ .../hadoop/hive/metastore/conf/MetastoreConf.java | 5 ++ .../hadoop/hive/metastore/MetaStoreDirectSql.java | 2 +- 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index df3ecf6f662..33171ebb0cb 100755 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -849,6 +849,62 @@ public class TestHive { } } + @Test + public void testGetPartitionsWithMaxLimit() throws Exception { + String dbName = Warehouse.DEFAULT_DATABASE_NAME; + String tableName = "table_for_get_partitions_with_max_limit"; + + try { + Map<String, String> part_spec = new HashMap<String, String>(); + + Table table = createPartitionedTable(dbName, tableName); + part_spec.clear(); + part_spec.put("ds", "2025-06-30"); + part_spec.put("hr", "11"); + hm.createPartition(table, part_spec); + + Thread.sleep(1); + part_spec.clear(); + part_spec.put("ds", "2023-04-15"); + part_spec.put("hr", "12"); + hm.createPartition(table, part_spec); + + Thread.sleep(1); + part_spec.clear(); + part_spec.put("ds", "2023-09-01"); + part_spec.put("hr", "10"); + hm.createPartition(table, part_spec); + + // Default + Assert.assertEquals( + ((List<Partition>) hm.getPartitions(table, new HashMap(), (short) 1)).get(0).getTPartition().getValues(), + Arrays.asList("2023-04-15", "12")); + + // Sort by "PARTITIONS"."CREATE_TIME" desc + hm.setMetaConf(MetastoreConf.ConfVars.PARTITION_ORDER_EXPR.getVarname(), "\"PARTITIONS\".\"CREATE_TIME\" desc"); + Assert.assertEquals( + ((List<Partition>) hm.getPartitions(table, new HashMap(), (short) 1)).get(0).getTPartition().getValues(), + Arrays.asList("2023-09-01", "10")); + + // Sort by "PART_NAME" desc + hm.setMetaConf(MetastoreConf.ConfVars.PARTITION_ORDER_EXPR.getVarname(), "\"PART_NAME\" desc"); + Assert.assertEquals( + ((List<Partition>) hm.getPartitions(table, new HashMap(), (short) 1)).get(0).getTPartition().getValues(), + Arrays.asList("2025-06-30", "11")); + + // Test MetaStoreClient + Assert.assertEquals( + hm.getMSC().listPartitions(table.getDbName(), table.getTableName(), (short) 1).get(0).getValues(), + Arrays.asList("2025-06-30", "11")); + } catch (Exception e) { + fail("Unexpected exception: " + StringUtils.stringifyException(e)); + } finally { + hm.setMetaConf(MetastoreConf.ConfVars.PARTITION_ORDER_EXPR.getVarname(), + MetastoreConf.ConfVars.PARTITION_ORDER_EXPR.getDefaultVal().toString()); + cleanUpTableQuietly(dbName, tableName); + } + } + private void checkPartitionsConsistency(Table tbl) throws Exception { Set<Partition> allParts = hm.getAllPartitionsOf(tbl); List<Partition> allParts2 = hm.getPartitions(tbl); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 323d953be12..6f597a1739d 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -251,6 +251,7 @@ public class MetastoreConf { ConfVars.TRY_DIRECT_SQL_DDL, ConfVars.CLIENT_SOCKET_TIMEOUT, ConfVars.PARTITION_NAME_WHITELIST_PATTERN, + ConfVars.PARTITION_ORDER_EXPR, ConfVars.CAPABILITY_CHECK, ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES, ConfVars.EXPRESSION_PROXY_CLASS @@ -1266,6 +1267,10 @@ public class MetastoreConf { PARTITION_NAME_WHITELIST_PATTERN("metastore.partition.name.whitelist.pattern", "hive.metastore.partition.name.whitelist.pattern", "", "Partition names will be checked against this regex pattern and rejected if not matched."), + PARTITION_ORDER_EXPR("metastore.partition.order.expr", + "hive.metastore.partition.order.expr", "\"PART_NAME\" asc", + "The default partition order if the metastore does not return all partitions. \n" + + "It can be sorted based on any column in the PARTITIONS table (e.g., \"PARTITIONS\".\"CREATE_TIME\" desc, \"PARTITIONS\".\"LAST_ACCESS_TIME\" desc etc)"), PART_INHERIT_TBL_PROPS("metastore.partition.inherit.table.properties", "hive.metastore.partition.inherit.table.properties", "", "List of comma separated keys occurring in table properties which will get inherited to newly created partitions. \n" + diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 9d884f2f48a..71de41e2e9e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -901,7 +901,7 @@ class MetaStoreDirectSql { final String catNameLcase = normalizeSpace(catName).toLowerCase(); // We have to be mindful of order during filtering if we are not returning all partitions. - String orderForFilter = (max != null) ? " order by \"PART_NAME\" asc" : ""; + String orderForFilter = (max != null) ? " order by " + MetastoreConf.getVar(conf, ConfVars.PARTITION_ORDER_EXPR) : ""; String queryText = "select " + PARTITIONS + ".\"PART_ID\" from " + PARTITIONS + ""