This is an automated email from the ASF dual-hosted git repository. vihangk1 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 6b3b000 HIVE-23767: Pass ValidWriteIdList in get_partition* API requests (Kishen Das, reviewed by Vihang Karajgaonkar and Peter Vary) 6b3b000 is described below commit 6b3b000c5741fc852a76c87625578c37339bf874 Author: Kishen Das <kis...@cloudera.com> AuthorDate: Tue Jul 14 12:06:41 2020 -0700 HIVE-23767: Pass ValidWriteIdList in get_partition* API requests (Kishen Das, reviewed by Vihang Karajgaonkar and Peter Vary) --- .../org/apache/hadoop/hive/ql/metadata/Hive.java | 91 ++++++++++++++++++---- .../ql/metadata/SessionHiveMetaStoreClient.java | 48 +++++++++++- .../hadoop/hive/metastore/TestMetastoreExpr.java | 3 +- ...HiveMetastoreClientListPartitionsTempTable.java | 27 ++++--- .../ql/parse/TestUpdateDeleteSemanticAnalyzer.java | 5 ++ .../hadoop/hive/metastore/HiveMetaStoreClient.java | 45 ++++++++++- .../hadoop/hive/metastore/IMetaStoreClient.java | 16 +++- .../hadoop/hive/metastore/HiveMetaStore.java | 11 ++- .../metastore/HiveMetaStoreClientPreCatalog.java | 13 +++- 9 files changed, 219 insertions(+), 40 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 14eec31..e17086f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.metadata; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -134,6 +135,10 @@ import org.apache.hadoop.hive.metastore.api.FireEventRequestData; import org.apache.hadoop.hive.metastore.api.ForeignKeysRequest; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; +import org.apache.hadoop.hive.metastore.api.GetPartitionNamesPsRequest; +import org.apache.hadoop.hive.metastore.api.GetPartitionNamesPsResponse; +import org.apache.hadoop.hive.metastore.api.GetPartitionsPsWithAuthRequest; +import org.apache.hadoop.hive.metastore.api.GetPartitionsPsWithAuthResponse; import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalRequest; import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalResponse; import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; @@ -148,6 +153,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.NotNullConstraintsRequest; import org.apache.hadoop.hive.metastore.api.PartitionSpec; import org.apache.hadoop.hive.metastore.api.PartitionWithoutSD; +import org.apache.hadoop.hive.metastore.api.PartitionsByExprRequest; import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest; import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; import org.apache.hadoop.hive.metastore.api.PrincipalType; @@ -1460,6 +1466,24 @@ public class Hive { } /** + * Get ValidWriteIdList for the current transaction. + * This fetches the ValidWriteIdList from the metastore for a given table if txnManager has an open transaction. + * + * @param dbName + * @param tableName + * @return + * @throws LockException + */ + private ValidWriteIdList getValidWriteIdList(String dbName, String tableName) throws LockException { + ValidWriteIdList validWriteIdList = null; + long txnId = SessionState.get().getTxnMgr() != null ? SessionState.get().getTxnMgr().getCurrentTxnId() : 0; + if (txnId > 0) { + validWriteIdList = AcidUtils.getTableValidWriteIdListWithTxnList(conf, dbName, tableName); + } + return validWriteIdList; + } + + /** * Get all table names for the current database. * @return List of table names * @throws HiveException @@ -3550,11 +3574,6 @@ private void constructOneLBLocationMap(FileStatus fSta, } } - public List<String> getPartitionNames(String tblName, short max) throws HiveException { - String[] names = Utilities.getDbTableName(tblName); - return getPartitionNames(names[0], names[1], max); - } - public List<String> getPartitionNames(String dbName, String tblName, short max) throws HiveException { List<String> names = null; @@ -3580,7 +3599,17 @@ private void constructOneLBLocationMap(FileStatus fSta, List<String> pvals = MetaStoreUtils.getPvals(t.getPartCols(), partSpec); try { - names = getMSC().listPartitionNames(dbName, tblName, pvals, max); + GetPartitionNamesPsRequest req = new GetPartitionNamesPsRequest(); + req.setTblName(tblName); + req.setDbName(dbName); + req.setPartValues(pvals); + req.setMaxParts(max); + if (AcidUtils.isTransactionalTable(t)) { + ValidWriteIdList validWriteIdList = getValidWriteIdList(dbName, tblName); + req.setValidWriteIdList(validWriteIdList != null ? validWriteIdList.toString() : null); + } + GetPartitionNamesPsResponse res = getMSC().listPartitionNamesRequest(req); + names = res.getNames(); } catch (NoSuchObjectException nsoe) { // this means no partition exists for the given partition spec // key value pairs - thrift cannot handle null return values, hence @@ -3603,8 +3632,22 @@ private void constructOneLBLocationMap(FileStatus fSta, } try { String defaultPartitionName = HiveConf.getVar(conf, ConfVars.DEFAULTPARTITIONNAME); - names = getMSC().listPartitionNames(tbl.getCatalogName(), tbl.getDbName(), - tbl.getTableName(), defaultPartitionName, exprBytes, order, maxParts); + PartitionsByExprRequest req = + new PartitionsByExprRequest(tbl.getDbName(), tbl.getTableName(), ByteBuffer.wrap(exprBytes)); + if (defaultPartitionName != null) { + req.setDefaultPartitionName(defaultPartitionName); + } + if (maxParts >= 0) { + req.setMaxParts(maxParts); + } + req.setOrder(order); + req.setCatName(tbl.getCatalogName()); + if (AcidUtils.isTransactionalTable(tbl)) { + ValidWriteIdList validWriteIdList = getValidWriteIdList(tbl.getDbName(), tbl.getTableName()); + req.setValidWriteIdList(validWriteIdList != null ? validWriteIdList.toString() : null); + } + names = getMSC().listPartitionNames(req); + } catch (NoSuchObjectException nsoe) { return Lists.newArrayList(); } catch (Exception e) { @@ -3625,8 +3668,19 @@ private void constructOneLBLocationMap(FileStatus fSta, if (tbl.isPartitioned()) { List<org.apache.hadoop.hive.metastore.api.Partition> tParts; try { - tParts = getMSC().listPartitionsWithAuthInfo(tbl.getDbName(), tbl.getTableName(), - (short) -1, getUserName(), getGroupNames()); + GetPartitionsPsWithAuthRequest req = new GetPartitionsPsWithAuthRequest(); + req.setTblName(tbl.getTableName()); + req.setDbName(tbl.getDbName()); + req.setUserName(getUserName()); + req.setMaxParts((short) -1); + req.setGroupNames(getGroupNames()); + if (AcidUtils.isTransactionalTable(tbl)) { + ValidWriteIdList validWriteIdList = getValidWriteIdList(tbl.getDbName(), tbl.getTableName()); + req.setValidWriteIdList(validWriteIdList != null ? validWriteIdList.toString() : null); + } + GetPartitionsPsWithAuthResponse res = getMSC().listPartitionsWithAuthInfoRequest(req); + tParts = res.getPartitions(); + } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); throw new HiveException(e); @@ -3911,20 +3965,27 @@ private void constructOneLBLocationMap(FileStatus fSta, * @param tbl The table containing the partitions. * @param expr A serialized expression for partition predicates. * @param conf Hive config. - * @param result the resulting list of partitions + * @param partitions the resulting list of partitions * @return whether the resulting list contains partitions which may or may not match the expr */ public boolean getPartitionsByExpr(Table tbl, ExprNodeGenericFuncDesc expr, HiveConf conf, - List<Partition> result) throws HiveException, TException { - assert result != null; + List<Partition> partitions) throws HiveException, TException { + + Preconditions.checkNotNull(partitions); byte[] exprBytes = SerializationUtilities.serializeExpressionToKryo(expr); String defaultPartitionName = HiveConf.getVar(conf, ConfVars.DEFAULTPARTITIONNAME); List<org.apache.hadoop.hive.metastore.api.PartitionSpec> msParts = new ArrayList<>(); + ValidWriteIdList validWriteIdList = null; + if (AcidUtils.isTransactionalTable(tbl)) { + validWriteIdList = getValidWriteIdList(tbl.getDbName(), tbl.getTableName()); + } boolean hasUnknownParts = getMSC().listPartitionsSpecByExpr(tbl.getDbName(), - tbl.getTableName(), exprBytes, defaultPartitionName, (short)-1, msParts); - result.addAll(convertFromPartSpec(msParts.iterator(), tbl)); + tbl.getTableName(), exprBytes, defaultPartitionName, (short)-1, msParts, + validWriteIdList != null ? validWriteIdList.toString() : null); + partitions.addAll(convertFromPartSpec(msParts.iterator(), tbl)); return hasUnknownParts; + } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 4e77bd4..ed0f141 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -55,6 +55,8 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.GetPartitionsPsWithAuthRequest; +import org.apache.hadoop.hive.metastore.api.GetPartitionsPsWithAuthResponse; import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; @@ -68,6 +70,7 @@ import org.apache.hadoop.hive.metastore.api.PartitionSpec; import org.apache.hadoop.hive.metastore.api.PartitionValuesRequest; import org.apache.hadoop.hive.metastore.api.PartitionValuesResponse; import org.apache.hadoop.hive.metastore.api.PartitionValuesRow; +import org.apache.hadoop.hive.metastore.api.PartitionsByExprRequest; import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.metastore.api.TableMeta; @@ -76,6 +79,7 @@ import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; import org.apache.hadoop.hive.metastore.parser.ExpressionTree; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; +import org.apache.hadoop.hive.metastore.utils.FilterUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; @@ -1099,6 +1103,24 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I List<Partition> partitions = tt.listPartitionsWithAuthInfo(userName, groupNames); return getPartitionsForMaxParts(tableName, partitions, maxParts); } + + @Override + public GetPartitionsPsWithAuthResponse listPartitionsWithAuthInfoRequest( + GetPartitionsPsWithAuthRequest req) + throws MetaException, TException, NoSuchObjectException { + org.apache.hadoop.hive.metastore.api.Table table = getTempTable(req.getDbName(), + req.getTblName()); + if (table == null) { + return super.listPartitionsWithAuthInfoRequest(req); + } + TempTable tt = getPartitionedTempTable(table); + List<Partition> partitions = tt + .listPartitionsWithAuthInfo(req.getUserName(), req.getGroupNames()); + GetPartitionsPsWithAuthResponse response = new GetPartitionsPsWithAuthResponse(); + response.setPartitions( + getPartitionsForMaxParts(req.getTblName(), partitions, req.getMaxParts())); + return response; + } @Override public List<String> listPartitionNames(String catName, String dbName, String tblName, @@ -1135,6 +1157,26 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I } @Override + public List<String> listPartitionNames(PartitionsByExprRequest request) + throws MetaException, TException, NoSuchObjectException { + + org.apache.hadoop.hive.metastore.api.Table table = getTempTable(request.getDbName(), request.getTblName()); + if (table == null) { + return super.listPartitionNames(request.getCatName(), request.getDbName(), request.getTblName(), + request.getMaxParts()); + } + TempTable tt = getPartitionedTempTable(table); + List<Partition> partitions = tt.listPartitions(); + List<String> result = new ArrayList<>(); + for (int i = 0; i < ((request.getMaxParts() < 0 || request.getMaxParts() > partitions.size()) ? partitions.size() + : request.getMaxParts()); i++) { + result.add(makePartName(table.getPartitionKeys(), partitions.get(i).getValues())); + } + Collections.sort(result); + return result; + } + + @Override public List<Partition> listPartitions(String catName, String dbName, String tblName, int maxParts) throws TException { org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbName, tblName); @@ -1182,10 +1224,12 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I @Override public boolean listPartitionsSpecByExpr(String catName, String dbName, String tblName, byte[] expr, - String defaultPartitionName, short maxParts, List<PartitionSpec> result) throws TException { + String defaultPartitionName, short maxParts, List<PartitionSpec> result, + String validWriteIdList) throws TException { org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbName, tblName); if (table == null) { - return super.listPartitionsSpecByExpr(catName, dbName, tblName, expr, defaultPartitionName, maxParts, result); + return super.listPartitionsSpecByExpr(catName, dbName, tblName, expr, + defaultPartitionName, maxParts, result, validWriteIdList); } assert result != null; diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java b/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java index ebbbfa6..7160b9f 100644 --- a/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java +++ b/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java @@ -186,7 +186,8 @@ public class TestMetastoreExpr { // check with partition spec as well List<PartitionSpec> partSpec = new ArrayList<>(); client.listPartitionsSpecByExpr(dbName, tblName, - SerializationUtilities.serializeExpressionToKryo(expr), null, (short)-1, partSpec); + SerializationUtilities.serializeExpressionToKryo(expr), + null, (short)-1, partSpec, null); int partSpecSize = 0; if(!partSpec.isEmpty()) { partSpecSize = partSpec.iterator().next().getSharedSDPartitionSpec().getPartitionsSize(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientListPartitionsTempTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientListPartitionsTempTable.java index 43c3023..f6e7136 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientListPartitionsTempTable.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientListPartitionsTempTable.java @@ -245,7 +245,8 @@ public class TestSessionHiveMetastoreClientListPartitionsTempTable createTable4PartColsParts(getClient()); TestMetastoreExpr.ExprBuilder e = new TestMetastoreExpr.ExprBuilder(TABLE_NAME); getClient().listPartitionsByExpr(DB_NAME, TABLE_NAME, SerializationUtilities.serializeExpressionToKryo( - e.strCol("yyyy").val("2017").pred("=", 2).build()), null, (short)-1, null); + e.strCol("yyyy").val("2017").pred("=", 2).build()), null, + (short)-1, null); } @Test @@ -333,7 +334,8 @@ public class TestSessionHiveMetastoreClientListPartitionsTempTable createTable4PartColsParts(getClient()); TestMetastoreExpr.ExprBuilder e = new TestMetastoreExpr.ExprBuilder(TABLE_NAME); getClient().listPartitionsSpecByExpr(DB_NAME, TABLE_NAME, SerializationUtilities.serializeExpressionToKryo( - e.strCol("yyyy").val("2017").pred("=", 2).build()), null, (short)-1, null); + e.strCol("yyyy").val("2017").pred("=", 2).build()), null, + (short)-1, null, null); } @@ -344,7 +346,8 @@ public class TestSessionHiveMetastoreClientListPartitionsTempTable TestMetastoreExpr.ExprBuilder e = new TestMetastoreExpr.ExprBuilder(TABLE_NAME); List<PartitionSpec> result = new ArrayList<>(); getClient().listPartitionsSpecByExpr(DB_NAME, TABLE_NAME, SerializationUtilities.serializeExpressionToKryo( - e.strCol("yyyy").val("2017").pred(">=", 2).build()), null, (short)3, result); + e.strCol("yyyy").val("2017").pred(">=", 2).build()), + null, (short)3, result, null); assertEquals(3, result.iterator().next().getSharedSDPartitionSpec().getPartitionsSize()); } @@ -354,7 +357,8 @@ public class TestSessionHiveMetastoreClientListPartitionsTempTable TestMetastoreExpr.ExprBuilder e = new TestMetastoreExpr.ExprBuilder(TABLE_NAME); List<PartitionSpec> result = new ArrayList<>(); getClient().listPartitionsSpecByExpr(DB_NAME, TABLE_NAME, SerializationUtilities.serializeExpressionToKryo( - e.strCol("yyyy").val("2017").pred(">=", 2).build()), null, (short)100, result); + e.strCol("yyyy").val("2017").pred(">=", 2).build()), + null, (short)100, result, null); assertEquals(4, result.iterator().next().getSharedSDPartitionSpec().getPartitionsSize()); } @@ -362,38 +366,38 @@ public class TestSessionHiveMetastoreClientListPartitionsTempTable public void testListPartitionsSpecByExprNoDb() throws Exception { getClient().dropDatabase(DB_NAME); getClient().listPartitionsSpecByExpr(DB_NAME, TABLE_NAME, new byte[] {'f', 'o', 'o'}, - null, (short)-1, new ArrayList<>()); + null, (short)-1, new ArrayList<>(), null); } @Test(expected = MetaException.class) public void testListPartitionsSpecByExprNoTbl() throws Exception { getClient().listPartitionsSpecByExpr(DB_NAME, TABLE_NAME, new byte[] {'f', 'o', 'o'}, - null, (short)-1, new ArrayList<>()); + null, (short)-1, new ArrayList<>(), null); } @Test(expected = MetaException.class) public void testListPartitionsSpecByExprEmptyDbName() throws Exception { getClient().listPartitionsSpecByExpr("", TABLE_NAME, new byte[] {'f', 'o', 'o'}, - null, (short)-1, new ArrayList<>()); + null, (short)-1, new ArrayList<>(), null); } @Test(expected = MetaException.class) public void testListPartitionsSpecByExprEmptyTblName() throws Exception { createTable3PartCols1Part(getClient()); getClient().listPartitionsSpecByExpr(DB_NAME, "", new byte[] {'f', 'o', 'o'}, - null, (short)-1, new ArrayList<>()); + null, (short)-1, new ArrayList<>(), null); } @Test(expected = MetaException.class) public void testListPartitionsSpecByExprNullDbName() throws Exception { getClient().listPartitionsSpecByExpr(null, TABLE_NAME, new byte[] {'f', 'o', 'o'}, - null, (short)-1, new ArrayList<>()); + null, (short)-1, new ArrayList<>(), null); } @Test(expected = MetaException.class) public void testListPartitionsSpecByExprNullTblName() throws Exception { getClient().listPartitionsSpecByExpr(DB_NAME, null, new byte[] {'f', 'o', 'o' }, - null, (short)-1, new ArrayList<>()); + null, (short)-1, new ArrayList<>(), null); } private void checkExprPartitionSpec(int numParts, ExprNodeGenericFuncDesc expr) throws Exception { @@ -404,7 +408,8 @@ public class TestSessionHiveMetastoreClientListPartitionsTempTable // check with partition spec as well List<PartitionSpec> partSpec = new ArrayList<>(); getClient().listPartitionsSpecByExpr(DB_NAME, TABLE_NAME, - SerializationUtilities.serializeExpressionToKryo(expr), null, (short)-1, partSpec); + SerializationUtilities.serializeExpressionToKryo(expr), null, + (short)-1, partSpec, null); int partSpecSize = 0; if(!partSpec.isEmpty()) { partSpecSize = partSpec.iterator().next().getSharedSDPartitionSpec().getPartitionsSize(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java index 3204669..a0cd60f 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java @@ -20,12 +20,15 @@ package org.apache.hadoop.hive.ql.parse; import java.io.File; import java.io.IOException; import java.util.Arrays; +import java.util.BitSet; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.ValidReadTxnList; +import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; @@ -232,6 +235,8 @@ public class TestUpdateDeleteSemanticAnalyzer { conf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict"); conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); conf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, true); + conf.set(ValidTxnList.VALID_TXNS_KEY, + new ValidReadTxnList(new long[0], new BitSet(), 1000, Long.MAX_VALUE).writeToString()); TxnDbUtil.prepDb(conf); } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 8041bc5..4a45913 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -1846,6 +1846,10 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { if (req.getValidWriteIdList() == null) { req.setValidWriteIdList(getValidWriteIdList(TableName.getDbTable(req.getDbName(), req.getTblName()))); } + if(req.getCatName() == null) { + req.setCatName(getDefaultCatalog(conf)); + } + req.setMaxParts(shrinkMaxtoShort(req.getMaxParts())); GetPartitionsPsWithAuthResponse res = client.get_partitions_ps_with_auth_req(req); List<Partition> parts = deepCopyPartitions( FilterUtils.filterPartitionsIfEnabled(isClientFilterEnabled, filterHook, res.getPartitions())); @@ -1932,6 +1936,11 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { assert result != null; PartitionsByExprRequest req = new PartitionsByExprRequest( db_name, tbl_name, ByteBuffer.wrap(expr)); + if( catName == null ) { + req.setCatName(getDefaultCatalog(conf)); + }else { + req.setCatName(catName); + } if (default_partition_name != null) { req.setDefaultPartitionName(default_partition_name); } @@ -1961,15 +1970,15 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { @Override public boolean listPartitionsSpecByExpr(String dbName, String tblName, - byte[] expr, String defaultPartName, short maxParts, List<PartitionSpec> result) + byte[] expr, String defaultPartName, short maxParts, List<PartitionSpec> result, String validWriteIdList) throws TException { return listPartitionsSpecByExpr(getDefaultCatalog(conf), dbName, tblName, expr, defaultPartName, - maxParts, result); + maxParts, result, validWriteIdList); } @Override public boolean listPartitionsSpecByExpr(String catName, String dbName, String tblName, byte[] expr, - String defaultPartitionName, short maxParts, List<PartitionSpec> result) + String defaultPartitionName, short maxParts, List<PartitionSpec> result, String validWriteIdList) throws TException { assert result != null; PartitionsByExprRequest req = new PartitionsByExprRequest( @@ -1980,6 +1989,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { if (maxParts >= 0) { req.setMaxParts(maxParts); } + req.setValidWriteIdList(validWriteIdList); PartitionsSpecByExprResult r; try { r = client.get_partitions_spec_by_expr(req); @@ -2155,6 +2165,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { req.setCapabilities(version); req.setGetColumnStats(getColumnStats); req.setValidWriteIdList(getValidWriteIdList(TableName.getDbTable(dbName, tableName))); + if (getColumnStats) { req.setEngine(engine); } @@ -2414,6 +2425,9 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { if (req.getValidWriteIdList() == null) { req.setValidWriteIdList(getValidWriteIdList(TableName.getDbTable(req.getDbName(), req.getTblName()))); } + if( req.getCatName() == null ) { + req.setCatName(getDefaultCatalog(conf)); + } GetPartitionNamesPsResponse res = client.get_partition_names_ps_req(req); List<String> partNames = FilterUtils.filterPartitionNamesIfEnabled( isClientFilterEnabled, filterHook, getDefaultCatalog(conf), req.getDbName(), @@ -2463,12 +2477,23 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { if (order != null) { req.setOrder(order); } - req.setCatName(catName); + if ( catName == null ) { + req.setCatName(getDefaultCatalog(conf)); + }else { + req.setCatName(catName); + } return FilterUtils.filterPartitionNamesIfEnabled(isClientFilterEnabled, filterHook, catName, dbName, tblName, client.get_partition_names_req(req)); } @Override + public List<String> listPartitionNames(PartitionsByExprRequest req) + throws MetaException, TException, NoSuchObjectException { + return FilterUtils.filterPartitionNamesIfEnabled(isClientFilterEnabled, filterHook, req.getCatName(), + req.getDbName(), req.getTblName(), client.get_partition_names_req(req)); + } + + @Override public int getNumPartitionsByFilter(String db_name, String tbl_name, String filter) throws TException { return getNumPartitionsByFilter(getDefaultCatalog(conf), db_name, tbl_name, filter); @@ -4175,6 +4200,18 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { return client.get_serde(new GetSerdeRequest(serDeName)); } + /** + * This method is called to get the ValidWriteIdList in order to send the same in HMS get_* APIs, + * if the validWriteIdList is not explicitly passed (as a method argument) to the HMS APIs. + * This method returns the ValidWriteIdList based on the VALID_TABLES_WRITEIDS_KEY key. + * Since, VALID_TABLES_WRITEIDS_KEY is set during the lock acquisition phase after query compilation + * ( DriverTxnHandler.acquireLocks -> recordValidWriteIds -> setValidWriteIds ), + * this only covers a subset of cases, where we invoke get_* APIs after query compilation, + * if the validWriteIdList is not explicitly passed (as a method argument) to the HMS APIs. + * + * @param fullTableName + * @return + */ private String getValidWriteIdList(String fullTableName) { if (conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY) == null) { return null; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index dbc6d14..1b17a41 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -670,6 +670,7 @@ public interface IMetaStoreClient { */ Table getTable(String dbName, String tableName, boolean getColumnStats, String engine) throws MetaException, TException, NoSuchObjectException; + /** * Get a table object. * @param catName catalog the table is in. @@ -1253,6 +1254,17 @@ public interface IMetaStoreClient { throws MetaException, TException, NoSuchObjectException; /** + * Get a list of partition names matching the specified filter and return in order if specified. + * @param request request + * @return list of matching partition names. + * @throws MetaException error accessing the RDBMS. + * @throws TException thrift transport error. + * @throws NoSuchObjectException no such table. + */ + List<String> listPartitionNames(PartitionsByExprRequest request) + throws MetaException, TException, NoSuchObjectException; + + /** * Get a list of partition values * @param request request * @return reponse @@ -1378,7 +1390,7 @@ public interface IMetaStoreClient { * @throws TException thrift transport error or error executing the filter. */ boolean listPartitionsSpecByExpr(String dbName, String tblName, - byte[] expr, String defaultPartName, short maxParts, List<PartitionSpec> result) + byte[] expr, String defaultPartName, short maxParts, List<PartitionSpec> result, String validWriteIdList) throws TException; /** @@ -1396,7 +1408,7 @@ public interface IMetaStoreClient { * @throws TException thrift transport error or error executing the filter. */ boolean listPartitionsSpecByExpr(String catName, String dbName, String tblName, - byte[] expr, String defaultPartitionName, short maxParts, List<PartitionSpec> result) + byte[] expr, String defaultPartitionName, short maxParts, List<PartitionSpec> result, String validWriteIdList) throws TException; /** diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index ee0f22e..a9175ce 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -6262,8 +6262,15 @@ public class HiveMetaStore extends ThriftHiveMetastore { public GetPartitionsPsWithAuthResponse get_partitions_ps_with_auth_req(GetPartitionsPsWithAuthRequest req) throws MetaException, NoSuchObjectException, TException { String dbName = MetaStoreUtils.prependCatalogToDbName(req.getCatName(), req.getDbName(), conf); - List<Partition> partitions = get_partitions_ps_with_auth(dbName, req.getTblName(), - req.getPartVals(), req.getMaxParts(), req.getUserName(), req.getGroupNames()); + List<Partition> partitions = null; + if (req.getPartVals() == null) { + partitions = get_partitions_with_auth(dbName, req.getTblName(), req.getMaxParts(), req.getUserName(), + req.getGroupNames()); + } else { + partitions = + get_partitions_ps_with_auth(dbName, req.getTblName(), req.getPartVals(), req.getMaxParts(), + req.getUserName(), req.getGroupNames()); + } GetPartitionsPsWithAuthResponse res = new GetPartitionsPsWithAuthResponse(); res.setPartitions(partitions); return res; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index d7b5a09..97a9507 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -1377,15 +1377,15 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos @Override public boolean listPartitionsSpecByExpr(String dbName, String tblName, - byte[] expr, String defaultPartName, short maxParts, List<PartitionSpec> result) + byte[] expr, String defaultPartName, short maxParts, List<PartitionSpec> result, String validWriteIdList) throws TException { return listPartitionsSpecByExpr(getDefaultCatalog(conf), dbName, tblName, expr, defaultPartName, - maxParts, result); + maxParts, result, validWriteIdList); } @Override public boolean listPartitionsSpecByExpr(String catName, String dbName, String tblName, byte[] expr, - String defaultPartitionName, short maxParts, List<PartitionSpec> result) + String defaultPartitionName, short maxParts, List<PartitionSpec> result, String validWriteIdList) throws TException { assert result != null; PartitionsByExprRequest req = new PartitionsByExprRequest(dbName, tblName, ByteBuffer.wrap(expr)); @@ -1395,6 +1395,7 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos if (maxParts >= 0) { req.setMaxParts(maxParts); } + req.setValidWriteIdList(validWriteIdList); PartitionsSpecByExprResult r; try { r = client.get_partitions_spec_by_expr(req); @@ -1487,6 +1488,12 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos } @Override + public List<String> listPartitionNames(PartitionsByExprRequest request) + throws MetaException, TException, NoSuchObjectException { + throw new UnsupportedOperationException(); + } + + @Override public PartitionValuesResponse listPartitionValues(PartitionValuesRequest request) throws MetaException, TException, NoSuchObjectException { return client.get_partition_values(request);