This is an automated email from the ASF dual-hosted git repository. ngangam pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 8f75876 HIVE-25753: Improving performance of getLatestCommittedCompactionInfo (Yu-Wen Lai reviewed by Karen Coppage) 8f75876 is described below commit 8f7587694ab9ba9dd152af274278074c31d0a453 Author: Yu-Wen Lai <yu-wen....@cloudera.com> AuthorDate: Tue Nov 30 12:21:48 2021 -0800 HIVE-25753: Improving performance of getLatestCommittedCompactionInfo (Yu-Wen Lai reviewed by Karen Coppage) --- .../metastore/txn/TestCompactionTxnHandler.java | 39 ++++++++ .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 42 ++++++-- .../src/gen/thrift/gen-cpp/hive_metastore_types.h | 12 ++- .../GetLatestCommittedCompactionInfoRequest.java | 110 ++++++++++++++++++++- .../GetLatestCommittedCompactionInfoRequest.php | 24 +++++ .../src/gen/thrift/gen-py/hive_metastore/ttypes.py | 14 ++- .../src/gen/thrift/gen-rb/hive_metastore_types.rb | 4 +- .../src/main/thrift/hive_metastore.thrift | 1 + .../hadoop/hive/metastore/txn/TxnHandler.java | 8 +- 9 files changed, 233 insertions(+), 21 deletions(-) diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java index 9bfc324..a3b814e 100644 --- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java +++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java @@ -330,6 +330,18 @@ public class TestCompactionTxnHandler { CompactionInfoStruct lci2 = response.getCompactions().stream(). filter(c -> c.getPartitionname().equals(anotherPartitionName)).findFirst().get(); assertEquals(3, lci2.getId()); + + // check the result is correct without setting partition names + rqst.unsetPartitionnames(); + response = txnHandler.getLatestCommittedCompactionInfo(rqst); + assertNotNull(response); + assertEquals("Expecting a single compaction record for each partition", 2, response.getCompactionsSize()); + lci1 = response.getCompactions().stream() + .filter(c -> c.getPartitionname().equals(partitionName)).findFirst().get(); + assertEquals(1, lci1.getId()); + lci2 = response.getCompactions().stream(). + filter(c -> c.getPartitionname().equals(anotherPartitionName)).findFirst().get(); + assertEquals(3, lci2.getId()); } @Test @@ -366,6 +378,33 @@ public class TestCompactionTxnHandler { } @Test + public void testGetLatestCompactionWithIdFilter() throws Exception { + final String dbName = "foo"; + final String tableName = "bar"; + final String partitionName = "ds=today"; + addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR); + addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR); + GetLatestCommittedCompactionInfoRequest rqst = new GetLatestCommittedCompactionInfoRequest(); + rqst.setDbname(dbName); + rqst.setTablename(tableName); + rqst.addToPartitionnames(partitionName); + GetLatestCommittedCompactionInfoResponse response = txnHandler.getLatestCommittedCompactionInfo(rqst); + + assertNotNull(response); + assertEquals("Expecting a single record", 1, response.getCompactionsSize()); + CompactionInfoStruct lci = response.getCompactions().get(0); + assertEquals("Expecting the second succeeded compaction record", 2, lci.getId()); + assertEquals(partitionName, lci.getPartitionname()); + assertEquals(CompactionType.MINOR, lci.getType()); + + // response should only include compaction with id > 2 + rqst.setLastCompactionId(2); + response = txnHandler.getLatestCommittedCompactionInfo(rqst); + assertNotNull(response); + assertEquals("Expecting no record", 0, response.getCompactionsSize()); + } + + @Test public void testGetNoCompaction() throws Exception { final String dbName = "foo"; final String tableName = "bar"; diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp index 85ad737..d525faf 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp @@ -28149,6 +28149,11 @@ void GetLatestCommittedCompactionInfoRequest::__set_partitionnames(const std::ve this->partitionnames = val; __isset.partitionnames = true; } + +void GetLatestCommittedCompactionInfoRequest::__set_lastCompactionId(const int64_t val) { + this->lastCompactionId = val; +__isset.lastCompactionId = true; +} std::ostream& operator<<(std::ostream& out, const GetLatestCommittedCompactionInfoRequest& obj) { obj.printTo(out); @@ -28215,6 +28220,14 @@ uint32_t GetLatestCommittedCompactionInfoRequest::read(::apache::thrift::protoco xfer += iprot->skip(ftype); } break; + case 4: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->lastCompactionId); + this->__isset.lastCompactionId = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -28257,6 +28270,11 @@ uint32_t GetLatestCommittedCompactionInfoRequest::write(::apache::thrift::protoc } xfer += oprot->writeFieldEnd(); } + if (this->__isset.lastCompactionId) { + xfer += oprot->writeFieldBegin("lastCompactionId", ::apache::thrift::protocol::T_I64, 4); + xfer += oprot->writeI64(this->lastCompactionId); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -28267,20 +28285,23 @@ void swap(GetLatestCommittedCompactionInfoRequest &a, GetLatestCommittedCompacti swap(a.dbname, b.dbname); swap(a.tablename, b.tablename); swap(a.partitionnames, b.partitionnames); + swap(a.lastCompactionId, b.lastCompactionId); swap(a.__isset, b.__isset); } -GetLatestCommittedCompactionInfoRequest::GetLatestCommittedCompactionInfoRequest(const GetLatestCommittedCompactionInfoRequest& other1007) { - dbname = other1007.dbname; - tablename = other1007.tablename; - partitionnames = other1007.partitionnames; - __isset = other1007.__isset; +GetLatestCommittedCompactionInfoRequest::GetLatestCommittedCompactionInfoRequest(const GetLatestCommittedCompactionInfoRequest& other1000) { + dbname = other1000.dbname; + tablename = other1000.tablename; + partitionnames = other1000.partitionnames; + lastCompactionId = other1000.lastCompactionId; + __isset = other1000.__isset; } -GetLatestCommittedCompactionInfoRequest& GetLatestCommittedCompactionInfoRequest::operator=(const GetLatestCommittedCompactionInfoRequest& other1008) { - dbname = other1008.dbname; - tablename = other1008.tablename; - partitionnames = other1008.partitionnames; - __isset = other1008.__isset; +GetLatestCommittedCompactionInfoRequest& GetLatestCommittedCompactionInfoRequest::operator=(const GetLatestCommittedCompactionInfoRequest& other1001) { + dbname = other1001.dbname; + tablename = other1001.tablename; + partitionnames = other1001.partitionnames; + lastCompactionId = other1001.lastCompactionId; + __isset = other1001.__isset; return *this; } void GetLatestCommittedCompactionInfoRequest::printTo(std::ostream& out) const { @@ -28289,6 +28310,7 @@ void GetLatestCommittedCompactionInfoRequest::printTo(std::ostream& out) const { out << "dbname=" << to_string(dbname); out << ", " << "tablename=" << to_string(tablename); out << ", " << "partitionnames="; (__isset.partitionnames ? (out << to_string(partitionnames)) : (out << "<null>")); + out << ", " << "lastCompactionId="; (__isset.lastCompactionId ? (out << to_string(lastCompactionId)) : (out << "<null>")); out << ")"; } diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h index d383d56..1e05408 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h @@ -10640,8 +10640,9 @@ void swap(ShowCompactResponse &a, ShowCompactResponse &b); std::ostream& operator<<(std::ostream& out, const ShowCompactResponse& obj); typedef struct _GetLatestCommittedCompactionInfoRequest__isset { - _GetLatestCommittedCompactionInfoRequest__isset() : partitionnames(false) {} + _GetLatestCommittedCompactionInfoRequest__isset() : partitionnames(false), lastCompactionId(false) {} bool partitionnames :1; + bool lastCompactionId :1; } _GetLatestCommittedCompactionInfoRequest__isset; class GetLatestCommittedCompactionInfoRequest : public virtual ::apache::thrift::TBase { @@ -10649,13 +10650,14 @@ class GetLatestCommittedCompactionInfoRequest : public virtual ::apache::thrift: GetLatestCommittedCompactionInfoRequest(const GetLatestCommittedCompactionInfoRequest&); GetLatestCommittedCompactionInfoRequest& operator=(const GetLatestCommittedCompactionInfoRequest&); - GetLatestCommittedCompactionInfoRequest() : dbname(), tablename() { + GetLatestCommittedCompactionInfoRequest() : dbname(), tablename(), lastCompactionId(0) { } virtual ~GetLatestCommittedCompactionInfoRequest() noexcept; std::string dbname; std::string tablename; std::vector<std::string> partitionnames; + int64_t lastCompactionId; _GetLatestCommittedCompactionInfoRequest__isset __isset; @@ -10665,6 +10667,8 @@ class GetLatestCommittedCompactionInfoRequest : public virtual ::apache::thrift: void __set_partitionnames(const std::vector<std::string> & val); + void __set_lastCompactionId(const int64_t val); + bool operator == (const GetLatestCommittedCompactionInfoRequest & rhs) const { if (!(dbname == rhs.dbname)) @@ -10675,6 +10679,10 @@ class GetLatestCommittedCompactionInfoRequest : public virtual ::apache::thrift: return false; else if (__isset.partitionnames && !(partitionnames == rhs.partitionnames)) return false; + if (__isset.lastCompactionId != rhs.__isset.lastCompactionId) + return false; + else if (__isset.lastCompactionId && !(lastCompactionId == rhs.lastCompactionId)) + return false; return true; } bool operator != (const GetLatestCommittedCompactionInfoRequest &rhs) const { diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetLatestCommittedCompactionInfoRequest.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetLatestCommittedCompactionInfoRequest.java index a3e71e5..09fc08a 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetLatestCommittedCompactionInfoRequest.java +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetLatestCommittedCompactionInfoRequest.java @@ -14,6 +14,7 @@ package org.apache.hadoop.hive.metastore.api; private static final org.apache.thrift.protocol.TField DBNAME_FIELD_DESC = new org.apache.thrift.protocol.TField("dbname", org.apache.thrift.protocol.TType.STRING, (short)1); private static final org.apache.thrift.protocol.TField TABLENAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tablename", org.apache.thrift.protocol.TType.STRING, (short)2); private static final org.apache.thrift.protocol.TField PARTITIONNAMES_FIELD_DESC = new org.apache.thrift.protocol.TField("partitionnames", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField LAST_COMPACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("lastCompactionId", org.apache.thrift.protocol.TType.I64, (short)4); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new GetLatestCommittedCompactionInfoRequestStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new GetLatestCommittedCompactionInfoRequestTupleSchemeFactory(); @@ -21,12 +22,14 @@ package org.apache.hadoop.hive.metastore.api; private @org.apache.thrift.annotation.Nullable java.lang.String dbname; // required private @org.apache.thrift.annotation.Nullable java.lang.String tablename; // required private @org.apache.thrift.annotation.Nullable java.util.List<java.lang.String> partitionnames; // optional + private long lastCompactionId; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { DBNAME((short)1, "dbname"), TABLENAME((short)2, "tablename"), - PARTITIONNAMES((short)3, "partitionnames"); + PARTITIONNAMES((short)3, "partitionnames"), + LAST_COMPACTION_ID((short)4, "lastCompactionId"); private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>(); @@ -48,6 +51,8 @@ package org.apache.hadoop.hive.metastore.api; return TABLENAME; case 3: // PARTITIONNAMES return PARTITIONNAMES; + case 4: // LAST_COMPACTION_ID + return LAST_COMPACTION_ID; default: return null; } @@ -89,7 +94,9 @@ package org.apache.hadoop.hive.metastore.api; } // isset id assignments - private static final _Fields optionals[] = {_Fields.PARTITIONNAMES}; + private static final int __LASTCOMPACTIONID_ISSET_ID = 0; + private byte __isset_bitfield = 0; + private static final _Fields optionals[] = {_Fields.PARTITIONNAMES,_Fields.LAST_COMPACTION_ID}; public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -100,6 +107,8 @@ package org.apache.hadoop.hive.metastore.api; tmpMap.put(_Fields.PARTITIONNAMES, new org.apache.thrift.meta_data.FieldMetaData("partitionnames", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.LAST_COMPACTION_ID, new org.apache.thrift.meta_data.FieldMetaData("lastCompactionId", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetLatestCommittedCompactionInfoRequest.class, metaDataMap); } @@ -120,6 +129,7 @@ package org.apache.hadoop.hive.metastore.api; * Performs a deep copy on <i>other</i>. */ public GetLatestCommittedCompactionInfoRequest(GetLatestCommittedCompactionInfoRequest other) { + __isset_bitfield = other.__isset_bitfield; if (other.isSetDbname()) { this.dbname = other.dbname; } @@ -130,6 +140,7 @@ package org.apache.hadoop.hive.metastore.api; java.util.List<java.lang.String> __this__partitionnames = new java.util.ArrayList<java.lang.String>(other.partitionnames); this.partitionnames = __this__partitionnames; } + this.lastCompactionId = other.lastCompactionId; } public GetLatestCommittedCompactionInfoRequest deepCopy() { @@ -141,6 +152,8 @@ package org.apache.hadoop.hive.metastore.api; this.dbname = null; this.tablename = null; this.partitionnames = null; + setLastCompactionIdIsSet(false); + this.lastCompactionId = 0; } @org.apache.thrift.annotation.Nullable @@ -231,6 +244,28 @@ package org.apache.hadoop.hive.metastore.api; } } + public long getLastCompactionId() { + return this.lastCompactionId; + } + + public void setLastCompactionId(long lastCompactionId) { + this.lastCompactionId = lastCompactionId; + setLastCompactionIdIsSet(true); + } + + public void unsetLastCompactionId() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __LASTCOMPACTIONID_ISSET_ID); + } + + /** Returns true if field lastCompactionId is set (has been assigned a value) and false otherwise */ + public boolean isSetLastCompactionId() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __LASTCOMPACTIONID_ISSET_ID); + } + + public void setLastCompactionIdIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __LASTCOMPACTIONID_ISSET_ID, value); + } + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { case DBNAME: @@ -257,6 +292,14 @@ package org.apache.hadoop.hive.metastore.api; } break; + case LAST_COMPACTION_ID: + if (value == null) { + unsetLastCompactionId(); + } else { + setLastCompactionId((java.lang.Long)value); + } + break; + } } @@ -272,6 +315,9 @@ package org.apache.hadoop.hive.metastore.api; case PARTITIONNAMES: return getPartitionnames(); + case LAST_COMPACTION_ID: + return getLastCompactionId(); + } throw new java.lang.IllegalStateException(); } @@ -289,6 +335,8 @@ package org.apache.hadoop.hive.metastore.api; return isSetTablename(); case PARTITIONNAMES: return isSetPartitionnames(); + case LAST_COMPACTION_ID: + return isSetLastCompactionId(); } throw new java.lang.IllegalStateException(); } @@ -333,6 +381,15 @@ package org.apache.hadoop.hive.metastore.api; return false; } + boolean this_present_lastCompactionId = true && this.isSetLastCompactionId(); + boolean that_present_lastCompactionId = true && that.isSetLastCompactionId(); + if (this_present_lastCompactionId || that_present_lastCompactionId) { + if (!(this_present_lastCompactionId && that_present_lastCompactionId)) + return false; + if (this.lastCompactionId != that.lastCompactionId) + return false; + } + return true; } @@ -352,6 +409,10 @@ package org.apache.hadoop.hive.metastore.api; if (isSetPartitionnames()) hashCode = hashCode * 8191 + partitionnames.hashCode(); + hashCode = hashCode * 8191 + ((isSetLastCompactionId()) ? 131071 : 524287); + if (isSetLastCompactionId()) + hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(lastCompactionId); + return hashCode; } @@ -393,6 +454,16 @@ package org.apache.hadoop.hive.metastore.api; return lastComparison; } } + lastComparison = java.lang.Boolean.compare(isSetLastCompactionId(), other.isSetLastCompactionId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetLastCompactionId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.lastCompactionId, other.lastCompactionId); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -439,6 +510,12 @@ package org.apache.hadoop.hive.metastore.api; } first = false; } + if (isSetLastCompactionId()) { + if (!first) sb.append(", "); + sb.append("lastCompactionId:"); + sb.append(this.lastCompactionId); + first = false; + } sb.append(")"); return sb.toString(); } @@ -466,6 +543,8 @@ package org.apache.hadoop.hive.metastore.api; private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); } catch (org.apache.thrift.TException te) { throw new java.io.IOException(te); @@ -524,6 +603,14 @@ package org.apache.hadoop.hive.metastore.api; org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 4: // LAST_COMPACTION_ID + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.lastCompactionId = iprot.readI64(); + struct.setLastCompactionIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -561,6 +648,11 @@ package org.apache.hadoop.hive.metastore.api; oprot.writeFieldEnd(); } } + if (struct.isSetLastCompactionId()) { + oprot.writeFieldBegin(LAST_COMPACTION_ID_FIELD_DESC); + oprot.writeI64(struct.lastCompactionId); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -584,7 +676,10 @@ package org.apache.hadoop.hive.metastore.api; if (struct.isSetPartitionnames()) { optionals.set(0); } - oprot.writeBitSet(optionals, 1); + if (struct.isSetLastCompactionId()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); if (struct.isSetPartitionnames()) { { oprot.writeI32(struct.partitionnames.size()); @@ -594,6 +689,9 @@ package org.apache.hadoop.hive.metastore.api; } } } + if (struct.isSetLastCompactionId()) { + oprot.writeI64(struct.lastCompactionId); + } } @Override @@ -603,7 +701,7 @@ package org.apache.hadoop.hive.metastore.api; struct.setDbnameIsSet(true); struct.tablename = iprot.readString(); struct.setTablenameIsSet(true); - java.util.BitSet incoming = iprot.readBitSet(1); + java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { { org.apache.thrift.protocol.TList _list877 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRING); @@ -617,6 +715,10 @@ package org.apache.hadoop.hive.metastore.api; } struct.setPartitionnamesIsSet(true); } + if (incoming.get(1)) { + struct.lastCompactionId = iprot.readI64(); + struct.setLastCompactionIdIsSet(true); + } } } diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetLatestCommittedCompactionInfoRequest.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetLatestCommittedCompactionInfoRequest.php index 2bbeda9..5eb0b47 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetLatestCommittedCompactionInfoRequest.php +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetLatestCommittedCompactionInfoRequest.php @@ -40,6 +40,11 @@ class GetLatestCommittedCompactionInfoRequest 'type' => TType::STRING, ), ), + 4 => array( + 'var' => 'lastCompactionId', + 'isRequired' => false, + 'type' => TType::I64, + ), ); /** @@ -54,6 +59,10 @@ class GetLatestCommittedCompactionInfoRequest * @var string[] */ public $partitionnames = null; + /** + * @var int + */ + public $lastCompactionId = null; public function __construct($vals = null) { @@ -67,6 +76,9 @@ class GetLatestCommittedCompactionInfoRequest if (isset($vals['partitionnames'])) { $this->partitionnames = $vals['partitionnames']; } + if (isset($vals['lastCompactionId'])) { + $this->lastCompactionId = $vals['lastCompactionId']; + } } } @@ -119,6 +131,13 @@ class GetLatestCommittedCompactionInfoRequest $xfer += $input->skip($ftype); } break; + case 4: + if ($ftype == TType::I64) { + $xfer += $input->readI64($this->lastCompactionId); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -155,6 +174,11 @@ class GetLatestCommittedCompactionInfoRequest $output->writeListEnd(); $xfer += $output->writeFieldEnd(); } + if ($this->lastCompactionId !== null) { + $xfer += $output->writeFieldBegin('lastCompactionId', TType::I64, 4); + $xfer += $output->writeI64($this->lastCompactionId); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py index 298b6ed..c4d1fc7 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -16100,14 +16100,16 @@ class GetLatestCommittedCompactionInfoRequest(object): - dbname - tablename - partitionnames + - lastCompactionId """ - def __init__(self, dbname=None, tablename=None, partitionnames=None,): + def __init__(self, dbname=None, tablename=None, partitionnames=None, lastCompactionId=None,): self.dbname = dbname self.tablename = tablename self.partitionnames = partitionnames + self.lastCompactionId = lastCompactionId def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -16138,6 +16140,11 @@ class GetLatestCommittedCompactionInfoRequest(object): iprot.readListEnd() else: iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.lastCompactionId = iprot.readI64() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -16163,6 +16170,10 @@ class GetLatestCommittedCompactionInfoRequest(object): oprot.writeString(iter773.encode('utf-8') if sys.version_info[0] == 2 else iter773) oprot.writeListEnd() oprot.writeFieldEnd() + if self.lastCompactionId is not None: + oprot.writeFieldBegin('lastCompactionId', TType.I64, 4) + oprot.writeI64(self.lastCompactionId) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -30484,6 +30495,7 @@ GetLatestCommittedCompactionInfoRequest.thrift_spec = ( (1, TType.STRING, 'dbname', 'UTF8', None, ), # 1 (2, TType.STRING, 'tablename', 'UTF8', None, ), # 2 (3, TType.LIST, 'partitionnames', (TType.STRING, 'UTF8', False), None, ), # 3 + (4, TType.I64, 'lastCompactionId', None, None, ), # 4 ) all_structs.append(GetLatestCommittedCompactionInfoResponse) GetLatestCommittedCompactionInfoResponse.thrift_spec = ( diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb index 5c78f49..f73250d 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -4644,11 +4644,13 @@ class GetLatestCommittedCompactionInfoRequest DBNAME = 1 TABLENAME = 2 PARTITIONNAMES = 3 + LASTCOMPACTIONID = 4 FIELDS = { DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'}, TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename'}, - PARTITIONNAMES => {:type => ::Thrift::Types::LIST, :name => 'partitionnames', :element => {:type => ::Thrift::Types::STRING}, :optional => true} + PARTITIONNAMES => {:type => ::Thrift::Types::LIST, :name => 'partitionnames', :element => {:type => ::Thrift::Types::STRING}, :optional => true}, + LASTCOMPACTIONID => {:type => ::Thrift::Types::I64, :name => 'lastCompactionId', :optional => true} } def struct_fields; FIELDS; end diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 20259cc..9edc43a 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -1342,6 +1342,7 @@ struct GetLatestCommittedCompactionInfoRequest { 1: required string dbname, 2: required string tablename, 3: optional list<string> partitionnames, + 4: optional i64 lastCompactionId, } struct GetLatestCommittedCompactionInfoResponse { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index caa3ae2..84a059c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -4011,7 +4011,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { .append(" FROM \"COMPACTION_QUEUE\"") .append(" WHERE \"CQ_STATE\" = " + quoteChar(READY_FOR_CLEANING)) .append(") AS compactions ") - .append(" WHERE \"CC_DATABASE\"=? AND \"CC_TABLE\"=?"); + .append(" WHERE \"CC_DATABASE\" = ? AND \"CC_TABLE\" = ?"); params.add(rqst.getDbname()); params.add(rqst.getTablename()); if (rqst.getPartitionnamesSize() > 0) { @@ -4020,8 +4020,10 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { Collections.nCopies(rqst.getPartitionnamesSize(), "?"))); sb.append(")"); params.addAll(rqst.getPartitionnames()); - } else { - sb.append(" AND \"CC_PARTITION\" IS NULL"); + } + if (rqst.isSetLastCompactionId()) { + sb.append(" AND \"CC_ID\" > ?"); + params.add(String.valueOf(rqst.getLastCompactionId())); } sb.append(" ORDER BY \"CC_ID\" DESC");