This is an automated email from the ASF dual-hosted git repository. vihangk1 pushed a commit to branch branch-3 in repository https://gitbox.apache.org/repos/asf/hive.git
commit 7426703e60112937ed3749526fcfbfeda2bf2ace Author: Vihang Karajgaonkar <vihan...@apache.org> AuthorDate: Mon Mar 30 16:48:08 2020 -0700 HIVE-21851: FireEventResponse should include event id when available (Vihang Karajgaonkar, reviewed by Naveen Gangam) --- .../listener/TestDbNotificationListener.java | 12 ++- .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 34 ++++++- .../src/gen/thrift/gen-cpp/hive_metastore_types.h | 15 ++- .../hive/metastore/api/FireEventResponse.java | 112 ++++++++++++++++++++- .../src/gen/thrift/gen-php/metastore/Types.php | 27 ++++- .../src/gen/thrift/gen-py/hive_metastore/ttypes.py | 19 ++++ .../src/gen/thrift/gen-rb/hive_metastore_types.rb | 3 +- .../hadoop/hive/metastore/HiveMetaStore.java | 9 +- .../src/main/thrift/hive_metastore.thrift | 2 +- 9 files changed, 219 insertions(+), 14 deletions(-) diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java index 37b2bd8..bfb1982 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FireEventRequest; import org.apache.hadoop.hive.metastore.api.FireEventRequestData; +import org.apache.hadoop.hive.metastore.api.FireEventResponse; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.FunctionType; import org.apache.hadoop.hive.metastore.api.InsertEventRequestData; @@ -98,6 +99,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.hcatalog.api.repl.ReplicationV1CompatRule; import org.apache.hive.hcatalog.data.Pair; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; @@ -1047,7 +1049,10 @@ public class TestDbNotificationListener { rqst.setDbName(defaultDbName); rqst.setTableName(tblName); // Event 2 - msClient.fireListenerEvent(rqst); + FireEventResponse response = msClient.fireListenerEvent(rqst); + assertTrue("Event id must be set in the fireEvent response", response.isSetEventId()); + Assert.assertNotNull(response.getEventId()); + Assert.assertTrue(response.getEventId() != -1); // Get notifications from metastore NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); @@ -1116,7 +1121,10 @@ public class TestDbNotificationListener { rqst.setTableName(tblName); rqst.setPartitionVals(partCol1Vals); // Event 3 - msClient.fireListenerEvent(rqst); + FireEventResponse response = msClient.fireListenerEvent(rqst); + assertTrue("Event id must be set in the fireEvent response", response.isSetEventId()); + Assert.assertNotNull(response.getEventId()); + Assert.assertTrue(response.getEventId() != -1); // Get notifications from metastore NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp index 5f2948e..f3e5d3a 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp +++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp @@ -22756,6 +22756,10 @@ FireEventResponse::~FireEventResponse() throw() { } +void FireEventResponse::__set_eventId(const int64_t val) { + this->eventId = val; +} + uint32_t FireEventResponse::read(::apache::thrift::protocol::TProtocol* iprot) { apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); @@ -22775,7 +22779,20 @@ uint32_t FireEventResponse::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_STOP) { break; } - xfer += iprot->skip(ftype); + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->eventId); + this->__isset.eventId = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } xfer += iprot->readFieldEnd(); } @@ -22789,6 +22806,10 @@ uint32_t FireEventResponse::write(::apache::thrift::protocol::TProtocol* oprot) apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); xfer += oprot->writeStructBegin("FireEventResponse"); + xfer += oprot->writeFieldBegin("eventId", ::apache::thrift::protocol::T_I64, 1); + xfer += oprot->writeI64(this->eventId); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -22796,20 +22817,23 @@ uint32_t FireEventResponse::write(::apache::thrift::protocol::TProtocol* oprot) void swap(FireEventResponse &a, FireEventResponse &b) { using ::std::swap; - (void) a; - (void) b; + swap(a.eventId, b.eventId); + swap(a.__isset, b.__isset); } FireEventResponse::FireEventResponse(const FireEventResponse& other887) { - (void) other887; + eventId = other887.eventId; + __isset = other887.__isset; } FireEventResponse& FireEventResponse::operator=(const FireEventResponse& other888) { - (void) other888; + eventId = other888.eventId; + __isset = other888.__isset; return *this; } void FireEventResponse::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "FireEventResponse("; + out << "eventId=" << to_string(eventId); out << ")"; } diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h index fb86c97..a3a28ed 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h +++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h @@ -9353,19 +9353,30 @@ inline std::ostream& operator<<(std::ostream& out, const FireEventRequest& obj) return out; } +typedef struct _FireEventResponse__isset { + _FireEventResponse__isset() : eventId(false) {} + bool eventId :1; +} _FireEventResponse__isset; class FireEventResponse { public: FireEventResponse(const FireEventResponse&); FireEventResponse& operator=(const FireEventResponse&); - FireEventResponse() { + FireEventResponse() : eventId(0) { } virtual ~FireEventResponse() throw(); + int64_t eventId; + + _FireEventResponse__isset __isset; + + void __set_eventId(const int64_t val); - bool operator == (const FireEventResponse & /* rhs */) const + bool operator == (const FireEventResponse & rhs) const { + if (!(eventId == rhs.eventId)) + return false; return true; } bool operator != (const FireEventResponse &rhs) const { diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FireEventResponse.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FireEventResponse.java index 9125d86..1b48ecb 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FireEventResponse.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FireEventResponse.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public class FireEventResponse implements org.apache.thrift.TBase<FireEventResponse, FireEventResponse._Fields>, java.io.Serializable, Cloneable, Comparable<FireEventResponse> { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("FireEventResponse"); + private static final org.apache.thrift.protocol.TField EVENT_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("eventId", org.apache.thrift.protocol.TType.I64, (short)1); private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>(); static { @@ -45,10 +46,11 @@ import org.slf4j.LoggerFactory; schemes.put(TupleScheme.class, new FireEventResponseTupleSchemeFactory()); } + private long eventId; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { -; + EVENT_ID((short)1, "eventId"); private static final Map<String, _Fields> byName = new HashMap<String, _Fields>(); @@ -63,6 +65,8 @@ import org.slf4j.LoggerFactory; */ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { + case 1: // EVENT_ID + return EVENT_ID; default: return null; } @@ -101,9 +105,15 @@ import org.slf4j.LoggerFactory; return _fieldName; } } + + // isset id assignments + private static final int __EVENTID_ISSET_ID = 0; + private byte __isset_bitfield = 0; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.EVENT_ID, new org.apache.thrift.meta_data.FieldMetaData("eventId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(FireEventResponse.class, metaDataMap); } @@ -111,10 +121,20 @@ import org.slf4j.LoggerFactory; public FireEventResponse() { } + public FireEventResponse( + long eventId) + { + this(); + this.eventId = eventId; + setEventIdIsSet(true); + } + /** * Performs a deep copy on <i>other</i>. */ public FireEventResponse(FireEventResponse other) { + __isset_bitfield = other.__isset_bitfield; + this.eventId = other.eventId; } public FireEventResponse deepCopy() { @@ -123,15 +143,50 @@ import org.slf4j.LoggerFactory; @Override public void clear() { + setEventIdIsSet(false); + this.eventId = 0; + } + + public long getEventId() { + return this.eventId; + } + + public void setEventId(long eventId) { + this.eventId = eventId; + setEventIdIsSet(true); + } + + public void unsetEventId() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __EVENTID_ISSET_ID); + } + + /** Returns true if field eventId is set (has been assigned a value) and false otherwise */ + public boolean isSetEventId() { + return EncodingUtils.testBit(__isset_bitfield, __EVENTID_ISSET_ID); + } + + public void setEventIdIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __EVENTID_ISSET_ID, value); } public void setFieldValue(_Fields field, Object value) { switch (field) { + case EVENT_ID: + if (value == null) { + unsetEventId(); + } else { + setEventId((Long)value); + } + break; + } } public Object getFieldValue(_Fields field) { switch (field) { + case EVENT_ID: + return getEventId(); + } throw new IllegalStateException(); } @@ -143,6 +198,8 @@ import org.slf4j.LoggerFactory; } switch (field) { + case EVENT_ID: + return isSetEventId(); } throw new IllegalStateException(); } @@ -160,6 +217,15 @@ import org.slf4j.LoggerFactory; if (that == null) return false; + boolean this_present_eventId = true; + boolean that_present_eventId = true; + if (this_present_eventId || that_present_eventId) { + if (!(this_present_eventId && that_present_eventId)) + return false; + if (this.eventId != that.eventId) + return false; + } + return true; } @@ -167,6 +233,11 @@ import org.slf4j.LoggerFactory; public int hashCode() { List<Object> list = new ArrayList<Object>(); + boolean present_eventId = true; + list.add(present_eventId); + if (present_eventId) + list.add(eventId); + return list.hashCode(); } @@ -178,6 +249,16 @@ import org.slf4j.LoggerFactory; int lastComparison = 0; + lastComparison = Boolean.valueOf(isSetEventId()).compareTo(other.isSetEventId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetEventId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.eventId, other.eventId); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -198,6 +279,9 @@ import org.slf4j.LoggerFactory; StringBuilder sb = new StringBuilder("FireEventResponse("); boolean first = true; + sb.append("eventId:"); + sb.append(this.eventId); + first = false; sb.append(")"); return sb.toString(); } @@ -217,6 +301,8 @@ import org.slf4j.LoggerFactory; private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); } catch (org.apache.thrift.TException te) { throw new java.io.IOException(te); @@ -241,6 +327,14 @@ import org.slf4j.LoggerFactory; break; } switch (schemeField.id) { + case 1: // EVENT_ID + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.eventId = iprot.readI64(); + struct.setEventIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -254,6 +348,9 @@ import org.slf4j.LoggerFactory; struct.validate(); oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(EVENT_ID_FIELD_DESC); + oprot.writeI64(struct.eventId); + oprot.writeFieldEnd(); oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -271,11 +368,24 @@ import org.slf4j.LoggerFactory; @Override public void write(org.apache.thrift.protocol.TProtocol prot, FireEventResponse struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetEventId()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetEventId()) { + oprot.writeI64(struct.eventId); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, FireEventResponse struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.eventId = iprot.readI64(); + struct.setEventIdIsSet(true); + } } } diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php index 688574e..00641e5 100644 --- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php +++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php @@ -22480,12 +22480,25 @@ class FireEventRequest { class FireEventResponse { static $_TSPEC; + /** + * @var int + */ + public $eventId = null; - public function __construct() { + public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { self::$_TSPEC = array( + 1 => array( + 'var' => 'eventId', + 'type' => TType::I64, + ), ); } + if (is_array($vals)) { + if (isset($vals['eventId'])) { + $this->eventId = $vals['eventId']; + } + } } public function getName() { @@ -22507,6 +22520,13 @@ class FireEventResponse { } switch ($fid) { + case 1: + if ($ftype == TType::I64) { + $xfer += $input->readI64($this->eventId); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -22520,6 +22540,11 @@ class FireEventResponse { public function write($output) { $xfer = 0; $xfer += $output->writeStructBegin('FireEventResponse'); + if ($this->eventId !== null) { + $xfer += $output->writeFieldBegin('eventId', TType::I64, 1); + $xfer += $output->writeI64($this->eventId); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py index e429718..3f12407 100644 --- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -15734,10 +15734,19 @@ class FireEventRequest: return not (self == other) class FireEventResponse: + """ + Attributes: + - eventId + """ thrift_spec = ( + None, # 0 + (1, TType.I64, 'eventId', None, None, ), # 1 ) + def __init__(self, eventId=None,): + self.eventId = eventId + def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) @@ -15747,6 +15756,11 @@ class FireEventResponse: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break + if fid == 1: + if ftype == TType.I64: + self.eventId = iprot.readI64() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -15757,6 +15771,10 @@ class FireEventResponse: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('FireEventResponse') + if self.eventId is not None: + oprot.writeFieldBegin('eventId', TType.I64, 1) + oprot.writeI64(self.eventId) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -15766,6 +15784,7 @@ class FireEventResponse: def __hash__(self): value = 17 + value = (value * 31) ^ hash(self.eventId) return value def __repr__(self): diff --git a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb index 674a94e..7cbca4b 100644 --- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -3506,9 +3506,10 @@ end class FireEventResponse include ::Thrift::Struct, ::Thrift::Struct_Union + EVENTID = 1 FIELDS = { - + EVENTID => {:type => ::Thrift::Types::I64, :name => 'eventId'} } def struct_fields; FIELDS; end diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index d7c8050..e07e225 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -7649,7 +7649,14 @@ public class HiveMetaStore extends ThriftHiveMetastore { MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.INSERT, event); MetaStoreListenerNotifier.notifyEvent(listeners, EventType.INSERT, event); - return new FireEventResponse(); + FireEventResponse response = new FireEventResponse(); + if (event.getParameters() != null && event.getParameters() + .containsKey( + MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)) { + response.setEventId(Long.valueOf(event.getParameters() + .get(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME))); + } + return response; default: throw new TException("Event type " + rqst.getData().getSetField().toString() diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift index b3ffcb7..fcd1e56 100644 --- a/standalone-metastore/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -1138,7 +1138,7 @@ struct FireEventRequest { } struct FireEventResponse { - // NOP for now, this is just a place holder for future responses + 1: i64 eventId } struct WriteNotificationLogRequest {