Repository: hive Updated Branches: refs/heads/master 3ad42415a -> 4a6a0eba4
HIVE-16759 : Add table type information to HMS log notifications (Janaki Lahorani, reviewed by Sergio Pena, Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4a6a0eba Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4a6a0eba Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4a6a0eba Branch: refs/heads/master Commit: 4a6a0eba46c31b100ff9c14fd74d3489cea2b732 Parents: 3ad4241 Author: Janaki Lahorani <jan...@cloudera.com> Authored: Fri Jul 28 09:31:46 2017 -0700 Committer: Vihang Karajgaonkar <vih...@cloudera.com> Committed: Fri Jul 28 09:49:24 2017 -0700 ---------------------------------------------------------------------- .../hcatalog/messaging/AddPartitionMessage.java | 2 ++ .../messaging/AlterPartitionMessage.java | 2 ++ .../hcatalog/messaging/AlterTableMessage.java | 1 + .../hcatalog/messaging/CreateTableMessage.java | 2 ++ .../messaging/DropPartitionMessage.java | 1 + .../hcatalog/messaging/DropTableMessage.java | 1 + .../hive/hcatalog/messaging/InsertMessage.java | 1 + .../hive/hcatalog/messaging/MessageFactory.java | 12 +++++++++ .../messaging/json/JSONAddPartitionMessage.java | 15 +++++++++-- .../json/JSONAlterPartitionMessage.java | 19 +++++++++++-- .../messaging/json/JSONAlterTableMessage.java | 18 +++++++++++-- .../messaging/json/JSONCreateTableMessage.java | 16 +++++++++-- .../json/JSONDropPartitionMessage.java | 15 +++++++++-- .../messaging/json/JSONDropTableMessage.java | 11 ++++++-- .../messaging/json/JSONInsertMessage.java | 14 +++++++++- .../messaging/json/JSONMessageFactory.java | 28 +++++++++++++------- .../listener/TestNotificationListener.java | 7 +++++ .../api/TestHCatClientNotification.java | 5 ++++ .../listener/TestDbNotificationListener.java | 18 ++++++++++++- .../messaging/AddPartitionMessage.java | 2 ++ .../messaging/AlterPartitionMessage.java | 2 ++ .../metastore/messaging/AlterTableMessage.java | 2 ++ .../metastore/messaging/CreateTableMessage.java | 2 ++ .../messaging/DropPartitionMessage.java | 2 ++ .../metastore/messaging/DropTableMessage.java | 2 ++ .../hive/metastore/messaging/InsertMessage.java | 2 ++ .../messaging/json/JSONAddPartitionMessage.java | 8 +++++- .../json/JSONAlterPartitionMessage.java | 8 +++++- .../messaging/json/JSONAlterTableMessage.java | 8 +++++- .../messaging/json/JSONCreateTableMessage.java | 18 ++++++++++--- .../json/JSONDropPartitionMessage.java | 15 +++++++++-- .../messaging/json/JSONDropTableMessage.java | 13 ++++++++- .../messaging/json/JSONInsertMessage.java | 8 +++++- .../messaging/json/JSONMessageFactory.java | 2 +- 34 files changed, 248 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AddPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AddPartitionMessage.java index 5b114b5..9dacbf6 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AddPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AddPartitionMessage.java @@ -37,6 +37,8 @@ public abstract class AddPartitionMessage extends HCatEventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + /** * Getter for list of partitions added. * @return List of maps, where each map identifies values for each partition-key, for every added partition. http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java index 10a300d..8d9575e 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java @@ -31,6 +31,8 @@ public abstract class AlterPartitionMessage extends HCatEventMessage { public abstract String getTable(); + public abstract String getTableType(); + public abstract Map<String,String> getKeyValues(); @Override http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java index 0b58f29..94f07cd 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java @@ -28,6 +28,7 @@ public abstract class AlterTableMessage extends HCatEventMessage { } public abstract String getTable(); + public abstract String getTableType(); @Override public HCatEventMessage checkValid() { http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateTableMessage.java index 6c8e2a4..6442340 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateTableMessage.java @@ -34,6 +34,8 @@ public abstract class CreateTableMessage extends HCatEventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + @Override public HCatEventMessage checkValid() { if (getTable() == null) http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropPartitionMessage.java index ee222ec..d7b74f7 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropPartitionMessage.java @@ -32,6 +32,7 @@ public abstract class DropPartitionMessage extends HCatEventMessage { } public abstract String getTable(); + public abstract String getTableType(); public abstract List<Map<String, String>> getPartitions (); @Override http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropTableMessage.java index e47b572..345345e 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropTableMessage.java @@ -33,6 +33,7 @@ public abstract class DropTableMessage extends HCatEventMessage { * @return Table-name (String). */ public abstract String getTable(); + public abstract String getTableType(); @Override public HCatEventMessage checkValid() { http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java index be7ea10..d2aae7b 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java @@ -36,6 +36,7 @@ public abstract class InsertMessage extends HCatEventMessage { * @return Table-name (String). */ public abstract String getTable(); + public abstract String getTableType(); /** * Get the map of partition keyvalues. Will be null if this insert is to a table and not a http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java index 44574fe..28026db 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java @@ -211,4 +211,16 @@ public abstract class MessageFactory { */ public abstract InsertMessage buildInsertMessage(String db, String table, Map<String,String> partVals, List<String> files); + + /** + * Factory method for building insert message + * @param db Name of the database the insert occurred in + * @param table Table the insert occurred in + * @param partVals Partition values for the partition that the insert occurred in, may be null + * if the insert was done into a non-partitioned table + * @param files List of files created as a result of the insert, may be null. + * @return instance of InsertMessage + */ + public abstract InsertMessage buildInsertMessage(String db, Table table, + Map<String,String> partVals, List<String> files); } http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java index ac7dcd9..5d201f4 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java @@ -31,7 +31,7 @@ import java.util.Map; public class JSONAddPartitionMessage extends AddPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -45,11 +45,17 @@ public class JSONAddPartitionMessage extends AddPartitionMessage { public JSONAddPartitionMessage() {} public JSONAddPartitionMessage(String server, String servicePrincipal, String db, String table, - List<Map<String,String>> partitions, Long timestamp) { + List<Map<String,String>> partitions, Long timestamp) { + this(server, servicePrincipal, db, table, null, partitions, timestamp); + } + + public JSONAddPartitionMessage(String server, String servicePrincipal, String db, String table, + String tableType, List<Map<String,String>> partitions, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.partitions = partitions; this.timestamp = timestamp; checkValid(); @@ -68,6 +74,11 @@ public class JSONAddPartitionMessage extends AddPartitionMessage { public String getTable() { return table; } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Long getTimestamp() { return timestamp; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java index 4f1d104..7ae7d75 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java @@ -34,7 +34,7 @@ import java.util.Map; public class JSONAlterPartitionMessage extends AlterPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -53,16 +53,26 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage { String table, Map<String,String> keyValues, Long timestamp) { + this(server, servicePrincipal, db, table, null, keyValues, timestamp); + } + + public JSONAlterPartitionMessage(String server, + String servicePrincipal, + String db, + String table, + String tableType, + Map<String,String> keyValues, + Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; this.keyValues = keyValues; checkValid(); } - @Override public String getServer() { return server; @@ -89,6 +99,11 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Map<String,String> getKeyValues() { return keyValues; } http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java index b057d4a..f23aedf 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java @@ -32,7 +32,7 @@ import java.util.List; public class JSONAlterTableMessage extends AlterTableMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -47,15 +47,24 @@ public class JSONAlterTableMessage extends AlterTableMessage { String db, String table, Long timestamp) { + this(server, servicePrincipal, db, table, null, timestamp); + } + + public JSONAlterTableMessage(String server, + String servicePrincipal, + String db, + String table, + String tableType, + Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; checkValid(); } - @Override public String getServer() { return server; @@ -82,6 +91,11 @@ public class JSONAlterTableMessage extends AlterTableMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String toString() { try { return JSONMessageDeserializer.mapper.writeValueAsString(this); http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java index 9c66730..327dc97 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java @@ -28,7 +28,7 @@ import org.codehaus.jackson.annotate.JsonProperty; public class JSONCreateTableMessage extends CreateTableMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -38,11 +38,18 @@ public class JSONCreateTableMessage extends CreateTableMessage { */ public JSONCreateTableMessage() {} - public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) { + public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, + Long timestamp) { + this(server, servicePrincipal, db, table, null, timestamp); + } + + public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, + String tableType, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; checkValid(); } @@ -63,6 +70,11 @@ public class JSONCreateTableMessage extends CreateTableMessage { public String getTable() { return table; } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String toString() { try { return JSONMessageDeserializer.mapper.writeValueAsString(this); http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java index a4d6400..ecdc39d 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java @@ -31,7 +31,7 @@ import java.util.Map; public class JSONDropPartitionMessage extends DropPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -45,11 +45,17 @@ public class JSONDropPartitionMessage extends DropPartitionMessage { public JSONDropPartitionMessage() {} public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, - List<Map<String,String>> partitions, Long timestamp) { + List<Map<String,String>> partitions, Long timestamp) { + this(server, servicePrincipal, db, table, null, partitions, timestamp); + } + + public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, + String tableType, List<Map<String,String>> partitions, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.partitions = partitions; this.timestamp = timestamp; checkValid(); @@ -69,6 +75,11 @@ public class JSONDropPartitionMessage extends DropPartitionMessage { public String getTable() { return table; } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Long getTimestamp() { return timestamp; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java index 3b62023..4300625 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java @@ -28,7 +28,7 @@ import org.codehaus.jackson.annotate.JsonProperty; public class JSONDropTableMessage extends DropTableMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -38,11 +38,13 @@ public class JSONDropTableMessage extends DropTableMessage { */ public JSONDropTableMessage() {} - public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) { + public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, + String tableType, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; checkValid(); } @@ -52,6 +54,11 @@ public class JSONDropTableMessage extends DropTableMessage { public String getTable() { return table; } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String getServer() { return server; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java index 8a4db15..667df4d 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java @@ -31,7 +31,7 @@ import java.util.Map; public class JSONInsertMessage extends InsertMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -49,10 +49,17 @@ public class JSONInsertMessage extends InsertMessage { public JSONInsertMessage(String server, String servicePrincipal, String db, String table, Map<String,String> partKeyVals, List<String> files, Long timestamp) { + this(server, servicePrincipal, db, table, null, partKeyVals, files, timestamp); + } + + public JSONInsertMessage(String server, String servicePrincipal, String db, String table, + String tableType, Map<String,String> partKeyVals, List<String> files, + Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; this.partKeyVals = partKeyVals; this.files = files; @@ -64,6 +71,11 @@ public class JSONInsertMessage extends InsertMessage { public String getTable() { return table; } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String getServer() { return server; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java index 251084f..7962732 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java @@ -95,37 +95,40 @@ public class JSONMessageFactory extends MessageFactory { @Override public CreateTableMessage buildCreateTableMessage(Table table) { return new JSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), now()); + table.getTableName(), table.getTableType(), now()); } @Override public AlterTableMessage buildAlterTableMessage(Table before, Table after) { return new JSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), - before.getTableName(), now()); + before.getTableName(), before.getTableType(), now()); } @Override public DropTableMessage buildDropTableMessage(Table table) { - return new JSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), table.getTableName(), - now()); + return new JSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), table.getTableType(), now()); } @Override public AddPartitionMessage buildAddPartitionMessage(Table table, Iterator<Partition> partitionsIterator) { return new JSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), getPartitionKeyValues(table, partitionsIterator), now()); + table.getTableName(), table.getTableType(), + getPartitionKeyValues(table, partitionsIterator), now()); } @Override public AlterPartitionMessage buildAlterPartitionMessage(Table table, Partition before, Partition after) { return new JSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, - before.getDbName(), before.getTableName(), getPartitionKeyValues(table,before),now()); + before.getDbName(), before.getTableName(), table.getTableType(), + getPartitionKeyValues(table,before),now()); } @Override public DropPartitionMessage buildDropPartitionMessage(Table table, Iterator<Partition> partitions) { return new JSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), getPartitionKeyValues(table, partitions), now()); + table.getTableName(), table.getTableType(), + getPartitionKeyValues(table, partitions), now()); } @Override @@ -161,8 +164,15 @@ public class JSONMessageFactory extends MessageFactory { @Override public InsertMessage buildInsertMessage(String db, String table, Map<String,String> partKeyVals, List<String> files) { - return new JSONInsertMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db, table, partKeyVals, - files, now()); + return new JSONInsertMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db, table, null, + partKeyVals, files, now()); + } + + @Override + public InsertMessage buildInsertMessage(String db, Table table, Map<String,String> partKeyVals, + List<String> files) { + return new JSONInsertMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), table.getTableType(), partKeyVals, files, now()); } private long now() { http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java b/hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java index ef7b575..3a33403 100644 --- a/hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java +++ b/hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; @@ -168,6 +169,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis CreateTableMessage message = deserializer.getCreateTableMessage(messageBody); Assert.assertEquals("mytbl", message.getTable()); Assert.assertEquals("mydb", message.getDB()); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof CreateTableMessage); Assert.assertEquals("mydb", message2.getDB()); @@ -181,6 +183,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis Assert.assertEquals("mydb", message.getDB()); Assert.assertEquals(1, message.getPartitions().size()); Assert.assertEquals("2011", message.getPartitions().get(0).get("b")); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof AddPartitionMessage); Assert.assertEquals("mydb", message2.getDB()); @@ -195,6 +198,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis Assert.assertEquals("mydb", message.getDB()); Assert.assertEquals(1, message.getKeyValues().size()); Assert.assertTrue(message.getKeyValues().values().contains("2011")); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof AlterPartitionMessage); Assert.assertEquals("mydb", message2.getDB()); @@ -210,6 +214,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis Assert.assertEquals("mydb", message.getDB()); Assert.assertEquals(1, message.getPartitions().size()); Assert.assertEquals("2011", message.getPartitions().get(0).get("b")); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof DropPartitionMessage); Assert.assertEquals("mydb", message2.getDB()); @@ -223,6 +228,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis DropTableMessage message = deserializer.getDropTableMessage(messageBody); Assert.assertEquals("mytbl", message.getTable()); Assert.assertEquals("mydb", message.getDB()); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof DropTableMessage); Assert.assertEquals("mydb", message2.getDB()); @@ -241,6 +247,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis AlterTableMessage message = deserializer.getAlterTableMessage(messageBody); Assert.assertEquals("mytbl", message.getTable()); Assert.assertEquals("mydb", message.getDB()); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof AlterTableMessage); Assert.assertEquals("mydb", message2.getDB()); http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java index c09e687..b9a3218 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hive.hcatalog.messaging.AddPartitionMessage; import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage; import org.apache.hive.hcatalog.messaging.CreateTableMessage; @@ -137,6 +138,7 @@ public class TestHCatClientNotification { CreateTableMessage createTableMessage = md.getCreateTableMessage(event.getMessage()); assertEquals(dbName, createTableMessage.getDB()); assertEquals(tableName, createTableMessage.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), createTableMessage.getTableType()); // fetch the table marked by the message and compare HCatTable createdTable = hCatClient.getTable(dbName,tableName); @@ -167,6 +169,7 @@ public class TestHCatClientNotification { DropTableMessage dropTableMessage = md.getDropTableMessage(event.getMessage()); assertEquals(dbName, dropTableMessage.getDB()); assertEquals(tableName, dropTableMessage.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), dropTableMessage.getTableType()); } @Test @@ -198,6 +201,7 @@ public class TestHCatClientNotification { AddPartitionMessage addPartitionMessage = md.getAddPartitionMessage(event.getMessage()); assertEquals(dbName, addPartitionMessage.getDB()); assertEquals(tableName, addPartitionMessage.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), addPartitionMessage.getTableType()); List<Map<String,String>> ptndescs = addPartitionMessage.getPartitions(); // fetch the partition referred to by the message and compare @@ -245,6 +249,7 @@ public class TestHCatClientNotification { DropPartitionMessage dropPartitionMessage = md.getDropPartitionMessage(event.getMessage()); assertEquals(dbName, dropPartitionMessage.getDB()); assertEquals(tableName, dropPartitionMessage.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), dropPartitionMessage.getTableType()); List<Map<String, String>> droppedPartSpecs = dropPartitionMessage.getPartitions(); assertNotNull(droppedPartSpecs); assertEquals(1,droppedPartSpecs.size()); http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java index d937c03..c36b632 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreEventListener; import org.apache.hadoop.hive.metastore.MetaStoreEventListenerConstants; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FireEventRequest; @@ -384,7 +385,7 @@ public class TestDbNotificationListener { emptyParameters); Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, null, - emptyParameters, null, null, null); + emptyParameters, null, null, TableType.MANAGED_TABLE.toString()); msClient.createTable(table); // Get notifications from metastore @@ -402,6 +403,7 @@ public class TestDbNotificationListener { assertEquals(defaultDbName, createTblMsg.getDB()); assertEquals(tblName, createTblMsg.getTable()); assertEquals(table, createTblMsg.getTableObj()); + assertEquals(TableType.MANAGED_TABLE.toString(), createTblMsg.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); @@ -461,6 +463,7 @@ public class TestDbNotificationListener { AlterTableMessage alterTableMessage = md.getAlterTableMessage(event.getMessage()); assertEquals(table, alterTableMessage.getTableObjAfter()); + assertEquals(TableType.MANAGED_TABLE.toString(), alterTableMessage.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); @@ -515,6 +518,7 @@ public class TestDbNotificationListener { DropTableMessage dropTblMsg = md.getDropTableMessage(event.getMessage()); assertEquals(defaultDbName, dropTblMsg.getDB()); assertEquals(tblName, dropTblMsg.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), dropTblMsg.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_TABLE, firstEventId + 2); @@ -584,6 +588,7 @@ public class TestDbNotificationListener { Iterator<Partition> ptnIter = addPtnMsg.getPartitionObjs().iterator(); assertTrue(ptnIter.hasNext()); assertEquals(partition, ptnIter.next()); + assertEquals(TableType.MANAGED_TABLE.toString(), addPtnMsg.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); @@ -654,6 +659,7 @@ public class TestDbNotificationListener { assertEquals(defaultDbName, alterPtnMsg.getDB()); assertEquals(tblName, alterPtnMsg.getTable()); assertEquals(newPart, alterPtnMsg.getPtnObjAfter()); + assertEquals(TableType.MANAGED_TABLE.toString(), alterPtnMsg.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); @@ -721,6 +727,7 @@ public class TestDbNotificationListener { assertEquals(table.getDbName(), tableObj.getDbName()); assertEquals(table.getTableName(), tableObj.getTableName()); assertEquals(table.getOwner(), tableObj.getOwner()); + assertEquals(TableType.MANAGED_TABLE.toString(), dropPtnMsg.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_PARTITION, firstEventId + 3); @@ -803,6 +810,8 @@ public class TestDbNotificationListener { assertEquals(dbName, addPtnMsg.getDB()); assertEquals(tab2.getTableName(), addPtnMsg.getTable()); Iterator<Partition> ptnIter = addPtnMsg.getPartitionObjs().iterator(); + assertEquals(TableType.MANAGED_TABLE.toString(), addPtnMsg.getTableType()); + assertTrue(ptnIter.hasNext()); Partition msgPart = ptnIter.next(); assertEquals(part1.getValues(), msgPart.getValues()); @@ -820,6 +829,7 @@ public class TestDbNotificationListener { DropPartitionMessage dropPtnMsg = md.getDropPartitionMessage(event.getMessage()); assertEquals(dbName, dropPtnMsg.getDB()); assertEquals(tab1.getTableName(), dropPtnMsg.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), dropPtnMsg.getTableType()); Iterator<Map<String, String>> parts = dropPtnMsg.getPartitions().iterator(); assertTrue(parts.hasNext()); assertEquals(part1.getValues(), Lists.newArrayList(parts.next().values())); @@ -1205,6 +1215,12 @@ public class TestDbNotificationListener { // Parse the message field verifyInsert(event, defaultDbName, tblName); + // Parse the message field + InsertMessage insertMessage = md.getInsertMessage(event.getMessage()); + assertEquals(defaultDbName, insertMessage.getDB()); + assertEquals(tblName, insertMessage.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), insertMessage.getTableType()); + // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.INSERT, firstEventId + 2); MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java index 28f1101..774c020 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java @@ -37,6 +37,8 @@ public abstract class AddPartitionMessage extends EventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + public abstract Table getTableObj() throws Exception; /** http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java index e9ed7e5..077c9f7 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java @@ -31,6 +31,8 @@ public abstract class AlterPartitionMessage extends EventMessage { public abstract String getTable(); + public abstract String getTableType(); + public abstract boolean getIsTruncateOp(); public abstract Map<String,String> getKeyValues(); http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java index 39a87bc..58f01fe 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java @@ -28,6 +28,8 @@ public abstract class AlterTableMessage extends EventMessage { public abstract String getTable(); + public abstract String getTableType(); + public abstract boolean getIsTruncateOp(); public abstract Table getTableObjBefore() throws Exception; http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java index 441fd84..b75caa6 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java @@ -33,6 +33,8 @@ public abstract class CreateTableMessage extends EventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + public abstract Table getTableObj() throws Exception; /** http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java index 0dd3e50..d254ad9 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java @@ -32,6 +32,8 @@ public abstract class DropPartitionMessage extends EventMessage { public abstract String getTable(); + public abstract String getTableType(); + public abstract Table getTableObj() throws Exception; public abstract List<Map<String, String>> getPartitions (); http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java index 64a8cc5..03f73f1 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java @@ -31,6 +31,8 @@ public abstract class DropTableMessage extends EventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + @Override public EventMessage checkValid() { if (getTable() == null) http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java index 6505c67..01fc0f2 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java @@ -37,6 +37,8 @@ public abstract class InsertMessage extends EventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + /** * Getter for the replace flag being insert into/overwrite * @return Replace flag to represent INSERT INTO or INSERT OVERWRITE (Boolean). http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java index a488205..db3431e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java @@ -43,7 +43,7 @@ import java.util.Map; public class JSONAddPartitionMessage extends AddPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table, tableObjJson; + String server, servicePrincipal, db, table, tableType, tableObjJson; @JsonProperty Long timestamp; @@ -73,6 +73,7 @@ public class JSONAddPartitionMessage extends AddPartitionMessage { this.servicePrincipal = servicePrincipal; this.db = tableObj.getDbName(); this.table = tableObj.getTableName(); + this.tableType = tableObj.getTableType(); this.timestamp = timestamp; partitions = new ArrayList<Map<String, String>>(); partitionListJson = new ArrayList<String>(); @@ -112,6 +113,11 @@ public class JSONAddPartitionMessage extends AddPartitionMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Table getTableObj() throws Exception { return (Table) JSONMessageFactory.getTObj(tableObjJson,Table.class); } http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java index bd7776c..2991e08 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java @@ -34,7 +34,7 @@ import java.util.Map; public class JSONAlterPartitionMessage extends AlterPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table, tableObjJson; + String server, servicePrincipal, db, table, tableType, tableObjJson; @JsonProperty String isTruncateOp; @@ -60,6 +60,7 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage { this.servicePrincipal = servicePrincipal; this.db = tableObj.getDbName(); this.table = tableObj.getTableName(); + this.tableType = tableObj.getTableType(); this.isTruncateOp = Boolean.toString(isTruncateOp); this.timestamp = timestamp; this.keyValues = JSONMessageFactory.getPartitionKeyValues(tableObj, partitionObjBefore); @@ -99,6 +100,11 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public boolean getIsTruncateOp() { return Boolean.parseBoolean(isTruncateOp); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java index 58eb1a7..1df9c8b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java @@ -29,7 +29,7 @@ import org.codehaus.jackson.annotate.JsonProperty; public class JSONAlterTableMessage extends AlterTableMessage { @JsonProperty - String server, servicePrincipal, db, table, tableObjBeforeJson, tableObjAfterJson; + String server, servicePrincipal, db, table, tableType, tableObjBeforeJson, tableObjAfterJson; @JsonProperty String isTruncateOp; @@ -49,6 +49,7 @@ public class JSONAlterTableMessage extends AlterTableMessage { this.servicePrincipal = servicePrincipal; this.db = tableObjBefore.getDbName(); this.table = tableObjBefore.getTableName(); + this.tableType = tableObjBefore.getTableType(); this.isTruncateOp = Boolean.toString(isTruncateOp); this.timestamp = timestamp; try { @@ -86,6 +87,11 @@ public class JSONAlterTableMessage extends AlterTableMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public boolean getIsTruncateOp() { return Boolean.parseBoolean(isTruncateOp); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java index dbc3dd4..9dd7a74 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java @@ -35,7 +35,7 @@ import com.google.common.collect.Lists; public class JSONCreateTableMessage extends CreateTableMessage { @JsonProperty - String server, servicePrincipal, db, table, tableObjJson; + String server, servicePrincipal, db, table, tableType, tableObjJson; @JsonProperty Long timestamp; @JsonProperty @@ -48,18 +48,25 @@ public class JSONCreateTableMessage extends CreateTableMessage { } public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, - Long timestamp) { + String tableType, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; checkValid(); } + public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, + Long timestamp) { + this(server, servicePrincipal, db, table, null, timestamp); + } + public JSONCreateTableMessage(String server, String servicePrincipal, Table tableObj, Iterator<String> fileIter, Long timestamp) { - this(server, servicePrincipal, tableObj.getDbName(), tableObj.getTableName(), timestamp); + this(server, servicePrincipal, tableObj.getDbName(), tableObj.getTableName(), + tableObj.getTableType(), timestamp); try { this.tableObjJson = JSONMessageFactory.createTableObjJson(tableObj); } catch (TException e) { @@ -94,6 +101,11 @@ public class JSONCreateTableMessage extends CreateTableMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Table getTableObj() throws Exception { return (Table) JSONMessageFactory.getTObj(tableObjJson,Table.class); } http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java index f1860af..576806c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java @@ -33,7 +33,7 @@ import java.util.Map; public class JSONDropPartitionMessage extends DropPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table, tableObjJson; + String server, servicePrincipal, db, table, tableType, tableObjJson; @JsonProperty Long timestamp; @@ -49,10 +49,16 @@ public class JSONDropPartitionMessage extends DropPartitionMessage { public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, List<Map<String, String>> partitions, Long timestamp) { + this(server, servicePrincipal, db, table, null, partitions, timestamp); + } + + public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, + String tableType, List<Map<String, String>> partitions, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.partitions = partitions; this.timestamp = timestamp; checkValid(); @@ -61,7 +67,7 @@ public class JSONDropPartitionMessage extends DropPartitionMessage { public JSONDropPartitionMessage(String server, String servicePrincipal, Table tableObj, List<Map<String, String>> partitionKeyValues, long timestamp) { this(server, servicePrincipal, tableObj.getDbName(), tableObj.getTableName(), - partitionKeyValues, timestamp); + tableObj.getTableType(), partitionKeyValues, timestamp); try { this.tableObjJson = JSONMessageFactory.createTableObjJson(tableObj); } catch (TException e) { @@ -90,6 +96,11 @@ public class JSONDropPartitionMessage extends DropPartitionMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Long getTimestamp() { return timestamp; } http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java index 635ab61..17f38ba 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java @@ -28,7 +28,7 @@ import org.codehaus.jackson.annotate.JsonProperty; public class JSONDropTableMessage extends DropTableMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -41,10 +41,16 @@ public class JSONDropTableMessage extends DropTableMessage { public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) { + this(server, servicePrincipal, db, table, null, timestamp); + } + + public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, + String tableType, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; checkValid(); } @@ -55,6 +61,11 @@ public class JSONDropTableMessage extends DropTableMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String getServer() { return server; } http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java index 18a15f5..1369fd2 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java @@ -36,7 +36,7 @@ import java.util.List; public class JSONInsertMessage extends InsertMessage { @JsonProperty - String server, servicePrincipal, db, table, tableObjJson, ptnObjJson; + String server, servicePrincipal, db, table, tableType, tableObjJson, ptnObjJson; @JsonProperty Long timestamp; @@ -64,6 +64,7 @@ public class JSONInsertMessage extends InsertMessage { this.db = tableObj.getDbName(); this.table = tableObj.getTableName(); + this.tableType = tableObj.getTableType(); try { this.tableObjJson = JSONMessageFactory.createTableObjJson(tableObj); @@ -89,6 +90,11 @@ public class JSONInsertMessage extends InsertMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String getServer() { return server; } http://git-wip-us.apache.org/repos/asf/hive/blob/4a6a0eba/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java index a4c31f2..b24d1e3 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java @@ -111,7 +111,7 @@ public class JSONMessageFactory extends MessageFactory { @Override public DropTableMessage buildDropTableMessage(Table table) { return new JSONDropTableMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), now()); + table.getTableName(), table.getTableType(), now()); } @Override