This is an automated email from the ASF dual-hosted git repository. sankarh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 2daaed7 HIVE-21307: Need to set GzipJSONMessageEncoder as default config for EVENT_MESSAGE_FACTORY (Sankar Hariappan, reviewed by Mahesh Kumar Behera) 2daaed7 is described below commit 2daaed73de09b4b7987fd2d682140d7ee5637640 Author: Sankar Hariappan <sank...@apache.org> AuthorDate: Sun Feb 24 13:44:53 2019 +0530 HIVE-21307: Need to set GzipJSONMessageEncoder as default config for EVENT_MESSAGE_FACTORY (Sankar Hariappan, reviewed by Mahesh Kumar Behera) Signed-off-by: Sankar Hariappan <sank...@apache.org> --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +- .../test/java/org/apache/hive/hcatalog/api/TestHCatClient.java | 5 ++++- .../apache/hive/hcatalog/api/TestHCatClientNotification.java | 6 ++++-- .../hive/hcatalog/listener/TestDbNotificationListener.java | 1 + .../hive/ql/parse/TestReplAcidTablesWithJsonMessage.java | 9 ++++++++- .../ql/parse/TestReplAcrossInstancesWithJsonMessageFormat.java | 10 ++++++++-- .../TestReplIncrementalLoadAcidTablesWithJsonMessage.java | 10 ++++++++-- .../hive/ql/parse/TestReplTableMigrationWithJsonFormat.java | 10 ++++++++-- .../hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java | 9 ++++++++- .../org/apache/hadoop/hive/metastore/conf/MetastoreConf.java | 2 +- 10 files changed, 51 insertions(+), 13 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 04166db..ae44b08 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1191,7 +1191,7 @@ public class HiveConf extends Configuration { */ @Deprecated METASTORE_EVENT_MESSAGE_FACTORY("hive.metastore.event.message.factory", - "org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder", + "org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder", "Factory class for making encoding and decoding messages in the events generated."), /** * @deprecated Use MetastoreConf.EXECUTE_SET_UGI diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index da08d2f..5b43323 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -41,6 +41,8 @@ import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.api.PartitionEventType; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; @@ -80,7 +82,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertArrayEquals; -import org.apache.hadoop.util.Shell; import javax.annotation.Nullable; @@ -120,6 +121,8 @@ public class TestHCatClient { System.setProperty(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS.varname, DbNotificationListener.class.getName()); // turn on db notification listener on metastore + System.setProperty(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(), + JSONMessageEncoder.class.getName()); msPort = MetaStoreTestUtils.startMetaStoreWithRetry(); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java index c4c8f50..ae7e572 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; @@ -66,8 +67,9 @@ public class TestHCatClientNotification { @BeforeClass public static void setupClient() throws Exception { - HiveConf conf = new HiveConf(); conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, - DbNotificationListener.class.getName()); + HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, DbNotificationListener.class.getName()); + conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_MESSAGE_FACTORY, JSONMessageEncoder.class.getName()); hCatClient = HCatClient.create(conf); md = MessageFactory.getInstance().getDeserializer(); } diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java index 9eacfd8..0c3e0466 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java @@ -274,6 +274,7 @@ public class TestDbNotificationListener { conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, DummyRawStoreFailEvent.class.getName()); MetastoreConf.setTimeVar(conf, MetastoreConf.ConfVars.EVENT_DB_LISTENER_CLEAN_INTERVAL, CLEANUP_SLEEP_TIME, TimeUnit.SECONDS); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY, JSONMessageEncoder.class.getName()); conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); SessionState.start(new CliSessionState(conf)); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcidTablesWithJsonMessage.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcidTablesWithJsonMessage.java index c16799d..747b7a8 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcidTablesWithJsonMessage.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcidTablesWithJsonMessage.java @@ -18,12 +18,16 @@ package org.apache.hadoop.hive.ql.parse; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TestRule; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; public class TestReplAcidTablesWithJsonMessage extends TestReplicationScenariosAcidTables { @@ -32,7 +36,10 @@ public class TestReplAcidTablesWithJsonMessage extends TestReplicationScenariosA @BeforeClass public static void classLevelSetup() throws Exception { - internalBeforeClassSetup(Collections.emptyMap(), TestReplAcidTablesWithJsonMessage.class); + Map<String, String> overrides = new HashMap<>(); + overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(), + JSONMessageEncoder.class.getCanonicalName()); + internalBeforeClassSetup(overrides, TestReplAcidTablesWithJsonMessage.class); } @Before diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcrossInstancesWithJsonMessageFormat.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcrossInstancesWithJsonMessageFormat.java index 0ec0275..05517c7 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcrossInstancesWithJsonMessageFormat.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcrossInstancesWithJsonMessageFormat.java @@ -18,13 +18,16 @@ package org.apache.hadoop.hive.ql.parse; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TestRule; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashMap; +import java.util.Map; public class TestReplAcrossInstancesWithJsonMessageFormat extends TestReplicationScenariosAcrossInstances { @@ -34,7 +37,10 @@ public class TestReplAcrossInstancesWithJsonMessageFormat @BeforeClass public static void classLevelSetup() throws Exception { - internalBeforeClassSetup(Collections.emptyMap(), TestReplicationScenarios.class); + Map<String, String> overrides = new HashMap<>(); + overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(), + JSONMessageEncoder.class.getCanonicalName()); + internalBeforeClassSetup(overrides, TestReplicationScenarios.class); } @Before diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplIncrementalLoadAcidTablesWithJsonMessage.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplIncrementalLoadAcidTablesWithJsonMessage.java index 422508d..cd273f0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplIncrementalLoadAcidTablesWithJsonMessage.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplIncrementalLoadAcidTablesWithJsonMessage.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; @@ -25,6 +27,8 @@ import org.junit.rules.TestRule; import org.junit.Ignore; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; @Ignore public class TestReplIncrementalLoadAcidTablesWithJsonMessage @@ -35,8 +39,10 @@ public class TestReplIncrementalLoadAcidTablesWithJsonMessage @BeforeClass public static void classLevelSetup() throws Exception { - internalBeforeClassSetup(Collections.emptyMap(), - TestReplIncrementalLoadAcidTablesWithJsonMessage.class); + Map<String, String> overrides = new HashMap<>(); + overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(), + JSONMessageEncoder.class.getCanonicalName()); + internalBeforeClassSetup(overrides, TestReplIncrementalLoadAcidTablesWithJsonMessage.class); } @Before diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplTableMigrationWithJsonFormat.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplTableMigrationWithJsonFormat.java index 0151ed0..5e7bf7e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplTableMigrationWithJsonFormat.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplTableMigrationWithJsonFormat.java @@ -17,13 +17,19 @@ */ package org.apache.hadoop.hive.ql.parse; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder; import org.junit.BeforeClass; -import java.util.Collections; +import java.util.HashMap; +import java.util.Map; public class TestReplTableMigrationWithJsonFormat extends TestReplicationWithTableMigration { @BeforeClass public static void classLevelSetup() throws Exception { - internalBeforeClassSetup(Collections.emptyMap()); + Map<String, String> overrides = new HashMap<>(); + overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(), + JSONMessageEncoder.class.getCanonicalName()); + internalBeforeClassSetup(overrides); } } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java index f76dc1d..6d9c31f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.ql.parse; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder; import org.apache.hive.hcatalog.api.repl.ReplicationV1CompatRule; import org.junit.BeforeClass; import org.junit.Rule; @@ -24,6 +26,8 @@ import org.junit.rules.TestRule; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; public class TestReplWithJsonMessageFormat extends TestReplicationScenarios { @Rule @@ -33,7 +37,10 @@ public class TestReplWithJsonMessageFormat extends TestReplicationScenarios { @BeforeClass public static void setUpBeforeClass() throws Exception { - internalBeforeClassSetup(Collections.emptyMap(), false); + Map<String, String> overrides = new HashMap<>(); + overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(), + JSONMessageEncoder.class.getCanonicalName()); + internalBeforeClassSetup(overrides, false); } } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 3a9912f..ce352e5 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -544,7 +544,7 @@ public class MetastoreConf { "Alternatively, configure hive.metastore.transactional.event.listeners to ensure both are invoked in same JDO transaction."), EVENT_MESSAGE_FACTORY("metastore.event.message.factory", "hive.metastore.event.message.factory", - "org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder", + "org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder", "Factory class for making encoding and decoding messages in the events generated."), EVENT_NOTIFICATION_PARAMETERS_EXCLUDE_PATTERNS("metastore.notification.parameters.exclude.patterns", "hive.metastore.notification.parameters.exclude.patterns", "",