Repository: hive Updated Branches: refs/heads/branch-3 a09c17416 -> 2859a12ea
HIVE-19921: Fix perf duration and queue name in HiveProtoLoggingHook (Harish JP, reviewd by Anishek Agarwal) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2859a12e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2859a12e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2859a12e Branch: refs/heads/branch-3 Commit: 2859a12eae788002e9c93b50d5915bf280d223cd Parents: a09c174 Author: Anishek Agarwal <anis...@gmail.com> Authored: Mon Jun 18 09:08:34 2018 -0700 Committer: Thejas M Nair <the...@hortonworks.com> Committed: Mon Jun 25 16:32:30 2018 -0700 ---------------------------------------------------------------------- .../hive/ql/hooks/HiveProtoLoggingHook.java | 6 +++- .../hive/ql/hooks/TestHiveProtoLoggingHook.java | 29 +++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2859a12e/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java index eef6ac9..bddca1a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java @@ -366,6 +366,7 @@ public class HiveProtoLoggingHook implements ExecuteWithHookContext { ApplicationId llapId = determineLlapId(conf, executionMode); if (llapId != null) { addMapEntry(builder, OtherInfoType.LLAP_APP_ID, llapId.toString()); + builder.setQueue(conf.get(HiveConf.ConfVars.LLAP_DAEMON_QUEUE_NAME.varname)); } conf.stripHiddenConfigurations(conf); @@ -391,7 +392,10 @@ public class HiveProtoLoggingHook implements ExecuteWithHookContext { builder.setOperationId(hookContext.getOperationId()); } addMapEntry(builder, OtherInfoType.STATUS, Boolean.toString(success)); - JSONObject perfObj = new JSONObject(hookContext.getPerfLogger().getEndTimes()); + JSONObject perfObj = new JSONObject(); + for (String key : hookContext.getPerfLogger().getEndTimes().keySet()) { + perfObj.put(key, hookContext.getPerfLogger().getDuration(key)); + } addMapEntry(builder, OtherInfoType.PERF, perfObj.toString()); return builder.build(); http://git-wip-us.apache.org/repos/asf/hive/blob/2859a12e/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java index 98b73e8..96fb73c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java @@ -22,6 +22,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; +import java.util.Map; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -46,6 +47,9 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + public class TestHiveProtoLoggingHook { @@ -106,6 +110,8 @@ public class TestHiveProtoLoggingHook { @Test public void testPostEventLog() throws Exception { context.setHookType(HookType.POST_EXEC_HOOK); + context.getPerfLogger().PerfLogBegin("test", "LogTest"); + context.getPerfLogger().PerfLogEnd("test", "LogTest"); EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance()); evtLogger.handle(context); @@ -119,7 +125,11 @@ public class TestHiveProtoLoggingHook { Assert.assertEquals("test_op_id", event.getOperationId()); assertOtherInfo(event, OtherInfoType.STATUS, Boolean.TRUE.toString()); - assertOtherInfo(event, OtherInfoType.PERF, null); + String val = findOtherInfo(event, OtherInfoType.PERF); + Map<String, Long> map = new ObjectMapper().readValue(val, + new TypeReference<Map<String, Long>>() {}); + // This should be really close to zero. + Assert.assertTrue("Expected LogTest in PERF", map.get("LogTest") < 100); } @Test @@ -158,15 +168,20 @@ public class TestHiveProtoLoggingHook { return event; } - private void assertOtherInfo(HiveHookEventProto event, OtherInfoType key, String value) { + private String findOtherInfo(HiveHookEventProto event, OtherInfoType key) { for (MapFieldEntry otherInfo : event.getOtherInfoList()) { if (otherInfo.getKey().equals(key.name())) { - if (value != null) { - Assert.assertEquals(value, otherInfo.getValue()); - } - return; + return otherInfo.getValue(); } } - Assert.fail("Cannot find key: " + key); + Assert.fail("Cannot find key " + key); + return null; + } + + private void assertOtherInfo(HiveHookEventProto event, OtherInfoType key, String value) { + String val = findOtherInfo(event, key); + if (value != null) { + Assert.assertEquals(value, val); + } } }
