HIVE-15834: Add unit tests for org.json usage on master (Daniel Voros via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2fa4dc27 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2fa4dc27 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2fa4dc27 Branch: refs/heads/hive-14535 Commit: 2fa4dc277e0cd28261602c392c2f55d040d22677 Parents: 811e599 Author: Daniel Voros <daniel.vo...@gmail.com> Authored: Thu May 25 21:16:44 2017 +0200 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Thu May 25 21:16:44 2017 +0200 ---------------------------------------------------------------------- .../hadoop/hive/common/jsonexplain/Op.java | 4 +- .../hadoop/hive/common/jsonexplain/TestOp.java | 81 +++++ .../hive/common/jsonexplain/TestStage.java | 194 ++++++++++++ .../hive/common/jsonexplain/TestVertex.java | 108 +++++++ .../jsonexplain/tez/TestTezJsonParser.java | 53 ++++ .../apache/hadoop/hive/ql/exec/ExplainTask.java | 19 +- .../apache/hadoop/hive/ql/hooks/ATSHook.java | 4 +- .../hadoop/hive/ql/exec/TestExplainTask.java | 293 ++++++++++++++++++- .../hadoop/hive/ql/hooks/TestATSHook.java | 59 ++++ 9 files changed, 796 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2fa4dc27/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java index 39c44f1..e9eb5a7 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java @@ -29,6 +29,7 @@ import java.util.Set; import org.apache.hadoop.hive.common.jsonexplain.Vertex.VertexType; import org.json.JSONArray; +import com.google.common.annotations.VisibleForTesting; import org.json.JSONException; import org.json.JSONObject; @@ -85,7 +86,8 @@ public final class Op { } } - private void inlineJoinOp() throws Exception { + @VisibleForTesting + void inlineJoinOp() throws Exception { // inline map join operator if (this.type == OpType.MAPJOIN) { // get the map for posToVertex http://git-wip-us.apache.org/repos/asf/hive/blob/2fa4dc27/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestOp.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestOp.java b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestOp.java new file mode 100644 index 0000000..eb5dca4 --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestOp.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hadoop.hive.common.jsonexplain.tez.TezJsonParser; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Test; + +import java.util.*; + +import static org.junit.Assert.assertEquals; + +public class TestOp { + + private ObjectMapper objectMapper = new ObjectMapper(); + private TezJsonParser tezJsonParser; + + @Before + public void setUp() throws Exception { + this.tezJsonParser = new TezJsonParser(); + } + + + @Test + public void testInlineJoinOpJsonHandling() throws Exception { + String jsonString = "{" + + "\"input vertices:\":{\"a\":\"AVERTEX\"}," + "\"condition map:\": [" + + "{\"c1\": \"{\\\"type\\\": \\\"type\\\", \\\"left\\\": \\\"left\\\", " + + "\\\"right\\\": \\\"right\\\"}\"}]," + + "\"keys:\":{\"left\":\"AKEY\", \"right\":\"BKEY\"}}"; + JSONObject mapJoin = new JSONObject(jsonString); + + Vertex vertexB = new Vertex("vertex-b", null, null, tezJsonParser); + Op dummyOp = new Op("Dummy Op", "dummy-id", "output-vertex-name", null, Collections.EMPTY_LIST, + null, mapJoin, null, tezJsonParser); + vertexB.outputOps.add(dummyOp); + + Vertex vertexC = new Vertex("vertex-c", null, null, tezJsonParser); + vertexC.outputOps.add(dummyOp); + + + Vertex vertexA = new Vertex("vertex-a", null, null, tezJsonParser); + vertexA.tagToInput = new HashMap<>(); + vertexA.tagToInput.put("left", "vertex-b"); + vertexA.tagToInput.put("right", "vertex-c"); + vertexA.parentConnections.add(new Connection("left", vertexB)); + vertexA.parentConnections.add(new Connection("right", vertexC)); + + + Map<String, String> attrs = new HashMap<>(); + + Op uut = new Op("Map Join Operator", "op-id", "output-vertex-name", null, Collections.EMPTY_LIST, + attrs, mapJoin, vertexA, tezJsonParser); + uut.inlineJoinOp(); + + assertEquals(1, attrs.size()); + + String result = attrs.get("Conds:"); + String expected = "dummy-id.AKEY=dummy-id.BKEY(type)"; + assertEquals(expected, result); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/2fa4dc27/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestStage.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestStage.java b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestStage.java new file mode 100644 index 0000000..e344eb3 --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestStage.java @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain; + +import org.apache.hadoop.hive.common.jsonexplain.tez.TezJsonParser; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class TestStage { + + private Stage uut; + private Stage stageA; + private Stage stageB; + private TezJsonParser tezJsonParser; + + @Before + public void setUp() { + this.tezJsonParser = new TezJsonParser(); + this.uut = new Stage("uut", tezJsonParser); + this.stageA = new Stage("stage-a", tezJsonParser); + this.stageB = new Stage("stage-b", tezJsonParser); + } + + @Test + public void testAddDependencyNonRoot() throws Exception { + Map<String, Stage> children = new LinkedHashMap<>(); + children.put("a", stageA); + children.put("b", stageB); + + + String jsonString = "{\"DEPENDENT STAGES\":\"a,b\"}"; + JSONObject names = new JSONObject(jsonString); + + uut.addDependency(names, children); + + assertEquals(2, uut.parentStages.size()); + assertEquals(stageA, uut.parentStages.get(0)); + assertEquals(stageB, uut.parentStages.get(1)); + + assertEquals(1, stageA.childStages.size()); + assertEquals(uut, stageA.childStages.get(0)); + + assertEquals(1, stageB.childStages.size()); + assertEquals(uut, stageB.childStages.get(0)); + } + + @Test + public void testAddDependencyRoot() throws Exception { + Map<String, Stage> children = new LinkedHashMap<>(); + children.put("a", stageA); + children.put("b", stageB); + + String jsonString = "{\"ROOT STAGE\":\"X\",\"DEPENDENT STAGES\":\"a,b\"}"; + JSONObject names = new JSONObject(jsonString); + + uut.addDependency(names, children); + + assertEquals(2, uut.parentStages.size()); + assertEquals(1, stageA.childStages.size()); + assertEquals(1, stageB.childStages.size()); + } + + + @Test + public void testExtractVertexNonTez() throws Exception { + String jsonString = "{\"OperatorName\":{\"a\":\"A\",\"b\":\"B\"}," + + "\"attr1\":\"ATTR1\"}"; + JSONObject object = new JSONObject(jsonString); + + uut.extractVertex(object); + + assertEquals("OperatorName", uut.op.name); + assertEquals(1, uut.attrs.size()); + assertEquals("ATTR1", uut.attrs.get("attr1")); + } + + @Test + public void testExtractVertexTezNoEdges() throws Exception { + String jsonString = "{\"Tez\":{\"a\":\"A\",\"Vertices:\":{\"v1\":{}}}}"; + JSONObject object = new JSONObject(jsonString); + uut.extractVertex(object); + + assertEquals(1, uut.vertexs.size()); + assertTrue(uut.vertexs.containsKey("v1")); + } + + @Test + public void testExtractVertexTezWithOneEdge() throws Exception { + String jsonString = "{\"Tez\":{\"a\":\"A\"," + + "\"Vertices:\":{\"v1\":{},\"v2\":{}}," + + "\"Edges:\":{\"v2\":{\"parent\":\"v1\",\"type\":\"TYPE\"}}}}"; + JSONObject object = new JSONObject(jsonString); + uut.extractVertex(object); + + assertEquals(2, uut.vertexs.size()); + assertTrue(uut.vertexs.containsKey("v1")); + assertTrue(uut.vertexs.containsKey("v2")); + + assertEquals(0, uut.vertexs.get("v1").parentConnections.size()); + assertEquals(1, uut.vertexs.get("v2").parentConnections.size()); + assertEquals("v1", uut.vertexs.get("v2").parentConnections.get(0).from.name); + assertEquals("TYPE", uut.vertexs.get("v2").parentConnections.get(0).type); + + } + + + @Test + public void testExtractVertexTezWithOneToManyEdge() throws Exception { + String jsonString = "{\"Tez\":{\"a\":\"A\"," + + "\"Vertices:\":{\"v1\":{},\"v2\":{},\"v3\":{}}," + + "\"Edges:\":{\"v1\":[{\"parent\":\"v2\",\"type\":\"TYPE1\"}," + + "{\"parent\":\"v3\",\"type\":\"TYPE2\"}]}}}"; + JSONObject object = new JSONObject(jsonString); + + uut.extractVertex(object); + + assertEquals(3, uut.vertexs.size()); + assertTrue(uut.vertexs.containsKey("v1")); + assertTrue(uut.vertexs.containsKey("v2")); + assertTrue(uut.vertexs.containsKey("v3")); + + assertEquals(2, uut.vertexs.get("v1").parentConnections.size()); + assertEquals(1, uut.vertexs.get("v2").children.size()); + assertEquals(1, uut.vertexs.get("v3").children.size()); + assertEquals("v1", uut.vertexs.get("v2").children.get(0).name); + assertEquals("v1", uut.vertexs.get("v3").children.get(0).name); + assertEquals("TYPE1", uut.vertexs.get("v1").parentConnections.get(0).type); + assertEquals("TYPE2", uut.vertexs.get("v1").parentConnections.get(1).type); + + } + + @Test + public void testExtractOpEmptyObject() throws Exception { + JSONObject object = new JSONObject(); + Op result = uut.extractOp("op-name", object); + + assertEquals("op-name", result.name); + assertEquals(0, result.attrs.size()); + assertNull(result.vertex); + } + + @Test + public void testExtractOpSimple() throws Exception { + String jsonString = "{\"a\":\"A\",\"b\":\"B\"}"; + JSONObject object = new JSONObject(jsonString); + + Op result = uut.extractOp("op-name", object); + + assertEquals("op-name", result.name); + assertEquals(2, result.attrs.size()); + assertNull(result.vertex); + } + + @Test + public void testExtract() throws Exception { + String jsonString = "{\"b\":{\"b2\":\"B2\",\"b1\":\"B1\"}," + + "\"Processor Tree:\":{\"a1\":{\"t1\":\"T1\"}}}"; + JSONObject object = new JSONObject(jsonString); + + Op result = uut.extractOp("op-name", object); + assertEquals("op-name", result.name); + assertEquals(2, result.attrs.size()); + + List<String> attrs = new ArrayList<>(); + assertEquals("B1", result.attrs.get("b1")); + assertEquals("B2", result.attrs.get("b2")); + assertNotNull(result.vertex); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/2fa4dc27/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestVertex.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestVertex.java b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestVertex.java new file mode 100644 index 0000000..4303be7 --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestVertex.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain; + +import org.apache.hadoop.hive.common.jsonexplain.tez.TezJsonParser; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestVertex { + + private TezJsonParser tezJsonParser; + + @Before + public void setUp() throws Exception { + this.tezJsonParser = new TezJsonParser(); + } + + + @Test + public void testExtractOpTree() throws Exception { + JSONObject object = new JSONObject("{\"Join:\":[{},{}]}"); + + Vertex uut = new Vertex("name", object, null, tezJsonParser); + uut.extractOpTree(); + + assertEquals(2, uut.mergeJoinDummyVertexs.size()); + } + + @Test + public void testExtractOpNonJsonChildrenShouldThrow() throws Exception { + String jsonString = "{\"opName\":{\"children\":\"not-json\"}}"; + JSONObject operator = new JSONObject(jsonString); + + Vertex uut = new Vertex("name", null, null, tezJsonParser); + + try { + uut.extractOp(operator, null); + } catch (Exception e) { + assertEquals("Unsupported operator name's children operator is neither a jsonobject nor a jsonarray", e.getMessage()); + } + } + + @Test + public void testExtractOpNoChildrenOperatorId() throws Exception { + String jsonString = "{\"opName\":{\"OperatorId:\":\"operator-id\"}}"; + JSONObject operator = new JSONObject(jsonString); + + Vertex uut = new Vertex("name", null, null, tezJsonParser); + + Op result = uut.extractOp(operator, null); + assertEquals("opName", result.name); + assertEquals("operator-id", result.operatorId); + assertEquals(0, result.children.size()); + assertEquals(0, result.attrs.size()); + } + + @Test + public void testExtractOpOneChild() throws Exception { + String jsonString = "{\"opName\":{\"children\":{\"childName\":" + + "{\"OperatorId:\":\"child-operator-id\"}}}}"; + JSONObject operator = new JSONObject(jsonString); + + Vertex uut = new Vertex("name", null, null, tezJsonParser); + + Op result = uut.extractOp(operator, null); + assertEquals("opName", result.name); + assertEquals(1, result.children.size()); + assertEquals("childName", result.children.get(0).name); + assertEquals("child-operator-id", result.children.get(0).operatorId); + } + + @Test + public void testExtractOpMultipleChildren() throws Exception { + String jsonString = "{\"opName\":{\"children\":[" + + "{\"childName1\":{\"OperatorId:\":\"child-operator-id1\"}}," + + "{\"childName2\":{\"OperatorId:\":\"child-operator-id2\"}}]}}"; + JSONObject operator = new JSONObject(jsonString); + + Vertex uut = new Vertex("name", null, null, tezJsonParser); + + Op result = uut.extractOp(operator, null); + assertEquals("opName", result.name); + assertEquals(2, result.children.size()); + assertEquals("childName1", result.children.get(0).name); + assertEquals("child-operator-id1", result.children.get(0).operatorId); + assertEquals("childName2", result.children.get(1).name); + assertEquals("child-operator-id2", result.children.get(1).operatorId); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/2fa4dc27/common/src/test/org/apache/hadoop/hive/common/jsonexplain/tez/TestTezJsonParser.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/tez/TestTezJsonParser.java b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/tez/TestTezJsonParser.java new file mode 100644 index 0000000..cf34ab8 --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/tez/TestTezJsonParser.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain.tez; + +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestTezJsonParser { + + private TezJsonParser uut; + + @Before + public void setUp() throws Exception { + this.uut = new TezJsonParser(); + } + + @Test + public void testExtractStagesAndPlans() throws Exception { + String jsonString = "{\"STAGE DEPENDENCIES\":{\"s1\":{\"ROOT STAGE\":\"\"}," + + "\"s2\":{\"DEPENDENT STAGES\":\"s1\"}},\"STAGE PLANS\":{}}"; + JSONObject input = new JSONObject(jsonString); + + uut.extractStagesAndPlans(input); + + assertEquals(2, uut.stages.size()); + assertEquals(1, uut.stages.get("s1").childStages.size()); + assertEquals("s2", uut.stages.get("s1").childStages.get(0).internalName); + assertEquals(0, uut.stages.get("s2").childStages.size()); + assertEquals(0, uut.stages.get("s1").parentStages.size()); + assertEquals(1, uut.stages.get("s2").parentStages.size()); + assertEquals("s1", uut.stages.get("s2").parentStages.get(0).internalName); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/2fa4dc27/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index 8ddb8d6..902664d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -42,6 +42,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.jsonexplain.JsonParser; @@ -132,7 +133,8 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable { * {"input_tables":[{"tablename": "default@test_sambavi_v1", "tabletype": "TABLE"}], * "input partitions":["default@srcpart@ds=2008-04-08/hr=11"]} */ - private static JSONObject getJSONDependencies(ExplainWork work) + @VisibleForTesting + static JSONObject getJSONDependencies(ExplainWork work) throws Exception { assert(work.getDependency()); @@ -202,7 +204,8 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable { private static String falseCondNameVectorizationEnabled = HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED.varname + " IS false"; - private ImmutablePair<Boolean, JSONObject> outputPlanVectorization(PrintStream out, boolean jsonOutput) + @VisibleForTesting + ImmutablePair<Boolean, JSONObject> outputPlanVectorization(PrintStream out, boolean jsonOutput) throws Exception { if (out != null) { @@ -422,7 +425,8 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable { } } - private JSONObject collectAuthRelatedEntities(PrintStream out, ExplainWork work) + @VisibleForTesting + JSONObject collectAuthRelatedEntities(PrintStream out, ExplainWork work) throws Exception { BaseSemanticAnalyzer analyzer = work.getAnalyzer(); @@ -486,7 +490,8 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable { return sb.toString(); } - private JSONObject outputMap(Map<?, ?> mp, boolean hasHeader, PrintStream out, + @VisibleForTesting + JSONObject outputMap(Map<?, ?> mp, boolean hasHeader, PrintStream out, boolean extended, boolean jsonOutput, int indent) throws Exception { TreeMap<Object, Object> tree = getBasictypeKeyedMap(mp); @@ -696,7 +701,8 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable { return outputPlan(work, out, extended, jsonOutput, indent, ""); } - private JSONObject outputPlan(Object work, PrintStream out, + @VisibleForTesting + JSONObject outputPlan(Object work, PrintStream out, boolean extended, boolean jsonOutput, int indent, String appendToHeader) throws Exception { // Check if work has an explain annotation Annotation note = AnnotationUtils.getAnnotation(work.getClass(), Explain.class); @@ -1072,7 +1078,8 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable { return null; } - private JSONObject outputDependencies(Task<?> task, + @VisibleForTesting + JSONObject outputDependencies(Task<?> task, PrintStream out, JSONObject parentJson, boolean jsonOutput, boolean taskType, int indent) throws Exception { http://git-wip-us.apache.org/repos/asf/hive/blob/2fa4dc27/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java index f44661e..84f992a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java @@ -32,6 +32,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService; @@ -78,7 +79,8 @@ public class ATSHook implements ExecuteWithHookContext { private static boolean defaultATSDomainCreated = false; private static final String DEFAULT_ATS_DOMAIN = "hive_default_ats"; - private enum OtherInfoTypes { + @VisibleForTesting + enum OtherInfoTypes { QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, SESSION_ID, THREAD_NAME, VERSION, CLIENT_IP_ADDRESS, HIVE_ADDRESS, HIVE_INSTANCE_TYPE, CONF, PERF, LLAP_APP_ID }; http://git-wip-us.apache.org/repos/asf/hive/blob/2fa4dc27/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java index 805bc5b..cac1fad 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java @@ -18,26 +18,52 @@ package org.apache.hadoop.hive.ql.exec; -import static org.junit.Assert.assertEquals; - -import java.io.PrintStream; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.hooks.Entity; +import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ExplainConfiguration; import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.*; import org.apache.hadoop.hive.ql.plan.Explain.Level; -import org.apache.hadoop.hive.ql.plan.ExplainWork; -import org.apache.hadoop.hive.ql.plan.TableScanDesc; -import org.junit.Ignore; +import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; +import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Before; import org.junit.Test; +import java.io.PrintStream; +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class TestExplainTask { + private static final String BACKUP_ID = "backup-id-mock"; + private static final String AST = "ast-mock"; + + private PrintStream out; + private ExplainTask uut; + private ObjectMapper objectMapper = new ObjectMapper(); + + @Before + public void setUp() { + uut = new ExplainTask(); + uut.conf = mock(HiveConf.class); + out = mock(PrintStream.class); + } + public static class DummyExplainDesc<K, V> extends TableScanDesc { private static final long serialVersionUID = 1L; private Map<K, V> explainResult; @@ -139,4 +165,249 @@ public class TestExplainTask { return baos.toString(); } + @Test + public void testGetJSONDependenciesJsonShhouldMatch() throws Exception { + ExplainWork work = mockExplainWork(); + + when(work.getDependency()).thenReturn(true); + + // Mock inputs + HashSet<ReadEntity> inputs = new HashSet<>(); + + // One input table + Table table = mock(Table.class); + when(table.getCompleteName()).thenReturn("table-name-mock"); + when(table.getTableType()).thenReturn(TableType.EXTERNAL_TABLE); + ReadEntity input1 = mock(ReadEntity.class); + when(input1.getType()).thenReturn(Entity.Type.TABLE); + when(input1.getTable()).thenReturn(table); + inputs.add(input1); + + // And one partition + Partition partition = mock(Partition.class); + when(partition.getCompleteName()).thenReturn("partition-name-mock"); + ReadEntity input2 = mock(ReadEntity.class); + when(input2.getType()).thenReturn(Entity.Type.PARTITION); + when(input2.getPartition()).thenReturn(partition); + inputs.add(input2); + + when(work.getInputs()).thenReturn(inputs); + + JsonNode result = objectMapper.readTree(ExplainTask.getJSONDependencies(work).toString()); + JsonNode expected = objectMapper.readTree("{\"input_partitions\":[{\"partitionName\":" + + "\"partition-name-mock\"}],\"input_tables\":[{\"tablename\":\"table-name-mock\"," + + "\"tabletype\":\"EXTERNAL_TABLE\"}]}"); + + assertEquals(expected, result); + } + + @Test + public void testGetJSONPlan() throws Exception { + when(uut.conf.getVar(HiveConf.ConfVars.HIVESTAGEIDREARRANGE)).thenReturn("EXECUTION"); + Task mockTask = mockTask(); + when(mockTask.getId()).thenReturn("mockTaskId"); + ExplainWork explainWorkMock = mockExplainWork(); + when(mockTask.getWork()).thenReturn(explainWorkMock); + List<Task<?>> tasks = Arrays.<Task<?>>asList(mockTask); + + + JsonNode result = objectMapper.readTree(uut.getJSONPlan(null, tasks, null, true, + false, false).toString()); + JsonNode expected = objectMapper.readTree("{\"STAGE DEPENDENCIES\":{\"mockTaskId\":" + + "{\"ROOT STAGE\":\"TRUE\",\"BACKUP STAGE\":\"backup-id-mock\"}},\"STAGE PLANS\":" + + "{\"mockTaskId\":{}}}"); + + assertEquals(expected, result); + } + + @Test + public void testOutputDependenciesJsonShouldMatch() throws Exception { + Task<? extends ExplainTask> task = mockTask(); + + JsonNode result = objectMapper.readTree( + uut.outputDependencies(task, out, null, true, true, 0).toString()); + JsonNode expected = objectMapper.readTree("{\"ROOT STAGE\":\"TRUE\",\"BACKUP STAGE\":" + + "\""+BACKUP_ID+"\",\"TASK TYPE\":\"EXPLAIN\"}"); + + assertEquals(expected, result); + } + + @Test + public void testGetJSONLogicalPlanJsonShouldMatch() throws Exception { + JsonNode result = objectMapper.readTree( + uut.getJSONLogicalPlan(null, mockExplainWork()).toString()); + JsonNode expected = objectMapper.readTree("{}"); + + assertEquals(expected, result); + } + + @Test + public void testOutputMapJsonShouldMatch() throws Exception { + Map<Object, Object> map = new LinkedHashMap<>(); + + // String + map.put("key-1", "value-1"); + + // SparkWork + map.put("spark-work", new SparkWork("spark-work")); + + // Empty list + List<Object> emptList = Collections.emptyList(); + map.put("empty-list", emptList); + + // List of TezWork.Dependency + List<Object> tezList1 = new ArrayList<>(Arrays.asList(new Object[] {mockTezWorkDependency()})); + map.put("tez-list-1", tezList1); + List<Object> tezList2 = new ArrayList<>( + Arrays.asList(new Object[] {mockTezWorkDependency(), mockTezWorkDependency()})); + map.put("tez-list-2", tezList2); + + // List of SparkWork.Dependency + List<Object> sparkList1 = new ArrayList<>( + Arrays.asList(new Object[]{mockSparkWorkDependency()})); + map.put("spark-list-1", sparkList1); + List<Object> sparkList2 = new ArrayList<>( + Arrays.asList(new Object[]{mockSparkWorkDependency(), mockSparkWorkDependency()})); + map.put("spark-list-2", sparkList2); + + // inner Map + Map<Object, Object> innerMap = new LinkedHashMap<>(); + innerMap.put("inner-key-1", "inner-value-1"); + innerMap.put("inner-key-2", tezList1); + map.put("map-1", innerMap); + + JsonNode result = objectMapper.readTree( + uut.outputMap(map, false, null, false, true, 0).toString()); + JsonNode expected = objectMapper.readTree("{\"key-1\":\"value-1\",\"tez-list-2\":" + + "[{\"parent\":\"name\"}," + "{\"parent\":\"name\"}],\"tez-list-1\":" + + "{\"parent\":\"name\"},\"empty-list\":\"[]\",\"spark-list-2\":" + + "[{\"parent\":\"mock-name\"},{\"parent\":\"mock-name\"}]," + + "\"spark-list-1\":{\"parent\":" + + "\"mock-name\"}, \"map-1\":\"{inner-key-1=inner-value-1, " + + "inner-key-2=[mock-tez-dependency]}\",\"spark-work\":" + + "{\"Spark\":{\"DagName:\":\"spark-work:2\"}}}"); + + assertEquals(expected, result); + } + + @Test + public void testOutputPlanJsonShouldMatch() throws Exception { + // SparkWork + SparkWork work = new SparkWork("spark-work"); + + JsonNode result = objectMapper.readTree( + uut.outputPlan(work, null, false, true, 0, null).toString()); + JsonNode expected = objectMapper.readTree("{\"Spark\":{\"DagName:\":\"spark-work:1\"}}"); + assertEquals(expected, result); + + // Operator with single child + CollectOperator parentCollectOperator1 = new CollectOperator(); + CollectOperator child1 = new CollectOperator(); + parentCollectOperator1.setChildOperators(new ArrayList<Operator<? extends OperatorDesc>>( + Arrays.asList(new CollectOperator[] {child1}))); + parentCollectOperator1.setConf(new CollectDesc()); + + result = objectMapper.readTree( + uut.outputPlan(parentCollectOperator1, null, false, true, 0, null).toString()); + expected = objectMapper.readTree("{\"Collect\":{\"children\":{}}}"); + assertEquals(expected, result); + + // Operator with 2 children + CollectOperator parentCollectOperator2 = new CollectOperator(); + CollectOperator child2 = new CollectOperator(); + parentCollectOperator2.setChildOperators(new ArrayList<Operator<? extends OperatorDesc>>( + Arrays.asList(new CollectOperator[] {child1, child2}))); + parentCollectOperator2.setConf(new CollectDesc()); + result = objectMapper.readTree( + uut.outputPlan(parentCollectOperator2, null, false, true, 0, null).toString()); + expected = objectMapper.readTree("{\"Collect\":{\"children\":[{},{}]}}"); + assertEquals(expected, result); + } + + @Test + public void testCollectAuthRelatedEntitiesJsonShouldMatch() throws Exception { + QueryState qs = mock(QueryState.class); + when(qs.getHiveOperation()).thenReturn(HiveOperation.EXPLAIN); + uut.queryState = qs; + + SessionState.start(new HiveConf(ExplainTask.class)); + // SessionState.get().setCommandType(HiveOperation.EXPLAIN); + HiveAuthenticationProvider authenticationProviderMock = mock(HiveAuthenticationProvider.class); + when(authenticationProviderMock.getUserName()).thenReturn("test-user"); + SessionState.get().setAuthenticator(authenticationProviderMock); + SessionState.get().setAuthorizer(mock(HiveAuthorizationProvider.class)); + ExplainWork work = mockExplainWork(); + + JsonNode result = objectMapper.readTree(uut.collectAuthRelatedEntities(null, work).toString()); + JsonNode expected = objectMapper.readTree("{\"CURRENT_USER\":\"test-user\"," + + "\"OPERATION\":\"EXPLAIN\",\"INPUTS\":[],\"OUTPUTS\":[]}"); + assertEquals(expected, result); + } + + @Test + public void testOutputPlanVectorizationJsonShouldMatch() throws Exception { + QueryState qs = mock(QueryState.class); + when(qs.getHiveOperation()).thenReturn(HiveOperation.EXPLAIN); + HiveConf hiveConf = new HiveConf(); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true); + when(qs.getConf()).thenReturn(hiveConf); + uut.queryState = qs; + + JsonNode result = objectMapper.readTree(uut.outputPlanVectorization(null, true).getRight().toString()); + JsonNode expected = objectMapper.readTree("{\"enabled\":true,\"enabledConditionsMet\":[\"hive.vectorized.execution.enabled IS true\"]}"); + assertEquals(expected, result); + + + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, false); + result = objectMapper.readTree(uut.outputPlanVectorization(null, true).getRight().toString()); + expected = objectMapper.readTree("{\"enabled\":false,\"enabledConditionsNotMet\":[\"hive.vectorized.execution.enabled IS false\"]}"); + assertEquals(expected, result); + + } + + private TezWork.Dependency mockTezWorkDependency() { + TezWork.Dependency dep = mock(TezWork.Dependency.class); + when(dep.getName()).thenReturn("name"); + when(dep.toString()).thenReturn("mock-tez-dependency"); + return dep; + } + + private SparkWork.Dependency mockSparkWorkDependency() { + SparkWork.Dependency dep = mock(SparkWork.Dependency.class); + when(dep.getName()).thenReturn("mock-name"); + when(dep.toString()).thenReturn("mock-spark-dependency"); + return dep; + } + + private ExplainWork mockExplainWork() { + ExplainWork explainWork = mock(ExplainWork.class); + + // Should produce JSON + when(explainWork.isFormatted()).thenReturn(true); + + // Should have some AST + // when(explainWork.getAstStringTree()).thenReturn(AST); + + when(explainWork.getAnalyzer()).thenReturn(mock(BaseSemanticAnalyzer.class)); + + return explainWork; + } + + private Task<ExplainTask> mockTask() { + Task<ExplainTask> task = mock(Task.class); + + // Explain type + when(task.getType()).thenReturn(StageType.EXPLAIN); + + // This is a root task + when(task.isRootTask()).thenReturn(true); + + // Set up backup task + Task backupTask = mock(Task.class); + when(backupTask.getId()).thenReturn(BACKUP_ID); + when(task.getBackupTask()).thenReturn(backupTask); + + return task; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/2fa4dc27/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestATSHook.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestATSHook.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestATSHook.java new file mode 100644 index 0000000..c484062 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestATSHook.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.hooks; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +public class TestATSHook { + + private ObjectMapper objectMapper = new ObjectMapper(); + private ATSHook uut; + + @Before + public void setUp() { + uut = new ATSHook(); + } + + @Test + public void testCreatePreHookEventJsonShhouldMatch() throws Exception { + TimelineEntity timelineEntity = uut.createPreHookEvent( + "test-query-id", "test-query", new org.json.JSONObject(), 0L, + "test-user", "test-request-user", 0, 0, "test-opid", + "client-ip-address", "hive-instance-address", "hive-instance-type", "session-id", "log-id", + "thread-id", "execution-mode", Collections.<String>emptyList(), Collections.<String>emptyList(), + new HiveConf(), null, "domain-id"); + String resultStr = (String) timelineEntity.getOtherInfo() + .get(ATSHook.OtherInfoTypes.QUERY.name()); + + JsonNode result = objectMapper.readTree(resultStr); + JsonNode expected = objectMapper.readTree("{\"queryText\":\"test-query\"," + + "\"queryPlan\":{}}"); + + assertEquals(expected, result); + } +}