HIVE-19274: Add an OpTreeSignature persistence checker hook (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/da10aabe Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/da10aabe Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/da10aabe Branch: refs/heads/storage-branch-2.6 Commit: da10aabe56edf8fbb26d89d64bedcc4afa84a305 Parents: 10699bf Author: Zoltan Haindrich <k...@rxd.hu> Authored: Wed Apr 25 17:41:57 2018 +0200 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Wed Apr 25 17:41:57 2018 +0200 ---------------------------------------------------------------------- data/conf/llap/hive-site.xml | 2 +- data/conf/perf-reg/tez/hive-site.xml | 2 +- .../RuntimeStatsPersistenceCheckerHook.java | 71 ++++++++++++++++++++ .../ql/optimizer/signature/SignatureUtils.java | 2 +- .../apache/hadoop/hive/ql/plan/JoinDesc.java | 6 +- 5 files changed, 77 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/da10aabe/data/conf/llap/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml index 990b473..1507a56 100644 --- a/data/conf/llap/hive-site.xml +++ b/data/conf/llap/hive-site.xml @@ -163,7 +163,7 @@ <property> <name>hive.exec.post.hooks</name> - <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value> + <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter, org.apache.hadoop.hive.ql.hooks.RuntimeStatsPersistenceCheckerHook</value> <description>Post Execute Hook for Tests</description> </property> http://git-wip-us.apache.org/repos/asf/hive/blob/da10aabe/data/conf/perf-reg/tez/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/perf-reg/tez/hive-site.xml b/data/conf/perf-reg/tez/hive-site.xml index e11f8f8..78a5481 100644 --- a/data/conf/perf-reg/tez/hive-site.xml +++ b/data/conf/perf-reg/tez/hive-site.xml @@ -162,7 +162,7 @@ <property> <name>hive.exec.post.hooks</name> - <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value> + <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter, org.apache.hadoop.hive.ql.hooks.RuntimeStatsPersistenceCheckerHook</value> <description>Post Execute Hook for Tests</description> </property> http://git-wip-us.apache.org/repos/asf/hive/blob/da10aabe/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java new file mode 100644 index 0000000..b0bdad3 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.hooks; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hive.ql.optimizer.signature.OpTreeSignature; +import org.apache.hadoop.hive.ql.optimizer.signature.RuntimeStatsPersister; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This hook adds a persistence loop-back ensure that runtime statistics could be used. + */ +public class RuntimeStatsPersistenceCheckerHook implements ExecuteWithHookContext { + + private static final Logger LOG = LoggerFactory.getLogger(RuntimeStatsPersistenceCheckerHook.class); + + @Override + public void run(HookContext hookContext) throws Exception { + + PlanMapper pm = ((PrivateHookContext) hookContext).getContext().getPlanMapper(); + + List<OpTreeSignature> sigs = pm.getAll(OpTreeSignature.class); + + for (OpTreeSignature sig : sigs) { + try { + OpTreeSignature sig2 = persistenceLoop(sig, OpTreeSignature.class); + sig.getSig().proveEquals(sig2.getSig()); + } catch (Exception e) { + throw new RuntimeException("while checking the signature of: " + sig.getSig(), e); + } + } + for (OpTreeSignature sig : sigs) { + try { + OpTreeSignature sig2 = persistenceLoop(sig, OpTreeSignature.class); + if (!sig.equals(sig2)) { + throw new RuntimeException("signature mismatch"); + } + } catch (Exception e) { + throw new RuntimeException("while checking the signature of: " + sig.getSig(), e); + } + } + LOG.info("signature checked: " + sigs.size()); + } + + private <T> T persistenceLoop(T sig, Class<T> clazz) throws IOException { + RuntimeStatsPersister sp = RuntimeStatsPersister.INSTANCE; + String stored = sp.encode(sig); + return sp.decode(stored, clazz); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/da10aabe/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/SignatureUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/SignatureUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/SignatureUtils.java index f599d33..6c86f9a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/SignatureUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/SignatureUtils.java @@ -65,7 +65,7 @@ public final class SignatureUtils { if (sigMethods.isEmpty()) { // by supplying using "o" this enforces identity/equls matching // which will most probably make the signature very unique - ret.put(classLabel, o); + ret.put(classLabel, System.identityHashCode(o)); } else { ret.put(classLabel, "1"); for (Method method : sigMethods) { http://git-wip-us.apache.org/repos/asf/hive/blob/da10aabe/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java index 95990b8..b5ffcd9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java @@ -269,12 +269,12 @@ public class JoinDesc extends AbstractOperatorDesc { */ @Explain(displayName = "filter predicates") @Signature - public Map<Byte, String> getFiltersStringMap() { + public Map<String, String> getFiltersStringMap() { if (getFilters() == null || getFilters().size() == 0) { return null; } - LinkedHashMap<Byte, String> ret = new LinkedHashMap<Byte, String>(); + LinkedHashMap<String, String> ret = new LinkedHashMap<>(); boolean filtersPresent = false; for (Map.Entry<Byte, List<ExprNodeDesc>> ent : getFilters().entrySet()) { @@ -295,7 +295,7 @@ public class JoinDesc extends AbstractOperatorDesc { sb.append("}"); } } - ret.put(ent.getKey(), sb.toString()); + ret.put(String.valueOf(ent.getKey()), sb.toString()); } if (filtersPresent) {