Repository: hive Updated Branches: refs/heads/master e51f7c9d2 -> 3df6bc28b
HIVE-18737 : add an option to disable LLAP IO ACID for non-original files (Sergey Shelukhin, reviewed by Gopal Vijayaraghavan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dbf38ed1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dbf38ed1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dbf38ed1 Branch: refs/heads/master Commit: dbf38ed1434656f487f942b2f9df4fea3e29e44a Parents: e51f7c9 Author: sergey <ser...@apache.org> Authored: Tue Feb 20 17:21:00 2018 -0800 Committer: sergey <ser...@apache.org> Committed: Tue Feb 20 17:21:00 2018 -0800 ---------------------------------------------------------------------- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 1 + ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/dbf38ed1/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index b51dc7e..38f6430 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3120,6 +3120,7 @@ public class HiveConf extends Configuration { false, "Use Tez cartesian product edge to speed up cross product"), // The default is different on the client and server, so it's null here. LLAP_IO_ENABLED("hive.llap.io.enabled", null, "Whether the LLAP IO layer is enabled."), + LLAP_IO_ACID_ENABLED("hive.llap.io.acid", true, "Whether the LLAP IO layer is enabled for ACID."), LLAP_IO_TRACE_SIZE("hive.llap.io.trace.size", "2Mb", new SizeValidator(0L, true, (long)Integer.MAX_VALUE, false), "The buffer size for a per-fragment LLAP debug trace. 0 to disable."), http://git-wip-us.apache.org/repos/asf/hive/blob/dbf38ed1/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java index 91d855b..96c5916 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java @@ -29,6 +29,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.io.AcidInputFormat; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.ColumnarSplit; @@ -234,13 +235,13 @@ public class OrcSplit extends FileSplit implements ColumnarSplit, LlapAwareSplit public boolean canUseLlapIo(Configuration conf) { final boolean hasDelta = deltas != null && !deltas.isEmpty(); final boolean isAcidRead = AcidUtils.isFullAcidScan(conf); - final boolean isVectorized = HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED); + final boolean isVectorized = HiveConf.getBoolVar(conf, ConfVars.HIVE_VECTORIZATION_ENABLED); Boolean isSplitUpdate = null; if (isAcidRead) { final AcidUtils.AcidOperationalProperties acidOperationalProperties = AcidUtils.getAcidOperationalProperties(conf); isSplitUpdate = acidOperationalProperties.isSplitUpdate(); + // TODO: this is brittle. Who said everyone has to upgrade using upgrade process? assert isSplitUpdate : "should be true in Hive 3.0"; } @@ -250,7 +251,8 @@ public class OrcSplit extends FileSplit implements ColumnarSplit, LlapAwareSplit return true; } } else { - if (isAcidRead && hasBase && isVectorized) { + boolean isAcidEnabled = HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_ACID_ENABLED); + if (isAcidEnabled && isAcidRead && hasBase && isVectorized) { if (hasDelta) { if (isSplitUpdate) { // Base with delete deltas
