Repository: hive Updated Branches: refs/heads/master fe14a9088 -> 415373bbd
HIVE-11483 : Add encoding and decoding for query string config (Rajat Khandelwal, reviewed by Amareshwari Sriramadasu Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/415373bb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/415373bb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/415373bb Branch: refs/heads/master Commit: 415373bbd1c58e7b8f00aa32c0bb7d485d4f91e5 Parents: fe14a90 Author: Rajat Khandelwal <pro...@apache.org> Authored: Wed Mar 9 14:17:21 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Wed Mar 9 14:17:21 2016 +0530 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 76 ++++++++++++++++---- .../apache/hadoop/hive/conf/TestHiveConf.java | 10 +++ .../java/org/apache/hadoop/hive/ql/Driver.java | 5 +- .../hive/ql/exec/errors/TaskLogProcessor.java | 2 +- .../hadoop/hive/ql/exec/mr/ExecDriver.java | 2 +- .../hadoop/hive/ql/exec/tez/DagUtils.java | 2 +- .../hadoop/hive/ql/session/SessionState.java | 6 +- .../ql/exec/errors/TestTaskLogProcessor.java | 8 +-- .../apache/hadoop/hive/ql/hooks/TestHooks.java | 2 +- 9 files changed, 85 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 5098851..27a56dd 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -40,12 +40,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.security.auth.login.LoginException; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; + +import java.io.*; import java.net.URL; +import java.net.URLDecoder; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -96,6 +95,35 @@ public class HiveConf extends Configuration { this.isSparkConfigUpdated = isSparkConfigUpdated; } + public interface EncoderDecoder<K, V> { + V encode(K key); + K decode(V value); + } + + public static class URLEncoderDecoder implements EncoderDecoder<String, String> { + private static final String UTF_8 = "UTF-8"; + @Override + public String encode(String key) { + try { + return URLEncoder.encode(key, UTF_8); + } catch (UnsupportedEncodingException e) { + return key; + } + } + + @Override + public String decode(String value) { + try { + return URLDecoder.decode(value, UTF_8); + } catch (UnsupportedEncodingException e) { + return value; + } + } + } + public static class EncoderDecoderFactory { + public static final URLEncoderDecoder URL_ENCODER_DECODER = new URLEncoderDecoder(); + } + static { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { @@ -3283,10 +3311,8 @@ public class HiveConf extends Configuration { public static String getVar(Configuration conf, ConfVars var) { assert (var.valClass == String.class) : var.varname; - if (var.altName != null) { - return conf.get(var.varname, conf.get(var.altName, var.defaultStrVal)); - } - return conf.get(var.varname, var.defaultStrVal); + return var.altName != null ? conf.get(var.varname, conf.get(var.altName, var.defaultStrVal)) + : conf.get(var.varname, var.defaultStrVal); } public static String getTrimmedVar(Configuration conf, ConfVars var) { @@ -3309,10 +3335,13 @@ public class HiveConf extends Configuration { } public static String getVar(Configuration conf, ConfVars var, String defaultVal) { - if (var.altName != null) { - return conf.get(var.varname, conf.get(var.altName, defaultVal)); - } - return conf.get(var.varname, defaultVal); + String ret = var.altName != null ? conf.get(var.varname, conf.get(var.altName, defaultVal)) + : conf.get(var.varname, defaultVal); + return ret; + } + + public static String getVar(Configuration conf, ConfVars var, EncoderDecoder<String, String> encoderDecoder) { + return encoderDecoder.decode(getVar(conf, var)); } public String getLogIdVar(String defaultValue) { @@ -3333,6 +3362,10 @@ public class HiveConf extends Configuration { assert (var.valClass == String.class) : var.varname; conf.set(var.varname, val); } + public static void setVar(Configuration conf, ConfVars var, String val, + EncoderDecoder<String, String> encoderDecoder) { + setVar(conf, var, encoderDecoder.encode(val)); + } public static ConfVars getConfVars(String name) { return vars.get(name); @@ -3350,6 +3383,21 @@ public class HiveConf extends Configuration { setVar(this, var, val); } + public String getQueryString() { + return getQueryString(this); + } + + public static String getQueryString(Configuration conf) { + return getVar(conf, ConfVars.HIVEQUERYSTRING, EncoderDecoderFactory.URL_ENCODER_DECODER); + } + + public void setQueryString(String query) { + setQueryString(this, query); + } + + public static void setQueryString(Configuration conf, String query) { + setVar(conf, ConfVars.HIVEQUERYSTRING, query, EncoderDecoderFactory.URL_ENCODER_DECODER); + } public void logVars(PrintStream ps) { for (ConfVars one : ConfVars.values()) { ps.println(one.varname + "=" + ((get(one.varname) != null) ? get(one.varname) : "")); @@ -3904,7 +3952,7 @@ public class HiveConf extends Configuration { } private static boolean isAllowed(Configuration conf, ConfVars setting) { - String mode = HiveConf.getVar(conf, ConfVars.HIVEMAPREDMODE, null); + String mode = HiveConf.getVar(conf, ConfVars.HIVEMAPREDMODE, (String)null); return (mode != null) ? !"strict".equals(mode) : !HiveConf.getBoolVar(conf, setting); } } http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java index 365d500..f88573f 100644 --- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java +++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java @@ -25,6 +25,8 @@ import org.apache.hive.common.util.HiveTestUtils; import org.junit.Assert; import org.junit.Test; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.util.concurrent.TimeUnit; @@ -156,4 +158,12 @@ public class TestHiveConf { conf.setSparkConfigUpdated(false); Assert.assertFalse(conf.getSparkConfigUpdated()); } + @Test + public void testEncodingDecoding() throws UnsupportedEncodingException { + HiveConf conf = new HiveConf(); + String query = "select blah, '\u0001' from random_table"; + conf.setQueryString(query); + Assert.assertEquals(URLEncoder.encode(query, "UTF-8"), conf.get(ConfVars.HIVEQUERYSTRING.varname)); + Assert.assertEquals(query, conf.getQueryString()); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 3253146..b50c5a2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -86,7 +86,6 @@ import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatter; import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; -import org.apache.hadoop.hive.ql.parse.CalcitePlanner; import org.apache.hadoop.hive.ql.parse.ColumnAccessInfo; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHook; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; @@ -499,7 +498,7 @@ public class Driver implements CommandProcessor { plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId, SessionState.get().getHiveOperation(), schema); - conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, queryStr); + conf.setQueryString(queryStr); conf.set("mapreduce.workflow.id", "hive_" + queryId); conf.set("mapreduce.workflow.name", queryStr); @@ -1484,7 +1483,7 @@ public class Driver implements CommandProcessor { String queryId = plan.getQueryId(); // Get the query string from the conf file as the compileInternal() method might // hide sensitive information during query redaction. - String queryStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING); + String queryStr = conf.getQueryString(); maxthreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.EXECPARALLETHREADNUMBER); http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java index 68123d4..b788d8f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java @@ -53,7 +53,7 @@ public class TaskLogProcessor { private final String query; public TaskLogProcessor(JobConf conf) { - query = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING); + query = HiveConf.getQueryString(conf); heuristics.put(new ScriptErrorHeuristic(), new HeuristicStats()); heuristics.put(new MapAggrMemErrorHeuristic(), new HeuristicStats()); http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index b184b4e..ce020a5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -568,7 +568,7 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop // Intentionally overwrites anything the user may have put here conf.setBoolean("hive.input.format.sorted", mWork.isInputFormatSorted()); - if (HiveConf.getVar(conf, ConfVars.HIVE_CURRENT_DATABASE, null) == null) { + if (HiveConf.getVar(conf, ConfVars.HIVE_CURRENT_DATABASE, (String)null) == null) { HiveConf.setVar(conf, ConfVars.HIVE_CURRENT_DATABASE, getCurrentDB()); } } http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 473dbd6..79da860 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -904,7 +904,7 @@ public class DagUtils { public FileStatus getHiveJarDirectory(Configuration conf) throws IOException, LoginException { FileStatus fstatus = null; - String hdfsDirPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_JAR_DIRECTORY, null); + String hdfsDirPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_JAR_DIRECTORY, (String)null); if (hdfsDirPathStr != null) { LOG.info("Hive jar directory is " + hdfsDirPathStr); fstatus = validateTargetDir(new Path(hdfsDirPathStr), conf); http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 109cd8c..78bbb1f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -373,11 +373,11 @@ public class SessionState { } public void setCmd(String cmdString) { - conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, cmdString); + conf.setQueryString(cmdString); } public String getCmd() { - return (conf.getVar(HiveConf.ConfVars.HIVEQUERYSTRING)); + return (conf.getQueryString()); } public String getQueryId() { @@ -1642,7 +1642,7 @@ public class SessionState { // Provide a facility to set current timestamp during tests if (conf.getBoolVar(ConfVars.HIVE_IN_TEST)) { String overrideTimestampString = - HiveConf.getVar(conf, HiveConf.ConfVars.HIVETESTCURRENTTIMESTAMP, null); + HiveConf.getVar(conf, HiveConf.ConfVars.HIVETESTCURRENTTIMESTAMP, (String)null); if (overrideTimestampString != null && overrideTimestampString.length() > 0) { queryCurrentTimestamp = Timestamp.valueOf(overrideTimestampString); } http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java index 67a86a6..477479d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java @@ -98,7 +98,7 @@ public class TestTaskLogProcessor { @Test public void testGetStackTraces() throws Exception { JobConf jobConf = new JobConf(); - jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;"); + HiveConf.setQueryString(jobConf, "select * from foo group by moo;"); final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf); @@ -150,7 +150,7 @@ public class TestTaskLogProcessor { @Test public void testScriptErrorHeuristic() throws Exception { JobConf jobConf = new JobConf(); - jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;"); + HiveConf.setQueryString(jobConf, "select * from foo group by moo;"); final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf); @@ -177,7 +177,7 @@ public class TestTaskLogProcessor { @Test public void testDataCorruptErrorHeuristic() throws Exception { JobConf jobConf = new JobConf(); - jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;"); + HiveConf.setQueryString(jobConf, "select * from foo group by moo;"); final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf); @@ -210,7 +210,7 @@ public class TestTaskLogProcessor { @Test public void testMapAggrMemErrorHeuristic() throws Exception { JobConf jobConf = new JobConf(); - jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;"); + HiveConf.setQueryString(jobConf, "select * from foo group by moo;"); final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf); http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java index 8d27762..4c14d8b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java @@ -73,7 +73,7 @@ public class TestHooks { Driver driver = createDriver(conf); int ret = driver.compile("select 'XXX' from t1"); assertEquals("Checking command success", 0, ret); - assertEquals("select 'AAA' from t1", HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING)); + assertEquals("select 'AAA' from t1", conf.getQueryString()); } public static class SimpleQueryRedactor extends Redactor {