This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit a6a0ba59dff73f7ad27bf02a20da1ce06085d0b2
Author: Daniel Voros <daniel.vo...@gmail.com>
AuthorDate: Tue Mar 17 17:00:17 2020 +0000
HIVE-22901: Variable substitution can lead to OOM on circular references
(Daniel Voros via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <k...@rxd.hu>
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++
.../apache/hadoop/hive/conf/SystemVariables.java | 10 ++++++++
.../hadoop/hive/conf/TestSystemVariables.java | 29 ++++++++++++++++++++++
.../org/apache/hive/jdbc/TestRestrictedList.java | 1 +
4 files changed, 43 insertions(+)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 54b33a3..d50912b 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4791,6 +4791,7 @@ public class HiveConf extends Configuration {
"hive.spark.client.rpc.max.size," +
"hive.spark.client.rpc.threads," +
"hive.spark.client.secret.bits," +
+ "hive.query.max.length," +
"hive.spark.client.rpc.server.address," +
"hive.spark.client.rpc.server.port," +
"hive.spark.client.rpc.sasl.mechanisms," +
@@ -4827,6 +4828,8 @@ public class HiveConf extends Configuration {
SPARK_CLIENT_TYPE.varname,
"Comma separated list of variables which are related to remote spark
context.\n" +
"Changing these variables will result in re-creating the spark
session."),
+ HIVE_QUERY_MAX_LENGTH("hive.query.max.length", "10Mb", new
SizeValidator(), "The maximum" +
+ " size of a query string. Enforced after variable substitutions."),
HIVE_QUERY_TIMEOUT_SECONDS("hive.query.timeout.seconds", "0s",
new TimeValidator(TimeUnit.SECONDS),
"Timeout for Running Query in seconds. A nonpositive value means
infinite. " +
diff --git a/common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java
b/common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java
index 695f3ec..89ea20e 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java
@@ -88,6 +88,10 @@ public class SystemVariables {
}
protected final String substitute(Configuration conf, String expr, int
depth) {
+ long maxLength = 0;
+ if (conf != null) {
+ maxLength = HiveConf.getSizeVar(conf,
HiveConf.ConfVars.HIVE_QUERY_MAX_LENGTH);
+ }
Matcher match = varPat.matcher("");
String eval = expr;
StringBuilder builder = new StringBuilder();
@@ -107,12 +111,18 @@ public class SystemVariables {
found = true;
}
builder.append(eval.substring(prev, match.start())).append(substitute);
+ if (maxLength > 0 && builder.length() > maxLength) {
+ throw new IllegalStateException("Query length longer than
hive.query.max.length ("+builder.length()+">"+maxLength+").");
+ }
prev = match.end();
}
if (!found) {
return eval;
}
builder.append(eval.substring(prev));
+ if (maxLength > 0 && builder.length() > maxLength) {
+ throw new IllegalStateException("Query length longer than
hive.query.max.length ("+builder.length()+">"+maxLength+").");
+ }
eval = builder.toString();
}
if (s > depth) {
diff --git
a/common/src/test/org/apache/hadoop/hive/conf/TestSystemVariables.java
b/common/src/test/org/apache/hadoop/hive/conf/TestSystemVariables.java
index 6004aba..3641020 100644
--- a/common/src/test/org/apache/hadoop/hive/conf/TestSystemVariables.java
+++ b/common/src/test/org/apache/hadoop/hive/conf/TestSystemVariables.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.conf;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
@@ -24,6 +25,7 @@ import org.junit.Test;
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertNull;
+import static org.junit.Assert.fail;
public class TestSystemVariables {
public static final String SYSTEM = "system";
@@ -74,4 +76,31 @@ public class TestSystemVariables {
System.setProperty("java.io.tmpdir", "");
assertEquals("", SystemVariables.substitute(systemJavaIoTmpDir));
}
+
+ @Test
+ public void test_SubstituteLongSelfReference() {
+ String randomPart = RandomStringUtils.random(100_000);
+ String reference = "${hiveconf:myTestVariable}";
+
+ StringBuilder longStringWithReferences = new StringBuilder();
+ for(int i = 0; i < 10; i ++) {
+ longStringWithReferences.append(randomPart).append(reference);
+ }
+
+ SystemVariables uut = new SystemVariables();
+
+ HiveConf conf = new HiveConf();
+ conf.set(HiveConf.ConfVars.HIVE_QUERY_MAX_LENGTH.varname, "100Kb");
+ conf.set("myTestVariable", longStringWithReferences.toString());
+
+ try {
+ uut.substitute(conf, longStringWithReferences.toString(), 40);
+ } catch (Exception e) {
+ if (!e.getMessage().startsWith("Query length longer than
hive.query.max.length")) {
+ fail("unexpected error message: " + e.getMessage());
+ }
+ return;
+ }
+ fail("should have thrown exception during substitution");
+ }
}
diff --git
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java
index 0890653..cc32a7e 100644
---
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java
+++
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java
@@ -93,6 +93,7 @@ public class TestRestrictedList {
addToExpectedRestrictedMap("hive.spark.client.rpc.server.address");
addToExpectedRestrictedMap("hive.spark.client.rpc.server.port");
addToExpectedRestrictedMap("hive.spark.client.rpc.sasl.mechanisms");
+ addToExpectedRestrictedMap("hive.query.max.length");
addToExpectedRestrictedMap("bonecp.test");
addToExpectedRestrictedMap("hive.druid.broker.address.default");
addToExpectedRestrictedMap("hive.druid.coordinator.address.default");
