This is an automated email from the ASF dual-hosted git repository.
saihemanth-cloudera pushed a commit to branch branch-4.2.1
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-4.2.1 by this push:
new e88e99f97e7 Few backports for Branch 4.2.1 (#6612)
e88e99f97e7 is described below
commit e88e99f97e794018cc17fa39419cb312b73f046b
Author: Sai Hemanth Gantasala
<[email protected]>
AuthorDate: Thu Jul 16 20:15:33 2026 -0700
Few backports for Branch 4.2.1 (#6612)
* HIVE-29622: Improve metastore direct-SQL partition lookup robustness
(#6509)
* HIVE-29653: Fix SAML bearer token authentication bypass in HiveServer2
(#6534)
* HIVE-29671: Harden Schema path authorization for avro serde (#6549)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 11 ++
.../auth/saml/TestHttpSamlAuthentication.java | 91 +++++++++++++-
.../org/apache/hadoop/hive/ql/plan/PlanUtils.java | 2 +
.../security/authorization/AuthorizationUtils.java | 91 +++++++++++++-
.../authorization/command/CommandAuthorizerV2.java | 8 ++
.../plugin/metastore/events/AlterTableEvent.java | 20 +++
.../plugin/metastore/events/CreateTableEvent.java | 22 +++-
.../TestAvroSchemaUrlAuthorizationUtils.java | 139 +++++++++++++++++++++
.../metastore/TestHiveMetaStoreAuthorizer.java | 77 +++++++++++-
.../llap/avro_extschema_insert.q.out | 4 +
.../llap/avro_tableproperty_optimize.q.out | 2 +
.../results/clientpositive/llap/avrotblsjoin.q.out | 2 +
.../clientpositive/llap/compustat_avro.q.out | 2 +
.../hadoop/hive/serde2/avro/AvroSerdeUtils.java | 138 +++++++++++++++++++-
.../hive/serde2/avro/TestAvroSerdeUtils.java | 101 ++++++++++++++-
.../auth/saml/HiveSamlAuthTokenGenerator.java | 12 +-
.../hive/service/cli/thrift/ThriftHttpServlet.java | 3 +-
.../hadoop/hive/metastore/DirectSqlUpdatePart.java | 55 ++++----
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 17 ++-
.../hadoop/hive/metastore/TestObjectStore.java | 74 +++++++++++
20 files changed, 818 insertions(+), 53 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 72519be0cda..e608628b895 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2290,6 +2290,17 @@ public static enum ConfVars {
"Whether to use former Java date/time APIs to convert between
timezones when writing timestamps in " +
"Avro files. Once data are written to the file the effect is permanent
(also reflected in the metadata)." +
"Changing the value of this property affects only new data written to
the file."),
+
HIVE_AVRO_SCHEMA_URL_ALLOWED_SCHEMES("hive.avro.schema.url.allowed.schemes",
+ "hdfs,s3,s3a,s3n,abfs,abfss,gs,wasb,wasbs,viewfs,o3fs,ofs",
+ "Comma-separated list of URI schemes permitted for avro.schema.url
when loading schemas from a remote " +
+ "location. HTTP/HTTPS and file:// are never allowed via this
setting. A URI with no scheme is " +
+ "resolved against the default filesystem."),
+
HIVE_AVRO_SCHEMA_URL_REMOTE_HTTP_ENABLED("hive.avro.schema.url.remote.http.enabled",
false,
+ "Whether to allow avro.schema.url values that use http or https. When
enabled, the host must also appear " +
+ "in hive.avro.schema.url.http.allowed.hosts. Disabled by default
to prevent server-side request forgery."),
+
HIVE_AVRO_SCHEMA_URL_HTTP_ALLOWED_HOSTS("hive.avro.schema.url.http.allowed.hosts",
"",
+ "Comma-separated list of hosts permitted for avro.schema.url when
hive.avro.schema.url.remote.http.enabled " +
+ "is true. HTTP/HTTPS schema fetch is rejected when this list is
empty."),
HIVE_INT_TIMESTAMP_CONVERSION_IN_SECONDS("hive.int.timestamp.conversion.in.seconds",
false,
"Boolean/tinyint/smallint/int/bigint value is interpreted as
milliseconds during the timestamp conversion.\n" +
"Set this flag to true to interpret the value as seconds to be
consistent with float/double." ),
diff --git
a/itests/hive-unit/src/test/java/org/apache/hive/service/auth/saml/TestHttpSamlAuthentication.java
b/itests/hive-unit/src/test/java/org/apache/hive/service/auth/saml/TestHttpSamlAuthentication.java
index 7d119e9372c..f58cf6a0adf 100644
---
a/itests/hive-unit/src/test/java/org/apache/hive/service/auth/saml/TestHttpSamlAuthentication.java
+++
b/itests/hive-unit/src/test/java/org/apache/hive/service/auth/saml/TestHttpSamlAuthentication.java
@@ -35,6 +35,7 @@
import java.net.InetAddress;
import java.net.ServerSocket;
import java.nio.charset.StandardCharsets;
+import java.util.Base64;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
@@ -137,9 +138,17 @@ public void cleanUpIdpEnv() {
idpContainer.stop();
idpContainer = null;
}
- if (miniHS2 != null) {
+ if (miniHS2 != null && miniHS2.isStarted()) {
miniHS2.stop();
}
+ HiveSamlAuthTokenGenerator.shutdown();
+ }
+
+ private static ISAMLAuthTokenGenerator createTokenGenerator(String tokenTtl)
{
+ HiveSamlAuthTokenGenerator.shutdown();
+ HiveConf conf = new HiveConf();
+ conf.setVar(ConfVars.HIVE_SERVER2_SAML_CALLBACK_TOKEN_TTL, tokenTtl);
+ return HiveSamlAuthTokenGenerator.get(conf);
}
private void setupIDP(boolean useSignedAssertions, String authMode) throws
Exception {
@@ -554,6 +563,86 @@ public void testTokenReuse() throws Exception {
}
}
+ @Test
+ public void testValidTokenRoundTrip() throws Exception {
+ ISAMLAuthTokenGenerator tokenGenerator = createTokenGenerator("30s");
+ String token = tokenGenerator.get("alice", "relay-state-1");
+ assertEquals("alice", tokenGenerator.validate(token));
+ }
+
+ @Test
+ public void testForgedSignatureRejected() throws Exception {
+ ISAMLAuthTokenGenerator tokenGenerator = createTokenGenerator("30s");
+ String forgedPayload = "u=alice;id=1337;time=" + System.currentTimeMillis()
+ + ";rs=deadbeef;sg=bogus";
+ try {
+ String forgedToken =
Base64.getEncoder().encodeToString(forgedPayload.getBytes(StandardCharsets.UTF_8));
+ tokenGenerator.validate(forgedToken);
+ fail("Expected forged token to be rejected");
+ } catch (HttpSamlAuthenticationException e) {
+ assertEquals("Token could not be verified", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testInvalidTokenRejected() throws Exception {
+ ISAMLAuthTokenGenerator tokenGenerator = createTokenGenerator("30s");
+ try {
+ tokenGenerator.validate("notAValidToken");
+ fail("Expected malformed base64 token to be rejected");
+ } catch (HttpSamlAuthenticationException e) {
+ assertEquals("Invalid token", e.getMessage());
+ }
+ String invalidStructure =
Base64.getEncoder().encodeToString("foo".getBytes());
+ try {
+ tokenGenerator.validate(invalidStructure);
+ fail("Expected invalid token structure to be rejected");
+ } catch (HttpSamlAuthenticationException e) {
+ assertEquals("Invalid token", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testExpiredTokenRejected() throws Exception {
+ ISAMLAuthTokenGenerator tokenGenerator = createTokenGenerator("1s");
+ String token = tokenGenerator.get("alice", "relay-state-1");
+ Thread.sleep(1100);
+ try {
+ tokenGenerator.validate(token);
+ fail("Expected expired token to be rejected");
+ } catch (HttpSamlAuthenticationException e) {
+ assertEquals("Token is expired", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testParseHandlesBase64PaddingInSignature() {
+ Map<String, String> kv = new HashMap<>();
+ String token = "u=alice;id=1;time=1000;rs=rs1;sg=YWJjZA==";
+ assertTrue(HiveSamlAuthTokenGenerator.parse(token, kv));
+ assertEquals("alice", kv.get("u"));
+ assertEquals("YWJjZA==", kv.get("sg"));
+ }
+
+ @Test
+ public void testParseRejectsEncodedBearerToken() {
+ Map<String, String> kv = new HashMap<>();
+ String encoded = Base64.getEncoder().encodeToString(
+ "u=alice;id=1;time=1000;rs=rs1;sg=abc".getBytes());
+ assertFalse(HiveSamlAuthTokenGenerator.parse(encoded, kv));
+ }
+
+ @Test
+ public void testParseDecodedTokenFromGenerator() throws Exception {
+ ISAMLAuthTokenGenerator tokenGenerator = createTokenGenerator("30s");
+ String encoded = tokenGenerator.get("bob", "relay-42");
+ String decoded = new String(Base64.getDecoder().decode(encoded),
StandardCharsets.UTF_8);
+ Map<String, String> kv = new HashMap<>();
+ assertTrue(HiveSamlAuthTokenGenerator.parse(decoded, kv));
+ assertEquals("bob", kv.get("u"));
+ assertEquals("relay-42", kv.get(HiveSamlAuthTokenGenerator.RELAY_STATE));
+ }
+
private static void assertLoggedInUser(HiveConnection connection, String
expectedUser)
throws SQLException {
Statement stmt = connection.createStatement();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
index 75f78221dc1..98e7bea564a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
@@ -65,6 +65,7 @@
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.security.authorization.AuthorizationUtils;
import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.util.NullOrdering;
@@ -1002,6 +1003,7 @@ public static ReadEntity addInput(Set<ReadEntity> inputs,
ReadEntity newInput, b
assert false;
} else {
inputs.add(newInput);
+ AuthorizationUtils.addAvroSchemaUrlInputForReadEntity(inputs, newInput);
return newInput;
}
// make compile happy
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationUtils.java
index fe0d2de242a..c92083b0b58 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationUtils.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationUtils.java
@@ -17,11 +17,17 @@
*/
package org.apache.hadoop.hive.ql.security.authorization;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
import org.apache.hadoop.hive.metastore.api.HiveObjectType;
@@ -31,12 +37,14 @@
import org.apache.hadoop.hive.ql.ddl.privilege.PrincipalDesc;
import org.apache.hadoop.hive.ql.ddl.privilege.PrivilegeDesc;
import org.apache.hadoop.hive.ql.ddl.privilege.PrivilegeObjectDesc;
-import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.hooks.Entity;
import org.apache.hadoop.hive.ql.hooks.Entity.Type;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity.WriteType;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.PlanUtils;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizationTranslator;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrincipal;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrincipal.HivePrincipalType;
@@ -46,6 +54,9 @@
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivObjectActionType;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils;
+import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.AvroTableProperties;
+import org.apache.hadoop.hive.serde2.avro.AvroSerDe;
/**
* Utility code shared by hive internal code and sql standard authorization
plugin implementation
@@ -307,4 +318,82 @@ public static HivePrivObjectActionType
getActionType(Entity privObject) {
return actionType;
}
+ /**
+ * When a table or partition is read, add a DFS ReadEntity for its
avro.schema.url if the
+ * schema would be fetched from a filesystem location at query time.
+ */
+ public static void addAvroSchemaUrlInputForReadEntity(Collection<ReadEntity>
inputs,
+ ReadEntity readEntity) {
+ if (readEntity == null || !readEntity.isDirect() ||
readEntity.isUpdateOrDelete()) {
+ return;
+ }
+ switch (readEntity.getTyp()) {
+ case TABLE:
+ if (readEntity.getTable() != null) {
+ addAvroSchemaUrlInputIfNeeded(inputs, readEntity.getTable());
+ }
+ break;
+ case PARTITION:
+ case DUMMYPARTITION:
+ if (readEntity.getPartition() != null) {
+ addAvroSchemaUrlInputIfNeeded(inputs,
readEntity.getPartition().getTable());
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ public static void addAvroSchemaUrlInputIfNeeded(Collection<ReadEntity>
inputs, Table table) {
+ String schemaUrl = getFilesystemAvroSchemaUrlToAuthorize(table);
+ if (schemaUrl == null) {
+ return;
+ }
+ ReadEntity schemaUrlInput = toAvroSchemaUrlReadEntity(schemaUrl);
+ if (inputs instanceof Set) {
+ PlanUtils.addInput((Set<ReadEntity>) inputs, schemaUrlInput);
+ } else if (!inputs.contains(schemaUrlInput)) {
+ inputs.add(schemaUrlInput);
+ }
+ }
+
+ private static ReadEntity toAvroSchemaUrlReadEntity(String schemaUrl) {
+ Path path = new Path(schemaUrl);
+ return new ReadEntity(path, isLocalFilesystemSchemaUrl(schemaUrl));
+ }
+
+ private static boolean isLocalFilesystemSchemaUrl(String schemaUrl) {
+ try {
+ String scheme = new URI(schemaUrl).getScheme();
+ return scheme != null && "file".equalsIgnoreCase(scheme);
+ } catch (URISyntaxException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Returns the avro.schema.url when it refers to a filesystem location that
will be read to
+ * resolve the schema, or null when no schema-url authorization is required.
+ */
+ public static String getFilesystemAvroSchemaUrlToAuthorize(Table table) {
+ if (table == null || table.isTemporary()) {
+ return null;
+ }
+ if (!AvroSerDe.class.getName().equals(table.getSerializationLib())) {
+ return null;
+ }
+ String literal =
table.getProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName());
+ if (StringUtils.isNotEmpty(literal) &&
!AvroSerdeUtils.SCHEMA_NONE.equals(literal)) {
+ return null;
+ }
+ String schemaUrl =
table.getProperty(AvroTableProperties.SCHEMA_URL.getPropName());
+ if (StringUtils.isEmpty(schemaUrl) ||
AvroSerdeUtils.SCHEMA_NONE.equals(schemaUrl)) {
+ return null;
+ }
+ if (!AvroSerdeUtils.isFilesystemSchemaUrl(schemaUrl)) {
+ return null;
+ }
+ return schemaUrl;
+ }
+
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV2.java
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV2.java
index a4c3fd9288c..41bfb120f55 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV2.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV2.java
@@ -72,6 +72,7 @@ static void doAuthorization(HiveOperation op,
BaseSemanticAnalyzer sem, SessionS
List<ReadEntity> inputList = new ArrayList<ReadEntity>(inputs);
List<WriteEntity> outputList = new ArrayList<WriteEntity>(outputs);
addPermanentFunctionEntities(ss, inputList);
+ enrichAvroSchemaUrlInputs(inputList);
List<HivePrivilegeObject> inputsHObjs = getHivePrivObjects(inputList,
selectTab2Cols, hiveOpType, sem);
List<HivePrivilegeObject> outputHObjs = getHivePrivObjects(outputList,
updateTab2Cols, hiveOpType, sem);
@@ -84,6 +85,13 @@ static void doAuthorization(HiveOperation op,
BaseSemanticAnalyzer sem, SessionS
ss.getAuthorizerV2().checkPrivileges(hiveOpType, inputsHObjs, outputHObjs,
authzContextBuilder.build());
}
+ private static void enrichAvroSchemaUrlInputs(List<ReadEntity> inputList) {
+ List<ReadEntity> snapshot = new ArrayList<>(inputList);
+ for (ReadEntity readEntity : snapshot) {
+ AuthorizationUtils.addAvroSchemaUrlInputForReadEntity(inputList,
readEntity);
+ }
+ }
+
private static void addPermanentFunctionEntities(SessionState ss,
List<ReadEntity> inputList) throws HiveException {
for (Entry<String, FunctionInfo> function :
ss.getCurrentFunctionsInUse().entrySet()) {
if (function.getValue().getFunctionType() != FunctionType.PERSISTENT) {
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/events/AlterTableEvent.java
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/events/AlterTableEvent.java
index 78793b74325..f58ff338303 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/events/AlterTableEvent.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/events/AlterTableEvent.java
@@ -25,6 +25,7 @@
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent;
import org.apache.hadoop.hive.metastore.events.PreEventContext;
+import org.apache.hadoop.hive.ql.security.authorization.AuthorizationUtils;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.metastore.HiveMetaStoreAuthorizableEvent;
@@ -78,6 +79,8 @@ private List<HivePrivilegeObject> getInputHObjs() {
ret.add(getHivePrivilegeObject(oldTable));
+ addAvroSchemaUrlInputAuth(ret, event);
+
COMMAND_STR = buildCommandString(COMMAND_STR, oldTable);
LOG.debug("<== AlterTableEvent.getInputHObjs(): ret={}", ret);
@@ -122,6 +125,23 @@ private List<HivePrivilegeObject> getOutputHObjs() {
return ret;
}
+ private void addAvroSchemaUrlInputAuth(List<HivePrivilegeObject> ret,
PreAlterTableEvent event) {
+ Table newTable = event.getNewTable();
+ Table oldTable = event.getOldTable();
+ String newSchemaUrl =
AuthorizationUtils.getFilesystemAvroSchemaUrlToAuthorize(
+ new org.apache.hadoop.hive.ql.metadata.Table(newTable));
+ if (newSchemaUrl == null) {
+ return;
+ }
+ String oldSchemaUrl = oldTable == null ? null
+ : AuthorizationUtils.getFilesystemAvroSchemaUrlToAuthorize(
+ new org.apache.hadoop.hive.ql.metadata.Table(oldTable));
+ if (StringUtils.equals(oldSchemaUrl, newSchemaUrl)) {
+ return;
+ }
+ ret.add(getHivePrivilegeObjectDfsUri(newSchemaUrl));
+ }
+
private String buildCommandString(String cmdStr, Table tbl) {
String ret = cmdStr;
if (tbl != null) {
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/events/CreateTableEvent.java
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/events/CreateTableEvent.java
index 69017f9d457..41c80304bdf 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/events/CreateTableEvent.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/events/CreateTableEvent.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent;
import org.apache.hadoop.hive.metastore.events.PreEventContext;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.ql.security.authorization.AuthorizationUtils;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType;
@@ -65,18 +66,27 @@ private List<HivePrivilegeObject> getInputHObjs() {
Database database = event.getDatabase();
String uri = getSdLocation(table.getSd());
- if (StringUtils.isEmpty(uri)) {
- return ret;
+ if (StringUtils.isNotEmpty(uri)) {
+ // Skip DFS_URI only if table location is under default db path
+ if (this.needDFSUriAuth(uri, this.getDefaultTablePath(database, table)))
{
+ ret.add(new HivePrivilegeObject(HivePrivilegeObjectType.DFS_URI, uri));
+ }
}
- // Skip DFS_URI only if table location is under default db path
- if (this.needDFSUriAuth(uri, this.getDefaultTablePath(database, table))) {
- ret.add(new HivePrivilegeObject(HivePrivilegeObjectType.DFS_URI, uri));
- }
+ addAvroSchemaUrlInputAuth(ret, table);
return ret;
}
+ private void addAvroSchemaUrlInputAuth(List<HivePrivilegeObject> ret, Table
table) {
+ String schemaUrl =
AuthorizationUtils.getFilesystemAvroSchemaUrlToAuthorize(
+ new org.apache.hadoop.hive.ql.metadata.Table(table));
+ if (schemaUrl == null) {
+ return;
+ }
+ ret.add(getHivePrivilegeObjectDfsUri(schemaUrl));
+ }
+
private List<HivePrivilegeObject> getOutputHObjs() {
LOG.debug("==> CreateTableEvent.getOutputHObjs()");
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/TestAvroSchemaUrlAuthorizationUtils.java
b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/TestAvroSchemaUrlAuthorizationUtils.java
new file mode 100644
index 00000000000..1a86ee507b0
--- /dev/null
+++
b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/TestAvroSchemaUrlAuthorizationUtils.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.security.authorization;
+
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.ql.hooks.Entity;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.AvroTableProperties;
+import org.apache.hadoop.hive.serde2.avro.AvroSerDe;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class TestAvroSchemaUrlAuthorizationUtils {
+
+ @Test
+ public void getFilesystemSchemaUrlToAuthorizeRequiresAvroSerde() {
+ org.apache.hadoop.hive.metastore.api.Table table =
avroTable("hdfs://nn/schema.avsc", null);
+
table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
+ assertNull(AuthorizationUtils.getFilesystemAvroSchemaUrlToAuthorize(new
Table(table)));
+ }
+
+ @Test
+ public void getFilesystemSchemaUrlToAuthorizeSkipsWhenLiteralPresent() {
+ org.apache.hadoop.hive.metastore.api.Table table =
avroTable("hdfs://nn/schema.avsc",
+ "{\"type\":\"record\",\"name\":\"r\",\"fields\":[]}");
+ assertNull(AuthorizationUtils.getFilesystemAvroSchemaUrlToAuthorize(new
Table(table)));
+ }
+
+ @Test
+ public void getFilesystemSchemaUrlToAuthorizeReturnsHdfsUrl() {
+ String schemaUrl = "hdfs://nn/schema.avsc";
+ org.apache.hadoop.hive.metastore.api.Table table = avroTable(schemaUrl,
null);
+ assertEquals(schemaUrl,
+ AuthorizationUtils.getFilesystemAvroSchemaUrlToAuthorize(new
Table(table)));
+ }
+
+ @Test
+ public void getFilesystemSchemaUrlToAuthorizeRejectsHttpUrl() {
+ org.apache.hadoop.hive.metastore.api.Table table =
avroTable("http://example.com/schema.avsc";, null);
+ assertNull(AuthorizationUtils.getFilesystemAvroSchemaUrlToAuthorize(new
Table(table)));
+ }
+
+ @Test
+ public void addSchemaUrlInputForReadEntityAddsDfsReadEntity() {
+ String schemaUrl = "hdfs://nn/schema.avsc";
+ Table table = new Table(avroTable(schemaUrl, null));
+ Set<ReadEntity> inputs = new LinkedHashSet<>();
+ ReadEntity tableInput = new ReadEntity(table);
+ inputs.add(tableInput);
+
+ AuthorizationUtils.addAvroSchemaUrlInputForReadEntity(inputs, tableInput);
+
+ assertEquals(2, inputs.size());
+ assertTrue(inputs.stream().anyMatch(input -> input.getTyp() ==
Entity.Type.DFS_DIR
+ && input.getD().toString().contains(schemaUrl)));
+ }
+
+ @Test
+ public void addSchemaUrlInputForReadEntitySkipsIndirectReads() {
+ Table table = new Table(avroTable("hdfs://nn/schema.avsc", null));
+ Set<ReadEntity> inputs = new LinkedHashSet<>();
+ ReadEntity tableInput = new ReadEntity(table, null, false);
+ inputs.add(tableInput);
+
+ AuthorizationUtils.addAvroSchemaUrlInputForReadEntity(inputs, tableInput);
+
+ assertEquals(1, inputs.size());
+ }
+
+ @Test
+ public void addSchemaUrlInputForReadEntityAddsDfsDirEntity() {
+ String schemaUrl = "hdfs://nn/schema.avsc";
+ Table table = new Table(avroTable(schemaUrl, null));
+ Set<ReadEntity> inputs = new LinkedHashSet<>();
+ ReadEntity tableInput = new ReadEntity(table);
+ inputs.add(tableInput);
+
+ AuthorizationUtils.addAvroSchemaUrlInputForReadEntity(inputs, tableInput);
+
+ boolean foundDfsInput = false;
+ for (ReadEntity input : inputs) {
+ if (input.getTyp() == Entity.Type.DFS_DIR) {
+ foundDfsInput = true;
+ assertTrue(input.getD().toString().contains("hdfs://nn/schema.avsc"));
+ }
+ }
+ assertTrue(foundDfsInput);
+ }
+
+ private static org.apache.hadoop.hive.metastore.api.Table avroTable(String
schemaUrl,
+ String schemaLiteral) {
+ org.apache.hadoop.hive.metastore.api.Table table = new
org.apache.hadoop.hive.metastore.api.Table();
+ table.setDbName("default");
+ table.setTableName("avro_tbl");
+ Map<String, String> params = new HashMap<>();
+ if (schemaUrl != null) {
+ params.put(AvroTableProperties.SCHEMA_URL.getPropName(), schemaUrl);
+ }
+ if (schemaLiteral != null) {
+ params.put(AvroTableProperties.SCHEMA_LITERAL.getPropName(),
schemaLiteral);
+ }
+ table.setParameters(params);
+
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setCols(new ArrayList<FieldSchema>());
+ SerDeInfo serdeInfo = new SerDeInfo();
+ serdeInfo.setSerializationLib(AvroSerDe.class.getName());
+ sd.setSerdeInfo(serdeInfo);
+ table.setSd(sd);
+ return table;
+ }
+}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/TestHiveMetaStoreAuthorizer.java
b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/TestHiveMetaStoreAuthorizer.java
index 8975c605c17..c974e4787be 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/TestHiveMetaStoreAuthorizer.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/TestHiveMetaStoreAuthorizer.java
@@ -46,6 +46,8 @@
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.metastore.filtercontext.TableFilterContext;
+import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.AvroTableProperties;
+import org.apache.hadoop.hive.serde2.avro.AvroSerDe;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.thrift.TException;
import org.junit.Before;
@@ -68,6 +70,7 @@
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.verify;
@@ -187,11 +190,13 @@ private Pair<List<HivePrivilegeObject>,
List<HivePrivilegeObject>> getHivePrivil
ArgumentCaptor<List<HivePrivilegeObject>> outputsCapturer = ArgumentCaptor
.forClass(class_listPrivObjects);
- verify(mockHiveAuthorizer).checkPrivileges(any(HiveOperationType.class),
+ verify(mockHiveAuthorizer,
atLeastOnce()).checkPrivileges(any(HiveOperationType.class),
inputsCapturer.capture(), outputsCapturer.capture(),
any(HiveAuthzContext.class));
- return new ImmutablePair<>(inputsCapturer.getValue(),
outputsCapturer.getValue());
+ int lastIdx = inputsCapturer.getAllValues().size() - 1;
+ return new ImmutablePair<>(inputsCapturer.getAllValues().get(lastIdx),
+ outputsCapturer.getAllValues().get(lastIdx));
}
@Test
@@ -834,6 +839,74 @@ public void testUnAuthorizedCause() {
}
}
+ @Test
+ public void testW_CreateAvroTable_SchemaUrlDfsUriPrivObject() throws
Exception {
+
UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(authorizedUser));
+ String schemaUrl = "hdfs://namenode:9000/path/to/schema.avsc";
+ try {
+ Table table = new TableBuilder()
+ .setTableName("avro_tbl")
+ .addCol("id", ColumnType.INT_TYPE_NAME)
+ .setSerdeLib(AvroSerDe.class.getName())
+ .addTableParam(AvroTableProperties.SCHEMA_URL.getPropName(),
schemaUrl)
+ .setOwner(authorizedUser)
+ .build(conf);
+ hmsHandler.create_table(table);
+
+ Pair<List<HivePrivilegeObject>, List<HivePrivilegeObject>> io =
getHivePrivilegeObjectsFromLastCall();
+ List<HivePrivilegeObject> inputs = io.getLeft();
+ List<HivePrivilegeObject> dfsUriInputs = inputs.stream()
+ .filter(o -> o.getType() ==
HivePrivilegeObject.HivePrivilegeObjectType.DFS_URI)
+ .collect(Collectors.toList());
+
+ assertTrue("Should authorize read access to avro.schema.url",
+ dfsUriInputs.stream().anyMatch(o ->
schemaUrl.equals(o.getObjectName())));
+ } finally {
+ try {
+ hmsHandler.drop_table("default", "avro_tbl", true);
+ } catch (Exception e) {
+ // Ignore cleanup errors
+ }
+ }
+ }
+
+ @Test
+ public void testX_AlterAvroTable_SchemaUrlDfsUriPrivObject() throws
Exception {
+
UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(authorizedUser));
+ String oldSchemaUrl = "hdfs://namenode:9000/path/to/old_schema.avsc";
+ String newSchemaUrl = "hdfs://namenode:9000/path/to/new_schema.avsc";
+ try {
+ Table table = new TableBuilder()
+ .setTableName("avro_tbl")
+ .addCol("id", ColumnType.INT_TYPE_NAME)
+ .setSerdeLib(AvroSerDe.class.getName())
+ .addTableParam(AvroTableProperties.SCHEMA_URL.getPropName(),
oldSchemaUrl)
+ .setOwner(authorizedUser)
+ .build(conf);
+ hmsHandler.create_table(table);
+ GetTableRequest request = new GetTableRequest("default", "avro_tbl");
+ request.setCatName("hive");
+ Table alteredTable = hmsHandler.get_table_core(request);
+
alteredTable.getParameters().put(AvroTableProperties.SCHEMA_URL.getPropName(),
newSchemaUrl);
+ hmsHandler.alter_table("default", "avro_tbl", alteredTable);
+
+ Pair<List<HivePrivilegeObject>, List<HivePrivilegeObject>> io =
getHivePrivilegeObjectsFromLastCall();
+ List<HivePrivilegeObject> inputs = io.getLeft();
+ List<HivePrivilegeObject> dfsUriInputs = inputs.stream()
+ .filter(o -> o.getType() ==
HivePrivilegeObject.HivePrivilegeObjectType.DFS_URI)
+ .collect(Collectors.toList());
+
+ assertTrue("Should authorize read access to updated avro.schema.url",
+ dfsUriInputs.stream().anyMatch(o ->
newSchemaUrl.equals(o.getObjectName())));
+ } finally {
+ try {
+ hmsHandler.drop_table("default", "avro_tbl", true);
+ } catch (Exception e) {
+ // Ignore cleanup errors
+ }
+ }
+ }
+
@Test
public void testDropTableNoTablePathWritePermissionShouldFail() throws
Exception {
UserGroupInformation.setLoginUser(
diff --git
a/ql/src/test/results/clientpositive/llap/avro_extschema_insert.q.out
b/ql/src/test/results/clientpositive/llap/avro_extschema_insert.q.out
index b083819fddc..aec2bc47adf 100644
--- a/ql/src/test/results/clientpositive/llap/avro_extschema_insert.q.out
+++ b/ql/src/test/results/clientpositive/llap/avro_extschema_insert.q.out
@@ -86,11 +86,13 @@ POSTHOOK: Lineage: avro_extschema_insert1
PARTITION(p1=part1).col2 SCRIPT []
POSTHOOK: Lineage: avro_extschema_insert1 PARTITION(p1=part1).col3 SCRIPT []
PREHOOK: query: insert overwrite table avro_extschema_insert2 partition (p1)
select * from avro_extschema_insert1
PREHOOK: type: QUERY
+#### A masked pattern was here ####
PREHOOK: Input: default@avro_extschema_insert1
PREHOOK: Input: default@avro_extschema_insert1@p1=part1
PREHOOK: Output: default@avro_extschema_insert2
POSTHOOK: query: insert overwrite table avro_extschema_insert2 partition (p1)
select * from avro_extschema_insert1
POSTHOOK: type: QUERY
+#### A masked pattern was here ####
POSTHOOK: Input: default@avro_extschema_insert1
POSTHOOK: Input: default@avro_extschema_insert1@p1=part1
POSTHOOK: Output: default@avro_extschema_insert2
@@ -100,11 +102,13 @@ POSTHOOK: Lineage: avro_extschema_insert2
PARTITION(p1=part1).col2 SIMPLE [(avro
POSTHOOK: Lineage: avro_extschema_insert2 PARTITION(p1=part1).col3 SIMPLE
[(avro_extschema_insert1)avro_extschema_insert1.FieldSchema(name:col3,
type:string, comment:), ]
PREHOOK: query: select * from avro_extschema_insert2
PREHOOK: type: QUERY
+#### A masked pattern was here ####
PREHOOK: Input: default@avro_extschema_insert2
PREHOOK: Input: default@avro_extschema_insert2@p1=part1
#### A masked pattern was here ####
POSTHOOK: query: select * from avro_extschema_insert2
POSTHOOK: type: QUERY
+#### A masked pattern was here ####
POSTHOOK: Input: default@avro_extschema_insert2
POSTHOOK: Input: default@avro_extschema_insert2@p1=part1
#### A masked pattern was here ####
diff --git
a/ql/src/test/results/clientpositive/llap/avro_tableproperty_optimize.q.out
b/ql/src/test/results/clientpositive/llap/avro_tableproperty_optimize.q.out
index 8f6c02f01c6..4283c91daa8 100644
--- a/ql/src/test/results/clientpositive/llap/avro_tableproperty_optimize.q.out
+++ b/ql/src/test/results/clientpositive/llap/avro_tableproperty_optimize.q.out
@@ -93,10 +93,12 @@ col3 string
#### A masked pattern was here ####
PREHOOK: query: SELECT * FROM avro_extschema_url_n0
PREHOOK: type: QUERY
+#### A masked pattern was here ####
PREHOOK: Input: default@avro_extschema_url_n0
#### A masked pattern was here ####
POSTHOOK: query: SELECT * FROM avro_extschema_url_n0
POSTHOOK: type: QUERY
+#### A masked pattern was here ####
POSTHOOK: Input: default@avro_extschema_url_n0
#### A masked pattern was here ####
s1 1 s2
diff --git a/ql/src/test/results/clientpositive/llap/avrotblsjoin.q.out
b/ql/src/test/results/clientpositive/llap/avrotblsjoin.q.out
index 907af5705fd..ea376b0da57 100644
--- a/ql/src/test/results/clientpositive/llap/avrotblsjoin.q.out
+++ b/ql/src/test/results/clientpositive/llap/avrotblsjoin.q.out
@@ -81,11 +81,13 @@ WARNING: Comparing bigint and string may result in loss of
information.
Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage
'Reducer 2' is a cross product
PREHOOK: query: select table1_n1.col1, table1_1.* from table1_n1 join table1_1
on table1_n1.col1=table1_1.col1 where table1_1.col1="1"
PREHOOK: type: QUERY
+#### A masked pattern was here ####
PREHOOK: Input: default@table1_1
PREHOOK: Input: default@table1_n1
#### A masked pattern was here ####
POSTHOOK: query: select table1_n1.col1, table1_1.* from table1_n1 join
table1_1 on table1_n1.col1=table1_1.col1 where table1_1.col1="1"
POSTHOOK: type: QUERY
+#### A masked pattern was here ####
POSTHOOK: Input: default@table1_1
POSTHOOK: Input: default@table1_n1
#### A masked pattern was here ####
diff --git a/ql/src/test/results/clientpositive/llap/compustat_avro.q.out
b/ql/src/test/results/clientpositive/llap/compustat_avro.q.out
index 3d80d716c60..b89b3fe093a 100644
--- a/ql/src/test/results/clientpositive/llap/compustat_avro.q.out
+++ b/ql/src/test/results/clientpositive/llap/compustat_avro.q.out
@@ -47,11 +47,13 @@ comment from deserializer
COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\",\"col4\":\"true\",\"col5\":\"true\",\"col6\":\"true\"}}
PREHOOK: query: analyze table testAvro compute statistics for columns col1,col3
PREHOOK: type: ANALYZE_TABLE
+#### A masked pattern was here ####
PREHOOK: Input: default@testavro
PREHOOK: Output: default@testavro
#### A masked pattern was here ####
POSTHOOK: query: analyze table testAvro compute statistics for columns
col1,col3
POSTHOOK: type: ANALYZE_TABLE
+#### A masked pattern was here ####
POSTHOOK: Input: default@testavro
POSTHOOK: Output: default@testavro
#### A masked pattern was here ####
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
index e468c58a6d5..2d3dd6236e6 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
@@ -43,10 +43,13 @@
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
import java.util.ArrayList;
+import java.util.Locale;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
/**
* Utilities useful only to the AvroSerde itself. Not mean to be used by
@@ -97,7 +100,8 @@ public String getPropName(){
+ AvroTableProperties.SCHEMA_LITERAL.getPropName() + " nor "
+ AvroTableProperties.SCHEMA_URL.getPropName() + " specified, can't
determine table schema";
-
+ private static final Set<String> HTTP_SCHEMES =
+ new HashSet<>(Arrays.asList("http", "https"));
/**
* Determine the schema to that's been provided for Avro serde work.
@@ -137,11 +141,26 @@ public static Schema
determineSchemaOrThrowException(Configuration conf, Propert
throw new AvroSerdeException(EXCEPTION_MESSAGE);
}
+ validateSchemaUrl(schemaString, conf);
+
try {
- Schema s = getSchemaFromFS(schemaString, conf);
- if (s == null) {
- //in case schema is not a file system
- return AvroSerdeUtils.getSchemaFor(new URL(schemaString));
+ Schema s;
+ if (requiresHttpFetch(schemaString)) {
+ URL url = new URL(schemaString);
+ java.net.URLConnection conn = url.openConnection();
+ if (conn instanceof java.net.HttpURLConnection) {
+ ((java.net.HttpURLConnection)
conn).setInstanceFollowRedirects(false);
+ }
+ try (InputStream in = conn.getInputStream()) {
+ s = getSchemaParser().parse(in);
+ }
+ } else {
+ s = getSchemaFromFS(schemaString, conf);
+ if (s == null) {
+ throw new AvroSerdeException("Unable to read Avro schema from: " +
schemaString
+ + ". avro.schema.url must refer to a Hadoop FileSystem URI (e.g.
hdfs://, s3a://). "
+ + "Consider using avro.schema.literal instead.");
+ }
}
return s;
} catch (IOException ioe) {
@@ -151,6 +170,115 @@ public static Schema
determineSchemaOrThrowException(Configuration conf, Propert
}
}
+ /**
+ * Validate that the avro.schema.url uses a permitted scheme and host.
+ */
+ static void validateSchemaUrl(String schemaUrl, Configuration conf) throws
AvroSerdeException {
+ final URI uri;
+ try {
+ uri = new URI(schemaUrl);
+ } catch (URISyntaxException e) {
+ throw new AvroSerdeException("Invalid avro.schema.url: " + schemaUrl, e);
+ }
+
+ final String scheme = uri.getScheme();
+ if (scheme == null || scheme.isEmpty()) {
+ return;
+ }
+
+ final String schemeLower = scheme.toLowerCase(Locale.ROOT);
+ if ("file".equals(schemeLower)) {
+ throw new AvroSerdeException("avro.schema.url scheme '" + scheme
+ + "' is not permitted. Use a Hadoop FileSystem URI (e.g. hdfs://,
s3a://) or "
+ + "avro.schema.literal instead.");
+ }
+ if (HTTP_SCHEMES.contains(schemeLower)) {
+ if (!HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_AVRO_SCHEMA_URL_REMOTE_HTTP_ENABLED)) {
+ throw new AvroSerdeException("avro.schema.url scheme '" + scheme
+ + "' is not permitted. Remote HTTP schema fetch is disabled. Set "
+ +
HiveConf.ConfVars.HIVE_AVRO_SCHEMA_URL_REMOTE_HTTP_ENABLED.varname
+ + " to true and configure "
+ + HiveConf.ConfVars.HIVE_AVRO_SCHEMA_URL_HTTP_ALLOWED_HOSTS.varname
+ + ", or use avro.schema.literal instead.");
+ }
+ final String host = uri.getHost();
+ if (host == null || host.isEmpty()) {
+ throw new AvroSerdeException("avro.schema.url must specify a host for
HTTP/HTTPS schemas.");
+ }
+ if (!isHttpHostAllowed(host, conf)) {
+ throw new AvroSerdeException("avro.schema.url host '" + host
+ + "' is not in the permitted host list configured by "
+ +
HiveConf.ConfVars.HIVE_AVRO_SCHEMA_URL_HTTP_ALLOWED_HOSTS.varname + ".");
+ }
+ return;
+ }
+
+ if (!getAllowedFilesystemSchemes(conf).contains(schemeLower)) {
+ throw new AvroSerdeException("avro.schema.url scheme '" + scheme
+ + "' is not permitted. Use a Hadoop FileSystem URI (e.g. hdfs://,
s3a://) or "
+ + "avro.schema.literal instead.");
+ }
+ }
+
+ /**
+ * Returns true if the schema URL refers to a Hadoop filesystem location
(including scheme-less URIs).
+ */
+ public static boolean isFilesystemSchemaUrl(String schemaUrl) {
+ if (schemaUrl == null || schemaUrl.isEmpty() ||
SCHEMA_NONE.equals(schemaUrl)) {
+ return false;
+ }
+ try {
+ URI uri = new URI(schemaUrl);
+ String scheme = uri.getScheme();
+ if (scheme == null || scheme.isEmpty()) {
+ return true;
+ }
+ String schemeLower = scheme.toLowerCase(Locale.ROOT);
+ return !HTTP_SCHEMES.contains(schemeLower) &&
!"file".equals(schemeLower);
+ } catch (URISyntaxException e) {
+ return false;
+ }
+ }
+
+ private static boolean requiresHttpFetch(String schemaUrl) throws
AvroSerdeException {
+ try {
+ URI uri = new URI(schemaUrl);
+ String scheme = uri.getScheme();
+ return scheme != null &&
HTTP_SCHEMES.contains(scheme.toLowerCase(Locale.ROOT));
+ } catch (URISyntaxException e) {
+ throw new AvroSerdeException("Invalid avro.schema.url: " + schemaUrl, e);
+ }
+ }
+
+ private static Set<String> getAllowedFilesystemSchemes(Configuration conf) {
+ String schemes = conf == null
+ ?
HiveConf.ConfVars.HIVE_AVRO_SCHEMA_URL_ALLOWED_SCHEMES.getDefaultValue()
+ : HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_AVRO_SCHEMA_URL_ALLOWED_SCHEMES);
+ Set<String> allowed = new HashSet<>();
+ for (String scheme : schemes.split(",")) {
+ String trimmed = scheme.trim().toLowerCase(Locale.ROOT);
+ if (!trimmed.isEmpty()) {
+ allowed.add(trimmed);
+ }
+ }
+ return allowed;
+ }
+
+ private static boolean isHttpHostAllowed(String host, Configuration conf) {
+ String allowedHosts = HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_AVRO_SCHEMA_URL_HTTP_ALLOWED_HOSTS);
+ if (allowedHosts == null || allowedHosts.trim().isEmpty()) {
+ return false;
+ }
+ String hostLower = host.toLowerCase(Locale.ROOT);
+ for (String allowedHost : allowedHosts.split(",")) {
+ String trimmed = allowedHost.trim().toLowerCase(Locale.ROOT);
+ if (!trimmed.isEmpty() && trimmed.equals(hostLower)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
// Protected for testing and so we can pass in a conf for testing.
protected static Schema getSchemaFromFS(String schemaFSUrl,
Configuration conf) throws IOException,
URISyntaxException {
diff --git
a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java
b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java
index 3ef802d605d..509138065e9 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java
@@ -22,6 +22,7 @@
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.junit.Test;
import java.io.IOException;
@@ -38,6 +39,7 @@
import static org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.isNullableType;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestAvroSerdeUtils {
@@ -151,19 +153,103 @@ public void determineSchemaFindsLiterals() throws
Exception {
}
@Test
- public void determineSchemaTriesToOpenUrl() throws AvroSerdeException,
IOException {
+ public void determineSchemaRejectsUnknownSchemeUrl() throws
AvroSerdeException, IOException {
Configuration conf = new Configuration();
Properties props = new Properties();
props.put(AvroTableProperties.SCHEMA_URL.getPropName(),
"not:///a.real.url");
try {
AvroSerdeUtils.determineSchemaOrThrowException(conf, props);
- fail("Should have tried to open that URL");
+ fail("Should have rejected unknown scheme URL");
} catch (AvroSerdeException e) {
- assertEquals("Unable to read schema from given path: not:///a.real.url",
e.getMessage());
+ assertTrue(e.getMessage().contains("not permitted"));
}
}
+ @Test
+ public void determineSchemaRejectsHttpByDefault() throws IOException {
+ Configuration conf = new Configuration();
+ Properties props = new Properties();
+ props.put(AvroTableProperties.SCHEMA_URL.getPropName(),
+ "http://169.254.169.254/latest/meta-data/";);
+
+ try {
+ determineSchemaOrThrowException(conf, props);
+ fail("Should have rejected HTTP schema URL");
+ } catch (AvroSerdeException e) {
+ assertTrue(e.getMessage().contains("not permitted"));
+ }
+ }
+
+ @Test
+ public void determineSchemaRejectsHttpsByDefault() throws IOException {
+ Configuration conf = new Configuration();
+ Properties props = new Properties();
+ props.put(AvroTableProperties.SCHEMA_URL.getPropName(),
"https://internal-service/admin";);
+
+ try {
+ determineSchemaOrThrowException(conf, props);
+ fail("Should have rejected HTTPS schema URL");
+ } catch (AvroSerdeException e) {
+ assertTrue(e.getMessage().contains("not permitted"));
+ }
+ }
+
+ @Test
+ public void determineSchemaRejectsFileUrl() throws IOException {
+ Configuration conf = new Configuration();
+ Properties props = new Properties();
+ props.put(AvroTableProperties.SCHEMA_URL.getPropName(),
"file:///etc/passwd");
+
+ try {
+ determineSchemaOrThrowException(conf, props);
+ fail("Should have rejected file:// schema URL");
+ } catch (AvroSerdeException e) {
+ assertTrue(e.getMessage().contains("not permitted"));
+ }
+ }
+
+ @Test
+ public void determineSchemaRejectsHttpWhenHostNotAllowlisted() throws
IOException {
+ HiveConf conf = new HiveConf();
+
conf.setBoolVar(HiveConf.ConfVars.HIVE_AVRO_SCHEMA_URL_REMOTE_HTTP_ENABLED,
true);
+ conf.setVar(HiveConf.ConfVars.HIVE_AVRO_SCHEMA_URL_HTTP_ALLOWED_HOSTS,
"schema.example.com");
+ Properties props = new Properties();
+ props.put(AvroTableProperties.SCHEMA_URL.getPropName(),
+ "http://169.254.169.254/latest/meta-data/";);
+
+ try {
+ determineSchemaOrThrowException(conf, props);
+ fail("Should have rejected HTTP host not in allowlist");
+ } catch (AvroSerdeException e) {
+ assertTrue(e.getMessage().contains("not in the permitted host list"));
+ }
+ }
+
+ @Test
+ public void determineSchemaAllowsHttpWhenHostAllowlisted() throws
IOException {
+ HiveConf conf = new HiveConf();
+
conf.setBoolVar(HiveConf.ConfVars.HIVE_AVRO_SCHEMA_URL_REMOTE_HTTP_ENABLED,
true);
+ conf.setVar(HiveConf.ConfVars.HIVE_AVRO_SCHEMA_URL_HTTP_ALLOWED_HOSTS,
"127.0.0.1");
+ Properties props = new Properties();
+ props.put(AvroTableProperties.SCHEMA_URL.getPropName(),
"http://127.0.0.1:1/schema.avsc";);
+
+ try {
+ determineSchemaOrThrowException(conf, props);
+ fail("Expected fetch failure after HTTP validation passed");
+ } catch (AvroSerdeException e) {
+ assertTrue(e.getMessage().contains("Unable to read schema from given
path"));
+ }
+ }
+
+ @Test
+ public void isFilesystemSchemaUrlIdentifiesFilesystemUrls() {
+ assertTrue(AvroSerdeUtils.isFilesystemSchemaUrl("hdfs://nn/schema.avsc"));
+ assertTrue(AvroSerdeUtils.isFilesystemSchemaUrl("/tmp/schema.avsc"));
+
assertTrue(!AvroSerdeUtils.isFilesystemSchemaUrl("http://example.com/schema.avsc";));
+ assertTrue(!AvroSerdeUtils.isFilesystemSchemaUrl("file:///etc/passwd"));
+ }
+
@Test
public void noneOptionWorksForSpecifyingSchemas() throws IOException,
AvroSerdeException {
Configuration conf = new Configuration();
@@ -195,9 +281,9 @@ public void noneOptionWorksForSpecifyingSchemas() throws
IOException, AvroSerdeE
props.put(AvroTableProperties.SCHEMA_URL.getPropName(),
"not:///a.real.url");
try {
determineSchemaOrThrowException(conf, props);
- fail("Should have tried to open that bogus URL");
+ fail("Should have rejected unknown scheme URL");
} catch (AvroSerdeException e) {
- assertEquals("Unable to read schema from given path: not:///a.real.url",
e.getMessage());
+ assertTrue(e.getMessage().contains("not permitted"));
}
}
@@ -221,6 +307,11 @@ public void determineSchemaCanReadSchemaFromHDFS() throws
IOException, AvroSerde
AvroSerdeUtils.getSchemaFromFS(onHDFS,
miniDfs.getFileSystem().getConf());
Schema expectedSchema = AvroSerdeUtils.getSchemaFor(schemaString);
assertEquals(expectedSchema, schemaFromHDFS);
+
+ Properties props = new Properties();
+ props.put(AvroTableProperties.SCHEMA_URL.getPropName(), onHDFS);
+ Schema resolved =
determineSchemaOrThrowException(miniDfs.getFileSystem().getConf(), props);
+ assertEquals(expectedSchema, resolved);
} finally {
if(miniDfs != null) miniDfs.shutdown();
}
diff --git
a/service/src/java/org/apache/hive/service/auth/saml/HiveSamlAuthTokenGenerator.java
b/service/src/java/org/apache/hive/service/auth/saml/HiveSamlAuthTokenGenerator.java
index 51cf646b01e..d9060f44dfc 100644
---
a/service/src/java/org/apache/hive/service/auth/saml/HiveSamlAuthTokenGenerator.java
+++
b/service/src/java/org/apache/hive/service/auth/saml/HiveSamlAuthTokenGenerator.java
@@ -19,6 +19,7 @@
package org.apache.hive.service.auth.saml;
import com.google.common.annotations.VisibleForTesting;
+import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
@@ -78,11 +79,11 @@ public String get(String username, String relayStateKey) {
}
private String encode(String token) {
- return Base64.getEncoder().encodeToString(token.getBytes());
+ return
Base64.getEncoder().encodeToString(token.getBytes(StandardCharsets.UTF_8));
}
private String decode(String encodedToken) {
- return new String(Base64.getDecoder().decode(encodedToken));
+ return new String(Base64.getDecoder().decode(encodedToken),
StandardCharsets.UTF_8);
}
private String getTokenStr(String username, String id, String timestamp,
@@ -100,7 +101,7 @@ private String getTokenStr(String username, String id,
String timestamp,
private String getSign(String input) {
try {
MessageDigest md = MessageDigest.getInstance("SHA-256");
- md.update(input.getBytes());
+ md.update(input.getBytes(StandardCharsets.UTF_8));
md.update(signatureSecret);
byte[] digest = md.digest();
return Base64.getEncoder().encodeToString(digest);
@@ -144,7 +145,8 @@ private boolean isExpired(long currentTime, long tokenTime)
{
}
private boolean signatureMatches(String origSign, String derivedSign) {
- return !MessageDigest.isEqual(origSign.getBytes(), derivedSign.getBytes());
+ return MessageDigest.isEqual(origSign.getBytes(StandardCharsets.UTF_8),
+ derivedSign.getBytes(StandardCharsets.UTF_8));
}
public static boolean parse(String token, Map<String, String> kv) {
@@ -153,7 +155,7 @@ public static boolean parse(String token, Map<String,
String> kv) {
return false;
}
for (String split : splits) {
- String[] pair = split.split(SEPARATOR);
+ String[] pair = split.split(SEPARATOR, 2);
if (pair.length != 2) {
return false;
}
diff --git
a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
index 67ebb605d90..43f963107fa 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
@@ -382,7 +382,8 @@ private String doSamlAuth(HttpServletRequest request,
HttpServletResponse respon
LOG.info("Successfully validated the token for user {}", user);
// token is valid; now confirm if the client identifier matches with the
relay state.
Map<String, String> keyValues = new HashMap<>();
- if (HiveSamlAuthTokenGenerator.parse(token, keyValues)) {
+ String decodedToken = new String(Base64.getDecoder().decode(token),
java.nio.charset.StandardCharsets.UTF_8);
+ if (HiveSamlAuthTokenGenerator.parse(decodedToken, keyValues)) {
String relayStateKey =
keyValues.get(HiveSamlAuthTokenGenerator.RELAY_STATE);
if (!HiveSamlRelayStateStore.get()
.validateClientIdentifier(relayStateKey, clientIdentifier)) {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdatePart.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdatePart.java
index 15924396932..cc601fef793 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdatePart.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdatePart.java
@@ -65,6 +65,7 @@
import java.util.Set;
import java.util.stream.Collectors;
+import static org.apache.commons.lang3.StringUtils.repeat;
import static
org.apache.hadoop.hive.common.StatsSetupConst.COLUMN_STATS_ACCURATE;
import static org.apache.hadoop.hive.metastore.HMSHandler.getPartValsFromName;
import static
org.apache.hadoop.hive.metastore.MetastoreDirectSqlUtils.executeWithArray;
@@ -112,6 +113,10 @@ static String quoteString(String input) {
return "'" + input + "'";
}
+ private static String makeParams(int size) {
+ return (size == 0) ? "" : repeat(",?", size).substring(1);
+ }
+
private void populateInsertUpdateMap(Map<PartitionInfo, ColumnStatistics>
statsPartInfoMap,
Map<PartColNameInfo,
MPartitionColumnStatistics> updateMap,
Map<PartColNameInfo,
MPartitionColumnStatistics>insertMap,
@@ -422,35 +427,39 @@ private Map<String, Map<String, String>>
updatePartitionParamTable(Connection db
private Map<PartitionInfo, ColumnStatistics> getPartitionInfo(Connection
dbConn, long tblId,
Map<String,
ColumnStatistics> partColStatsMap)
- throws SQLException, MetaException {
- List<String> queries = new ArrayList<>();
- StringBuilder prefix = new StringBuilder();
- StringBuilder suffix = new StringBuilder();
+ throws MetaException {
Map<PartitionInfo, ColumnStatistics> partitionInfoMap = new HashMap<>();
+ List<String> partNames = new ArrayList<>(partColStatsMap.keySet());
+ if (partNames.isEmpty()) {
+ return partitionInfoMap;
+ }
- List<String> partKeys = partColStatsMap.keySet().stream().map(
- e -> quoteString(e)).collect(Collectors.toList()
- );
-
- prefix.append("select \"PART_ID\", \"WRITE_ID\", \"PART_NAME\" from
\"PARTITIONS\" where ");
- suffix.append(" and \"TBL_ID\" = " + tblId);
- TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix, suffix,
- partKeys, "\"PART_NAME\"", true, false);
-
- try (Statement statement = dbConn.createStatement()) {
- for (String query : queries) {
+ Batchable.runBatched(maxBatchSize, partNames, new Batchable<String,
Void>() {
+ @Override
+ public List<Void> run(List<String> input) throws Exception {
+ String placeholders = makeParams(input.size());
+ String query = "select \"PART_ID\", \"WRITE_ID\", \"PART_NAME\" from
\"PARTITIONS\" where "
+ + "\"PART_NAME\" in (" + placeholders + ") and \"TBL_ID\" = ?";
// Select for update makes sure that the partitions are not modified
while the stats are getting updated.
query = sqlGenerator.addForUpdateClause(query);
LOG.debug("Execute query: " + query);
- try (ResultSet rs = statement.executeQuery(query)) {
- while (rs.next()) {
- PartitionInfo partitionInfo = new PartitionInfo(rs.getLong(1),
- rs.getLong(2), rs.getString(3));
- partitionInfoMap.put(partitionInfo,
partColStatsMap.get(rs.getString(3)));
+ try (PreparedStatement ps = dbConn.prepareStatement(query)) {
+ int paramIndex = 1;
+ for (String partName : input) {
+ ps.setString(paramIndex++, partName);
+ }
+ ps.setLong(paramIndex, tblId);
+ try (ResultSet rs = ps.executeQuery()) {
+ while (rs.next()) {
+ String partName = rs.getString(3);
+ PartitionInfo partitionInfo = new PartitionInfo(rs.getLong(1),
rs.getLong(2), partName);
+ partitionInfoMap.put(partitionInfo,
partColStatsMap.get(partName));
+ }
}
}
+ return Collections.emptyList();
}
- }
+ });
return partitionInfoMap;
}
@@ -483,6 +492,10 @@ public Map<String, Map<String, String>>
updatePartitionColumnStatistics(Map<Stri
Map<PartitionInfo, ColumnStatistics> partitionInfoMap =
getPartitionInfo(dbConn, tbl.getId(), partColStatsMap);
+ if (partitionInfoMap.isEmpty()) {
+ return Collections.emptyMap();
+ }
+
result = updatePartitionParamTable(dbConn, partitionInfoMap,
validWriteIds,
writeId, TxnUtils.isAcidTable(tbl));
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 415d7c7f557..06c7fd5b442 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -721,7 +721,7 @@ public List<Partition> getPartitionsViaPartNames(final
String catName, final Str
return Batchable.runBatched(batchSize, partNames, new Batchable<String,
Partition>() {
@Override
public List<Partition> run(List<String> input) throws MetaException {
- return getPartitionsByNames(catName, dbName, tblName, partNames,
false, args);
+ return getPartitionsByNames(catName, dbName, tblName, input, false,
args);
}
});
}
@@ -1011,9 +1011,7 @@ private List<Partition> getPartitionsByNames(String
catName, String dbName,
throws MetaException {
// Get most of the fields for the partNames provided.
// Assume db and table names are the same for all partition, as provided
in arguments.
- String quotedPartNames = partNameList.stream()
- .map(DirectSqlUpdatePart::quoteString)
- .collect(Collectors.joining(","));
+ String partNameParams = makeParams(partNameList.size());
String queryText =
"select " + PARTITIONS + ".\"PART_ID\"," + SDS + ".\"SD_ID\"," + SDS +
".\"CD_ID\","
@@ -1026,11 +1024,18 @@ private List<Partition> getPartitionsByNames(String
catName, String dbName,
+ " left outer join " + SERDES + " on " + SDS + ".\"SERDE_ID\" = " +
SERDES + ".\"SERDE_ID\" "
+ " inner join " + TBLS + " on " + TBLS + ".\"TBL_ID\" = " +
PARTITIONS + ".\"TBL_ID\" "
+ " inner join " + DBS + " on " + DBS + ".\"DB_ID\" = " + TBLS +
".\"DB_ID\" "
- + " where \"PART_NAME\" in (" + quotedPartNames + ") "
+ + " where " + PARTITIONS + ".\"PART_NAME\" in (" + partNameParams + ")
"
+ " and " + TBLS + ".\"TBL_NAME\" = ? and " + DBS + ".\"NAME\" = ? and
" + DBS
+ ".\"CTLG_NAME\" = ? order by \"PART_NAME\" asc";
- Object[] params = new Object[]{tblName, dbName, catName};
+ Object[] params = new Object[partNameList.size() + 3];
+ int i = 0;
+ for (String partName : partNameList) {
+ params[i++] = partName;
+ }
+ params[i++] = tblName;
+ params[i++] = dbName;
+ params[i] = catName;
return getPartitionsByQuery(catName, dbName, tblName, queryText, params,
isAcidTable, args);
}
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 07af321300e..880669533a0 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -145,6 +145,9 @@ public class TestObjectStore {
private static final String USER1 = "testobjectstoreuser1";
private static final String ROLE1 = "testobjectstorerole1";
private static final String ROLE2 = "testobjectstorerole2";
+ private static final String SQLI_PART_NAME = "test_part_col=missing') OR 1=1
-- ";
+ private static final List<String> ALL_PART_NAMES =
+ Arrays.asList("test_part_col=a0", "test_part_col=a1",
"test_part_col=a2");
private static final Logger LOG =
LoggerFactory.getLogger(TestObjectStore.class.getName());
private static final class LongSupplier implements Supplier<Long> {
@@ -798,6 +801,21 @@ public void testDirectSQLDropPartitionsCacheInSession()
Assert.assertEquals(1, partitions.size());
}
+ @Test
+ public void testDirectSQLDropPartitionsRejectsSqlInjectionInPartName()
+ throws Exception {
+ createPartitionedTable(false, false);
+
+ objectStore.dropPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ Collections.singletonList(SQLI_PART_NAME), true, false);
+
+ List<Partition> partitions;
+ try (AutoCloseable c = deadline()) {
+ partitions = objectStore.getPartitionsByNames(DEFAULT_CATALOG_NAME, DB1,
TABLE1, ALL_PART_NAMES);
+ }
+ Assert.assertEquals(3, partitions.size());
+ }
+
/**
* Checks if the JDO cache is able to handle directSQL partition drops cross
sessions.
*/
@@ -1020,6 +1038,62 @@ public void
testDeletePartitionColumnStatisticsWhenEngineHasSpecialCharacter() t
List.of("test_part_col=a2"), null, "special '");
}
+ @Test
+ public void testGetPartitionsByNamesRejectsSqlInjectionInPartName() throws
Exception {
+ createPartitionedTable(true, true);
+ List<Partition> partitions;
+ try (AutoCloseable c = deadline()) {
+ partitions = objectStore.getPartitionsByNames(DEFAULT_CATALOG_NAME, DB1,
TABLE1,
+ Collections.singletonList(SQLI_PART_NAME));
+ }
+ Assert.assertEquals(0, partitions.size());
+ try (AutoCloseable c = deadline()) {
+ partitions = objectStore.getPartitionsByNames(DEFAULT_CATALOG_NAME, DB1,
TABLE1, ALL_PART_NAMES);
+ }
+ Assert.assertEquals(3, partitions.size());
+ }
+
+ @Test
+ public void
testUpdatePartitionColumnStatisticsInBatchRejectsSqlInjectionInPartName()
+ throws Exception {
+ createPartitionedTable(true, true);
+ Table tbl = objectStore.getTable(DEFAULT_CATALOG_NAME, DB1, TABLE1);
+
+ List<List<ColumnStatistics>> baseline;
+ try (AutoCloseable c = deadline()) {
+ baseline =
objectStore.getPartitionColumnStatistics(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ ALL_PART_NAMES, Collections.singletonList("test_part_col"));
+ }
+ Assert.assertEquals(1, baseline.size());
+ Assert.assertEquals(3, baseline.get(0).size());
+ long baselineNumNulls =
baseline.get(0).get(0).getStatsObj().get(0).getStatsData()
+ .getLongStats().getNumNulls();
+
+ ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, DB1,
TABLE1);
+ statsDesc.setCatName(DEFAULT_CATALOG_NAME);
+ statsDesc.setPartName(SQLI_PART_NAME);
+ ColumnStatisticsData injectedData = new
ColStatsBuilder<>(long.class).numNulls(999).numDVs(2)
+ .low(3L).high(4L).build();
+ ColumnStatisticsObj statsObj = new ColumnStatisticsObj("test_part_col",
"int", injectedData);
+ ColumnStatistics maliciousStats = new ColumnStatistics(statsDesc,
+ Collections.singletonList(statsObj));
+ maliciousStats.setEngine(ENGINE);
+
+ Map<String, ColumnStatistics> statsMap = new HashMap<>();
+ statsMap.put(SQLI_PART_NAME, maliciousStats);
+ objectStore.updatePartitionColumnStatisticsInBatch(statsMap, tbl, null,
null, -1);
+
+ List<List<ColumnStatistics>> after;
+ try (AutoCloseable c = deadline()) {
+ after = objectStore.getPartitionColumnStatistics(DEFAULT_CATALOG_NAME,
DB1, TABLE1,
+ ALL_PART_NAMES, Collections.singletonList("test_part_col"));
+ }
+ Assert.assertEquals(3, after.get(0).size());
+ for (ColumnStatistics cs : after.get(0)) {
+ Assert.assertEquals(baselineNumNulls,
+ cs.getStatsObj().get(0).getStatsData().getLongStats().getNumNulls());
+ }
+ }
@Test
public void testAggrStatsUseDB() throws Exception {
