This is an automated email from the ASF dual-hosted git repository.
daijy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new fcadd38 HIVE-21833: Ranger Authorization in Hive based on object
ownership (Sam An, reviewed by Daniel Dai)
fcadd38 is described below
commit fcadd388df2a7c12378febab167ee75a36520f6e
Author: Daniel Dai <da...@cloudera.com>
AuthorDate: Mon Jun 17 16:07:48 2019 -0700
HIVE-21833: Ranger Authorization in Hive based on object ownership (Sam An,
reviewed by Daniel Dai)
---
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 9 +-
.../authorization/plugin/HivePrivilegeObject.java | 41 ++++-
.../TestHivePrivilegeObjectOwnerNameAndType.java | 170 +++++++++++++++++++++
.../hadoop/hive/metastore/cache/CachedStore.java | 4 +-
4 files changed, 217 insertions(+), 7 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 255c65a..18438aa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.LockComponent;
import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.cache.results.CacheUsage;
@@ -1395,15 +1396,21 @@ public class Driver implements IDriver {
List<String> partKeys = null;
List<String> columns = null;
String className = null;
+ String ownerName = null;
+ PrincipalType ownerType = null;
switch(privObject.getType()){
case DATABASE:
dbname = privObject.getDatabase().getName();
+ ownerName = privObject.getDatabase().getOwnerName();
+ ownerType = privObject.getDatabase().getOwnerType();
break;
case TABLE:
dbname = privObject.getTable().getDbName();
objName = privObject.getTable().getTableName();
columns = tableName2Cols == null ? null :
tableName2Cols.get(Table.getCompleteName(dbname, objName));
+ ownerName = privObject.getTable().getOwner();
+ ownerType = privObject.getTable().getOwnerType();
break;
case DFS_DIR:
case LOCAL_DIR:
@@ -1428,7 +1435,7 @@ public class Driver implements IDriver {
}
HivePrivObjectActionType actionType =
AuthorizationUtils.getActionType(privObject);
HivePrivilegeObject hPrivObject = new HivePrivilegeObject(privObjType,
dbname, objName,
- partKeys, columns, actionType, null, className);
+ partKeys, columns, actionType, null, className, ownerName,
ownerType);
hivePrivobjs.add(hPrivObject);
}
return hivePrivobjs;
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java
index 87d2e68..b08f7ca 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import
org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
/**
* Represents the object on which privilege is being granted/revoked, and
objects
@@ -65,6 +66,16 @@ public class HivePrivilegeObject implements
Comparable<HivePrivilegeObject> {
(o.className != null ? className.compareTo(o.className) : 1) :
(o.className != null ? -1 : 0);
}
+ if (compare == 0) {
+ compare = ownerName != null?
+ (o.ownerName != null ? ownerName.compareTo(o.ownerName) : 1) :
+ (o.ownerName != null ? -1 : 0);
+ }
+ if (compare == 0) {
+ compare = ownerType != null?
+ (o.ownerType != null ? ownerType.compareTo(o.ownerType) : 1) :
+ (o.ownerType != null ? -1 : 0);
+ }
return compare;
}
@@ -118,6 +129,8 @@ public class HivePrivilegeObject implements
Comparable<HivePrivilegeObject> {
private final List<String> columns;
private final HivePrivObjectActionType actionType;
private final String className;
+ private final String ownerName;
+ private final PrincipalType ownerType;
// cellValueTransformers is corresponding to the columns.
// Its size should be the same as columns.
// For example, if a table has two columns, "key" and "value"
@@ -164,9 +177,14 @@ public class HivePrivilegeObject implements
Comparable<HivePrivilegeObject> {
this(HivePrivilegeObjectType.TABLE_OR_VIEW, dbname, objectName, null,
columns, null);
}
- public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname,
String objectName,
- List<String> partKeys, List<String> columns, HivePrivObjectActionType
actionType,
- List<String> commandParams, String className) {
+ public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname,
String objectName, List<String> partKeys,
+ List<String> columns, HivePrivObjectActionType actionType, List<String>
commandParams, String className) {
+ this(type, dbname, objectName, partKeys, columns, actionType,
commandParams, className, null, null);
+ }
+
+ public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname,
String objectName, List<String> partKeys,
+ List<String> columns, HivePrivObjectActionType actionType, List<String>
commandParams, String className,
+ String ownerName, PrincipalType ownerType) {
this.type = type;
this.dbname = dbname;
this.objectName = objectName;
@@ -175,6 +193,8 @@ public class HivePrivilegeObject implements
Comparable<HivePrivilegeObject> {
this.actionType = actionType;
this.commandParams = commandParams;
this.className = className;
+ this.ownerName = ownerName;
+ this.ownerType = ownerType;
}
public HivePrivilegeObjectType getType() {
@@ -271,10 +291,23 @@ public class HivePrivilegeObject implements
Comparable<HivePrivilegeObject> {
default:
}
}
-
return "Object [type=" + type + ", name=" + name + actionTypeStr + "]";
}
+ /**
+ * @return ownerName of the object
+ */
+ public String getOwnerName() {
+ return this.ownerName;
+ }
+
+ /**
+ * @return principal type of the owner
+ */
+ public PrincipalType getOwnerType() {
+ return this.ownerType;
+ }
+
private String getDbObjectName(String dbname2, String objectName2) {
return (dbname == null ? "" : dbname + ".") + objectName;
}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHivePrivilegeObjectOwnerNameAndType.java
b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHivePrivilegeObjectOwnerNameAndType.java
new file mode 100644
index 0000000..9b50a0d
--- /dev/null
+++
b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHivePrivilegeObjectOwnerNameAndType.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.security.authorization.plugin;
+
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verify;
+
+/**
+ * Test HiveAuthorizer api invocation.
+ */
+public class TestHivePrivilegeObjectOwnerNameAndType {
+ protected static HiveConf conf;
+ protected static Driver driver;
+ private static final String TABLE_NAME =
TestHivePrivilegeObjectOwnerNameAndType.class.getSimpleName() + "Table";
+ static HiveAuthorizer mockedAuthorizer;
+
+ /**
+ * This factory creates a mocked HiveAuthorizer class. Use the mocked class
to
+ * capture the argument passed to it in the test case.
+ */
+ static class MockedHiveAuthorizerFactory implements HiveAuthorizerFactory {
+ @Override
+ public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory
metastoreClientFactory,
+ HiveConf conf, HiveAuthenticationProvider authenticator,
HiveAuthzSessionContext ctx) {
+ TestHivePrivilegeObjectOwnerNameAndType.mockedAuthorizer =
Mockito.mock(HiveAuthorizer.class);
+ return TestHivePrivilegeObjectOwnerNameAndType.mockedAuthorizer;
+ }
+
+ }
+
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+
UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser("hive"));
+ conf = new HiveConf();
+
+ // Turn on mocked authorization
+ conf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER,
MockedHiveAuthorizerFactory.class.getName());
+ //conf.setVar(ConfVars.HIVE_AUTHENTICATOR_MANAGER,
SessionStateUserAuthenticator.class.getName());
+ conf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true);
+ conf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false);
+ conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, true);
+ conf.setVar(ConfVars.HIVE_TXN_MANAGER, DbTxnManager.class.getName());
+ conf.setVar(ConfVars.HIVEMAPREDMODE, "nonstrict");
+
+ SessionState.start(conf);
+ driver = new Driver(conf);
+ runCmd("create table " + TABLE_NAME + " (i int, j int, k string)
partitioned by (city string, `date` string) ");
+ }
+
+ private static void runCmd(String cmd) throws Exception {
+ CommandProcessorResponse resp = driver.run(cmd);
+ assertEquals(0, resp.getResponseCode());
+ }
+
+ @AfterClass
+ public static void afterTests() throws Exception {
+ // Drop the tables when we're done. This makes the test work inside an IDE
+ runCmd("drop table if exists " + TABLE_NAME);
+ driver.close();
+ }
+
+ @Test
+ public void testOwnerNames() throws Exception {
+ reset(mockedAuthorizer);
+ driver.compile("create table default.t1 (name string)");
+
+ Pair<List<HivePrivilegeObject>, List<HivePrivilegeObject>> io =
getHivePrivilegeObjectInputs();
+ boolean containsDBOwnerName = false;
+ boolean containsTblOwnerName = false;
+ for (HivePrivilegeObject hpo : io.getLeft()) {
+ if (hpo.getType() ==
HivePrivilegeObject.HivePrivilegeObjectType.DATABASE && hpo.getOwnerName() !=
null) {
+ containsDBOwnerName = true;
+ }
+ if (hpo.getType() ==
HivePrivilegeObject.HivePrivilegeObjectType.TABLE_OR_VIEW && hpo.getOwnerName()
!= null) {
+ containsTblOwnerName = true;
+ }
+ }
+ for (HivePrivilegeObject hpo : io.getRight()) {
+ if (hpo.getType() ==
HivePrivilegeObject.HivePrivilegeObjectType.DATABASE && hpo.getOwnerName() !=
null) {
+ containsDBOwnerName = true;
+ }
+ if (hpo.getType() ==
HivePrivilegeObject.HivePrivilegeObjectType.TABLE_OR_VIEW && hpo.getOwnerName()
!= null) {
+ containsTblOwnerName = true;
+ }
+ }
+ if (!containsTblOwnerName || !containsDBOwnerName) {
+ String errorMessage = "Ownername is not present in HivePrivilegeObject";
+ throw new HiveAuthzPluginException(errorMessage);
+ }
+ }
+
+ @Test
+ public void testOwnerType() throws Exception {
+ reset(mockedAuthorizer);
+ driver.compile("create table default.t1 (name string)");
+
+ Pair<List<HivePrivilegeObject>, List<HivePrivilegeObject>> io =
getHivePrivilegeObjectInputs();
+ boolean containsOwnerType = false;
+ for (HivePrivilegeObject hpo : io.getLeft()) {
+ if (hpo.getOwnerType() != null) {
+ containsOwnerType = true;
+ }
+ }
+ for (HivePrivilegeObject hpo : io.getRight()) {
+ if (hpo.getOwnerType() != null) {
+ containsOwnerType = true;
+ }
+ }
+ Assert.assertTrue(containsOwnerType);
+ }
+
+ /**
+ * @return pair with left value as inputs and right value as outputs,
+ * passed in current call to authorizer.checkPrivileges
+ * @throws HiveAuthzPluginException
+ * @throws HiveAccessControlException
+ */
+ private Pair<List<HivePrivilegeObject>, List<HivePrivilegeObject>>
getHivePrivilegeObjectInputs()
+ throws HiveAuthzPluginException, HiveAccessControlException {
+ // Create argument capturer
+ // a class variable cast to this generic of generic class
+ Class<List<HivePrivilegeObject>> classListPrivObjects = (Class) List.class;
+ ArgumentCaptor<List<HivePrivilegeObject>> inputsCapturer =
ArgumentCaptor.forClass(classListPrivObjects);
+ ArgumentCaptor<List<HivePrivilegeObject>> outputsCapturer =
ArgumentCaptor.forClass(classListPrivObjects);
+
+ verify(mockedAuthorizer)
+ .checkPrivileges(any(HiveOperationType.class),
inputsCapturer.capture(), outputsCapturer.capture(),
+ any(HiveAuthzContext.class));
+
+ return new ImmutablePair(inputsCapturer.getValue(),
outputsCapturer.getValue());
+ }
+
+}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 1552ea0..07f325d 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -494,7 +494,7 @@ public class CachedStore implements RawStore, Configurable {
AggrStats aggrStatsAllButDefaultPartition = null;
if (!table.getPartitionKeys().isEmpty()) {
Deadline.startTimer("getPartitions");
- partitions = rawStore.getPartitions(catName, dbName, tblName,
Integer.MAX_VALUE);
+ partitions = rawStore.getPartitions(catName, dbName, tblName,
-1);
Deadline.stopTimer();
List<String> partNames = new ArrayList<>(partitions.size());
for (Partition p : partitions) {
@@ -862,7 +862,7 @@ public class CachedStore implements RawStore, Configurable {
dbName, tblName);
try {
Deadline.startTimer("getPartitions");
- List<Partition> partitions = rawStore.getPartitions(catName, dbName,
tblName, Integer.MAX_VALUE);
+ List<Partition> partitions = rawStore.getPartitions(catName, dbName,
tblName, -1);
Deadline.stopTimer();
sharedCache.refreshPartitionsInCache(StringUtils.normalizeIdentifier(catName),
StringUtils.normalizeIdentifier(dbName),
StringUtils.normalizeIdentifier(tblName), partitions);
