This is an automated email from the ASF dual-hosted git repository. daijy pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new fcadd38 HIVE-21833: Ranger Authorization in Hive based on object ownership (Sam An, reviewed by Daniel Dai) fcadd38 is described below commit fcadd388df2a7c12378febab167ee75a36520f6e Author: Daniel Dai <da...@cloudera.com> AuthorDate: Mon Jun 17 16:07:48 2019 -0700 HIVE-21833: Ranger Authorization in Hive based on object ownership (Sam An, reviewed by Daniel Dai) --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 9 +- .../authorization/plugin/HivePrivilegeObject.java | 41 ++++- .../TestHivePrivilegeObjectOwnerNameAndType.java | 170 +++++++++++++++++++++ .../hadoop/hive/metastore/cache/CachedStore.java | 4 +- 4 files changed, 217 insertions(+), 7 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 255c65a..18438aa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -63,6 +63,7 @@ import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockType; +import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.cache.results.CacheUsage; @@ -1395,15 +1396,21 @@ public class Driver implements IDriver { List<String> partKeys = null; List<String> columns = null; String className = null; + String ownerName = null; + PrincipalType ownerType = null; switch(privObject.getType()){ case DATABASE: dbname = privObject.getDatabase().getName(); + ownerName = privObject.getDatabase().getOwnerName(); + ownerType = privObject.getDatabase().getOwnerType(); break; case TABLE: dbname = privObject.getTable().getDbName(); objName = privObject.getTable().getTableName(); columns = tableName2Cols == null ? null : tableName2Cols.get(Table.getCompleteName(dbname, objName)); + ownerName = privObject.getTable().getOwner(); + ownerType = privObject.getTable().getOwnerType(); break; case DFS_DIR: case LOCAL_DIR: @@ -1428,7 +1435,7 @@ public class Driver implements IDriver { } HivePrivObjectActionType actionType = AuthorizationUtils.getActionType(privObject); HivePrivilegeObject hPrivObject = new HivePrivilegeObject(privObjType, dbname, objName, - partKeys, columns, actionType, null, className); + partKeys, columns, actionType, null, className, ownerName, ownerType); hivePrivobjs.add(hPrivObject); } return hivePrivobjs; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java index 87d2e68..b08f7ca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; +import org.apache.hadoop.hive.metastore.api.PrincipalType; /** * Represents the object on which privilege is being granted/revoked, and objects @@ -65,6 +66,16 @@ public class HivePrivilegeObject implements Comparable<HivePrivilegeObject> { (o.className != null ? className.compareTo(o.className) : 1) : (o.className != null ? -1 : 0); } + if (compare == 0) { + compare = ownerName != null? + (o.ownerName != null ? ownerName.compareTo(o.ownerName) : 1) : + (o.ownerName != null ? -1 : 0); + } + if (compare == 0) { + compare = ownerType != null? + (o.ownerType != null ? ownerType.compareTo(o.ownerType) : 1) : + (o.ownerType != null ? -1 : 0); + } return compare; } @@ -118,6 +129,8 @@ public class HivePrivilegeObject implements Comparable<HivePrivilegeObject> { private final List<String> columns; private final HivePrivObjectActionType actionType; private final String className; + private final String ownerName; + private final PrincipalType ownerType; // cellValueTransformers is corresponding to the columns. // Its size should be the same as columns. // For example, if a table has two columns, "key" and "value" @@ -164,9 +177,14 @@ public class HivePrivilegeObject implements Comparable<HivePrivilegeObject> { this(HivePrivilegeObjectType.TABLE_OR_VIEW, dbname, objectName, null, columns, null); } - public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname, String objectName, - List<String> partKeys, List<String> columns, HivePrivObjectActionType actionType, - List<String> commandParams, String className) { + public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname, String objectName, List<String> partKeys, + List<String> columns, HivePrivObjectActionType actionType, List<String> commandParams, String className) { + this(type, dbname, objectName, partKeys, columns, actionType, commandParams, className, null, null); + } + + public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname, String objectName, List<String> partKeys, + List<String> columns, HivePrivObjectActionType actionType, List<String> commandParams, String className, + String ownerName, PrincipalType ownerType) { this.type = type; this.dbname = dbname; this.objectName = objectName; @@ -175,6 +193,8 @@ public class HivePrivilegeObject implements Comparable<HivePrivilegeObject> { this.actionType = actionType; this.commandParams = commandParams; this.className = className; + this.ownerName = ownerName; + this.ownerType = ownerType; } public HivePrivilegeObjectType getType() { @@ -271,10 +291,23 @@ public class HivePrivilegeObject implements Comparable<HivePrivilegeObject> { default: } } - return "Object [type=" + type + ", name=" + name + actionTypeStr + "]"; } + /** + * @return ownerName of the object + */ + public String getOwnerName() { + return this.ownerName; + } + + /** + * @return principal type of the owner + */ + public PrincipalType getOwnerType() { + return this.ownerType; + } + private String getDbObjectName(String dbname2, String objectName2) { return (dbname == null ? "" : dbname + ".") + objectName; } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHivePrivilegeObjectOwnerNameAndType.java b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHivePrivilegeObjectOwnerNameAndType.java new file mode 100644 index 0000000..9b50a0d --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHivePrivilegeObjectOwnerNameAndType.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.security.authorization.plugin; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; + +/** + * Test HiveAuthorizer api invocation. + */ +public class TestHivePrivilegeObjectOwnerNameAndType { + protected static HiveConf conf; + protected static Driver driver; + private static final String TABLE_NAME = TestHivePrivilegeObjectOwnerNameAndType.class.getSimpleName() + "Table"; + static HiveAuthorizer mockedAuthorizer; + + /** + * This factory creates a mocked HiveAuthorizer class. Use the mocked class to + * capture the argument passed to it in the test case. + */ + static class MockedHiveAuthorizerFactory implements HiveAuthorizerFactory { + @Override + public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreClientFactory, + HiveConf conf, HiveAuthenticationProvider authenticator, HiveAuthzSessionContext ctx) { + TestHivePrivilegeObjectOwnerNameAndType.mockedAuthorizer = Mockito.mock(HiveAuthorizer.class); + return TestHivePrivilegeObjectOwnerNameAndType.mockedAuthorizer; + } + + } + + @BeforeClass + public static void beforeTest() throws Exception { + UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser("hive")); + conf = new HiveConf(); + + // Turn on mocked authorization + conf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER, MockedHiveAuthorizerFactory.class.getName()); + //conf.setVar(ConfVars.HIVE_AUTHENTICATOR_MANAGER, SessionStateUserAuthenticator.class.getName()); + conf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true); + conf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, true); + conf.setVar(ConfVars.HIVE_TXN_MANAGER, DbTxnManager.class.getName()); + conf.setVar(ConfVars.HIVEMAPREDMODE, "nonstrict"); + + SessionState.start(conf); + driver = new Driver(conf); + runCmd("create table " + TABLE_NAME + " (i int, j int, k string) partitioned by (city string, `date` string) "); + } + + private static void runCmd(String cmd) throws Exception { + CommandProcessorResponse resp = driver.run(cmd); + assertEquals(0, resp.getResponseCode()); + } + + @AfterClass + public static void afterTests() throws Exception { + // Drop the tables when we're done. This makes the test work inside an IDE + runCmd("drop table if exists " + TABLE_NAME); + driver.close(); + } + + @Test + public void testOwnerNames() throws Exception { + reset(mockedAuthorizer); + driver.compile("create table default.t1 (name string)"); + + Pair<List<HivePrivilegeObject>, List<HivePrivilegeObject>> io = getHivePrivilegeObjectInputs(); + boolean containsDBOwnerName = false; + boolean containsTblOwnerName = false; + for (HivePrivilegeObject hpo : io.getLeft()) { + if (hpo.getType() == HivePrivilegeObject.HivePrivilegeObjectType.DATABASE && hpo.getOwnerName() != null) { + containsDBOwnerName = true; + } + if (hpo.getType() == HivePrivilegeObject.HivePrivilegeObjectType.TABLE_OR_VIEW && hpo.getOwnerName() != null) { + containsTblOwnerName = true; + } + } + for (HivePrivilegeObject hpo : io.getRight()) { + if (hpo.getType() == HivePrivilegeObject.HivePrivilegeObjectType.DATABASE && hpo.getOwnerName() != null) { + containsDBOwnerName = true; + } + if (hpo.getType() == HivePrivilegeObject.HivePrivilegeObjectType.TABLE_OR_VIEW && hpo.getOwnerName() != null) { + containsTblOwnerName = true; + } + } + if (!containsTblOwnerName || !containsDBOwnerName) { + String errorMessage = "Ownername is not present in HivePrivilegeObject"; + throw new HiveAuthzPluginException(errorMessage); + } + } + + @Test + public void testOwnerType() throws Exception { + reset(mockedAuthorizer); + driver.compile("create table default.t1 (name string)"); + + Pair<List<HivePrivilegeObject>, List<HivePrivilegeObject>> io = getHivePrivilegeObjectInputs(); + boolean containsOwnerType = false; + for (HivePrivilegeObject hpo : io.getLeft()) { + if (hpo.getOwnerType() != null) { + containsOwnerType = true; + } + } + for (HivePrivilegeObject hpo : io.getRight()) { + if (hpo.getOwnerType() != null) { + containsOwnerType = true; + } + } + Assert.assertTrue(containsOwnerType); + } + + /** + * @return pair with left value as inputs and right value as outputs, + * passed in current call to authorizer.checkPrivileges + * @throws HiveAuthzPluginException + * @throws HiveAccessControlException + */ + private Pair<List<HivePrivilegeObject>, List<HivePrivilegeObject>> getHivePrivilegeObjectInputs() + throws HiveAuthzPluginException, HiveAccessControlException { + // Create argument capturer + // a class variable cast to this generic of generic class + Class<List<HivePrivilegeObject>> classListPrivObjects = (Class) List.class; + ArgumentCaptor<List<HivePrivilegeObject>> inputsCapturer = ArgumentCaptor.forClass(classListPrivObjects); + ArgumentCaptor<List<HivePrivilegeObject>> outputsCapturer = ArgumentCaptor.forClass(classListPrivObjects); + + verify(mockedAuthorizer) + .checkPrivileges(any(HiveOperationType.class), inputsCapturer.capture(), outputsCapturer.capture(), + any(HiveAuthzContext.class)); + + return new ImmutablePair(inputsCapturer.getValue(), outputsCapturer.getValue()); + } + +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 1552ea0..07f325d 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -494,7 +494,7 @@ public class CachedStore implements RawStore, Configurable { AggrStats aggrStatsAllButDefaultPartition = null; if (!table.getPartitionKeys().isEmpty()) { Deadline.startTimer("getPartitions"); - partitions = rawStore.getPartitions(catName, dbName, tblName, Integer.MAX_VALUE); + partitions = rawStore.getPartitions(catName, dbName, tblName, -1); Deadline.stopTimer(); List<String> partNames = new ArrayList<>(partitions.size()); for (Partition p : partitions) { @@ -862,7 +862,7 @@ public class CachedStore implements RawStore, Configurable { dbName, tblName); try { Deadline.startTimer("getPartitions"); - List<Partition> partitions = rawStore.getPartitions(catName, dbName, tblName, Integer.MAX_VALUE); + List<Partition> partitions = rawStore.getPartitions(catName, dbName, tblName, -1); Deadline.stopTimer(); sharedCache.refreshPartitionsInCache(StringUtils.normalizeIdentifier(catName), StringUtils.normalizeIdentifier(dbName), StringUtils.normalizeIdentifier(tblName), partitions);