Repository: incubator-sentry Updated Branches: refs/heads/master c57c53179 -> 3071da2fc
SENTRY-277: Add Pig+HCat test for Metastore auth plugin (Prasad Mujumdar via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/3071da2f Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/3071da2f Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/3071da2f Branch: refs/heads/master Commit: 3071da2fca0a5e9123622334cdab8de22c5cee89 Parents: c57c531 Author: Jarek Jarcec Cecho <[email protected]> Authored: Tue Jun 10 07:04:59 2014 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Tue Jun 10 07:04:59 2014 -0700 ---------------------------------------------------------------------- pom.xml | 16 +++ .../metastore/MetastoreAuthzBinding.java | 18 ++- sentry-tests/sentry-tests-hive/pom.xml | 15 +++ .../apache/sentry/tests/e2e/hive/Context.java | 25 +++- ...actMetastoreTestWithStaticConfiguration.java | 31 +++++ .../e2e/metastore/TestMetaStoreWithPigHCat.java | 113 +++++++++++++++++++ 6 files changed, 211 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 564a2ad..05943c6 100644 --- a/pom.xml +++ b/pom.xml @@ -81,6 +81,7 @@ limitations under the License. <solr.version>4.8.0</solr.version> <solr.sentry.handlers.version>4.4.0-cdh5.1.0-SNAPSHOT</solr.sentry.handlers.version> <zookeeper.version>3.4.5-cdh5.1.0-SNAPSHOT</zookeeper.version> + <pig.version>0.12.0-cdh5.1.0-SNAPSHOT</pig.version> </properties> <dependencyManagement> @@ -126,6 +127,16 @@ limitations under the License. <version>${hive.version}</version> </dependency> <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-core</artifactId> + <version>${hive.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-pig-adapter</artifactId> + <version>${hive.version}</version> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> @@ -363,6 +374,11 @@ limitations under the License. <artifactId>mockito-all</artifactId> <version>${mockito.version}</version> </dependency> + <dependency> + <groupId>org.apache.pig</groupId> + <artifactId>pig</artifactId> + <version>${pig.version}</version> + </dependency> </dependencies> </dependencyManagement> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/MetastoreAuthzBinding.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/MetastoreAuthzBinding.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/MetastoreAuthzBinding.java index 2737793..197017b 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/MetastoreAuthzBinding.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/MetastoreAuthzBinding.java @@ -240,13 +240,21 @@ public class MetastoreAuthzBinding extends MetaStorePreEventListener { } private void authorizeAddPartition(PreAddPartitionEvent context) - throws InvalidOperationException, MetaException { - // check if we need to validate URI permissions when storage location is - // non-default + throws InvalidOperationException, MetaException, NoSuchObjectException { HierarcyBuilder inputBuilder = new HierarcyBuilder(); - if (!StringUtils.isEmpty(context.getPartition().getSd().getLocation())) { - inputBuilder.addUriToOutput(getAuthServer(), context.getPartition() + + // check if we need to validate URI permissions when storage location is + // non-default, ie something not under the parent table + String partitionLocation = context.getPartition().getSd().getLocation(); + if (!StringUtils.isEmpty(partitionLocation)) { + String tableLocation = context + .getHandler() + .get_table(context.getPartition().getDbName(), + context.getPartition().getTableName()).getSd().getLocation(); + if (!partitionLocation.startsWith(tableLocation)) { + inputBuilder.addUriToOutput(getAuthServer(), context.getPartition() .getSd().getLocation()); + } } authorizeMetastoreAccess(HiveOperation.ALTERTABLE_ADDPARTS, inputBuilder.build(), http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/sentry-tests/sentry-tests-hive/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/pom.xml b/sentry-tests/sentry-tests-hive/pom.xml index 03802a3..cd95252 100644 --- a/sentry-tests/sentry-tests-hive/pom.xml +++ b/sentry-tests/sentry-tests-hive/pom.xml @@ -102,6 +102,16 @@ limitations under the License. <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-pig-adapter</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <scope>test</scope> @@ -173,6 +183,11 @@ limitations under the License. <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.pig</groupId> + <artifactId>pig</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>${zookeeper.version}</version> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java index 83fe397..4985912 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java @@ -43,6 +43,8 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.pig.ExecType; +import org.apache.pig.PigServer; import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -251,6 +253,20 @@ public class Context { return client; } + public PigServer getPigServer(String userName, final ExecType exType) + throws Exception { + UserGroupInformation clientUgi = UserGroupInformation + .createRemoteUser(userName); + PigServer pigServer = (PigServer) ShimLoader.getHadoopShims().doAs( + clientUgi, new PrivilegedExceptionAction<Object>() { + @Override + public PigServer run() throws Exception { + return new PigServer(exType, new HiveConf()); + } + }); + return pigServer; + } + /** * Execute "set x" and extract value from key=val format result Verify the * extracted value @@ -270,8 +286,13 @@ public class Context { resultValues[1]); } - public static void verifyMetastoreAuthException(MetaException e) + public static void verifyMetastoreAuthException(Throwable e) throws Exception { - assertTrue(e.getMessage().contains(METASTORE_AUTH_ERROR_MSG)); + if (e instanceof MetaException) { + assertTrue(e.getMessage().contains(METASTORE_AUTH_ERROR_MSG)); + } else { + throw new Exception("Excepted MetaException but got " + + e.getClass().getName(), e); + } } } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java index 2e70b14..fe10f5d 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java @@ -17,10 +17,13 @@ */ package org.apache.sentry.tests.e2e.metastore; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -29,7 +32,12 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.pig.PigServer; import org.apache.sentry.tests.e2e.dbprovider.PolicyProviderForTest; import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration; import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory.HiveServer2Type; @@ -153,4 +161,27 @@ public abstract class AbstractMetastoreTestWithStaticConfiguration extends client.createDatabase(db); } + public void execHiveSQL(String sqlStmt, String userName) throws Exception { + HiveConf hiveConf = new HiveConf(); + Driver driver = new Driver(hiveConf); + SessionState.start(new CliSessionState(hiveConf)); + driver.run(sqlStmt); + driver.close(); + SessionState.get().close(); + } + + public void execPigLatin(String userName, final PigServer pigServer, + final String pigLatin) throws Exception { + UserGroupInformation clientUgi = UserGroupInformation + .createRemoteUser(userName); + ShimLoader.getHadoopShims().doAs(clientUgi, + new PrivilegedExceptionAction<Object>() { + @Override + public Void run() throws Exception { + pigServer.registerQuery(pigLatin); + return null; + } + }); + } + } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetaStoreWithPigHCat.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetaStoreWithPigHCat.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetaStoreWithPigHCat.java new file mode 100644 index 0000000..46feace --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetaStoreWithPigHCat.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.tests.e2e.metastore; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hive.hcatalog.pig.HCatStorer; +import org.apache.pig.ExecType; +import org.apache.pig.PigServer; +import org.apache.sentry.tests.e2e.dbprovider.PolicyProviderForTest; +import org.apache.sentry.tests.e2e.hive.Context; +import org.apache.sentry.tests.e2e.hive.StaticUserGroup; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.io.Resources; + +public class TestMetaStoreWithPigHCat extends + AbstractMetastoreTestWithStaticConfiguration { + private PolicyProviderForTest policyFile; + private File dataFile; + private static final String dbName = "db_1"; + private static final String db_all_role = "all_db1"; + private static final String uri_role = "uri_role"; + + @Before + public void setup() throws Exception { + context = createContext(); + dataFile = new File(dataDir, SINGLE_TYPE_DATA_FILE_NAME); + FileOutputStream to = new FileOutputStream(dataFile); + Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to); + to.close(); + + policyFile = PolicyProviderForTest.setAdminOnServer1(ADMINGROUP); + policyFile + .addRolesToGroup(USERGROUP1, db_all_role) + .addRolesToGroup(USERGROUP2, "read_db_role") + .addPermissionsToRole(db_all_role, "server=server1->db=" + dbName) + .addPermissionsToRole("read_db_role", + "server=server1->db=" + dbName + "->table=*->action=SELECT") + .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + writePolicyFile(policyFile); + + HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1); + client.dropDatabase(dbName, true, true, true); + createMetastoreDB(client, dbName); + client.close(); + } + + /** + * Verify add partition via Pig+HCatStore + */ + @Test + public void testPartionLoad() throws Exception { + String tabName = "tab1"; + execHiveSQL("CREATE TABLE " + dbName + "." + tabName + + " (id int) PARTITIONED BY (part_col STRING)", USER1_1); + + // user with ALL on DB should be able to add partion using Pig/HCatStore + PigServer pigServer = context.getPigServer(USER1_1, ExecType.LOCAL); + execPigLatin(USER1_1, pigServer, "A = load '" + dataFile.getPath() + + "' as (id:int);"); + execPigLatin(USER1_1, pigServer, "store A into '" + dbName + "." + tabName + + "' using " + HCatStorer.class.getName() + " ('part_col=part1');"); + HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1); + assertEquals(1, client.listPartitionNames(dbName, tabName, (short) 10) + .size()); + client.close(); + + // user without ALL on DB should NOT be able to add partition with + // Pig/HCatStore + pigServer = context.getPigServer(USER2_1, ExecType.LOCAL); + execPigLatin(USER2_1, pigServer, "A = load '" + dataFile.getPath() + + "' as (id:int);"); + try { + execPigLatin(USER2_1, pigServer, "store A into '" + dbName + "." + + tabName + + "' using " + HCatStorer.class.getName() + " ('part_col=part2');"); + // TODO: The HCatStore seems to be swallowing the exception. Thus we + // manually verify that partition is not created by above call. + client = context.getMetaStoreClient(ADMIN1); + assertEquals(1, client.listPartitionNames(dbName, tabName, (short)10).size()); + client.close(); + // fail("HCatStore should fail for non-privilege user"); + + } catch (IOException e) { + Context.verifyMetastoreAuthException(e.getCause()); + } + + } + +}
