http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithHiveServer.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithHiveServer.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithHiveServer.java deleted file mode 100644 index a314c0d..0000000 --- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithHiveServer.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.tests.e2e.hive; - -import java.io.File; -import java.util.Map; - -import org.junit.Assert; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServer; -import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory; -import org.junit.AfterClass; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.io.Files; - -public abstract class AbstractTestWithHiveServer { - private static final Logger LOGGER = LoggerFactory - .getLogger(AbstractTestWithHiveServer.class); - protected static File baseDir; - protected static File logDir; - protected static File confDir; - protected static File dataDir; - protected static File policyFile; - protected static HiveServer hiveServer; - protected static FileSystem fileSystem; - - protected static final String ADMIN1 = StaticUserGroup.ADMIN1, - ADMINGROUP = StaticUserGroup.ADMINGROUP, - USER1_1 = StaticUserGroup.USER1_1, - USER2_1 = StaticUserGroup.USER2_1, - USER3_1 = StaticUserGroup.USER3_1, - USERGROUP1 = StaticUserGroup.USERGROUP1, - USERGROUP2 = StaticUserGroup.USERGROUP2, - USERGROUP3 = StaticUserGroup.USERGROUP3; - - public static Context createContext(Map<String, String> properties) - throws Exception { - fileSystem = FileSystem.get(new Configuration()); - baseDir = Files.createTempDir(); - LOGGER.info("BaseDir = " + baseDir); - logDir = assertCreateDir(new File(baseDir, "log")); - confDir = assertCreateDir(new File(baseDir, "etc")); - dataDir = assertCreateDir(new File(baseDir, "data")); - policyFile = new File(confDir, HiveServerFactory.AUTHZ_PROVIDER_FILENAME); - hiveServer = HiveServerFactory.create(properties, baseDir, confDir, logDir, policyFile.getPath(), fileSystem); - hiveServer.start(); - return new Context(hiveServer, fileSystem, - baseDir, dataDir, policyFile); - } - - protected static File assertCreateDir(File dir) { - if(!dir.isDirectory()) { - Assert.assertTrue("Failed creating " + dir, dir.mkdirs()); - } - return dir; - } - - protected FileSystem getFileSystem() { - return fileSystem; - } - - @AfterClass - public static void tearDownWithHiveServer() throws Exception { - if(hiveServer != null) { - hiveServer.shutdown(); - hiveServer = null; - } - if(baseDir != null) { - if(System.getProperty(HiveServerFactory.KEEP_BASEDIR) == null) { - FileUtils.deleteQuietly(baseDir); - } - baseDir = null; - } - } -}
http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java deleted file mode 100644 index 38c1c8b..0000000 --- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java +++ /dev/null @@ -1,723 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.tests.e2e.hive; - -import static org.apache.sentry.core.common.utils.SentryConstants.AUTHORIZABLE_SPLITTER; -import static org.apache.sentry.core.common.utils.SentryConstants.PRIVILEGE_PREFIX; -import static org.apache.sentry.core.common.utils.SentryConstants.ROLE_SPLITTER; - -import java.io.File; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.HashSet; - -import com.google.common.collect.Sets; -import org.junit.Assert; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.sentry.binding.hive.v2.SentryHiveAuthorizationTaskFactoryImplV2; -import org.apache.sentry.binding.hive.v2.metastore.SentryMetastorePostEventListenerV2; -import org.apache.sentry.core.model.db.DBModelAction; -import org.apache.sentry.core.model.db.DBModelAuthorizable; -import org.apache.sentry.provider.db.SimpleDBProviderBackend; -import org.apache.sentry.core.model.db.DBModelAuthorizables; -import org.apache.sentry.api.service.thrift.SentryPolicyServiceClient; -import org.apache.sentry.provider.file.PolicyFile; -import org.apache.sentry.service.thrift.KerberosConfiguration; -import org.apache.sentry.service.thrift.SentryServiceClientFactory; -import org.apache.sentry.service.thrift.ServiceConstants.ClientConfig; -import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; -import org.apache.sentry.tests.e2e.hive.fs.DFS; -import org.apache.sentry.tests.e2e.hive.fs.DFSFactory; -import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServer; -import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory; -import org.apache.sentry.tests.e2e.minisentry.SentrySrvFactory; -import org.apache.sentry.tests.e2e.minisentry.SentrySrvFactory.SentrySrvType; -import org.apache.sentry.tests.e2e.minisentry.SentrySrv; -import org.apache.tools.ant.util.StringUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Maps; -import com.google.common.io.Files; - -import javax.security.auth.Subject; -import javax.security.auth.kerberos.KerberosPrincipal; -import javax.security.auth.login.LoginContext; - -public abstract class AbstractTestWithStaticConfiguration { - private static final Logger LOGGER = LoggerFactory - .getLogger(AbstractTestWithStaticConfiguration.class); - protected static final String SINGLE_TYPE_DATA_FILE_NAME = "kv1.dat"; - protected static final String ALL_DB1 = "server=server1->db=db_1", - ALL_DB2 = "server=server1->db=db_2", - SELECT_DB1_TBL1 = "server=server1->db=db_1->table=tb_1->action=select", - SELECT_DB1_TBL2 = "server=server1->db=db_1->table=tb_2->action=select", - SELECT_DB1_NONTABLE = "server=server1->db=db_1->table=blahblah->action=select", - INSERT_DB1_TBL1 = "server=server1->db=db_1->table=tb_1->action=insert", - SELECT_DB2_TBL2 = "server=server1->db=db_2->table=tb_2->action=select", - INSERT_DB2_TBL1 = "server=server1->db=db_2->table=tb_1->action=insert", - SELECT_DB1_VIEW1 = "server=server1->db=db_1->table=view_1->action=select", - ADMIN1 = StaticUserGroup.ADMIN1, - ADMINGROUP = StaticUserGroup.ADMINGROUP, - USER1_1 = StaticUserGroup.USER1_1, - USER1_2 = StaticUserGroup.USER1_2, - USER2_1 = StaticUserGroup.USER2_1, - USER3_1 = StaticUserGroup.USER3_1, - USER4_1 = StaticUserGroup.USER4_1, - USERGROUP1 = StaticUserGroup.USERGROUP1, - USERGROUP2 = StaticUserGroup.USERGROUP2, - USERGROUP3 = StaticUserGroup.USERGROUP3, - USERGROUP4 = StaticUserGroup.USERGROUP4, - GROUP1_ROLE = "group1_role", - DB1 = "db_1", - DB2 = "db_2", - DB3 = "db_3", - TBL1 = "tb_1", - TBL2 = "tb_2", - TBL3 = "tb_3", - VIEW1 = "view_1", - VIEW2 = "view_2", - VIEW3 = "view_3", - INDEX1 = "index_1", - DEFAULT = "default"; - - protected static final String SERVER_HOST = "localhost"; - private static final String EXTERNAL_SENTRY_SERVICE = "sentry.e2etest.external.sentry"; - protected static final String EXTERNAL_HIVE_LIB = "sentry.e2etest.hive.lib"; - - protected static boolean policyOnHdfs = false; - protected static boolean useSentryService = false; - protected static boolean setMetastoreListener = true; - protected static String testServerType = null; - protected static boolean enableHiveConcurrency = false; - // indicate if the database need to be clear for every test case in one test class - protected static boolean clearDbPerTest = true; - - protected static File baseDir; - protected static File logDir; - protected static File confDir; - protected static File dataDir; - protected static File policyFileLocation; - protected static HiveServer hiveServer; - protected static FileSystem fileSystem; - protected static HiveServerFactory.HiveServer2Type hiveServer2Type; - protected static DFS dfs; - protected static Map<String, String> properties; - protected static SentrySrv sentryServer; - protected static Configuration sentryConf; - protected static Context context; - protected final String semanticException = "Exception No valid privileges"; - - protected static boolean clientKerberos = false; - protected static String REALM = System.getProperty("sentry.service.realm", "EXAMPLE.COM"); - protected static final String SERVER_KERBEROS_NAME = "sentry/" + SERVER_HOST + "@" + REALM; - protected static final String SERVER_KEY_TAB = System.getProperty("sentry.service.server.keytab"); - - private static LoginContext clientLoginContext; - protected static SentryPolicyServiceClient client; - - /** - * Get sentry client with authenticated Subject - * (its security-related attributes(for example, kerberos principal and key) - * @param clientShortName - * @param clientKeyTabDir - * @return client's Subject - */ - public static Subject getClientSubject(String clientShortName, String clientKeyTabDir) { - String clientKerberosPrincipal = clientShortName + "@" + REALM; - File clientKeyTabFile = new File(clientKeyTabDir); - Subject clientSubject = new Subject(false, Sets.newHashSet( - new KerberosPrincipal(clientKerberosPrincipal)), new HashSet<Object>(), - new HashSet<Object>()); - try { - clientLoginContext = new LoginContext("", clientSubject, null, - KerberosConfiguration.createClientConfig(clientKerberosPrincipal, clientKeyTabFile)); - clientLoginContext.login(); - } catch (Exception ex) { - LOGGER.error("Exception: " + ex); - } - clientSubject = clientLoginContext.getSubject(); - return clientSubject; - } - - public static void createContext() throws Exception { - context = new Context(hiveServer, fileSystem, - baseDir, dataDir, policyFileLocation); - } - protected void dropDb(String user, String...dbs) throws Exception { - Connection connection = context.createConnection(user); - Statement statement = connection.createStatement(); - for(String db : dbs) { - statement.execute("DROP DATABASE IF EXISTS " + db + " CASCADE"); - } - statement.close(); - connection.close(); - } - protected void createDb(String user, String...dbs) throws Exception { - Connection connection = context.createConnection(user); - Statement statement = connection.createStatement(); - ArrayList<String> allowedDBs = new ArrayList<String>(Arrays.asList(DB1, DB2, DB3)); - for(String db : dbs) { - Assert.assertTrue(db + " is not part of known test dbs which will be cleaned up after the test", allowedDBs.contains(db)); - statement.execute("CREATE DATABASE " + db); - } - statement.close(); - connection.close(); - } - - protected void createTable(String user, String db, File dataFile, String...tables) - throws Exception { - Connection connection = context.createConnection(user); - Statement statement = connection.createStatement(); - statement.execute("USE " + db); - for(String table : tables) { - statement.execute("DROP TABLE IF EXISTS " + table); - statement.execute("create table " + table - + " (under_col int comment 'the under column', value string)"); - if(dataFile != null) { - statement.execute("load data local inpath '" + dataFile.getPath() - + "' into table " + table); - ResultSet res = statement.executeQuery("select * from " + table); - Assert.assertTrue("Table should have data after load", res.next()); - res.close(); - } - } - statement.close(); - connection.close(); - } - - protected static File assertCreateDir(File dir) { - if(!dir.isDirectory()) { - Assert.assertTrue("Failed creating " + dir, dir.mkdirs()); - } - return dir; - } - - @BeforeClass - public static void setupTestStaticConfiguration() throws Exception { - LOGGER.info("AbstractTestWithStaticConfiguration setupTestStaticConfiguration"); - properties = Maps.newHashMap(); - if(!policyOnHdfs) { - policyOnHdfs = Boolean.valueOf(System.getProperty("sentry.e2etest.policyonhdfs", "false")); - } - if (testServerType != null) { - properties.put("sentry.e2etest.hiveServer2Type", testServerType); - } - baseDir = Files.createTempDir(); - LOGGER.info("BaseDir = " + baseDir); - logDir = assertCreateDir(new File(baseDir, "log")); - confDir = assertCreateDir(new File(baseDir, "etc")); - dataDir = assertCreateDir(new File(baseDir, "data")); - policyFileLocation = new File(confDir, HiveServerFactory.AUTHZ_PROVIDER_FILENAME); - - String dfsType = System.getProperty(DFSFactory.FS_TYPE); - dfs = DFSFactory.create(dfsType, baseDir, testServerType); - fileSystem = dfs.getFileSystem(); - - PolicyFile policyFile = PolicyFile.setAdminOnServer1(ADMIN1) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); - policyFile.write(policyFileLocation); - - String policyURI; - if (policyOnHdfs) { - String dfsUri = FileSystem.getDefaultUri(fileSystem.getConf()).toString(); - LOGGER.error("dfsUri " + dfsUri); - policyURI = dfsUri + System.getProperty("sentry.e2etest.hive.policy.location", - "/user/hive/sentry"); - policyURI += "/" + HiveServerFactory.AUTHZ_PROVIDER_FILENAME; - } else { - policyURI = policyFileLocation.getPath(); - } - - boolean startSentry = Boolean.valueOf(System.getProperty(EXTERNAL_SENTRY_SERVICE, "false")); - - if (useSentryService && (!startSentry)) { - setupSentryService(); - } - - if (enableHiveConcurrency) { - properties.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true"); - properties.put(HiveConf.ConfVars.HIVE_TXN_MANAGER.varname, - "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager"); - properties.put(HiveConf.ConfVars.HIVE_LOCK_MANAGER.varname, - "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); - } - - properties.put(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT.varname, "0"); - properties.put(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL.varname, "true"); - hiveServer = create(properties, baseDir, confDir, logDir, policyURI, fileSystem); - hiveServer.start(); - createContext(); - - // Create tmp as scratch dir if it doesn't exist - Path tmpPath = new Path("/tmp"); - if (!fileSystem.exists(tmpPath)) { - fileSystem.mkdirs(tmpPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); - } - } - - public static HiveServer create(Map<String, String> properties, - File baseDir, File confDir, File logDir, String policyFile, - FileSystem fileSystem) throws Exception { - String type = properties.get(HiveServerFactory.HIVESERVER2_TYPE); - if(type == null) { - type = System.getProperty(HiveServerFactory.HIVESERVER2_TYPE); - } - if(type == null) { - type = HiveServerFactory.HiveServer2Type.InternalHiveServer2.name(); - } - hiveServer2Type = HiveServerFactory.HiveServer2Type.valueOf(type.trim()); - return HiveServerFactory.create(hiveServer2Type, properties, - baseDir, confDir, logDir, policyFile, fileSystem); - } - - protected static void writePolicyFile(PolicyFile policyFile) throws Exception { - policyFile.write(context.getPolicyFile()); - if(policyOnHdfs) { - LOGGER.info("use policy file on HDFS"); - dfs.writePolicyFile(context.getPolicyFile()); - } else if(useSentryService) { - LOGGER.info("use sentry service, granting permissions"); - grantPermissions(policyFile); - } - } - - private static void grantPermissions(PolicyFile policyFile) throws Exception { - Connection connection = context.createConnection(ADMIN1); - Statement statement = context.createStatement(connection); - - // remove existing metadata - ResultSet resultSet = statement.executeQuery("SHOW ROLES"); - while( resultSet.next()) { - Statement statement1 = context.createStatement(connection); - String roleName = resultSet.getString(1).trim(); - if(!roleName.equalsIgnoreCase("admin_role")) { - LOGGER.info("Dropping role :" + roleName); - statement1.execute("DROP ROLE " + roleName); - } - } - - // create roles and add privileges - for (Map.Entry<String, Collection<String>> roleEntry : policyFile.getRolesToPermissions() - .asMap().entrySet()) { - String roleName = roleEntry.getKey(); - if(!roleEntry.getKey().equalsIgnoreCase("admin_role")){ - LOGGER.info("Creating role : " + roleName); - statement.execute("CREATE ROLE " + roleName); - for (String privilege : roleEntry.getValue()) { - addPrivilege(roleEntry.getKey(), privilege, statement); - } - } - } - // grant roles to groups - for (Map.Entry<String, Collection<String>> groupEntry : policyFile.getGroupsToRoles().asMap() - .entrySet()) { - for (String roleNames : groupEntry.getValue()) { - for (String roleName : roleNames.split(",")) { - String sql = "GRANT ROLE " + roleName + " TO GROUP " + groupEntry.getKey(); - LOGGER.info("Granting role to group: " + sql); - statement.execute(sql); - } - } - } - } - - private static void addPrivilege(String roleName, String privileges, Statement statement) - throws IOException, SQLException{ - String serverName = null, dbName = null, tableName = null, uriPath = null, columnName = null; - String action = "ALL";//AccessConstants.ALL; - for (String privilege : ROLE_SPLITTER.split(privileges)) { - for(String section : AUTHORIZABLE_SPLITTER.split(privilege)) { - // action is not an authorizeable - if(!section.toLowerCase().startsWith(PRIVILEGE_PREFIX)) { - DBModelAuthorizable dbAuthorizable = DBModelAuthorizables.from(section); - if(dbAuthorizable == null) { - throw new IOException("Unknown Auth type " + section); - } - - if (DBModelAuthorizable.AuthorizableType.Server.equals(dbAuthorizable.getAuthzType())) { - serverName = dbAuthorizable.getName(); - } else if (DBModelAuthorizable.AuthorizableType.Db.equals(dbAuthorizable.getAuthzType())) { - dbName = dbAuthorizable.getName(); - } else if (DBModelAuthorizable.AuthorizableType.Table.equals(dbAuthorizable.getAuthzType())) { - tableName = dbAuthorizable.getName(); - } else if (DBModelAuthorizable.AuthorizableType.Column.equals(dbAuthorizable.getAuthzType())) { - columnName = dbAuthorizable.getName(); - } else if (DBModelAuthorizable.AuthorizableType.URI.equals(dbAuthorizable.getAuthzType())) { - uriPath = dbAuthorizable.getName(); - } else { - throw new IOException("Unsupported auth type " + dbAuthorizable.getName() - + " : " + dbAuthorizable.getTypeName()); - } - } else { - action = DBModelAction.valueOf( - StringUtils.removePrefix(section, PRIVILEGE_PREFIX).toUpperCase()) - .toString(); - } - } - - LOGGER.info("addPrivilege"); - if (columnName != null) { - statement.execute("CREATE DATABASE IF NOT EXISTS " + dbName); - statement.execute("USE " + dbName); - String sql = "GRANT " + action + " ( " + columnName + " ) ON TABLE " + tableName + " TO ROLE " + roleName; - LOGGER.info("Granting column level privilege: database = " + dbName + ", sql = " + sql); - statement.execute(sql); - } else if (tableName != null) { - statement.execute("CREATE DATABASE IF NOT EXISTS " + dbName); - statement.execute("USE " + dbName); - String sql = "GRANT " + action + " ON TABLE " + tableName + " TO ROLE " + roleName; - LOGGER.info("Granting table level privilege: database = " + dbName + ", sql = " + sql); - statement.execute(sql); - } else if (dbName != null) { - String sql = "GRANT " + action + " ON DATABASE " + dbName + " TO ROLE " + roleName; - LOGGER.info("Granting db level privilege: " + sql); - statement.execute(sql); - } else if (uriPath != null) { - String sql = "GRANT " + action + " ON URI '" + uriPath + "' TO ROLE " + roleName; - LOGGER.info("Granting uri level privilege: " + sql); - statement.execute(sql);//ALL? - } else if (serverName != null) { - String sql = "GRANT ALL ON SERVER " + serverName + " TO ROLE " + roleName; - LOGGER.info("Granting server level privilege: " + sql); - statement.execute(sql); - } - } - } - - private static void setupSentryService() throws Exception { - - sentryConf = new Configuration(false); - - properties.put(HiveServerFactory.AUTHZ_PROVIDER_BACKEND, - SimpleDBProviderBackend.class.getName()); - properties.put(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY.varname, - SentryHiveAuthorizationTaskFactoryImplV2.class.getName()); - properties - .put(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS.varname, "2"); - properties.put(ServerConfig.SECURITY_MODE, ServerConfig.SECURITY_MODE_NONE); - properties.put(ServerConfig.ADMIN_GROUPS, ADMINGROUP); - properties.put(ServerConfig.RPC_ADDRESS, SERVER_HOST); - properties.put(ServerConfig.RPC_PORT, String.valueOf(0)); - properties.put(ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, "false"); - - properties.put(ServerConfig.SENTRY_STORE_JDBC_URL, - "jdbc:derby:;databaseName=" + baseDir.getPath() - + "/sentrystore_db;create=true"); - properties.put(ServerConfig.SENTRY_STORE_JDBC_PASS, "dummy"); - properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING, ServerConfig.SENTRY_STORE_LOCAL_GROUP_MAPPING); - properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFileLocation.getPath()); - properties.put(ServerConfig.RPC_MIN_THREADS, "3"); - for (Map.Entry<String, String> entry : properties.entrySet()) { - sentryConf.set(entry.getKey(), entry.getValue()); - } - sentryServer = SentrySrvFactory.create( - SentrySrvType.INTERNAL_SERVER, sentryConf, 1); - properties.put(ClientConfig.SERVER_RPC_ADDRESS, sentryServer.get(0) - .getAddress() - .getHostName()); - sentryConf.set(ClientConfig.SERVER_RPC_ADDRESS, sentryServer.get(0) - .getAddress() - .getHostName()); - properties.put(ClientConfig.SERVER_RPC_PORT, - String.valueOf(sentryServer.get(0).getAddress().getPort())); - sentryConf.set(ClientConfig.SERVER_RPC_PORT, - String.valueOf(sentryServer.get(0).getAddress().getPort())); - - startSentryService(); - if (setMetastoreListener) { - LOGGER.info("setMetastoreListener is enabled"); - properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname, - SentryMetastorePostEventListenerV2.class.getName()); - } - - } - - private static void startSentryService() throws Exception { - sentryServer.startAll(); - } - - public static SentryPolicyServiceClient getSentryClient() throws Exception { - if (sentryServer == null) { - throw new IllegalAccessException("Sentry service not initialized"); - } - return SentryServiceClientFactory.create(sentryServer.get(0).getConf()); - } - - /** - * Get Sentry authorized client to communicate with sentry server, - * the client can be for a minicluster, real distributed cluster, - * sentry server can use policy file or it's a service. - * @param clientShortName: principal prefix string - * @param clientKeyTabDir: authorization key path - * @return sentry client to talk to sentry server - * @throws Exception - */ - public static SentryPolicyServiceClient getSentryClient(String clientShortName, - String clientKeyTabDir) throws Exception { - if (!useSentryService) { - LOGGER.info("Running on a minicluser env."); - return getSentryClient(); - } - - if (clientKerberos) { - if (sentryConf == null ) { - sentryConf = new Configuration(false); - } - final String SENTRY_HOST = System.getProperty("sentry.host", SERVER_HOST); - final String SERVER_KERBEROS_PRINCIPAL = "sentry/" + SENTRY_HOST + "@" + REALM; - sentryConf.set(ServerConfig.PRINCIPAL, SERVER_KERBEROS_PRINCIPAL); - sentryConf.set(ServerConfig.KEY_TAB, SERVER_KEY_TAB); - sentryConf.set(ServerConfig.ALLOW_CONNECT, "hive"); - sentryConf.set(ServerConfig.SECURITY_USE_UGI_TRANSPORT, "false"); - sentryConf.set(ClientConfig.SERVER_RPC_ADDRESS, - System.getProperty("sentry.service.server.rpc.address")); - sentryConf.set(ClientConfig.SERVER_RPC_PORT, - System.getProperty("sentry.service.server.rpc.port", "8038")); - sentryConf.set(ClientConfig.SERVER_RPC_CONN_TIMEOUT, "720000"); //millis - Subject clientSubject = getClientSubject(clientShortName, clientKeyTabDir); - client = Subject.doAs(clientSubject, - new PrivilegedExceptionAction<SentryPolicyServiceClient>() { - @Override - public SentryPolicyServiceClient run() throws Exception { - return SentryServiceClientFactory.create(sentryConf); - } - }); - } else { - client = getSentryClient(); - } - return client; - } - - @Before - public void setup() throws Exception{ - LOGGER.info("AbstractTestStaticConfiguration setup"); - dfs.createBaseDir(); - if (clearDbPerTest) { - LOGGER.info("Before per test run clean up"); - clearAll(true); - } - } - - @After - public void clearAfterPerTest() throws Exception { - LOGGER.info("AbstractTestStaticConfiguration clearAfterPerTest"); - if (clearDbPerTest) { - LOGGER.info("After per test run clean up"); - clearAll(true); - } - } - - protected static void clearAll(boolean clearDb) throws Exception { - LOGGER.info("About to run clearAll"); - ResultSet resultSet; - Connection connection = context.createConnection(ADMIN1); - Statement statement = context.createStatement(connection); - - if (clearDb) { - LOGGER.info("About to clear all databases and default database tables"); - resultSet = execQuery(statement, "SHOW DATABASES"); - while (resultSet.next()) { - String db = resultSet.getString(1); - if (!db.equalsIgnoreCase("default")) { - try (Statement statement1 = context.createStatement(connection)) { - exec(statement1, "DROP DATABASE IF EXISTS " + db + " CASCADE"); - } catch (Exception ex) { - // For database and tables managed by other processes than Sentry - // drop them might run into exception - LOGGER.error("Exception: " + ex); - } - } - } - if (resultSet != null) { resultSet.close(); } - exec(statement, "USE default"); - resultSet = execQuery(statement, "SHOW TABLES"); - while (resultSet.next()) { - try (Statement statement1 = context.createStatement(connection)) { - exec(statement1, "DROP TABLE IF EXISTS " + resultSet.getString(1)); - } catch (Exception ex) { - // For table managed by other processes than Sentry - // drop it might run into exception - LOGGER.error("Exception: " + ex); - } - } - if (resultSet != null) { resultSet.close(); } - } - - if(useSentryService) { - LOGGER.info("About to clear all roles"); - resultSet = execQuery(statement, "SHOW ROLES"); - while (resultSet.next()) { - try (Statement statement1 = context.createStatement(connection)) { - String role = resultSet.getString(1); - if (!role.toLowerCase().contains("admin")) { - exec(statement1, "DROP ROLE " + role); - } - } - } - if (resultSet != null) { resultSet.close(); } - } - - if (statement != null) { statement.close(); } - if (connection != null) { connection.close(); } - } - - protected static void setupAdmin() throws Exception { - if(useSentryService) { - LOGGER.info("setupAdmin to create admin_role"); - Connection connection = context.createConnection(ADMIN1); - Statement statement = connection.createStatement(); - try { - statement.execute("CREATE ROLE admin_role"); - } catch ( Exception e) { - //It is ok if admin_role already exists - } - statement.execute("GRANT ALL ON SERVER " - + HiveServerFactory.DEFAULT_AUTHZ_SERVER_NAME + " TO ROLE admin_role"); - statement.execute("GRANT ROLE admin_role TO GROUP " + ADMINGROUP); - statement.close(); - connection.close(); - } - } - - protected PolicyFile setupPolicy() throws Exception { - LOGGER.info("Pre create policy file with admin group mapping"); - PolicyFile policyFile = PolicyFile.setAdminOnServer1(ADMINGROUP); - policyFile.setUserGroupMapping(StaticUserGroup.getStaticMapping()); - writePolicyFile(policyFile); - return policyFile; - } - - @AfterClass - public static void tearDownTestStaticConfiguration() throws Exception { - if(hiveServer != null) { - hiveServer.shutdown(); - hiveServer = null; - } - - if (sentryServer != null) { - sentryServer.close(); - sentryServer = null; - } - - if(baseDir != null) { - if(System.getProperty(HiveServerFactory.KEEP_BASEDIR) == null) { - FileUtils.deleteQuietly(baseDir); - } - baseDir = null; - } - if(dfs != null) { - try { - dfs.tearDown(); - } catch (Exception e) { - LOGGER.info("Exception shutting down dfs", e); - } - } - if (context != null) { - context.close(); - } - } - - public static SentrySrv getSentrySrv() { - return sentryServer; - } - - /** - * A convenience method to validate: - * if expected is equivalent to returned; - * Firstly check if each expected item is in the returned list; - * Secondly check if each returned item in in the expected list. - */ - protected void validateReturnedResult(List<String> expected, List<String> returned) { - for (String obj : expected) { - Assert.assertTrue("expected " + obj + " not found in the returned list: " + returned.toString(), - returned.contains(obj)); - } - for (String obj : returned) { - Assert.assertTrue("returned " + obj + " not found in the expected list: " + expected.toString(), - expected.contains(obj)); - } - } - - /** - * A convenient function to run a sequence of sql commands - * @param user - * @param sqls - * @throws Exception - */ - protected static void execBatch(String user, List<String> sqls) throws Exception { - Connection conn = context.createConnection(user); - Statement stmt = context.createStatement(conn); - for (String sql : sqls) { - exec(stmt, sql); - } - if (stmt != null) { - stmt.close(); - } - if (conn != null) { - conn.close(); - } - } - - /** - * A convenient funciton to run one sql with log - * @param stmt - * @param sql - * @throws Exception - */ - protected static void exec(Statement stmt, String sql) throws Exception { - if (stmt == null) { - LOGGER.error("Statement is null"); - return; - } - LOGGER.info("Running [" + sql + "]"); - stmt.execute(sql); - } - - /** - * A convenient funciton to execute query with log then return ResultSet - * @param stmt - * @param sql - * @return ResetSet - * @throws Exception - */ - protected static ResultSet execQuery(Statement stmt, String sql) throws Exception { - LOGGER.info("Running [" + sql + "]"); - return stmt.executeQuery(sql); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java deleted file mode 100644 index 2e508d6..0000000 --- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java +++ /dev/null @@ -1,321 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.tests.e2e.hive; - -import org.apache.sentry.binding.hive.conf.InvalidConfigurationException; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.net.URI; -import java.security.PrivilegedExceptionAction; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Set; - -import org.junit.Assert; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.pig.ExecType; -import org.apache.pig.PigServer; -import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Charsets; -import com.google.common.base.Strings; -import com.google.common.collect.Sets; -import com.google.common.io.Files; - -public class Context { - - private static final Logger LOGGER = LoggerFactory - .getLogger(Context.class); - - public static final String AUTHZ_EXCEPTION_SQL_STATE = "42000"; - public static final String AUTHZ_LINK_FAILURE_SQL_STATE = "08S01"; - public static final String AUTHZ_EXCEPTION_ERROR_MSG = "No valid privileges"; - private static final String METASTORE_AUTH_ERROR_MSG = "does not have privileges"; - private static final String INVALID_CONFIG_ERROR_MSG = InvalidConfigurationException.class.getSimpleName(); - - private final HiveServer hiveServer; - private final FileSystem fileSystem; - private final File baseDir; - private final File dataDir; - - private File policyFile; - private final Set<Connection> connections; - private final Set<Statement> statements; - - public Context(HiveServer hiveServer, FileSystem fileSystem, - File baseDir, File dataDir, File policyFile) throws Exception { - this.hiveServer = hiveServer; - this.fileSystem = fileSystem; - this.baseDir = baseDir; - this.dataDir = dataDir; - this.policyFile = policyFile; - connections = Sets.newHashSet(); - statements = Sets.newHashSet(); - } - - public Connection createConnection(String username) throws Exception { - - String password = username; - Connection connection = hiveServer.createConnection(username, password); - connections.add(connection); - assertNotNull("Connection is null", connection); - assertFalse("Connection should not be closed", connection.isClosed()); - Statement statement = connection.createStatement(); - statement.close(); - return connection; - } - - public Statement createStatement(Connection connection) - throws Exception { - Statement statement = connection.createStatement(); - assertNotNull("Statement is null", statement); - statements.add(statement); - return statement; - } - /** - * Deprecated} use append() - */ - public void writePolicyFile(String buf) throws IOException { - FileOutputStream out = new FileOutputStream(policyFile); - out.write(buf.getBytes(Charsets.UTF_8)); - out.close(); - } - /** - * Deprecated} use append() - */ - @Deprecated - public void appendToPolicyFileWithNewLine(String line) throws IOException { - append(line); - } - public void append(String...lines) throws IOException { - StringBuffer buffer = new StringBuffer(); - for(String line : lines) { - buffer.append(line).append("\n"); - } - Files.append(buffer, policyFile, Charsets.UTF_8); - } - - public boolean deletePolicyFile() throws IOException { - return policyFile.delete(); - } - /** - * Deprecated} use append() - */ - public void makeNewPolicy(String policyLines[]) throws FileNotFoundException { - PrintWriter policyWriter = new PrintWriter (policyFile.toString()); - for (String line : policyLines) { - policyWriter.println(line); - } - policyWriter.close(); - assertFalse(policyWriter.checkError()); - } - - public void close() { - for(Statement statement : statements) { - try { - statement.close(); - } catch (Exception exception) { - LOGGER.warn("Error closing " + statement, exception); - } - } - statements.clear(); - - for(Connection connection : connections) { - try { - connection.close(); - } catch (Exception exception) { - LOGGER.warn("Error closing " + connection, exception); - } - } - connections.clear(); - } - - public void assertAuthzException(Statement statement, String query) - throws SQLException { - try { - statement.execute(query); - Assert.fail("Expected SQLException for '" + query + "'"); - } catch (SQLException e) { - verifyAuthzException(e); - } - } - - public void assertAuthzExecHookException(Statement statement, String query, - String expectedMsg) throws SQLException { - try { - statement.execute(query); - Assert.fail("Expected SQLException for '" + query + "'"); - } catch (SQLException e) { - verifyAuthzExecHookException(e); - assertNotNull(e.getMessage()); - assertTrue(e.getMessage().contains(expectedMsg)); - } - } - - public void assertSentryException(Statement statement, String query, String exceptionType) - throws SQLException { - try { - statement.execute(query); - Assert.fail("Expected SQLException for '" + query + "'"); - } catch (SQLException e) { - verifyAuthzExceptionForState(e, AUTHZ_LINK_FAILURE_SQL_STATE); - assertTrue("Expected " + exceptionType + " : " + e.getMessage(), - Strings.nullToEmpty(e.getMessage()).contains(exceptionType)); - } - } - - public void assertSentrySemanticException(Statement statement, String query, String exceptionType) - throws SQLException { - try { - statement.execute(query); - Assert.fail("Expected SQLException for '" + query + "'"); - } catch (SQLException e) { - verifyAuthzExceptionForState(e, AUTHZ_EXCEPTION_SQL_STATE); - assertTrue("Expected " + exceptionType + " : " + e.getMessage(), - Strings.nullToEmpty(e.getMessage()).contains(exceptionType)); - } - } - // verify that the sqlexception is due to authorization failure - public void verifyAuthzException(SQLException sqlException) throws SQLException{ - verifyAuthzExceptionForState(sqlException, AUTHZ_EXCEPTION_SQL_STATE); - } - - // verify that the sqlexception is due to authorization failure due to exec hooks - public void verifyAuthzExecHookException(SQLException sqlException) throws SQLException{ - verifyAuthzExceptionForState(sqlException, AUTHZ_LINK_FAILURE_SQL_STATE); - } - - // verify that the sqlexception is due to authorization failure - private void verifyAuthzExceptionForState(SQLException sqlException, - String expectedSqlState) throws SQLException { - if (!expectedSqlState.equals(sqlException.getSQLState())) { - throw sqlException; - } - } - - public File getBaseDir() { - return baseDir; - } - - public File getDataDir() { - return dataDir; - } - - public File getPolicyFile() { - return policyFile; - } - - public void setPolicyFile(File policyFile) { - this.policyFile = policyFile; - } - - @SuppressWarnings("static-access") - public URI getDFSUri() throws IOException { - return fileSystem.getDefaultUri(fileSystem.getConf()); - } - - public String getProperty(String propName) { - return hiveServer.getProperty(propName); - } - - public String getConnectionURL() { - return hiveServer.getURL(); - } - - // TODO: Handle kerberos login - public HiveMetaStoreClient getMetaStoreClient(String userName) throws Exception { - UserGroupInformation clientUgi = UserGroupInformation.createRemoteUser(userName); - HiveMetaStoreClient client = (HiveMetaStoreClient) clientUgi. - doAs(new PrivilegedExceptionAction<Object> () { - @Override - public HiveMetaStoreClient run() throws Exception { - return new HiveMetaStoreClient(new HiveConf()); - } - }); - return client; - } - - public PigServer getPigServer(String userName, final ExecType exType) - throws Exception { - UserGroupInformation clientUgi = UserGroupInformation - .createRemoteUser(userName); - PigServer pigServer = (PigServer) clientUgi. - doAs(new PrivilegedExceptionAction<Object>() { - @Override - public PigServer run() throws Exception { - return new PigServer(exType, new HiveConf()); - } - }); - return pigServer; - } - - /** - * Execute "set x" and extract value from key=val format result Verify the - * extracted value - * - * @param stmt - * @return - * @throws Exception - */ - public void verifySessionConf(Connection con, String key, String expectedVal) - throws Exception { - Statement stmt = con.createStatement(); - ResultSet res = stmt.executeQuery("set " + key); - assertTrue(res.next()); - String resultValues[] = res.getString(1).split("="); // "key=val" - assertEquals("Result not in key = val format", 2, resultValues.length); - assertEquals("Conf value should be set by execute()", expectedVal, - resultValues[1]); - } - - public static void verifyMetastoreAuthException(Throwable e) - throws Exception { - if (e instanceof MetaException) { - assertTrue(e.getMessage().contains(METASTORE_AUTH_ERROR_MSG)); - } else { - throw new Exception("Excepted MetaException but got " - + e.getClass().getName(), e); - } - } - public void verifyInvalidConfigurationException(Throwable e) - throws Exception { - if (e instanceof SQLException) { - assertTrue(e.getMessage().contains(INVALID_CONFIG_ERROR_MSG)); - } else { - throw new Exception("Excepted " + INVALID_CONFIG_ERROR_MSG + " but got: " - + e.getClass().getName(), e); - } - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/DummySentryOnFailureHook.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/DummySentryOnFailureHook.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/DummySentryOnFailureHook.java deleted file mode 100644 index 99614e0..0000000 --- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/DummySentryOnFailureHook.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.tests.e2e.hive; - -import org.apache.hadoop.hive.ql.metadata.AuthorizationException; -import org.apache.hadoop.hive.ql.plan.HiveOperation; -import org.apache.sentry.binding.hive.SentryOnFailureHook; -import org.apache.sentry.binding.hive.SentryOnFailureHookContext; -import org.apache.sentry.core.model.db.Database; -import org.apache.sentry.core.model.db.Table; - -public class DummySentryOnFailureHook implements SentryOnFailureHook { - - public static boolean invoked = false; - public static HiveOperation hiveOp; - public static Database db; - public static Table table; - public static AuthorizationException exception; - - @Override - public void run(SentryOnFailureHookContext failureHookContext) - throws Exception { - invoked = true; - hiveOp = failureHookContext.getHiveOp(); - db = failureHookContext.getDatabase(); - table = failureHookContext.getTable(); - exception = failureHookContext.getException(); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/PolicyFileEditor.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/PolicyFileEditor.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/PolicyFileEditor.java deleted file mode 100644 index 1c9511d..0000000 --- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/PolicyFileEditor.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.tests.e2e.hive; - - - - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import com.google.common.base.Charsets; -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import com.google.common.io.Files; - -/** - * Deprecated} use Context.append() - */ -public class PolicyFileEditor { - - private static final String NL = System.getProperty("line.separator", "\n"); - - private File policy; - - public PolicyFileEditor (File policy) throws IOException { - policy.delete(); - policy.createNewFile(); - this.policy = policy; - } - - public void clearOldPolicy() throws IOException { - policy.delete(); - policy.createNewFile(); - } - - public void addPolicy(String line, String cat) throws IOException { - List<String> result = new ArrayList<String>(); - boolean exist = false; - for(String s : Files.readLines(policy, Charsets.UTF_8)) { - result.add(s); - if (s.equals("[" + cat + "]")) { - result.add(line); - exist = true; - } - } - if (!exist) { - result.add("[" + cat + "]"); - result.add(line); - } - Files.write(Joiner.on(NL).join(result), policy, Charsets.UTF_8); - } - public void removePolicy(String line) throws IOException { - List<String> result = Lists.newArrayList(); - for(String s : Files.readLines(policy, Charsets.UTF_8)) { - if (!s.equals(line)) { - result.add(s); - } - } - Files.write(Joiner.on(NL).join(result), policy, Charsets.UTF_8); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/PrivilegeResultSet.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/PrivilegeResultSet.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/PrivilegeResultSet.java deleted file mode 100644 index 9861a59..0000000 --- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/PrivilegeResultSet.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.tests.e2e.hive; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; - -/** - * This class holds ResultSet after query sentry privileges - * header: contain result header information, which is a array of string - * privilegeResultSet: contain privilege results from query - */ -public class PrivilegeResultSet { - private static final Logger LOGGER = LoggerFactory - .getLogger(PrivilegeResultSet.class); - - protected int colNum = 0; - protected List<String> header; - protected List<ArrayList<String>> privilegeResultSet; - - public PrivilegeResultSet() { - header = new ArrayList<String>(); - privilegeResultSet = new ArrayList<ArrayList<String>>(); - } - - public PrivilegeResultSet(Statement stmt, String query) { - LOGGER.info("Getting result set for " + query); - this.header = new ArrayList<String>(); - this.privilegeResultSet = new ArrayList<ArrayList<String>>(); - ResultSet rs = null; - try { - rs = stmt.executeQuery(query); - ResultSetMetaData rsmd = rs.getMetaData(); - this.colNum = rsmd.getColumnCount(); - for (int i = 1; i <= this.colNum; i++) { - this.header.add(rsmd.getColumnName(i).trim()); - } - while (rs.next()) { - ArrayList<String> row = new ArrayList<String>(); - for (int i = 1; i <= colNum; i++) { - row.add(rs.getString(i).trim()); - } - this.privilegeResultSet.add(row); - } - } catch (Exception ex) { - LOGGER.info("Exception when executing query: " + ex); - } finally { - try { - rs.close(); - } catch (Exception ex) { - LOGGER.error("failed to close result set: ", ex.getStackTrace()); - } - } - } - - public List<ArrayList<String>> getResultSet() { - return this.privilegeResultSet; - } - - public List<String> getHeader() { - return this.header; - } - - /** - * Given a column name, validate if one of its values equals to given colVal - */ - public boolean verifyResultSetColumn(String colName, String colVal) { - for (int i = 0; i < this.colNum; i ++) { - if (this.header.get(i).equalsIgnoreCase(colName)) { - for (int j = 0; j < this.privilegeResultSet.size(); j ++) { - if (this.privilegeResultSet.get(j).get(i).equalsIgnoreCase(colVal)) { - LOGGER.info("Found " + colName + " contains a value = " + colVal); - return true; - } - } - } - } - LOGGER.error("Failed to detect " + colName + " contains a value = " + colVal); - return false; - } - - /** - * Unmarshall ResultSet into a string - */ - @Override - public String toString() { - String prettyPrintString = new String("\n"); - for (String h : this.header) { - prettyPrintString += h + ","; - } - prettyPrintString += "\n"; - for (ArrayList<String> row : this.privilegeResultSet) { - for (String val : row) { - if (val.isEmpty()) { - val = "null"; - } - prettyPrintString += val + ","; - } - prettyPrintString += "\n"; - } - return prettyPrintString; - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/StaticUserGroup.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/StaticUserGroup.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/StaticUserGroup.java deleted file mode 100644 index 8306e95..0000000 --- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/StaticUserGroup.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.tests.e2e.hive; - -import java.util.HashMap; -import java.util.Map; - -public class StaticUserGroup { - public static String ADMIN1,ADMINGROUP; - public static final String - USER1_1 = "user1_1", - USER1_2 = "user1_2", - USER2_1 = "user2_1", - USER3_1 = "user3_1", - USER4_1 = "user4_1", - HIVE = "hive", - USERGROUP1 = "user_group1", - USERGROUP2 = "user_group2", - USERGROUP3 = "user_group3", - USERGROUP4 = "user_group4"; - private static final Map<String, String> staticMapping; - - static { - - ADMIN1 = System.getProperty("sentry.e2etest.admin.user", "admin1"); - ADMINGROUP = System.getProperty("sentry.e2etest.admin.group", "admin"); - staticMapping = new HashMap<String, String>(); - staticMapping.put(ADMIN1, ADMINGROUP); - staticMapping.put(HIVE, HIVE); - staticMapping.put(USER1_1, USERGROUP1); - staticMapping.put(USER1_2, USERGROUP1); - staticMapping.put(USER2_1, USERGROUP2); - staticMapping.put(USER3_1, USERGROUP3); - staticMapping.put(USER4_1, USERGROUP4); - } - - public static Map<String, String> getStaticMapping(){ - return staticMapping; - } - -} http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/TestConfigTool.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/TestConfigTool.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/TestConfigTool.java deleted file mode 100644 index 9423c23..0000000 --- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hive/TestConfigTool.java +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.tests.e2e.hive; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.PrintStream; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Set; - -import org.apache.sentry.binding.hive.authz.SentryConfigTool; -import org.apache.sentry.binding.hive.conf.HiveAuthzConf; -import org.apache.sentry.core.common.Subject; -import org.apache.sentry.core.common.exception.SentryConfigurationException; -import org.apache.sentry.provider.file.PolicyFile; -import org.junit.Before; -import org.junit.Test; - -public class TestConfigTool extends AbstractTestWithStaticConfiguration { - private static final String DB2_POLICY_FILE = "db2-policy-file.ini"; - private static String prefix; - - private PolicyFile policyFile; - private SentryConfigTool configTool; - - @Before - public void setup() throws Exception { - policyFile = PolicyFile.setAdminOnServer1(ADMINGROUP); - configTool = new SentryConfigTool(); - String hiveServer2 = System.getProperty("sentry.e2etest.hiveServer2Type", - "InternalHiveServer2"); - String policyOnHDFS = System.getProperty( - "sentry.e2etest.hive.policyOnHDFS", "true"); - if (policyOnHDFS.trim().equalsIgnoreCase("true") - && (hiveServer2.equals("UnmanagedHiveServer2"))) { - String policyLocation = System.getProperty( - "sentry.e2etest.hive.policy.location", "/user/hive/sentry"); - prefix = "hdfs://" + policyLocation + "/"; - } else { - prefix = "file://" + context.getPolicyFile().getParent() + "/"; - } - - } - - /** - * Verify errors/warnings from malformed policy file - * @throws Exception - */ - @Test - public void testInvalidPolicy() throws Exception { - // policy file, missing insert_tab2 and select_tab3 role definition - policyFile - .addRolesToGroup(USERGROUP1, "select_tab1", "insert_tab2") - .addRolesToGroup(USERGROUP2, "select_tab3") - .addPermissionsToRole("select_tab1", - "server=server1->db=db1->table=tab1->action=select") - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); - policyFile.write(context.getPolicyFile()); - - configTool.setPolicyFile(context.getPolicyFile().getPath()); - configTool.setupConfig(); - try { - configTool.getSentryProvider().validateResource(true); - fail("Policy validation should fail for malformed policy"); - } catch (SentryConfigurationException e) { - assertTrue(e - .getConfigWarnings() - .get(0) - .contains( - "Role select_tab3 for group " + USERGROUP2 + " does not exist")); - assertTrue(e - .getConfigWarnings() - .get(1) - .contains( - "Role insert_tab2 for group " + USERGROUP1 + " does not exist")); - } - } - - /** - * Verify errors/warnings from malformed policy file with per-DB policy - * @throws Exception - */ - @Test - public void testInvalidPerDbPolicy() throws Exception { - PolicyFile db2PolicyFile = new PolicyFile(); - File db2PolicyFileHandle = new File(context.getPolicyFile().getParent(), - DB2_POLICY_FILE); - // invalid db2 policy file with missing roles - db2PolicyFile - .addRolesToGroup(USERGROUP2, "select_tbl2", "insert_db2_tab2") - .addPermissionsToRole("select_tbl2", - "server=server1->db=db2->table=tbl2->action=select") - .write(db2PolicyFileHandle); - - policyFile - .addRolesToGroup(USERGROUP1, "select_tbl1") - .addRolesToGroup(USERGROUP2, "select_tbl3") - .addPermissionsToRole("select_tbl1", - "server=server1->db=db1->table=tbl1->action=select") - .addDatabase("db2", prefix + db2PolicyFileHandle.getName()) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()) - .write(context.getPolicyFile()); - - configTool.setPolicyFile(context.getPolicyFile().getPath()); - configTool.setupConfig(); - try { - configTool.getSentryProvider().validateResource(true); - fail("Policy validation should fail for malformed policy"); - } catch (SentryConfigurationException e) { - assertTrue(e - .getConfigWarnings() - .get(0) - .contains( - "Role select_tbl3 for group " + USERGROUP2 + " does not exist")); - assertTrue(e.getConfigWarnings().get(0) - .contains(context.getPolicyFile().getName())); - assertTrue(e - .getConfigWarnings() - .get(1) - .contains( - "Role insert_db2_tab2 for group " + USERGROUP2 - + " does not exist")); - assertTrue(e.getConfigWarnings().get(1) - .contains(db2PolicyFileHandle.getName())); - } - } - - /** - * Validate user permissions listing - * @throws Exception - */ - @Test - public void testUserPermissions() throws Exception { - policyFile - .addRolesToGroup(USERGROUP1, "select_tab1", "insert_tab2") - .addRolesToGroup(USERGROUP2, "select_tab3") - .addRolesToGroup(USERGROUP3, "select_colA") - .addRolesToGroup(USERGROUP4, "select_colAB") - .addPermissionsToRole("select_tab1", - "server=server1->db=db1->table=tab1->action=select") - .addPermissionsToRole("insert_tab2", - "server=server1->db=db1->table=tab2->action=insert") - .addPermissionsToRole("select_tab3", - "server=server1->db=db1->table=tab3->action=select") - .addPermissionsToRole("select_colA", - "server=server1->db=db1->table=tab3->column=a->action=select") - .addPermissionsToRole("select_colAB", - "server=server1->db=db1->table=tab3->column=a->action=select") - .addPermissionsToRole("select_colAB", - "server=server1->db=db1->table=tab3->column=b->action=select") - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); - policyFile.write(context.getPolicyFile()); - - configTool.setPolicyFile(context.getPolicyFile().getPath()); - configTool.setupConfig(); - configTool.validatePolicy(); - - Set<String> permList = configTool.getSentryProvider() - .listPrivilegesForSubject(new Subject(USER1_1)); - assertTrue(permList - .contains("server=server1->db=db1->table=tab1->action=select")); - assertTrue(permList - .contains("server=server1->db=db1->table=tab2->action=insert")); - - permList = configTool.getSentryProvider().listPrivilegesForSubject( - new Subject(USER2_1)); - assertTrue(permList - .contains("server=server1->db=db1->table=tab3->action=select")); - - permList = configTool.getSentryProvider().listPrivilegesForSubject( - new Subject(USER3_1)); - assertTrue(permList - .contains("server=server1->db=db1->table=tab3->column=a->action=select")); - - permList = configTool.getSentryProvider().listPrivilegesForSubject( - new Subject(USER4_1)); - assertTrue(permList - .contains("server=server1->db=db1->table=tab3->column=a->action=select")); - assertTrue(permList - .contains("server=server1->db=db1->table=tab3->column=b->action=select")); - - permList = configTool.getSentryProvider().listPrivilegesForSubject( - new Subject(ADMIN1)); - assertTrue(permList.contains("server=server1")); - } - - /*** - * Verify the mock compilation config setting forces query to abort - * @throws Exception - */ - @Test - public void testMockCompilation() throws Exception { - policyFile.setUserGroupMapping(StaticUserGroup.getStaticMapping()); - policyFile.write(context.getPolicyFile()); - // setup db objects needed by the test - Connection connection = context.createConnection(ADMIN1); - Statement statement = context.createStatement(connection); - - statement.execute("DROP TABLE IF EXISTS tab1"); - statement.execute("CREATE TABLE tab1(B INT, A STRING) " - + " row format delimited fields terminated by '|' stored as textfile"); - statement.execute("SELECT * FROM tab1"); - - statement.execute("SET " + HiveAuthzConf.HIVE_SENTRY_MOCK_COMPILATION - + "=true"); - try { - statement.execute("SELECT * FROM tab1"); - fail("Query should fail with mock error config enabled"); - } catch (SQLException e) { - assertTrue(e.getMessage().contains(HiveAuthzConf.HIVE_SENTRY_MOCK_ERROR)); - } - statement.close(); - - } - - /** - * verify missing permissions for query using remote query validation - * @throws Exception - */ - @Test - public void testQueryPermissions() throws Exception { - policyFile - .addRolesToGroup(USERGROUP1, "select_tab1", "insert_tab2") - .addRolesToGroup(USERGROUP2, "select_tab3") - .addRolesToGroup(USERGROUP3, "select_colAB") - .addRolesToGroup(USERGROUP4, "select_colA") - .addPermissionsToRole("select_tab1", - "server=server1->db=default->table=tab1->action=select") - .addPermissionsToRole("insert_tab2", - "server=server1->db=default->table=tab2->action=insert") - .addPermissionsToRole("select_tab3", - "server=server1->db=default->table=tab3->action=select") - .addPermissionsToRole("select_colA", - "server=server1->db=default->table=tab1->column=a->action=select") - .addPermissionsToRole("select_colAB", - "server=server1->db=default->table=tab1->column=a->action=select") - .addPermissionsToRole("select_colAB", - "server=server1->db=default->table=tab1->column=b->action=select") - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); - policyFile.write(context.getPolicyFile()); - - // setup db objects needed by the test - Connection connection = context.createConnection(ADMIN1); - Statement statement = context.createStatement(connection); - - statement.execute("DROP TABLE IF EXISTS tab1"); - statement.execute("DROP TABLE IF EXISTS tab2"); - statement.execute("DROP TABLE IF EXISTS tab3"); - statement.execute("CREATE TABLE tab1(b INT, a STRING) " - + " row format delimited fields terminated by '|' stored as textfile"); - statement.execute("CREATE TABLE tab2(b INT, a STRING) " - + " row format delimited fields terminated by '|' stored as textfile"); - statement.execute("CREATE TABLE tab3(b INT, a STRING) " - + " row format delimited fields terminated by '|' stored as textfile"); - statement.close(); - connection.close(); - - configTool.setPolicyFile(context.getPolicyFile().getPath()); - configTool.setJdbcURL(context.getConnectionURL()); - configTool.setUser(USER1_1); - configTool.setupConfig(); - ByteArrayOutputStream errBuffer = new ByteArrayOutputStream(); - - // user1_1 can query table1 - configTool.setUser(USER1_1); - configTool.verifyRemoteQuery("SELECT COUNT(*) FROM tab1"); - - // user1_1 can't select from tab3 - try { - System.setOut(new PrintStream(errBuffer)); - configTool.setUser(USER1_1); - configTool.verifyRemoteQuery("SELECT COUNT(*) FROM tab3"); - fail("Query should have failed with insufficient perms"); - } catch (SQLException e) { - assertTrue(errBuffer.toString().contains( - "Server=server1->Db=default->Table=tab3->action=select")); - errBuffer.flush(); - } - - // user2_1 can select from tab3, but can't insert into tab2 - try { - configTool.setUser(USER2_1); - configTool - .verifyRemoteQuery("INSERT OVERWRITE TABLE tab2 SELECT * FROM tab3"); - fail("Query should have failed with insufficient perms"); - } catch (SQLException e) { - assertTrue(errBuffer.toString().contains( - "Server=server1->Db=default->Table=tab2->action=insert")); - errBuffer.flush(); - } - - // user3_1 can select A from tab1, but can't select B from tab1 - configTool.setUser(USER3_1); - configTool.verifyRemoteQuery("SELECT a FROM tab1"); - configTool.verifyRemoteQuery("SELECT COUNT(a) FROM tab1"); - configTool.verifyRemoteQuery("SELECT b FROM tab1"); - configTool.verifyRemoteQuery("SELECT COUNT(b) FROM tab1"); - configTool.verifyRemoteQuery("SELECT a,b FROM tab1"); - - // user4_1 can select A from tab1, but can't select B from tab1 - configTool.setUser(USER4_1); - configTool.verifyRemoteQuery("SELECT a FROM tab1"); - configTool.verifyRemoteQuery("SELECT COUNT(a) FROM tab1"); - - try { - configTool.setUser(USER4_1); - configTool - .verifyRemoteQuery("SELECT b FROM tab1"); - fail("Query should have failed with insufficient perms"); - } catch (SQLException e) { - assertTrue(errBuffer.toString().contains( - "Server=server1->Db=default->Table=tab1->Column=b->action=select")); - errBuffer.flush(); - } - - try { - configTool.setUser(USER4_1); - configTool - .verifyRemoteQuery("SELECT COUNT(b) FROM tab1"); - fail("Query should have failed with insufficient perms"); - } catch (SQLException e) { - assertTrue(errBuffer.toString().contains( - "Server=server1->Db=default->Table=tab1->Column=b->action=select")); - } - } -}
