SENTRY-755: HDFS access of data files should be disabled for user with privileges only on some columns (Sravya Tirukkovalur, Reviewed by: Lenni Kuff)
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/a5b37c7e Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/a5b37c7e Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/a5b37c7e Branch: refs/heads/hive_plugin_v2 Commit: a5b37c7e122d0126a4d2a4f57ecf0359feadf0d5 Parents: 100e239 Author: Sravya Tirukkovalur <[email protected]> Authored: Fri Jul 24 13:35:23 2015 -0700 Committer: Sravya Tirukkovalur <[email protected]> Committed: Fri Jul 24 15:44:53 2015 -0700 ---------------------------------------------------------------------- .../hdfs/SentryAuthorizationProvider.java | 10 +- .../org/apache/sentry/hdfs/SentryPlugin.java | 8 +- .../SentryPolicyServiceClientDefaultImpl.java | 8 +- .../tests/e2e/hdfs/TestHDFSIntegration.java | 138 ++++++++++++++++++- 4 files changed, 153 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/a5b37c7e/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java index f3d8aac..d167183 100644 --- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java +++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java @@ -300,7 +300,15 @@ public class SentryAuthorizationProvider builder.setName(null); return list; } - + /* + Returns hadoop acls if + - Not managed + - Not stale and not an auth obj + Returns hive:hive + - If stale + Returns sentry acls + - Otherwise, if not stale and auth obj + */ @Override public AclFeature getAclFeature(INodeAuthorizationInfo node, int snapshotId) { AclFeature f = null; http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/a5b37c7e/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java index 221c397..7587a1d 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java @@ -165,7 +165,9 @@ public class SentryPlugin implements SentryPolicyStorePlugin { if (request.isSetPrivileges()) { String roleName = request.getRoleName(); for (TSentryPrivilege privilege : request.getPrivileges()) { - onAlterSentryRoleGrantPrivilegeCore(roleName, privilege); + if(!("COLUMN".equalsIgnoreCase(privilege.getPrivilegeScope()))) { + onAlterSentryRoleGrantPrivilegeCore(roleName, privilege); + } } } } @@ -202,7 +204,9 @@ public class SentryPlugin implements SentryPolicyStorePlugin { if (request.isSetPrivileges()) { String roleName = request.getRoleName(); for (TSentryPrivilege privilege : request.getPrivileges()) { - onAlterSentryRoleRevokePrivilegeCore(roleName, privilege); + if(!("COLUMN".equalsIgnoreCase(privilege.getPrivilegeScope()))) { + onAlterSentryRoleRevokePrivilegeCore(roleName, privilege); + } } } } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/a5b37c7e/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java index c3c1907..533a28c 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java @@ -529,7 +529,7 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService ImmutableList.Builder<String> listBuilder = ImmutableList.builder(); listBuilder.add(columnName); revokePrivilege(requestorUserName, roleName, - PrivilegeScope.TABLE, server, null, + PrivilegeScope.COLUMN, server, null, db, table, listBuilder.build(), action); } @@ -539,7 +539,7 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService ImmutableList.Builder<String> listBuilder = ImmutableList.builder(); listBuilder.add(columnName); revokePrivilege(requestorUserName, roleName, - PrivilegeScope.TABLE, server, null, + PrivilegeScope.COLUMN, server, null, db, table, listBuilder.build(), action, grantOption); } @@ -547,7 +547,7 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService String server, String db, String table, List<String> columns, String action) throws SentryUserException { revokePrivilege(requestorUserName, roleName, - PrivilegeScope.TABLE, server, null, + PrivilegeScope.COLUMN, server, null, db, table, columns, action); } @@ -555,7 +555,7 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService String server, String db, String table, List<String> columns, String action, Boolean grantOption) throws SentryUserException { revokePrivilege(requestorUserName, roleName, - PrivilegeScope.TABLE, server, null, + PrivilegeScope.COLUMN, server, null, db, table, columns, action, grantOption); } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/a5b37c7e/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java index 35a9213..786150b 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java @@ -105,6 +105,7 @@ public class TestHDFSIntegration { private static final Logger LOGGER = LoggerFactory .getLogger(TestHDFSIntegration.class); + public static class WordCountMapper extends MapReduceBase implements Mapper<LongWritable, Text, String, Long> { @@ -149,6 +150,8 @@ public class TestHDFSIntegration { protected static SentrySrv sentryServer; protected static boolean testSentryHA = false; private static final long STALE_THRESHOLD = 5000; + private static final long CACHE_REFRESH = 100; //Default is 500, but we want it to be low + // in our tests so that changes reflect soon private static String fsURI; private static int hmsPort; @@ -273,9 +276,9 @@ public class TestHDFSIntegration { out.close(); Reflection.staticField("hiveSiteURL") - .ofType(URL.class) - .in(HiveConf.class) - .set(hiveSite.toURI().toURL()); + .ofType(URL.class) + .in(HiveConf.class) + .set(hiveSite.toURI().toURL()); metastore = new InternalMetastoreServer(hiveConf); new Thread() { @@ -361,6 +364,8 @@ public class TestHDFSIntegration { conf.set("sentry.authorization-provider.hdfs-path-prefixes", "/user/hive/warehouse,/tmp/external"); conf.set("sentry.authorization-provider.cache-refresh-retry-wait.ms", "5000"); + conf.set("sentry.authorization-provider.cache-refresh-interval.ms", String.valueOf(CACHE_REFRESH)); + conf.set("sentry.authorization-provider.cache-stale-threshold.ms", String.valueOf(STALE_THRESHOLD)); conf.set("sentry.hdfs.service.security.mode", "none"); @@ -486,7 +491,7 @@ public class TestHDFSIntegration { conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); for( String role:roles) { - stmt.execute("drop role " + role); + stmt.execute("drop role " + role); } stmt.close(); conn.close(); @@ -911,6 +916,114 @@ public class TestHDFSIntegration { } + @Test + public void testColumnPrivileges() throws Throwable { + String dbName = "db2"; + + tmpHDFSDir = new Path("/tmp/external"); + dbNames = new String[]{dbName}; + roles = new String[]{"admin_role", "tab_role", "db_role", "col_role"}; + admin = StaticUserGroup.ADMIN1; + + Connection conn; + Statement stmt; + + conn = hiveServer2.createConnection("hive", "hive"); + stmt = conn.createStatement(); + stmt.execute("create role admin_role"); + stmt.execute("grant all on server server1 to role admin_role with grant option"); + stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); + + conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); + stmt = conn.createStatement(); + stmt.execute("create database " + dbName); + stmt.execute("use "+ dbName); + stmt.execute("create table p1 (s string) partitioned by (month int, day int)"); + stmt.execute("alter table p1 add partition (month=1, day=1)"); + stmt.execute("alter table p1 add partition (month=1, day=2)"); + stmt.execute("alter table p1 add partition (month=2, day=1)"); + stmt.execute("alter table p1 add partition (month=2, day=2)"); + loadData(stmt); + + stmt.execute("create role db_role"); + stmt.execute("grant select on database " + dbName + " to role db_role"); + stmt.execute("create role tab_role"); + stmt.execute("grant select on p1 to role tab_role"); + stmt.execute("create role col_role"); + stmt.execute("grant select(s) on p1 to role col_role"); + + stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP1); + + stmt.execute("grant role tab_role to group "+ StaticUserGroup.USERGROUP2); + stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP2); + + stmt.execute("grant role db_role to group "+ StaticUserGroup.USERGROUP3); + stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP3); + + stmt.execute("grant role col_role to group " + StaticUserGroup.ADMINGROUP); + + Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode + + //User with just column level privileges cannot read HDFS + verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.USERGROUP1, false); + + //User with permissions on table and column can read HDFS file + verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true); + + //User with permissions on db and column can read HDFS file + verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP3, true); + + //User with permissions on server and column cannot read HDFS file + //TODO:SENTRY-751 + verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.ADMINGROUP, false); + + stmt.close(); + conn.close(); + + } + + /* + TODO:SENTRY-819 + */ + @Test + public void testAllColumn() throws Throwable { + String dbName = "db2"; + + tmpHDFSDir = new Path("/tmp/external"); + dbNames = new String[]{dbName}; + roles = new String[]{"admin_role", "col_role"}; + admin = StaticUserGroup.ADMIN1; + + Connection conn; + Statement stmt; + + conn = hiveServer2.createConnection("hive", "hive"); + stmt = conn.createStatement(); + stmt.execute("create role admin_role"); + stmt.execute("grant all on server server1 to role admin_role with grant option"); + stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); + + conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); + stmt = conn.createStatement(); + stmt.execute("create database " + dbName); + stmt.execute("use "+ dbName); + stmt.execute("create table p1 (c1 string, c2 string) partitioned by (month int, day int)"); + stmt.execute("alter table p1 add partition (month=1, day=1)"); + loadDataTwoCols(stmt); + + stmt.execute("create role col_role"); + stmt.execute("grant select(c1,c2) on p1 to role col_role"); + stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP1); + Thread.sleep(100); + + //User with privileges on all columns of the data cannot still read the HDFS files + verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.USERGROUP1, false); + + stmt.close(); + conn.close(); + + } + private void verifyQuery(Statement stmt, String table, int n) throws Throwable { verifyQuery(stmt, table, n, NUM_RETRIES); } @@ -956,6 +1069,23 @@ public class TestHDFSIntegration { rs.close(); } + private void loadDataTwoCols(Statement stmt) throws IOException, SQLException { + FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path("/tmp/f2.txt")); + f1.writeChars("m1d1_t1, m1d1_t2\n"); + f1.writeChars("m1d1_t2, m1d1_t2\n"); + f1.writeChars("m1d1_t3, m1d1_t2\n"); + f1.flush(); + f1.close(); + stmt.execute("load data inpath \'/tmp/f2.txt\' overwrite into table p1 partition (month=1, day=1)"); + ResultSet rs = stmt.executeQuery("select * from p1"); + List<String> vals = new ArrayList<String>(); + while (rs.next()) { + vals.add(rs.getString(1)); + } + Assert.assertEquals(3, vals.size()); + rs.close(); + } + private void writeToPath(String path, int numRows, String user, String group) throws IOException { Path p = new Path(path); miniDFS.getFileSystem().mkdirs(p);
