This is an automated email from the ASF dual-hosted git repository.
veghlaci05 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 34b8acb19ea HIVE-27643: Exclude compaction queries from ranger
policies (Laszlo Vegh, reviewed by Denys Kuzmenko, Krisztian Kasa, Simhadri
Govindappa, Laszlo Bodor)
34b8acb19ea is described below
commit 34b8acb19ea4b59280e32fae0fe3c90d104d27a0
Author: veghlaci05 <veghlac...@gmail.com>
AuthorDate: Thu Aug 31 14:41:15 2023 +0200
HIVE-27643: Exclude compaction queries from ranger policies (Laszlo Vegh,
reviewed by Denys Kuzmenko, Krisztian Kasa, Simhadri Govindappa, Laszlo Bodor)
---
.../test/resources/testconfiguration.properties | 1 +
.../apache/hadoop/hive/cli/control/CliConfigs.java | 1 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 5 +-
.../hadoop/hive/ql/session/SessionState.java | 14 +++
.../hive/ql/txn/compactor/QueryCompactor.java | 4 +
.../compaction_query_based_masking.q | 29 +++++
.../llap/compaction_query_based_masking.q.out | 134 +++++++++++++++++++++
7 files changed, 186 insertions(+), 2 deletions(-)
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index f08b2c00e2a..46d76e8b40d 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -430,6 +430,7 @@ compaction.query.files=\
compaction_query_based_insert_only_partitioned_clustered.q,\
compaction_query_based_insert_only_partitioned_clustered_minor.q,\
compaction_query_based_insert_only_partitioned_minor.q,\
+ compaction_query_based_masking.q,\
compaction_query_based_minor.q,\
compaction_query_based_partitioned.q,\
compaction_query_based_partitioned_minor.q
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 4026f3a980c..7288eaeb969 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -269,6 +269,7 @@ public class CliConfigs {
customConfigValueMap.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY,
"true");
customConfigValueMap.put(HiveConf.ConfVars.HIVE_TXN_MANAGER,
"org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
customConfigValueMap.put(HiveConf.ConfVars.HIVE_COMPACTOR_GATHER_STATS,
"false");
+ customConfigValueMap.put(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest");
return customConfigValueMap;
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index a331f893d0d..76fb4b3ec21 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -12488,8 +12488,9 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
if (needRewritePrivObjs != null && !needRewritePrivObjs.isEmpty()) {
for (HivePrivilegeObject privObj : needRewritePrivObjs) {
MaskAndFilterInfo info = basicInfos.get(privObj);
- // First we check whether entity actually needs masking or filtering
- if (tableMask.needsMaskingOrFiltering(privObj)) {
+ // First we check whether entity actually needs masking or filtering.
Query based Compaction related queries are
+ // excluded from all masking and filtering.
+ if (tableMask.needsMaskingOrFiltering(privObj) &&
!SessionState.get().isCompaction()) {
if (info == null) {
// This is a table used by a materialized view
// Currently we do not support querying directly a materialized
view
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index c721068d9a7..9614d95e395 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -335,6 +335,12 @@ public class SessionState implements ISessionAuthState{
private Hive hiveDb;
private final Map<String, QueryState> queryStateMap = new HashMap<>();
+ /**
+ * Marker flag to indicate that the current SessionState (and Driver)
instance is used for executing compaction queries only.
+ * It is required to exclude compaction related queries from all Ranger
policies that would otherwise apply.
+ */
+ private boolean compaction = false;
+
public QueryState getQueryState(String queryId) {
return queryStateMap.get(queryId);
}
@@ -434,6 +440,14 @@ public class SessionState implements ISessionAuthState{
this.isHiveServerQuery = isHiveServerQuery;
}
+ public boolean isCompaction() {
+ return compaction;
+ }
+
+ public void setCompaction(boolean compaction) {
+ this.compaction = compaction;
+ }
+
public SessionState(HiveConf conf) {
this(conf, null);
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
index 6f0a0726360..01b24404beb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
@@ -97,6 +97,7 @@ abstract class QueryCompactor implements Compactor {
Util.overrideConfProps(conf, compactionInfo, tblProperties);
String user = compactionInfo.runAs;
SessionState sessionState = DriverUtils.setUpSessionState(conf, user,
true);
+ sessionState.setCompaction(true);
long compactorTxnId = Compactor.getCompactorTxnId(conf);
try {
for (String query : createQueries) {
@@ -144,6 +145,9 @@ abstract class QueryCompactor implements Compactor {
LOG.error("Unable to drop temp table {} which was created for running
{} compaction", tmpTableName,
compactionInfo.type);
LOG.error(ExceptionUtils.getStackTrace(e));
+ } finally {
+ //restore sessionState
+ sessionState.setCompaction(false);
}
}
}
diff --git
a/ql/src/test/queries/clientpositive/compaction_query_based_masking.q
b/ql/src/test/queries/clientpositive/compaction_query_based_masking.q
new file mode 100644
index 00000000000..a3df804888f
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/compaction_query_based_masking.q
@@ -0,0 +1,29 @@
+set hive.mapred.mode=nonstrict;
+set hive.security.authorization.enabled=true;
+set
hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+
+drop table masking_test_n_compact;
+drop table check_real_data;
+
+create table masking_test_n_compact (key int, value string) stored as orc
TBLPROPERTIES('transactional'='true');
+
+insert into masking_test_n_compact values('1', 'text1');
+insert into masking_test_n_compact values('2', 'text2');
+insert into masking_test_n_compact values('3', 'text3');
+
+select * from masking_test_n_compact;
+
+-- the rules are applied based on the table name
+alter table masking_test_n_compact rename to check_real_data;
+
+select * from check_real_data;
+
+alter table check_real_data rename to masking_test_n_compact;
+
+alter table masking_test_n_compact compact 'MAJOR' and wait;
+
+select * from masking_test_n_compact;
+
+alter table masking_test_n_compact rename to check_real_data;
+
+select * from check_real_data;
diff --git
a/ql/src/test/results/clientpositive/llap/compaction_query_based_masking.q.out
b/ql/src/test/results/clientpositive/llap/compaction_query_based_masking.q.out
new file mode 100644
index 00000000000..7f6d64b59ac
--- /dev/null
+++
b/ql/src/test/results/clientpositive/llap/compaction_query_based_masking.q.out
@@ -0,0 +1,134 @@
+PREHOOK: query: drop table masking_test_n_compact
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: drop table masking_test_n_compact
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: drop table check_real_data
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: drop table check_real_data
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: create table masking_test_n_compact (key int, value string)
stored as orc TBLPROPERTIES('transactional'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test_n_compact
+POSTHOOK: query: create table masking_test_n_compact (key int, value string)
stored as orc TBLPROPERTIES('transactional'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test_n_compact
+PREHOOK: query: insert into masking_test_n_compact values('1', 'text1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@masking_test_n_compact
+POSTHOOK: query: insert into masking_test_n_compact values('1', 'text1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@masking_test_n_compact
+POSTHOOK: Lineage: masking_test_n_compact.key SCRIPT []
+POSTHOOK: Lineage: masking_test_n_compact.value SCRIPT []
+PREHOOK: query: insert into masking_test_n_compact values('2', 'text2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@masking_test_n_compact
+POSTHOOK: query: insert into masking_test_n_compact values('2', 'text2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@masking_test_n_compact
+POSTHOOK: Lineage: masking_test_n_compact.key SCRIPT []
+POSTHOOK: Lineage: masking_test_n_compact.value SCRIPT []
+PREHOOK: query: insert into masking_test_n_compact values('3', 'text3')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@masking_test_n_compact
+POSTHOOK: query: insert into masking_test_n_compact values('3', 'text3')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@masking_test_n_compact
+POSTHOOK: Lineage: masking_test_n_compact.key SCRIPT []
+POSTHOOK: Lineage: masking_test_n_compact.value SCRIPT []
+PREHOOK: query: select * from masking_test_n_compact
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test_n_compact
+#### A masked pattern was here ####
+POSTHOOK: query: select * from masking_test_n_compact
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test_n_compact
+#### A masked pattern was here ####
+2 2txet
+PREHOOK: query: alter table masking_test_n_compact rename to check_real_data
+PREHOOK: type: ALTERTABLE_RENAME
+PREHOOK: Input: default@masking_test_n_compact
+PREHOOK: Output: database:default
+PREHOOK: Output: default@check_real_data
+PREHOOK: Output: default@masking_test_n_compact
+POSTHOOK: query: alter table masking_test_n_compact rename to check_real_data
+POSTHOOK: type: ALTERTABLE_RENAME
+POSTHOOK: Input: default@masking_test_n_compact
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@check_real_data
+POSTHOOK: Output: default@masking_test_n_compact
+PREHOOK: query: select * from check_real_data
+PREHOOK: type: QUERY
+PREHOOK: Input: default@check_real_data
+#### A masked pattern was here ####
+POSTHOOK: query: select * from check_real_data
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@check_real_data
+#### A masked pattern was here ####
+1 text1
+2 text2
+3 text3
+PREHOOK: query: alter table check_real_data rename to masking_test_n_compact
+PREHOOK: type: ALTERTABLE_RENAME
+PREHOOK: Input: default@check_real_data
+PREHOOK: Output: database:default
+PREHOOK: Output: default@check_real_data
+PREHOOK: Output: default@masking_test_n_compact
+POSTHOOK: query: alter table check_real_data rename to masking_test_n_compact
+POSTHOOK: type: ALTERTABLE_RENAME
+POSTHOOK: Input: default@check_real_data
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@check_real_data
+POSTHOOK: Output: default@masking_test_n_compact
+PREHOOK: query: alter table masking_test_n_compact compact 'MAJOR' and wait
+PREHOOK: type: ALTERTABLE_COMPACT
+PREHOOK: Input: default@masking_test_n_compact
+PREHOOK: Output: default@masking_test_n_compact
+POSTHOOK: query: alter table masking_test_n_compact compact 'MAJOR' and wait
+POSTHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: Input: default@masking_test_n_compact
+POSTHOOK: Output: default@masking_test_n_compact
+PREHOOK: query: select * from masking_test_n_compact
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test_n_compact
+#### A masked pattern was here ####
+POSTHOOK: query: select * from masking_test_n_compact
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test_n_compact
+#### A masked pattern was here ####
+2 2txet
+PREHOOK: query: alter table masking_test_n_compact rename to check_real_data
+PREHOOK: type: ALTERTABLE_RENAME
+PREHOOK: Input: default@masking_test_n_compact
+PREHOOK: Output: database:default
+PREHOOK: Output: default@check_real_data
+PREHOOK: Output: default@masking_test_n_compact
+POSTHOOK: query: alter table masking_test_n_compact rename to check_real_data
+POSTHOOK: type: ALTERTABLE_RENAME
+POSTHOOK: Input: default@masking_test_n_compact
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@check_real_data
+POSTHOOK: Output: default@masking_test_n_compact
+PREHOOK: query: select * from check_real_data
+PREHOOK: type: QUERY
+PREHOOK: Input: default@check_real_data
+#### A masked pattern was here ####
+POSTHOOK: query: select * from check_real_data
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@check_real_data
+#### A masked pattern was here ####
+1 text1
+2 text2
+3 text3
