Repository: hive Updated Branches: refs/heads/master fc05556ce -> 8d3108c41
HIVE-10629 Dropping table in an encrypted zone does not drop warehouse directory(Eugene Koifman, reviewed by Sergio Pena) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8d3108c4 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8d3108c4 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8d3108c4 Branch: refs/heads/master Commit: 8d3108c41802d7d1766c0d85bc81e895ad348dfb Parents: fc05556 Author: Eugene Koifman <ekoif...@hortonworks.com> Authored: Wed May 20 10:05:31 2015 -0700 Committer: Eugene Koifman <ekoif...@hortonworks.com> Committed: Wed May 20 10:05:31 2015 -0700 ---------------------------------------------------------------------- data/scripts/q_test_cleanup_for_encryption.sql | 2 +- data/scripts/q_test_init_for_encryption.sql | 2 +- .../test/resources/testconfiguration.properties | 3 +- .../org/apache/hadoop/hive/ql/QTestUtil.java | 3 +- .../hadoop/hive/metastore/HiveMetaStore.java | 36 +++++++++---- .../clientpositive/encryption_drop_table.q | 18 +++++++ .../encryption_insert_partition_dynamic.q | 8 +-- .../encryption_insert_partition_static.q | 8 +-- .../encryption_join_unencrypted_tbl.q | 4 +- ...yption_join_with_different_encryption_keys.q | 8 +-- .../encryption_load_data_to_encrypted_tables.q | 4 +- .../clientpositive/encryption_move_tbl.q | 4 +- .../encryption_select_read_only_encrypted_tbl.q | 4 +- .../encrypted/encryption_drop_table.q.out | 56 ++++++++++++++++++++ .../encryption_insert_partition_dynamic.q.out | 16 +++--- .../encryption_insert_partition_static.q.out | 30 +++++------ .../encryption_join_unencrypted_tbl.q.out | 8 +-- ...on_join_with_different_encryption_keys.q.out | 16 +++--- ...cryption_load_data_to_encrypted_tables.q.out | 8 +-- .../encrypted/encryption_move_tbl.q.out | 12 ++++- ...ryption_select_read_only_encrypted_tbl.q.out | 8 +-- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 3 ++ 22 files changed, 184 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/data/scripts/q_test_cleanup_for_encryption.sql ---------------------------------------------------------------------- diff --git a/data/scripts/q_test_cleanup_for_encryption.sql b/data/scripts/q_test_cleanup_for_encryption.sql index 08264ee..070cdbe 100644 --- a/data/scripts/q_test_cleanup_for_encryption.sql +++ b/data/scripts/q_test_cleanup_for_encryption.sql @@ -1 +1 @@ -DROP TABLE IF EXISTS src; +DROP TABLE IF EXISTS src PURGE; http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/data/scripts/q_test_init_for_encryption.sql ---------------------------------------------------------------------- diff --git a/data/scripts/q_test_init_for_encryption.sql b/data/scripts/q_test_init_for_encryption.sql index 1822ebb..56b44e0 100644 --- a/data/scripts/q_test_init_for_encryption.sql +++ b/data/scripts/q_test_init_for_encryption.sql @@ -1,4 +1,4 @@ -DROP TABLE IF EXISTS src; +DROP TABLE IF EXISTS src PURGE; CREATE TABLE src(key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index a485408..b9d85f6 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -349,7 +349,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\ encryption_select_read_only_unencrypted_tbl.q,\ encryption_load_data_to_encrypted_tables.q, \ encryption_unencrypted_nonhdfs_external_tables.q \ - encryption_move_tbl.q + encryption_move_tbl.q \ + encryption_drop_table.q beeline.positive.exclude=add_part_exist.q,\ alter1.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 2b4bd85..39d5d9e 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -372,6 +372,7 @@ public class QTestUtil { // Set the security key provider so that the MiniDFS cluster is initialized // with encryption conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI()); + conf.setInt("fs.trash.interval", 50); dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); fs = dfs.getFileSystem(); @@ -681,7 +682,7 @@ public class QTestUtil { if(tblObj.isIndexTable()) { continue; } - db.dropTable(dbName, tblName); + db.dropTable(dbName, tblName, true, true, clusterType == MiniClusterType.encrypted); } else { // this table is defined in srcTables, drop all indexes on it List<Index> indexes = db.getIndexes(dbName, tblName, (short)-1); http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index e796048..5d8718c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -55,6 +55,7 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.LogUtils; @@ -198,6 +199,7 @@ import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.txn.TxnHandler; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge; @@ -1478,6 +1480,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { Path tblPath = null; List<Path> partPaths = null; Table tbl = null; + boolean ifPurge = false; try { ms.openTransaction(); // drop any partitions @@ -1489,6 +1492,17 @@ public class HiveMetaStore extends ThriftHiveMetastore { throw new MetaException("Table metadata is corrupted"); } + /** + * Trash may be skipped iff: + * 1. deleteData == true, obviously. + * 2. tbl is external. + * 3. Either + * 3.1. User has specified PURGE from the commandline, and if not, + * 3.2. User has set the table to auto-purge. + */ + ifPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge"))) + || (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge"))); + firePreEvent(new PreDropTableEvent(tbl, deleteData, this)); boolean isIndexTable = isIndexTable(tbl); @@ -1521,6 +1535,19 @@ public class HiveMetaStore extends ThriftHiveMetastore { } } + if(!ifPurge) { + String trashInterval = hiveConf.get("fs.trash.interval"); + boolean trashEnabled = trashInterval != null && trashInterval.length() > 0 + && Float.parseFloat(trashInterval) > 0; + if (trashEnabled) { + HadoopShims.HdfsEncryptionShim shim = + ShimLoader.getHadoopShims().createHdfsEncryptionShim(FileSystem.get(hiveConf), hiveConf); + if (shim.isPathEncrypted(tblPath)) { + throw new MetaException("Unable to drop table because it is in an encryption zone" + + " and trash is enabled. Use PURGE option to skip trash."); + } + } + } // Drop the partitions and get a list of locations which need to be deleted partPaths = dropPartitionsAndGetLocations(ms, dbname, name, tblPath, tbl.getPartitionKeys(), deleteData && !isExternal); @@ -1536,15 +1563,6 @@ public class HiveMetaStore extends ThriftHiveMetastore { ms.rollbackTransaction(); } else if (deleteData && !isExternal) { // Data needs deletion. Check if trash may be skipped. - // Trash may be skipped iff: - // 1. deleteData == true, obviously. - // 2. tbl is external. - // 3. Either - // 3.1. User has specified PURGE from the commandline, and if not, - // 3.2. User has set the table to auto-purge. - boolean ifPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge"))) - || - (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge"))); // Delete the data in the partitions which have other locations deletePartitionData(partPaths, ifPurge); // Delete the data in the table http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_drop_table.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/encryption_drop_table.q b/ql/src/test/queries/clientpositive/encryption_drop_table.q new file mode 100644 index 0000000..193beea --- /dev/null +++ b/ql/src/test/queries/clientpositive/encryption_drop_table.q @@ -0,0 +1,18 @@ +-- SORT_QUERY_RESULTS; + +-- we're setting this so that TestNegaiveCliDriver.vm doesn't stop processing after DROP TABLE fails; + +set hive.cli.errors.ignore=true; + +DROP TABLE IF EXISTS encrypted_table PURGE; +CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; +CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; +CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; + +INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src; +SHOW TABLES; +DROP TABLE default.encrypted_table; +SHOW TABLES; +DROP TABLE default.encrypted_table PURGE; +SHOW TABLES; +CRYPTO DELETE_KEY --keyName key_128; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q b/ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q index 8674972..9556eed 100644 --- a/ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q +++ b/ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q @@ -6,8 +6,8 @@ set hive.enforce.bucketing=true; -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable; -drop table IF EXISTS unencryptedTable; +drop table IF EXISTS encryptedTable PURGE; +drop table IF EXISTS unencryptedTable PURGE; create table encryptedTable(value string) partitioned by (key string) clustered by (value) into 2 buckets stored as orc @@ -52,6 +52,6 @@ insert into table unencryptedTable partition (key) select * from unencryptedTable order by key; -- clean up -drop table encryptedTable; +drop table encryptedTable PURGE; CRYPTO DELETE_KEY --keyName key_1; -drop table unencryptedTable; \ No newline at end of file +drop table unencryptedTable PURGE; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q b/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q index 10fb965..9bff0bd 100644 --- a/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q +++ b/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q @@ -5,8 +5,8 @@ set hive.enforce.bucketing=true; -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable; -drop table IF EXISTS unencryptedTable; +drop table IF EXISTS encryptedTable PURGE; +drop table IF EXISTS unencryptedTable PURGE; create table encryptedTable(key string, value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc @@ -57,6 +57,6 @@ insert into table unencryptedTable partition select * from unencryptedTable order by key; -- clean up -drop table encryptedTable; +drop table encryptedTable PURGE; CRYPTO DELETE_KEY --keyName key_1; -drop table unencryptedTable; \ No newline at end of file +drop table unencryptedTable PURGE; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q b/ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q index adf82de..cf4d040 100644 --- a/ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q +++ b/ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q @@ -1,6 +1,6 @@ --SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table; +DROP TABLE IF EXISTS encrypted_table PURGE; CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; @@ -11,5 +11,5 @@ SELECT * FROM encrypted_table; EXPLAIN EXTENDED SELECT * FROM src t1 JOIN encrypted_table t2 WHERE t1.key = t2.key; -drop table encrypted_table; +drop table encrypted_table PURGE; CRYPTO DELETE_KEY --keyName key_128; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q b/ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q index c830247..dfd3406 100644 --- a/ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q +++ b/ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q @@ -1,12 +1,12 @@ --SORT_QUERY_RESULTS -- Java JCE must be installed in order to hava a key length of 256 bits -DROP TABLE IF EXISTS table_key_1; +DROP TABLE IF EXISTS table_key_1 PURGE; CREATE TABLE table_key_1 (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/table_key_1'; CRYPTO CREATE_KEY --keyName key_1 --bitLength 128; CRYPTO CREATE_ZONE --keyName key_1 --path ${hiveconf:hive.metastore.warehouse.dir}/table_key_1; -DROP TABLE IF EXISTS table_key_2; +DROP TABLE IF EXISTS table_key_2 PURGE; CREATE TABLE table_key_2 (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/table_key_2'; CRYPTO CREATE_KEY --keyName key_2 --bitLength 256; CRYPTO CREATE_ZONE --keyName key_2 --path ${hiveconf:hive.metastore.warehouse.dir}/table_key_2; @@ -17,8 +17,8 @@ INSERT OVERWRITE TABLE table_key_2 SELECT * FROM src; EXPLAIN EXTENDED SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key); SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key); -DROP TABLE table_key_1; -DROP TABLE table_key_2; +DROP TABLE table_key_1 PURGE; +DROP TABLE table_key_2 PURGE; CRYPTO DELETE_KEY --keyName key_1; CRYPTO DELETE_KEY --keyName key_2; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q b/ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q index e4d3a4c..a96807e 100644 --- a/ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q +++ b/ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q @@ -1,4 +1,4 @@ -DROP TABLE IF EXISTS encrypted_table; +DROP TABLE IF EXISTS encrypted_table PURGE; CREATE TABLE encrypted_table (key STRING, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/encrypted_table'; @@ -15,6 +15,6 @@ dfs -copyFromLocal ../../data/files/kv1.txt hdfs:///tmp/kv1.txt; LOAD DATA INPATH '/tmp/kv1.txt' OVERWRITE INTO TABLE encrypted_table; SELECT * FROM encrypted_table; -DROP TABLE encrypted_table; +DROP TABLE encrypted_table PURGE; crypto delete_key --keyName key1; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_move_tbl.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/encryption_move_tbl.q b/ql/src/test/queries/clientpositive/encryption_move_tbl.q index 1168c6c..5a8c036 100644 --- a/ql/src/test/queries/clientpositive/encryption_move_tbl.q +++ b/ql/src/test/queries/clientpositive/encryption_move_tbl.q @@ -4,7 +4,7 @@ set hive.cli.errors.ignore=true; -DROP TABLE IF EXISTS encrypted_table; +DROP TABLE IF EXISTS encrypted_table PURGE; CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; @@ -14,5 +14,7 @@ SHOW TABLES; ALTER TABLE default.encrypted_table RENAME TO default.plain_table; SHOW TABLES; +DROP TABLE encrypted_table PURGE; + CRYPTO DELETE_KEY --keyName key_128; http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q b/ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q index c013425..c9ac035 100644 --- a/ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q +++ b/ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q @@ -1,6 +1,6 @@ -- SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table; +DROP TABLE IF EXISTS encrypted_table PURGE; CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; @@ -12,5 +12,5 @@ dfs -chmod -R 555 ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_tab SELECT count(*) FROM encrypted_table; -drop table encrypted_table; +drop table encrypted_table PURGE; CRYPTO DELETE_KEY --keyName key_128; http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out new file mode 100644 index 0000000..8f51fb9 --- /dev/null +++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out @@ -0,0 +1,56 @@ +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encrypted_table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encrypted_table +Encryption key created: 'key_128' +Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key: 'key_128' +PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@encrypted_table +POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +encrypted_table +src +PREHOOK: query: DROP TABLE default.encrypted_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop table because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.) +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +encrypted_table +src +PREHOOK: query: DROP TABLE default.encrypted_table PURGE +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: DROP TABLE default.encrypted_table PURGE +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +src http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out index 22cbce9..cb6dc5c 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out @@ -1,16 +1,16 @@ PREHOOK: query: -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable +drop table IF EXISTS encryptedTable PURGE PREHOOK: type: DROPTABLE POSTHOOK: query: -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable +drop table IF EXISTS encryptedTable PURGE POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table IF EXISTS unencryptedTable +PREHOOK: query: drop table IF EXISTS unencryptedTable PURGE PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table IF EXISTS unencryptedTable +POSTHOOK: query: drop table IF EXISTS unencryptedTable PURGE POSTHOOK: type: DROPTABLE PREHOOK: query: create table encryptedTable(value string) partitioned by (key string) clustered by (value) into 2 buckets stored as orc @@ -882,21 +882,21 @@ val_501 501 val_502 502 val_86 86 PREHOOK: query: -- clean up -drop table encryptedTable +drop table encryptedTable PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@encryptedtable PREHOOK: Output: default@encryptedtable POSTHOOK: query: -- clean up -drop table encryptedTable +drop table encryptedTable PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encryptedtable POSTHOOK: Output: default@encryptedtable Encryption key deleted: 'key_1' -PREHOOK: query: drop table unencryptedTable +PREHOOK: query: drop table unencryptedTable PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@unencryptedtable PREHOOK: Output: default@unencryptedtable -POSTHOOK: query: drop table unencryptedTable +POSTHOOK: query: drop table unencryptedTable PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@unencryptedtable POSTHOOK: Output: default@unencryptedtable http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out index 96f8b6a..8966608 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out @@ -1,16 +1,16 @@ PREHOOK: query: -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable +drop table IF EXISTS encryptedTable PURGE PREHOOK: type: DROPTABLE POSTHOOK: query: -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable +drop table IF EXISTS encryptedTable PURGE POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table IF EXISTS unencryptedTable +PREHOOK: query: drop table IF EXISTS unencryptedTable PURGE PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table IF EXISTS unencryptedTable +POSTHOOK: query: drop table IF EXISTS unencryptedTable PURGE POSTHOOK: type: DROPTABLE PREHOOK: query: create table encryptedTable(key string, value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc @@ -555,16 +555,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: encryptedtable - Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: string), _col1 (type: string) auto parallelism: false @@ -595,7 +595,7 @@ STAGE PLANS: serialization.ddl struct encryptedtable { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 1385 + totalSize 1392 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -643,7 +643,7 @@ STAGE PLANS: serialization.ddl struct encryptedtable { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 1382 + totalSize 1385 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -675,14 +675,14 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging NumFilesPerFileSink: 1 Static Partition Specification: ds=today/ - Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE #### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat @@ -775,21 +775,21 @@ POSTHOOK: Input: default@unencryptedtable@ds=today 502 val_502 today 86 val_86 today PREHOOK: query: -- clean up -drop table encryptedTable +drop table encryptedTable PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@encryptedtable PREHOOK: Output: default@encryptedtable POSTHOOK: query: -- clean up -drop table encryptedTable +drop table encryptedTable PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encryptedtable POSTHOOK: Output: default@encryptedtable Encryption key deleted: 'key_1' -PREHOOK: query: drop table unencryptedTable +PREHOOK: query: drop table unencryptedTable PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@unencryptedtable PREHOOK: Output: default@unencryptedtable -POSTHOOK: query: drop table unencryptedTable +POSTHOOK: query: drop table unencryptedTable PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@unencryptedtable POSTHOOK: Output: default@unencryptedtable http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out index 0caf694..9b6f750 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out @@ -1,10 +1,10 @@ PREHOOK: query: --SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table +DROP TABLE IF EXISTS encrypted_table PURGE PREHOOK: type: DROPTABLE POSTHOOK: query: --SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table +DROP TABLE IF EXISTS encrypted_table PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -752,11 +752,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: drop table encrypted_table +PREHOOK: query: drop table encrypted_table PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@encrypted_table PREHOOK: Output: default@encrypted_table -POSTHOOK: query: drop table encrypted_table +POSTHOOK: query: drop table encrypted_table PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encrypted_table POSTHOOK: Output: default@encrypted_table http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out index 7d8ffce..40f6898 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out @@ -1,12 +1,12 @@ PREHOOK: query: --SORT_QUERY_RESULTS -- Java JCE must be installed in order to hava a key length of 256 bits -DROP TABLE IF EXISTS table_key_1 +DROP TABLE IF EXISTS table_key_1 PURGE PREHOOK: type: DROPTABLE POSTHOOK: query: --SORT_QUERY_RESULTS -- Java JCE must be installed in order to hava a key length of 256 bits -DROP TABLE IF EXISTS table_key_1 +DROP TABLE IF EXISTS table_key_1 PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -20,9 +20,9 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@table_key_1 Encryption key created: 'key_1' Encryption zone created: '/build/ql/test/data/warehouse/table_key_1' using key: 'key_1' -PREHOOK: query: DROP TABLE IF EXISTS table_key_2 +PREHOOK: query: DROP TABLE IF EXISTS table_key_2 PURGE PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS table_key_2 +POSTHOOK: query: DROP TABLE IF EXISTS table_key_2 PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -1310,19 +1310,19 @@ POSTHOOK: Input: default@table_key_2 498 val_498 498 val_498 498 val_498 498 val_498 498 val_498 498 val_498 -PREHOOK: query: DROP TABLE table_key_1 +PREHOOK: query: DROP TABLE table_key_1 PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@table_key_1 PREHOOK: Output: default@table_key_1 -POSTHOOK: query: DROP TABLE table_key_1 +POSTHOOK: query: DROP TABLE table_key_1 PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@table_key_1 POSTHOOK: Output: default@table_key_1 -PREHOOK: query: DROP TABLE table_key_2 +PREHOOK: query: DROP TABLE table_key_2 PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@table_key_2 PREHOOK: Output: default@table_key_2 -POSTHOOK: query: DROP TABLE table_key_2 +POSTHOOK: query: DROP TABLE table_key_2 PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@table_key_2 POSTHOOK: Output: default@table_key_2 http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out index 46f36d8..35de14f 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out @@ -1,6 +1,6 @@ -PREHOOK: query: DROP TABLE IF EXISTS encrypted_table +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -1048,11 +1048,11 @@ POSTHOOK: Input: default@encrypted_table 400 val_400 200 val_200 97 val_97 -PREHOOK: query: DROP TABLE encrypted_table +PREHOOK: query: DROP TABLE encrypted_table PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@encrypted_table PREHOOK: Output: default@encrypted_table -POSTHOOK: query: DROP TABLE encrypted_table +POSTHOOK: query: DROP TABLE encrypted_table PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encrypted_table POSTHOOK: Output: default@encrypted_table http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out index d18827e..e0098f3 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out @@ -1,6 +1,6 @@ -PREHOOK: query: DROP TABLE IF EXISTS encrypted_table +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -41,3 +41,11 @@ POSTHOOK: query: SHOW TABLES POSTHOOK: type: SHOWTABLES encrypted_table src +PREHOOK: query: DROP TABLE encrypted_table PURGE +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: DROP TABLE encrypted_table PURGE +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out index c4c0dde..5d4d2af 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out @@ -1,10 +1,10 @@ PREHOOK: query: -- SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table +DROP TABLE IF EXISTS encrypted_table PURGE PREHOOK: type: DROPTABLE POSTHOOK: query: -- SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table +DROP TABLE IF EXISTS encrypted_table PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -35,11 +35,11 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@encrypted_table #### A masked pattern was here #### 500 -PREHOOK: query: drop table encrypted_table +PREHOOK: query: drop table encrypted_table PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@encrypted_table PREHOOK: Output: default@encrypted_table -POSTHOOK: query: drop table encrypted_table +POSTHOOK: query: drop table encrypted_table PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encrypted_table POSTHOOK: Output: default@encrypted_table http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java ---------------------------------------------------------------------- diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index d349068..19324b8 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -1215,6 +1215,9 @@ public class Hadoop23Shims extends HadoopShimsSecure { } else { fullPath = path.getFileSystem(conf).makeQualified(path); } + if(!"hdfs".equalsIgnoreCase(path.toUri().getScheme())) { + return false; + } return (hdfsAdmin.getEncryptionZoneForPath(fullPath) != null); }