This is an automated email from the ASF dual-hosted git repository. ngangam pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 5c73a847cd3 HIVE-27728: Renaming partition from legacy tables should move the partition to the new location (#4752) 5c73a847cd3 is described below commit 5c73a847cd385424cdc8a7804176fbae7834c90f Author: dengzh <dengzhhu...@gmail.com> AuthorDate: Thu Oct 5 00:33:53 2023 +0800 HIVE-27728: Renaming partition from legacy tables should move the partition to the new location (#4752) --- .../clientpositive/translated_external_rename5.q | 26 ++++ .../llap/translated_external_rename5.q.out | 160 +++++++++++++++++++++ .../hadoop/hive/metastore/HiveAlterHandler.java | 4 +- 3 files changed, 189 insertions(+), 1 deletion(-) diff --git a/ql/src/test/queries/clientpositive/translated_external_rename5.q b/ql/src/test/queries/clientpositive/translated_external_rename5.q new file mode 100644 index 00000000000..033de9b30ab --- /dev/null +++ b/ql/src/test/queries/clientpositive/translated_external_rename5.q @@ -0,0 +1,26 @@ +--! qt:replace:/^(?!LOCATION|.*HOOK).*metadata_table_test1/### TABLE DIRECTORY ###/ + +set metastore.metadata.transformer.class=org.apache.hadoop.hive.metastore.MetastoreDefaultTransformer; +set hive.create.as.external.legacy=true; + +CREATE TABLE metadata_test1( + emp_number int, + emp_name string, + city string) +PARTITIONED BY(state string) +LOCATION 'pfile://${system:test.tmp.dir}/metadata_table_test1'; + +DESC FORMATTED metadata_test1; + +INSERT INTO metadata_test1 PARTITION(state='A') VALUES (11, 'ABC', 'AA'); +INSERT INTO metadata_test1 PARTITION(state='B') VALUES (12, 'XYZ', 'BX'), (13, 'UVW', 'BU'); +select * from default.metadata_test1; +select "======================== list table directory ========================="; +dfs -ls ${system:test.tmp.dir}/metadata_table_test1/; + +ALTER TABLE default.metadata_test1 PARTITION (state='A') RENAME TO PARTITION (state='C'); +select * from metadata_test1; +select "======================== list table directory ========================="; +dfs -ls ${system:test.tmp.dir}/metadata_table_test1/; + +drop table metadata_test1; diff --git a/ql/src/test/results/clientpositive/llap/translated_external_rename5.q.out b/ql/src/test/results/clientpositive/llap/translated_external_rename5.q.out new file mode 100644 index 00000000000..5519b54c79a --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/translated_external_rename5.q.out @@ -0,0 +1,160 @@ +PREHOOK: query: CREATE TABLE metadata_test1( + emp_number int, + emp_name string, + city string) +PARTITIONED BY(state string) +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@metadata_test1 +POSTHOOK: query: CREATE TABLE metadata_test1( + emp_number int, + emp_name string, + city string) +PARTITIONED BY(state string) +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@metadata_test1 +PREHOOK: query: DESC FORMATTED metadata_test1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@metadata_test1 +POSTHOOK: query: DESC FORMATTED metadata_test1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@metadata_test1 +# col_name data_type comment +emp_number int +emp_name string +city string + +# Partition Information +# col_name data_type comment +state string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +### TABLE DIRECTORY ### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + EXTERNAL TRUE + TRANSLATED_TO_EXTERNAL TRUE + bucketing_version 2 + external.table.purge TRUE + numFiles 0 + numPartitions 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: INSERT INTO metadata_test1 PARTITION(state='A') VALUES (11, 'ABC', 'AA') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@metadata_test1@state=A +POSTHOOK: query: INSERT INTO metadata_test1 PARTITION(state='A') VALUES (11, 'ABC', 'AA') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@metadata_test1@state=A +POSTHOOK: Lineage: metadata_test1 PARTITION(state=A).city SCRIPT [] +POSTHOOK: Lineage: metadata_test1 PARTITION(state=A).emp_name SCRIPT [] +POSTHOOK: Lineage: metadata_test1 PARTITION(state=A).emp_number SCRIPT [] +PREHOOK: query: INSERT INTO metadata_test1 PARTITION(state='B') VALUES (12, 'XYZ', 'BX'), (13, 'UVW', 'BU') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@metadata_test1@state=B +POSTHOOK: query: INSERT INTO metadata_test1 PARTITION(state='B') VALUES (12, 'XYZ', 'BX'), (13, 'UVW', 'BU') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@metadata_test1@state=B +POSTHOOK: Lineage: metadata_test1 PARTITION(state=B).city SCRIPT [] +POSTHOOK: Lineage: metadata_test1 PARTITION(state=B).emp_name SCRIPT [] +POSTHOOK: Lineage: metadata_test1 PARTITION(state=B).emp_number SCRIPT [] +PREHOOK: query: select * from default.metadata_test1 +PREHOOK: type: QUERY +PREHOOK: Input: default@metadata_test1 +PREHOOK: Input: default@metadata_test1@state=A +PREHOOK: Input: default@metadata_test1@state=B +#### A masked pattern was here #### +POSTHOOK: query: select * from default.metadata_test1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@metadata_test1 +POSTHOOK: Input: default@metadata_test1@state=A +POSTHOOK: Input: default@metadata_test1@state=B +#### A masked pattern was here #### +11 ABC AA A +12 XYZ BX B +13 UVW BU B +PREHOOK: query: select "======================== list table directory =========================" +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select "======================== list table directory =========================" +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +======================== list table directory ========================= +Found 2 items +### TABLE DIRECTORY ###/state=A +### TABLE DIRECTORY ###/state=B +PREHOOK: query: ALTER TABLE default.metadata_test1 PARTITION (state='A') RENAME TO PARTITION (state='C') +PREHOOK: type: ALTERTABLE_RENAMEPART +PREHOOK: Input: default@metadata_test1 +PREHOOK: Output: default@metadata_test1@state=A +POSTHOOK: query: ALTER TABLE default.metadata_test1 PARTITION (state='A') RENAME TO PARTITION (state='C') +POSTHOOK: type: ALTERTABLE_RENAMEPART +POSTHOOK: Input: default@metadata_test1 +POSTHOOK: Input: default@metadata_test1@state=A +POSTHOOK: Output: default@metadata_test1@state=A +POSTHOOK: Output: default@metadata_test1@state=C +PREHOOK: query: select * from metadata_test1 +PREHOOK: type: QUERY +PREHOOK: Input: default@metadata_test1 +PREHOOK: Input: default@metadata_test1@state=B +PREHOOK: Input: default@metadata_test1@state=C +#### A masked pattern was here #### +POSTHOOK: query: select * from metadata_test1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@metadata_test1 +POSTHOOK: Input: default@metadata_test1@state=B +POSTHOOK: Input: default@metadata_test1@state=C +#### A masked pattern was here #### +12 XYZ BX B +13 UVW BU B +11 ABC AA C +PREHOOK: query: select "======================== list table directory =========================" +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select "======================== list table directory =========================" +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +======================== list table directory ========================= +Found 2 items +### TABLE DIRECTORY ###/state=B +### TABLE DIRECTORY ###/state=C +PREHOOK: query: drop table metadata_test1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@metadata_test1 +PREHOOK: Output: database:default +PREHOOK: Output: default@metadata_test1 +POSTHOOK: query: drop table metadata_test1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@metadata_test1 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@metadata_test1 diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 571f5aae361..9bce99ad680 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -679,7 +679,9 @@ public class HiveAlterHandler implements AlterHandler { // 1) partition SD Location // 2) partition column stats if there are any because of part_name field in HMS table PART_COL_STATS // 3) rename the partition directory if it is not an external table - if (!tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) { + boolean shouldMoveData = !(MetaStoreUtils.isExternalTable(tbl) && + !MetaStoreUtils.isPropertyTrue(tbl.getParameters(), "TRANSLATED_TO_EXTERNAL")); + if (shouldMoveData) { // TODO: refactor this into a separate method after master merge, this one is too big. try { db = msdb.getDatabase(catName, dbname);