This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 13dfae1c0a7 HIVE-28580: Create table using import command gives ddl
pointing to incorrect location (#5512) (Zhihua Deng, reviewed by Denys
Kuzmenko, Sai Hemanth Gantasala)
13dfae1c0a7 is described below
commit 13dfae1c0a7d4540f4bc5edc50bc922f0cfc83e8
Author: dengzh <[email protected]>
AuthorDate: Fri Oct 25 14:27:34 2024 +0800
HIVE-28580: Create table using import command gives ddl pointing to
incorrect location (#5512) (Zhihua Deng, reviewed by Denys Kuzmenko, Sai
Hemanth Gantasala)
---
.../hive/ql/parse/ImportSemanticAnalyzer.java | 45 ++++++-----
.../hadoop/hive/ql/plan/ImportTableDesc.java | 3 +-
.../clientpositive/import_exported_table1.q | 20 +++++
.../llap/import_exported_table1.q.out | 91 ++++++++++++++++++++++
4 files changed, 137 insertions(+), 22 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 193abf3de7c..4d4956fbec1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -766,21 +766,8 @@ public class ImportSemanticAnalyzer extends
BaseSemanticAnalyzer {
*/
return;
}
- Path tgtPath;
- if (tblDesc.getLocation() == null) {
- if (table.getDataLocation() != null) {
- tgtPath = new Path(table.getDataLocation().toString(),
- Warehouse.makePartPath(partSpec.getPartSpec()));
- } else {
- Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName());
- tgtPath = new Path(
- wh.getDefaultTablePath( parentDb, tblDesc.getTableName(),
tblDesc.isExternal()),
- Warehouse.makePartPath(partSpec.getPartSpec()));
- }
- } else {
- tgtPath = new Path(tblDesc.getLocation(),
- Warehouse.makePartPath(partSpec.getPartSpec()));
- }
+ Path tableLocation = getTableDataLocation(wh, table, tblDesc, x);
+ Path tgtPath = new Path(tableLocation,
Warehouse.makePartPath(partSpec.getPartSpec()));
FileSystem tgtFs = FileSystem.get(tgtPath.toUri(), x.getConf());
checkTargetLocationEmpty(tgtFs, tgtPath, replicationSpec, x.getLOG());
partSpec.setLocation(tgtPath.toString());
@@ -1104,12 +1091,7 @@ public class ImportSemanticAnalyzer extends
BaseSemanticAnalyzer {
Path dataPath = new Path(fromURI.toString(),
EximUtil.DATA_PATH_NAME);
tblDesc.setLocation(dataPath.toString());
} else {
- Path tablePath = null;
- if (tblDesc.getLocation() != null) {
- tablePath = new Path(tblDesc.getLocation());
- } else {
- tablePath = wh.getDefaultTablePath(parentDb,
tblDesc.getTableName(), tblDesc.isExternal());
- }
+ Path tablePath = getTableDataLocation(wh, table, tblDesc, x);
FileSystem tgtFs = FileSystem.get(tablePath.toUri(), x.getConf());
checkTargetLocationEmpty(tgtFs, tablePath,
replicationSpec,x.getLOG());
t.addDependentTask(loadTable(fromURI, tblDesc, false, tablePath,
replicationSpec, x,
@@ -1403,4 +1385,25 @@ public class ImportSemanticAnalyzer extends
BaseSemanticAnalyzer {
}
}
+ private static Path getTableDataLocation(Warehouse wh, Table destTable,
ImportTableDesc tblDesc,
+ EximUtil.SemanticAnalyzerWrapperContext x) throws HiveException,
MetaException {
+ if (tblDesc.getLocation() != null) {
+ // If the import has specified the target location, use it.
+ // The partition can be imported to a customized location.
+ return new Path(tblDesc.getLocation());
+ } else if (destTable != null && destTable.getDataLocation() != null) {
+ // If the import table is existing, use the table location
+ return destTable.getDataLocation();
+ } else {
+ // For import new table
+ Table translatedTable =
x.getHive().getTranslateTableDryrun(tblDesc.toTable(x.getConf()).getTTable());
+ Path tablePath = translatedTable.getDataLocation();
+ if (tablePath == null) {
+ Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName());
+ tablePath = wh.getDefaultTablePath(parentDb, tblDesc.getTableName(),
tblDesc.isExternal());
+ }
+ return tablePath;
+ }
+ }
+
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
index 31cccff8dcd..b12cda11398 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.HiveTableName;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
@@ -175,7 +176,7 @@ public class ImportTableDesc {
return TableType.MANAGED_TABLE;
}
- public Table toTable(HiveConf conf) throws Exception {
+ public Table toTable(HiveConf conf) throws HiveException {
return createTblDesc.toTable(conf);
}
diff --git a/ql/src/test/queries/clientpositive/import_exported_table1.q
b/ql/src/test/queries/clientpositive/import_exported_table1.q
new file mode 100644
index 00000000000..3033d263292
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/import_exported_table1.q
@@ -0,0 +1,20 @@
+--! qt:replace:/^(?!LOCATION|.*HOOK).*(external|managed).test_b/### TABLE
DIRECTORY ###/
+set
metastore.metadata.transformer.class=org.apache.hadoop.hive.metastore.MetastoreDefaultTransformer;
+
+
+dfs ${system:test.dfs.mkdir} /tmp/test_import_exported_table1/;
+create database import_export location
'/tmp/test_import_exported_table1/external' managedlocation
'/tmp/test_import_exported_table1/managed';
+use import_export;
+
+create external table test_a (val string) partitioned by (pt string);
+insert into test_a partition (pt ='1111') values ("asfd");
+export table test_a partition (pt='1111') to '/tmp/test_a';
+import table test_b from '/tmp/test_a';
+show create table test_b;
+
+dfs -ls /tmp/test_import_exported_table1/external/test_b/pt=1111;
+select "============list managed directory===================";
+dfs -ls -R /tmp/test_import_exported_table1/managed;
+dfs -rmr /tmp/test_import_exported_table1;
+
+drop database import_export cascade;
diff --git
a/ql/src/test/results/clientpositive/llap/import_exported_table1.q.out
b/ql/src/test/results/clientpositive/llap/import_exported_table1.q.out
new file mode 100644
index 00000000000..83990824ee4
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/import_exported_table1.q.out
@@ -0,0 +1,91 @@
+#### A masked pattern was here ####
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:import_export
+#### A masked pattern was here ####
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:import_export
+#### A masked pattern was here ####
+PREHOOK: query: use import_export
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:import_export
+POSTHOOK: query: use import_export
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:import_export
+PREHOOK: query: create external table test_a (val string) partitioned by (pt
string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:import_export
+PREHOOK: Output: import_export@test_a
+POSTHOOK: query: create external table test_a (val string) partitioned by (pt
string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:import_export
+POSTHOOK: Output: import_export@test_a
+PREHOOK: query: insert into test_a partition (pt ='1111') values ("asfd")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: import_export@test_a@pt=1111
+POSTHOOK: query: insert into test_a partition (pt ='1111') values ("asfd")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: import_export@test_a@pt=1111
+POSTHOOK: Lineage: test_a PARTITION(pt=1111).val SCRIPT []
+#### A masked pattern was here ####
+PREHOOK: type: EXPORT
+#### A masked pattern was here ####
+POSTHOOK: type: EXPORT
+#### A masked pattern was here ####
+PREHOOK: type: IMPORT
+#### A masked pattern was here ####
+PREHOOK: Output: database:import_export
+#### A masked pattern was here ####
+POSTHOOK: type: IMPORT
+#### A masked pattern was here ####
+POSTHOOK: Output: database:import_export
+POSTHOOK: Output: import_export@test_b
+POSTHOOK: Output: import_export@test_b@pt=1111
+PREHOOK: query: show create table test_b
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: import_export@test_b
+POSTHOOK: query: show create table test_b
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: import_export@test_b
+CREATE EXTERNAL TABLE `test_b`(
+ `val` string)
+PARTITIONED BY (
+ `pt` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+### TABLE DIRECTORY ###'
+TBLPROPERTIES (
+ 'TRANSLATED_TO_EXTERNAL'='TRUE',
+ 'bucketing_version'='2',
+ 'external.table.purge'='TRUE',
+#### A masked pattern was here ####
+Found 1 items
+### TABLE DIRECTORY ###/pt=1111/000000_0
+PREHOOK: query: select "============list managed directory==================="
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select "============list managed directory==================="
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+============list managed directory===================
+#### A masked pattern was here ####
+PREHOOK: query: drop database import_export cascade
+PREHOOK: type: DROPDATABASE
+PREHOOK: Input: database:import_export
+PREHOOK: Output: database:import_export
+PREHOOK: Output: import_export@test_a
+PREHOOK: Output: import_export@test_b
+POSTHOOK: query: drop database import_export cascade
+POSTHOOK: type: DROPDATABASE
+POSTHOOK: Input: database:import_export
+POSTHOOK: Output: database:import_export
+POSTHOOK: Output: import_export@test_a
+POSTHOOK: Output: import_export@test_b
