This is an automated email from the ASF dual-hosted git repository.
chinnaraol pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 51781a8701c HIVE-27995: Fix inconsistent behavior of LOAD DATA command
for partitioned and non-partitioned tables (Shivangi Jha, Reviewed by Ayush
Saxena,Denys Kuzmenko, Chinna Rao Lalam)
51781a8701c is described below
commit 51781a8701c12ea0efbbae0845de62764047d2df
Author: Shivangi Jha <[email protected]>
AuthorDate: Fri Jul 19 10:42:49 2024 +0530
HIVE-27995: Fix inconsistent behavior of LOAD DATA command for partitioned
and non-partitioned tables (Shivangi Jha, Reviewed by Ayush Saxena,Denys
Kuzmenko, Chinna Rao Lalam)
---
.../java/org/apache/hive/jdbc/TestJdbcDriver2.java | 22 ++++++++++++++++++++++
.../hadoop/hive/ql/parse/LoadSemanticAnalyzer.java | 22 +++++++++++++++++++---
.../org/apache/hadoop/hive/ql/TestTxnLoadData.java | 12 ++++++------
.../queries/clientnegative/load_data_partition.q | 6 ++++++
.../queries/clientnegative/load_wrong_noof_part.q | 2 +-
.../clientnegative/load_data_partition.q.out | 15 +++++++++++++++
.../clientnegative/load_wrong_noof_part.q.out | 2 +-
7 files changed, 70 insertions(+), 11 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
index 1fa69877013..a752c673183 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
@@ -3428,4 +3428,26 @@ public class TestJdbcDriver2 {
stmt.close();
}
}
+ /**
+ * These test methods validate the error handling when attempting to load
data from a non-existent file into different types of Hive tables.
+ * Each test creates a specific type of table ( dynamically partitioned, or
bucketed), and then attempts to load data from a non-existent file.
+ * The tests pass if an exception is thrown with a message containing the
string "Invalid path".
+ */
+ @Test
+ public void testLoadDataNegativeForDynamicPartition() throws Exception {
+ HiveStatement stmt = (HiveStatement) con.createStatement();
+ try {
+ stmt.execute("drop table if exists T");
+ stmt.execute("create table T (a int, b int) partitioned by (p int)
stored as orc");
+ try {
+ stmt.execute("load data local inpath '/path/to/nonexistent/file.txt'
into table T");
+ fail("Expected an exception to be thrown");
+ } catch (Exception e) {
+ assertTrue("load data inpath",
+ e.getMessage() != null && e.getMessage().contains("Invalid
path"));
+ }
+ } finally {
+ stmt.close();
+ }
+ }
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index bbc652c7052..949aae6b343 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -155,7 +155,12 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer
{
return new URI(fromScheme, fromAuthority, path, null, null);
}
- private List<FileStatus> applyConstraintsAndGetFiles(URI fromURI, Table
table) throws SemanticException {
+ /**
+ * Verifies the file path existence. If the path is a directory, then it is
+ * verified to not be a glob pattern. If the path is a file, then it is
+ * verified to be a valid file name.
+ */
+ private FileStatus[] verifyFilePathExists(URI fromURI) throws
SemanticException {
FileStatus[] srcs;
@@ -166,7 +171,6 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer {
ErrorMsg.ILLEGAL_PATH.getMsg(), fromTree,
"Source file system should be \"file\" if \"local\" is specified"));
}
-
try {
FileSystem fileSystem = FileSystem.get(fromURI, conf);
srcs = matchFilesOrDir(fileSystem, new Path(fromURI));
@@ -175,6 +179,15 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer
{
ErrorMsg.INVALID_PATH.getMsg(), fromTree,
"No files matching path " + fromURI));
}
+ } catch (IOException e) {
+ throw new SemanticException(ASTErrorUtils.getMsg(
+ ErrorMsg.INVALID_PATH.getMsg(), fromTree), e);
+ }
+ return srcs;
+ }
+ private List<FileStatus> applyConstraintsAndGetFiles(URI fromURI, Table
table, FileStatus[] srcs)
+ throws SemanticException {
+ try {
for (FileStatus oneSrc : srcs) {
if (oneSrc.isDir()) {
@@ -182,6 +195,7 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer {
return null;
}
}
+ FileSystem fileSystem = FileSystem.get(fromURI, conf);
AcidUtils.validateAcidFiles(table, srcs, fileSystem);
// Do another loop if table is bucketed
List<String> bucketCols = table.getBucketCols();
@@ -284,6 +298,8 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer {
throw new SemanticException(ASTErrorUtils.getMsg(
ErrorMsg.INVALID_PATH.getMsg(), fromTree, e.getMessage()), e);
}
+ // validate the arguments
+ FileStatus[] srcs = verifyFilePathExists(fromURI);
// initialize destination table/partition
TableSpec ts = new TableSpec(db, conf, (ASTNode) tableTree);
@@ -338,7 +354,7 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer {
}
// make sure the arguments make sense
- List<FileStatus> files = applyConstraintsAndGetFiles(fromURI,
ts.tableHandle);
+ List<FileStatus> files = applyConstraintsAndGetFiles(fromURI,
ts.tableHandle, srcs);
if (queryReWritten) {
return;
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
index adba9ff8aa7..9fdb208f60b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
@@ -391,16 +391,16 @@ public class TestTxnLoadData extends
TxnCommandsBaseForTests {
*/
@Test
public void testValidations() throws Exception {
- runStatementOnDriver("drop table if exists T");
- runStatementOnDriver("drop table if exists Tstage");
- runStatementOnDriver("create table T (a int, b int) clustered by (a) into
2 buckets stored as orc tblproperties('transactional'='true')");
+ runStatementOnDriver("drop table if exists t");
+ runStatementOnDriver("drop table if exists tstage");
+ runStatementOnDriver("create table t (a int, b int) clustered by (a) into
2 buckets stored as orc tblproperties('transactional'='true')");
File createdFile= folder.newFile("myfile.txt");
FileUtils.writeStringToFile(createdFile, "hello world");
- runStatementOnDriver("create table Tstage (a int, b int) stored as orc
tblproperties('transactional'='false')");
+ runStatementOnDriver("create table tstage (a int, b int) stored as orc
tblproperties('transactional'='false')");
//this creates an ORC data file with correct schema under table root
- runStatementOnDriver("insert into Tstage values(1,2),(3,4)");
+ runStatementOnDriver("insert into tstage values(1,2),(3,4)");
// This will work with the new support of rewriting load into IAS.
- runStatementOnDriver("load data local inpath '" + getWarehouseDir() +
"/Tstage' into table T");
+ runStatementOnDriver("load data local inpath '" + getWarehouseDir() +
"/tstage' into table t");
}
private void checkExpected(List<String> rs, String[][] expected, String msg)
{
diff --git a/ql/src/test/queries/clientnegative/load_data_partition.q
b/ql/src/test/queries/clientnegative/load_data_partition.q
new file mode 100644
index 00000000000..e4090dffae2
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/load_data_partition.q
@@ -0,0 +1,6 @@
+drop table if exists validate_load_data;
+CREATE TABLE validate_load_data(key int, value string) partitioned by (hr int)
STORED AS TEXTFILE;
+LOAD DATA INPATH '/tmp/kv1.txt' INTO TABLE validate_load_data;
+SELECT * FROM validate_load_data;
+DROP TABLE validate_load_data;
+``````
\ No newline at end of file
diff --git a/ql/src/test/queries/clientnegative/load_wrong_noof_part.q
b/ql/src/test/queries/clientnegative/load_wrong_noof_part.q
index 7f5ad754142..c50a7c8e38c 100644
--- a/ql/src/test/queries/clientnegative/load_wrong_noof_part.q
+++ b/ql/src/test/queries/clientnegative/load_wrong_noof_part.q
@@ -1,3 +1,3 @@
CREATE TABLE loadpart1(a STRING, b STRING) PARTITIONED BY (ds STRING,ds1
STRING);
-LOAD DATA LOCAL INPATH '../../data1/files/kv1.txt' INTO TABLE loadpart1
PARTITION(ds='2009-05-05');
+LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE loadpart1
PARTITION(ds='2009-05-05');
diff --git a/ql/src/test/results/clientnegative/load_data_partition.q.out
b/ql/src/test/results/clientnegative/load_data_partition.q.out
new file mode 100644
index 00000000000..61b156e9835
--- /dev/null
+++ b/ql/src/test/results/clientnegative/load_data_partition.q.out
@@ -0,0 +1,15 @@
+PREHOOK: query: drop table if exists validate_load_data
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: drop table if exists validate_load_data
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE TABLE validate_load_data(key int, value string)
partitioned by (hr int) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@validate_load_data
+POSTHOOK: query: CREATE TABLE validate_load_data(key int, value string)
partitioned by (hr int) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@validate_load_data
+#### A masked pattern was here ####
diff --git a/ql/src/test/results/clientnegative/load_wrong_noof_part.q.out
b/ql/src/test/results/clientnegative/load_wrong_noof_part.q.out
index cf11cdc1366..491a85f09fe 100644
--- a/ql/src/test/results/clientnegative/load_wrong_noof_part.q.out
+++ b/ql/src/test/results/clientnegative/load_wrong_noof_part.q.out
@@ -6,4 +6,4 @@ POSTHOOK: query: CREATE TABLE loadpart1(a STRING, b STRING)
PARTITIONED BY (ds S
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@loadpart1
-FAILED: SemanticException [Error 10006]: Line 2:82 Partition not found
''2009-05-05''
+FAILED: SemanticException [Error 10006]: Line 2:81 Partition not found
''2009-05-05''