This is an automated email from the ASF dual-hosted git repository.

chinnaraol pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 51781a8701c HIVE-27995: Fix inconsistent behavior of LOAD DATA command 
for partitioned and non-partitioned tables (Shivangi Jha, Reviewed by Ayush 
Saxena,Denys Kuzmenko, Chinna Rao Lalam)
51781a8701c is described below

commit 51781a8701c12ea0efbbae0845de62764047d2df
Author: Shivangi Jha <[email protected]>
AuthorDate: Fri Jul 19 10:42:49 2024 +0530

    HIVE-27995: Fix inconsistent behavior of LOAD DATA command for partitioned 
and non-partitioned tables (Shivangi Jha, Reviewed by Ayush Saxena,Denys 
Kuzmenko, Chinna Rao Lalam)
---
 .../java/org/apache/hive/jdbc/TestJdbcDriver2.java | 22 ++++++++++++++++++++++
 .../hadoop/hive/ql/parse/LoadSemanticAnalyzer.java | 22 +++++++++++++++++++---
 .../org/apache/hadoop/hive/ql/TestTxnLoadData.java | 12 ++++++------
 .../queries/clientnegative/load_data_partition.q   |  6 ++++++
 .../queries/clientnegative/load_wrong_noof_part.q  |  2 +-
 .../clientnegative/load_data_partition.q.out       | 15 +++++++++++++++
 .../clientnegative/load_wrong_noof_part.q.out      |  2 +-
 7 files changed, 70 insertions(+), 11 deletions(-)

diff --git 
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java 
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
index 1fa69877013..a752c673183 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
@@ -3428,4 +3428,26 @@ public class TestJdbcDriver2 {
       stmt.close();
     }
   }
+  /**
+   * These test methods validate the error handling when attempting to load 
data from a non-existent file into different types of Hive tables.
+   * Each test creates a specific type of table ( dynamically partitioned, or 
bucketed), and then attempts to load data from a non-existent file.
+   * The tests pass if an exception is thrown with a message containing the 
string "Invalid path".
+   */
+  @Test
+  public void testLoadDataNegativeForDynamicPartition() throws Exception {
+    HiveStatement stmt = (HiveStatement) con.createStatement();
+    try {
+      stmt.execute("drop table if exists T");
+      stmt.execute("create table T (a int, b int) partitioned by (p int) 
stored as orc");
+      try {
+        stmt.execute("load data local inpath '/path/to/nonexistent/file.txt' 
into table T");
+        fail("Expected an exception to be thrown");
+      } catch (Exception e) {
+        assertTrue("load data inpath",
+                e.getMessage() != null && e.getMessage().contains("Invalid 
path"));
+      }
+    } finally {
+      stmt.close();
+    }
+  }
 }
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index bbc652c7052..949aae6b343 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -155,7 +155,12 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer 
{
     return new URI(fromScheme, fromAuthority, path, null, null);
   }
 
-  private List<FileStatus> applyConstraintsAndGetFiles(URI fromURI, Table 
table) throws SemanticException {
+  /**
+   * Verifies the file path existence. If the path is a directory, then it is
+   * verified to not be a glob pattern. If the path is a file, then it is
+   * verified to be a valid file name.
+   */
+  private FileStatus[] verifyFilePathExists(URI fromURI) throws 
SemanticException {
 
     FileStatus[] srcs;
 
@@ -166,7 +171,6 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer {
           ErrorMsg.ILLEGAL_PATH.getMsg(), fromTree,
           "Source file system should be \"file\" if \"local\" is specified"));
     }
-
     try {
       FileSystem fileSystem = FileSystem.get(fromURI, conf);
       srcs = matchFilesOrDir(fileSystem, new Path(fromURI));
@@ -175,6 +179,15 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer 
{
             ErrorMsg.INVALID_PATH.getMsg(), fromTree,
             "No files matching path " + fromURI));
       }
+    } catch (IOException e) {
+      throw new SemanticException(ASTErrorUtils.getMsg(
+              ErrorMsg.INVALID_PATH.getMsg(), fromTree), e);
+    }
+    return srcs;
+  }
+  private List<FileStatus> applyConstraintsAndGetFiles(URI fromURI, Table 
table, FileStatus[] srcs)
+          throws SemanticException {
+    try {
 
       for (FileStatus oneSrc : srcs) {
         if (oneSrc.isDir()) {
@@ -182,6 +195,7 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer {
           return null;
         }
       }
+      FileSystem fileSystem = FileSystem.get(fromURI, conf);
       AcidUtils.validateAcidFiles(table, srcs, fileSystem);
       // Do another loop if table is bucketed
       List<String> bucketCols = table.getBucketCols();
@@ -284,6 +298,8 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer {
       throw new SemanticException(ASTErrorUtils.getMsg(
           ErrorMsg.INVALID_PATH.getMsg(), fromTree, e.getMessage()), e);
     }
+    // validate the arguments
+    FileStatus[] srcs = verifyFilePathExists(fromURI);
 
     // initialize destination table/partition
     TableSpec ts = new TableSpec(db, conf, (ASTNode) tableTree);
@@ -338,7 +354,7 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer {
     }
 
     // make sure the arguments make sense
-    List<FileStatus> files = applyConstraintsAndGetFiles(fromURI, 
ts.tableHandle);
+    List<FileStatus> files = applyConstraintsAndGetFiles(fromURI, 
ts.tableHandle, srcs);
     if (queryReWritten) {
       return;
     }
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
index adba9ff8aa7..9fdb208f60b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
@@ -391,16 +391,16 @@ public class TestTxnLoadData extends 
TxnCommandsBaseForTests {
    */
   @Test
   public void testValidations() throws Exception {
-    runStatementOnDriver("drop table if exists T");
-    runStatementOnDriver("drop table if exists Tstage");
-    runStatementOnDriver("create table T (a int, b int) clustered by (a) into 
2 buckets stored as orc tblproperties('transactional'='true')");
+    runStatementOnDriver("drop table if exists t");
+    runStatementOnDriver("drop table if exists tstage");
+    runStatementOnDriver("create table t (a int, b int) clustered by (a) into 
2 buckets stored as orc tblproperties('transactional'='true')");
     File createdFile= folder.newFile("myfile.txt");
     FileUtils.writeStringToFile(createdFile, "hello world");
-    runStatementOnDriver("create table Tstage (a int, b int) stored as orc 
tblproperties('transactional'='false')");
+    runStatementOnDriver("create table tstage (a int, b int) stored as orc 
tblproperties('transactional'='false')");
     //this creates an ORC data file with correct schema under table root
-    runStatementOnDriver("insert into Tstage values(1,2),(3,4)");
+    runStatementOnDriver("insert into tstage values(1,2),(3,4)");
     // This will work with the new support of rewriting load into IAS.
-    runStatementOnDriver("load data local inpath '" + getWarehouseDir() + 
"/Tstage' into table T");
+    runStatementOnDriver("load data local inpath '" + getWarehouseDir() + 
"/tstage' into table t");
   }
 
   private void checkExpected(List<String> rs, String[][] expected, String msg) 
{
diff --git a/ql/src/test/queries/clientnegative/load_data_partition.q 
b/ql/src/test/queries/clientnegative/load_data_partition.q
new file mode 100644
index 00000000000..e4090dffae2
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/load_data_partition.q
@@ -0,0 +1,6 @@
+drop table if exists validate_load_data;
+CREATE TABLE validate_load_data(key int, value string) partitioned by (hr int) 
STORED AS TEXTFILE;
+LOAD DATA INPATH '/tmp/kv1.txt' INTO TABLE validate_load_data;
+SELECT * FROM validate_load_data;
+DROP TABLE validate_load_data;
+``````
\ No newline at end of file
diff --git a/ql/src/test/queries/clientnegative/load_wrong_noof_part.q 
b/ql/src/test/queries/clientnegative/load_wrong_noof_part.q
index 7f5ad754142..c50a7c8e38c 100644
--- a/ql/src/test/queries/clientnegative/load_wrong_noof_part.q
+++ b/ql/src/test/queries/clientnegative/load_wrong_noof_part.q
@@ -1,3 +1,3 @@
 
 CREATE TABLE loadpart1(a STRING, b STRING) PARTITIONED BY (ds STRING,ds1 
STRING);
-LOAD DATA LOCAL INPATH '../../data1/files/kv1.txt' INTO TABLE loadpart1 
PARTITION(ds='2009-05-05');
+LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE loadpart1 
PARTITION(ds='2009-05-05');
diff --git a/ql/src/test/results/clientnegative/load_data_partition.q.out 
b/ql/src/test/results/clientnegative/load_data_partition.q.out
new file mode 100644
index 00000000000..61b156e9835
--- /dev/null
+++ b/ql/src/test/results/clientnegative/load_data_partition.q.out
@@ -0,0 +1,15 @@
+PREHOOK: query: drop table if exists validate_load_data
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: drop table if exists validate_load_data
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE TABLE validate_load_data(key int, value string) 
partitioned by (hr int) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@validate_load_data
+POSTHOOK: query: CREATE TABLE validate_load_data(key int, value string) 
partitioned by (hr int) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@validate_load_data
+#### A masked pattern was here ####
diff --git a/ql/src/test/results/clientnegative/load_wrong_noof_part.q.out 
b/ql/src/test/results/clientnegative/load_wrong_noof_part.q.out
index cf11cdc1366..491a85f09fe 100644
--- a/ql/src/test/results/clientnegative/load_wrong_noof_part.q.out
+++ b/ql/src/test/results/clientnegative/load_wrong_noof_part.q.out
@@ -6,4 +6,4 @@ POSTHOOK: query: CREATE TABLE loadpart1(a STRING, b STRING) 
PARTITIONED BY (ds S
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@loadpart1
-FAILED: SemanticException [Error 10006]: Line 2:82 Partition not found 
''2009-05-05''
+FAILED: SemanticException [Error 10006]: Line 2:81 Partition not found 
''2009-05-05''

Reply via email to