This is an automated email from the ASF dual-hosted git repository.

sbadhya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new e92b681e846 HIVE-28530: Set files in thread safe manner in 
HiveSequenceFileInputFormat (#5461) (Sourabh Badhya reviewed by Denys Kuzmenko)
e92b681e846 is described below

commit e92b681e8467547cf126956b4af4bd50e3784c1a
Author: Sourabh Badhya <[email protected]>
AuthorDate: Wed Sep 25 21:38:28 2024 +0530

    HIVE-28530: Set files in thread safe manner in HiveSequenceFileInputFormat 
(#5461) (Sourabh Badhya reviewed by Denys Kuzmenko)
---
 .../hive/ql/io/HiveSequenceFileInputFormat.java    | 24 +++++++++++++---------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileInputFormat.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileInputFormat.java
index 0f679f6110e..cd35ccc481c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileInputFormat.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.io;
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
 import org.apache.hadoop.io.LongWritable;
@@ -40,24 +41,27 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
 public class HiveSequenceFileInputFormat<K extends LongWritable, V extends 
BytesRefArrayWritable>
     extends SequenceFileInputFormat<K, V> {
 
+  private static final ThreadLocal<Set<FileStatus>> fileStatuses = new 
ThreadLocal<>();
+
   public HiveSequenceFileInputFormat() {
     setMinSplitSize(SequenceFile.SYNC_INTERVAL);
   }
 
-  private Set<FileStatus> fileStatuses = null;
-
-  public void setFiles(Set<FileStatus> fileStatuses) {
-    this.fileStatuses= fileStatuses;
+  public void setFiles(Set<FileStatus> files) {
+    fileStatuses.set(files);
   }
 
   @Override
   protected FileStatus[] listStatus(JobConf job) throws IOException {
-    if(fileStatuses== null || fileStatuses.isEmpty()) {
-      // In cases where list of files to fetch is not provided we will use 
SequenceFileInputFormat
-      // e.g. SELECT without a job
-      return super.listStatus(job);
+    try {
+      if (CollectionUtils.isEmpty(fileStatuses.get())) {
+        // In cases where list of files to fetch is not provided we will use 
SequenceFileInputFormat
+        // e.g. SELECT without a job
+        return super.listStatus(job);
+      }
+      return fileStatuses.get().toArray(new FileStatus[0]);
+    } finally {
+      fileStatuses.remove();
     }
-    FileStatus[] fsStatusArray = new FileStatus[fileStatuses.size()];
-    return fileStatuses.toArray(fsStatusArray);
   }
 }

Reply via email to