This is an automated email from the ASF dual-hosted git repository.
sbadhya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e92b681e846 HIVE-28530: Set files in thread safe manner in
HiveSequenceFileInputFormat (#5461) (Sourabh Badhya reviewed by Denys Kuzmenko)
e92b681e846 is described below
commit e92b681e8467547cf126956b4af4bd50e3784c1a
Author: Sourabh Badhya <[email protected]>
AuthorDate: Wed Sep 25 21:38:28 2024 +0530
HIVE-28530: Set files in thread safe manner in HiveSequenceFileInputFormat
(#5461) (Sourabh Badhya reviewed by Denys Kuzmenko)
---
.../hive/ql/io/HiveSequenceFileInputFormat.java | 24 +++++++++++++---------
1 file changed, 14 insertions(+), 10 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileInputFormat.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileInputFormat.java
index 0f679f6110e..cd35ccc481c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileInputFormat.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.io;
import java.io.IOException;
import java.util.Set;
+import org.apache.commons.collections4.CollectionUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
import org.apache.hadoop.io.LongWritable;
@@ -40,24 +41,27 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
public class HiveSequenceFileInputFormat<K extends LongWritable, V extends
BytesRefArrayWritable>
extends SequenceFileInputFormat<K, V> {
+ private static final ThreadLocal<Set<FileStatus>> fileStatuses = new
ThreadLocal<>();
+
public HiveSequenceFileInputFormat() {
setMinSplitSize(SequenceFile.SYNC_INTERVAL);
}
- private Set<FileStatus> fileStatuses = null;
-
- public void setFiles(Set<FileStatus> fileStatuses) {
- this.fileStatuses= fileStatuses;
+ public void setFiles(Set<FileStatus> files) {
+ fileStatuses.set(files);
}
@Override
protected FileStatus[] listStatus(JobConf job) throws IOException {
- if(fileStatuses== null || fileStatuses.isEmpty()) {
- // In cases where list of files to fetch is not provided we will use
SequenceFileInputFormat
- // e.g. SELECT without a job
- return super.listStatus(job);
+ try {
+ if (CollectionUtils.isEmpty(fileStatuses.get())) {
+ // In cases where list of files to fetch is not provided we will use
SequenceFileInputFormat
+ // e.g. SELECT without a job
+ return super.listStatus(job);
+ }
+ return fileStatuses.get().toArray(new FileStatus[0]);
+ } finally {
+ fileStatuses.remove();
}
- FileStatus[] fsStatusArray = new FileStatus[fileStatuses.size()];
- return fileStatuses.toArray(fsStatusArray);
}
}