pvary commented on a change in pull request #1339:
URL: https://github.com/apache/hive/pull/1339#discussion_r463151233
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/io/AcidInputFormat.java
##########
@@ -118,70 +123,183 @@
*/
private long visibilityTxnId;
+ private List<DeltaFileMetaData> deltaFiles;
+
public DeltaMetaData() {
- this(0,0,new ArrayList<Integer>(), 0);
+ this(0, 0, new ArrayList<>(), 0, new ArrayList<>());
}
/**
+ * @param minWriteId min writeId of the delta directory
+ * @param maxWriteId max writeId of the delta directory
* @param stmtIds delta dir suffixes when a single txn writes > 1 delta in
the same partition
* @param visibilityTxnId maybe 0, if the dir name didn't have it.
txnid:0 is always visible
+ * @param deltaFileStatuses bucketFiles in the directory
*/
- DeltaMetaData(long minWriteId, long maxWriteId, List<Integer> stmtIds,
long visibilityTxnId) {
+ DeltaMetaData(long minWriteId, long maxWriteId, List<Integer> stmtIds,
long visibilityTxnId,
+ List<HadoopShims.HdfsFileStatusWithId> deltaFileStatuses) {
this.minWriteId = minWriteId;
this.maxWriteId = maxWriteId;
if (stmtIds == null) {
throw new IllegalArgumentException("stmtIds == null");
}
this.stmtIds = stmtIds;
this.visibilityTxnId = visibilityTxnId;
+ this.deltaFiles = new ArrayList<>();
+ for(HadoopShims.HdfsFileStatusWithId fileStatus : deltaFileStatuses) {
+ deltaFiles.add(new DeltaFileMetaData(fileStatus));
+ }
}
+
long getMinWriteId() {
return minWriteId;
}
+
long getMaxWriteId() {
return maxWriteId;
}
+
List<Integer> getStmtIds() {
return stmtIds;
}
+
long getVisibilityTxnId() {
return visibilityTxnId;
}
+
+ public List<DeltaFileMetaData> getDeltaFiles() {
+ return deltaFiles;
+ }
+
@Override
public void write(DataOutput out) throws IOException {
out.writeLong(minWriteId);
out.writeLong(maxWriteId);
out.writeInt(stmtIds.size());
- for(Integer id : stmtIds) {
+ for (Integer id : stmtIds) {
out.writeInt(id);
}
out.writeLong(visibilityTxnId);
+ out.writeInt(deltaFiles.size());
+ for (DeltaFileMetaData fileMeta : deltaFiles) {
+ fileMeta.write(out);
+ }
}
+
@Override
public void readFields(DataInput in) throws IOException {
minWriteId = in.readLong();
maxWriteId = in.readLong();
stmtIds.clear();
int numStatements = in.readInt();
- for(int i = 0; i < numStatements; i++) {
+ for (int i = 0; i < numStatements; i++) {
stmtIds.add(in.readInt());
}
visibilityTxnId = in.readLong();
+
+ deltaFiles.clear();
+ int numFiles = in.readInt();
+ for(int i = 0; i< numFiles; i++) {
+ DeltaFileMetaData file = new DeltaFileMetaData();
+ file.readFields(in);
+ deltaFiles.add(file);
+ }
}
- String getName() {
+ private String getName() {
assert stmtIds.isEmpty() : "use getName(int)";
return AcidUtils.addVisibilitySuffix(AcidUtils
.deleteDeltaSubdir(minWriteId, maxWriteId), visibilityTxnId);
}
- String getName(int stmtId) {
+ private String getName(int stmtId) {
assert !stmtIds.isEmpty() : "use getName()";
return AcidUtils.addVisibilitySuffix(AcidUtils
.deleteDeltaSubdir(minWriteId, maxWriteId, stmtId), visibilityTxnId);
}
+
+ public List<Path> getPaths(Path root) {
+ if (stmtIds.isEmpty()) {
+ return Collections.singletonList(new Path(root, getName()));
+ } else {
+ // To support multistatement transactions we may have multiple
directories corresponding to one DeltaMetaData
+ return getStmtIds().stream().map(stmtId -> new Path(root,
getName(stmtId))).collect(Collectors.toList());
+ }
+ }
@Override
Review comment:
Nit: newline
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For additional commands, e-mail: gitbox-h...@hive.apache.org
