This is an automated email from the ASF dual-hosted git repository. mengtao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new 21bcbfd792 [HUDI-5237] Support for HoodieUnMergedLogRecordScanner with InternalSchema (#7237) 21bcbfd792 is described below commit 21bcbfd7923c65acd7e458d6ca2a05f1ff9df109 Author: Alexander Trushev <trushev.a...@gmail.com> AuthorDate: Mon Nov 21 10:26:16 2022 +0700 [HUDI-5237] Support for HoodieUnMergedLogRecordScanner with InternalSchema (#7237) * [HUDI-5237] Support for HoodieUnMergedLogRecordScanner with InternalSchema --- .../common/table/log/AbstractHoodieLogRecordReader.java | 2 ++ .../common/table/log/HoodieMergedLogRecordScanner.java | 3 ++- .../common/table/log/HoodieUnMergedLogRecordScanner.java | 15 ++++++++++++--- .../metadata/HoodieMetadataMergedLogRecordReader.java | 5 +++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java index 88da6aa1f0..0c8d8b3f6c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java @@ -817,6 +817,8 @@ public abstract class AbstractHoodieLogRecordReader { public abstract Builder withReaderSchema(Schema schema); + public abstract Builder withInternalSchema(InternalSchema internalSchema); + public abstract Builder withLatestInstantTime(String latestInstantTime); public abstract Builder withReadBlocksLazily(boolean readBlocksLazily); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java index e846637493..708015b1c1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java @@ -308,8 +308,9 @@ public class HoodieMergedLogRecordScanner extends AbstractHoodieLogRecordReader return this; } + @Override public Builder withInternalSchema(InternalSchema internalSchema) { - this.internalSchema = internalSchema == null ? InternalSchema.getEmptyInternalSchema() : internalSchema; + this.internalSchema = internalSchema; return this; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java index 7ddb9f1236..b0d127c562 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java @@ -40,9 +40,10 @@ public class HoodieUnMergedLogRecordScanner extends AbstractHoodieLogRecordReade private HoodieUnMergedLogRecordScanner(FileSystem fs, String basePath, List<String> logFilePaths, Schema readerSchema, String latestInstantTime, boolean readBlocksLazily, boolean reverseReader, int bufferSize, - LogRecordScannerCallback callback, Option<InstantRange> instantRange, boolean useScanV2) { + LogRecordScannerCallback callback, Option<InstantRange> instantRange, InternalSchema internalSchema, + boolean useScanV2) { super(fs, basePath, logFilePaths, readerSchema, latestInstantTime, readBlocksLazily, reverseReader, bufferSize, instantRange, - false, true, Option.empty(), InternalSchema.getEmptyInternalSchema(), useScanV2); + false, true, Option.empty(), internalSchema, useScanV2); this.callback = callback; } @@ -81,6 +82,7 @@ public class HoodieUnMergedLogRecordScanner extends AbstractHoodieLogRecordReade private String basePath; private List<String> logFilePaths; private Schema readerSchema; + private InternalSchema internalSchema; private String latestInstantTime; private boolean readBlocksLazily; private boolean reverseReader; @@ -112,6 +114,12 @@ public class HoodieUnMergedLogRecordScanner extends AbstractHoodieLogRecordReade return this; } + @Override + public Builder withInternalSchema(InternalSchema internalSchema) { + this.internalSchema = internalSchema; + return this; + } + public Builder withLatestInstantTime(String latestInstantTime) { this.latestInstantTime = latestInstantTime; return this; @@ -151,7 +159,8 @@ public class HoodieUnMergedLogRecordScanner extends AbstractHoodieLogRecordReade @Override public HoodieUnMergedLogRecordScanner build() { return new HoodieUnMergedLogRecordScanner(fs, basePath, logFilePaths, readerSchema, - latestInstantTime, readBlocksLazily, reverseReader, bufferSize, callback, instantRange, useScanV2); + latestInstantTime, readBlocksLazily, reverseReader, bufferSize, callback, instantRange, + internalSchema, useScanV2); } } } diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java index b844f219cd..afcd726a0b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java @@ -168,6 +168,11 @@ public class HoodieMetadataMergedLogRecordReader extends HoodieMergedLogRecordSc return this; } + @Override + public Builder withInternalSchema(InternalSchema internalSchema) { + throw new UnsupportedOperationException(); + } + @Override public Builder withLatestInstantTime(String latestInstantTime) { this.latestInstantTime = latestInstantTime;