suxiaogang223 commented on code in PR #46911:
URL: https://github.com/apache/doris/pull/46911#discussion_r1966984470
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java:
##########
@@ -245,38 +246,34 @@ public List<Split> getSplits(int numBackends) throws
UserException {
splitStat.setType(SplitReadType.NATIVE);
splitStat.setRawFileConvertable(true);
List<RawFile> rawFiles = optRawFiles.get();
- if (optDeletionFiles.isPresent()) {
- List<DeletionFile> deletionFiles =
optDeletionFiles.get();
- for (int i = 0; i < rawFiles.size(); i++) {
- RawFile file = rawFiles.get(i);
- DeletionFile deletionFile = deletionFiles.get(i);
- LocationPath locationPath = new
LocationPath(file.path(),
- source.getCatalog().getProperties());
- try {
- List<Split> dorisSplits =
FileSplitter.splitFile(
- locationPath,
- getRealFileSplitSize(0),
- null,
- file.length(),
- -1,
- true,
- null,
-
PaimonSplit.PaimonSplitCreator.DEFAULT);
- for (Split dorisSplit : dorisSplits) {
- // the element in DeletionFiles might be
null
- if (deletionFile != null) {
- splitStat.setHasDeletionVector(true);
- ((PaimonSplit)
dorisSplit).setDeletionFile(deletionFile);
- }
- splits.add(dorisSplit);
+ for (int i = 0; i < rawFiles.size(); i++) {
+ RawFile file = rawFiles.get(i);
+ LocationPath locationPath = new
LocationPath(file.path(),
+ source.getCatalog().getProperties());
+ try {
+ List<Split> dorisSplits = FileSplitter.splitFile(
+ locationPath,
+ // if applyCountPushdown is true, we can't
to split the file
+ // becasue the raw file and deletion
vector is one-to-one mapping
+ getRealFileSplitSize(applyCountPushdown ?
Long.MAX_VALUE : 0),
+ null,
+ file.length(),
+ -1,
+ true,
+ null,
+ PaimonSplit.PaimonSplitCreator.DEFAULT);
+ for (Split dorisSplit : dorisSplits) {
+ // try to set deletion file
+ if (optDeletionFiles.isPresent() &&
optDeletionFiles.get().get(i) != null) {
+ ((PaimonSplit)
dorisSplit).setDeletionFile(optDeletionFiles.get().get(i));
+ splitStat.setHasDeletionVector(true);
}
- ++rawFileSplitNum;
- } catch (IOException e) {
- throw new UserException("Paimon error to split
file: " + e.getMessage(), e);
}
+ splits.addAll(dorisSplits);
+ ++rawFileSplitNum;
+ } catch (IOException e) {
+ throw new UserException("Paimon error to split
file: " + e.getMessage(), e);
}
- } else {
- createRawFileSplits(rawFiles, splits,
applyCountPushdown ? Long.MAX_VALUE : 0);
Review Comment:
We can put the conditional branch about whether there is a deletionFile in
the for loop.The modified logic is equivalent to the previous one.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
