This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fa43afcae3d [fix](hudi) return empty if there is no commit implemented
(#37702)
fa43afcae3d is described below
commit fa43afcae3d0adaa9f166b8ee4739cd8093adc9e
Author: Ashin Gau <[email protected]>
AuthorDate: Fri Jul 12 22:46:51 2024 +0800
[fix](hudi) return empty if there is no commit implemented (#37702)
## Proposed changes
Return empty set when there is no commit implemented, instead of
throwing error:
```
HoodieException("No instants to incrementally pull")
```
---
.../hudi/source/EmptyIncrementalRelation.java | 76 ++++++++++++++++++++++
.../trees/plans/logical/LogicalHudiScan.java | 5 +-
2 files changed, 80 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/EmptyIncrementalRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/EmptyIncrementalRelation.java
new file mode 100644
index 00000000000..c483bc46cfc
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/EmptyIncrementalRelation.java
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.hudi.source;
+
+import org.apache.doris.spi.Split;
+
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.exception.HoodieException;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class EmptyIncrementalRelation implements IncrementalRelation {
+ private static final String EMPTY_TS = "000";
+
+ private final Map<String, String> optParams;
+
+ public EmptyIncrementalRelation(Map<String, String> optParams) {
+ this.optParams = optParams;
+ }
+
+ @Override
+ public List<FileSlice> collectFileSlices() throws HoodieException {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<Split> collectSplits() throws HoodieException {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Map<String, String> getHoodieParams() {
+ optParams.put("hoodie.datasource.read.incr.operation", "true");
+ optParams.put("hoodie.datasource.read.begin.instanttime", EMPTY_TS);
+ optParams.put("hoodie.datasource.read.end.instanttime", EMPTY_TS);
+ optParams.put("hoodie.datasource.read.incr.includeStartTime", "true");
+ return optParams;
+ }
+
+ @Override
+ public boolean fallbackFullTableScan() {
+ return false;
+ }
+
+ @Override
+ public boolean isIncludeStartTime() {
+ return true;
+ }
+
+ @Override
+ public String getStartTs() {
+ return EMPTY_TS;
+ }
+
+ @Override
+ public String getEndTs() {
+ return EMPTY_TS;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
index 8659ad3d9c3..deeca65efcb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
@@ -23,6 +23,7 @@ import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper;
import org.apache.doris.datasource.hudi.source.COWIncrementalRelation;
+import org.apache.doris.datasource.hudi.source.EmptyIncrementalRelation;
import org.apache.doris.datasource.hudi.source.IncrementalRelation;
import org.apache.doris.datasource.hudi.source.MORIncrementalRelation;
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -206,7 +207,9 @@ public class LogicalHudiScan extends LogicalFileScan {
LOG.warn("Execute incremental read on RO table: {}",
table.getFullQualifiers());
}
}
- if (isCowOrRoTable) {
+ if
(hudiClient.getCommitsTimeline().filterCompletedInstants().countInstants() ==
0) {
+ newIncrementalRelation = Optional.of(new
EmptyIncrementalRelation(optParams));
+ } else if (isCowOrRoTable) {
newIncrementalRelation = Optional.of(new
COWIncrementalRelation(
optParams,
HiveMetaStoreClientHelper.getConfiguration(table), hudiClient));
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
