This is an automated email from the ASF dual-hosted git repository.
zhangzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 321a6023c9 [CORE] Add iceberg equality delete file proto definition
(#8778)
321a6023c9 is described below
commit 321a6023c91f6571d1b53c8fd02cfb0fc57b1739
Author: Joey <[email protected]>
AuthorDate: Tue Feb 25 14:48:12 2025 +0800
[CORE] Add iceberg equality delete file proto definition (#8778)
[CORE] Add iceberg equality delete file proto definition
---
.../org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java | 3 +++
.../src/main/resources/substrait/proto/substrait/algebra.proto | 10 ++++++++++
2 files changed, 13 insertions(+)
diff --git
a/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java
b/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java
index 85c3593e90..1823330622 100644
---
a/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java
+++
b/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java
@@ -117,6 +117,9 @@ public class IcebergLocalFilesNode extends LocalFilesNode {
throw new UnsupportedOperationException(
"Unsupported format " + delete.format().name() + " for delete
file.");
}
+ if (delete.equalityFieldIds() != null &&
!delete.equalityFieldIds().isEmpty()) {
+ deleteFileBuilder.addAllEqualityFieldIds(delete.equalityFieldIds());
+ }
icebergBuilder.addDeleteFiles(deleteFileBuilder);
}
fileBuilder.setIceberg(icebergBuilder);
diff --git
a/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
b/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
index d53a9aef9d..8c1468117b 100644
---
a/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
+++
b/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
@@ -173,6 +173,13 @@ message ReadRel {
EQUALITY_DELETES = 2;
}
message DeleteFile {
+ message Map {
+ message KeyValue {
+ int32 key = 1;
+ string value = 2;
+ }
+ repeated KeyValue key_values = 1;
+ }
FileContent fileContent = 1;
string filePath = 2;
uint64 fileSize = 3;
@@ -181,6 +188,9 @@ message ReadRel {
ParquetReadOptions parquet = 5;
OrcReadOptions orc = 6;
}
+ repeated int32 equalityFieldIds = 7;
+ Map lowerBounds = 8;
+ Map upperBounds = 9;
}
oneof file_format {
ParquetReadOptions parquet = 1;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
