nastra commented on code in PR #12706:
URL: https://github.com/apache/iceberg/pull/12706#discussion_r2026343357
##########
spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java:
##########
@@ -351,6 +353,7 @@ private OverwriteByFilter(Expression overwriteExpr) {
public void commit(WriterCommitMessage[] messages) {
OverwriteFiles overwriteFiles = table.newOverwrite();
overwriteFiles.overwriteByRowFilter(overwriteExpr);
+ overwriteFiles.caseSensitive(caseSensitive);
Review Comment:
I don't think this is the only place that needs to be fixed. I did a quick
check and I think there are a few other places that also need to be fixed. We
should also add some additional tests for the other places that fail without
the fix and pass with it.
```
---
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeltaWrite.java
+++
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeltaWrite.java
@@ -111,6 +111,7 @@ class SparkPositionDeltaWrite implements DeltaWrite,
RequiresDistributionAndOrde
private final SparkWriteRequirements writeRequirements;
private final Context context;
private final Map<String, String> writeProperties;
+ private final boolean caseSensitive;
private boolean cleanupOnAbort = false;
@@ -136,6 +137,7 @@ class SparkPositionDeltaWrite implements DeltaWrite,
RequiresDistributionAndOrde
this.writeRequirements = writeConf.positionDeltaRequirements(command);
this.context = new Context(dataSchema, writeConf, info,
writeRequirements);
this.writeProperties = writeConf.writeProperties();
+ this.caseSensitive = writeConf.caseSensitive();
}
@Override
@@ -205,6 +207,7 @@ class SparkPositionDeltaWrite implements DeltaWrite,
RequiresDistributionAndOrde
@Override
public void commit(WriterCommitMessage[] messages) {
RowDelta rowDelta = table.newRowDelta();
+ rowDelta.caseSensitive(caseSensitive);
CharSequenceSet referencedDataFiles = CharSequenceSet.empty();
diff --git
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
index bbc743413..de08611fb 100644
---
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
+++
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
@@ -374,6 +374,7 @@ public class SparkTable
DeleteFiles deleteFiles =
icebergTable
.newDelete()
+ // .caseSensitive(...)
.set("spark.app.id",
sparkSession().sparkContext().applicationId())
.deleteFromRowFilter(deleteExpr);
diff --git
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
index b15fe2576..f74cf74ca 100644
---
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
+++
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
@@ -351,6 +351,7 @@ abstract class SparkWrite implements Write,
RequiresDistributionAndOrdering {
public void commit(WriterCommitMessage[] messages) {
OverwriteFiles overwriteFiles = table.newOverwrite();
overwriteFiles.overwriteByRowFilter(overwriteExpr);
+ overwriteFiles.caseSensitive(writeConf.caseSensitive());
int numFiles = 0;
for (DataFile file : files(messages)) {
@@ -412,6 +413,7 @@ abstract class SparkWrite implements Write,
RequiresDistributionAndOrdering {
@Override
public void commit(WriterCommitMessage[] messages) {
OverwriteFiles overwriteFiles = table.newOverwrite();
+ overwriteFiles.caseSensitive(writeConf.caseSensitive());
List<DataFile> overwrittenFiles = overwrittenFiles();
int numOverwrittenFiles = overwrittenFiles.size();
@@ -589,6 +591,8 @@ abstract class SparkWrite implements Write,
RequiresDistributionAndOrdering {
public void doCommit(long epochId, WriterCommitMessage[] messages) {
OverwriteFiles overwriteFiles = table.newOverwrite();
overwriteFiles.overwriteByRowFilter(Expressions.alwaysTrue());
+ overwriteFiles.caseSensitive(writeConf.caseSensitive());
+
int numFiles = 0;
for (DataFile file : files(messages)) {
overwriteFiles.addFile(file);
```
It's possible that there are a few more places in Spark as I haven't done a
thorough analysis, so it would be good to fix those places as well and
double-check whether there are any other places that need to be updated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
