aokolnychyi commented on code in PR #15374:
URL: https://github.com/apache/iceberg/pull/15374#discussion_r2831374094
##########
spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java:
##########
@@ -44,182 +48,181 @@
import org.apache.spark.sql.connector.write.WriteBuilder;
import org.apache.spark.sql.connector.write.streaming.StreamingWrite;
import org.apache.spark.sql.sources.Filter;
-import org.apache.spark.sql.types.LongType$;
+import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
class SparkWriteBuilder implements WriteBuilder, SupportsDynamicOverwrite,
SupportsOverwrite {
private final SparkSession spark;
private final Table table;
private final SparkWriteConf writeConf;
- private final LogicalWriteInfo writeInfo;
- private final StructType dsSchema;
- private final String overwriteMode;
- private boolean overwriteDynamic = false;
- private boolean overwriteByFilter = false;
- private Expression overwriteExpr = null;
- private boolean overwriteFiles = false;
- private SparkCopyOnWriteScan copyOnWriteScan = null;
- private Command copyOnWriteCommand = null;
- private IsolationLevel copyOnWriteIsolationLevel = null;
+ private final LogicalWriteInfo info;
+ private final boolean caseSensitive;
+ private final boolean checkNullability;
+ private final boolean checkOrdering;
+ private final boolean mergeSchema;
+ private Mode mode = null;
SparkWriteBuilder(SparkSession spark, Table table, String branch,
LogicalWriteInfo info) {
this.spark = spark;
this.table = table;
this.writeConf = new SparkWriteConf(spark, table, branch, info.options());
- this.writeInfo = info;
- this.dsSchema = info.schema();
- this.overwriteMode = writeConf.overwriteMode();
+ this.info = info;
+ this.caseSensitive = writeConf.caseSensitive();
+ this.checkNullability = writeConf.checkNullability();
+ this.checkOrdering = writeConf.checkOrdering();
+ this.mergeSchema = writeConf.mergeSchema();
}
public WriteBuilder overwriteFiles(Scan scan, Command command,
IsolationLevel isolationLevel) {
- Preconditions.checkState(!overwriteByFilter, "Cannot overwrite individual
files and by filter");
Review Comment:
We started simple but then added more modes. Relying on different boolean
flags and validating their combinations is tricky.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
