rdblue commented on a change in pull request #1508:
URL: https://github.com/apache/iceberg/pull/1508#discussion_r711781150
##########
File path:
spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java
##########
@@ -1112,6 +1126,239 @@ public void testPartitionsTable() {
}
}
+ @Test
+ public synchronized void testSnapshotReadAfterAddColumn() {
+ TableIdentifier tableIdentifier = TableIdentifier.of("db", "table");
+ Table table = createTable(tableIdentifier, SCHEMA,
PartitionSpec.unpartitioned());
+
+ List<Row> originalRecords = Lists.newArrayList(
+ RowFactory.create(1, "x"),
+ RowFactory.create(2, "y"),
+ RowFactory.create(3, "z"));
+
+ StructType originalSparkSchema = SparkSchemaUtil.convert(SCHEMA);
+ Dataset<Row> inputDf = spark.createDataFrame(originalRecords,
originalSparkSchema);
+ inputDf.select("id", "data").write()
+ .format("iceberg")
+ .mode(SaveMode.Append)
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+
+ Dataset<Row> resultDf = spark.read()
+ .format("iceberg")
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords = resultDf.orderBy("id")
+ .collectAsList();
+
+ Assert.assertEquals("Records should match", originalRecords,
actualRecords);
+ Snapshot snapshot1 = table.currentSnapshot();
+
+ table.updateSchema().addColumn("category",
Types.StringType.get()).commit();
+
+ List<Row> newRecords = Lists.newArrayList(
+ RowFactory.create(4, "xy", "B"),
+ RowFactory.create(5, "xyz", "C"));
+
+ StructType newSparkSchema = SparkSchemaUtil.convert(SCHEMA2);
+ Dataset<Row> inputDf2 = spark.createDataFrame(newRecords, newSparkSchema);
+ inputDf2.select("id", "data", "category").write()
+ .format("iceberg")
+ .mode(SaveMode.Append)
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+
+ Dataset<Row> resultDf2 = spark.read()
+ .format("iceberg")
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords2 = resultDf2.orderBy("id")
+ .collectAsList();
+
+ List<Row> updatedRecords = Lists.newArrayList(
+ RowFactory.create(1, "x", null),
+ RowFactory.create(2, "y", null),
+ RowFactory.create(3, "z", null),
+ RowFactory.create(4, "xy", "B"),
+ RowFactory.create(5, "xyz", "C"));
+ Assert.assertEquals("Records should match", updatedRecords,
actualRecords2);
+
+ Dataset<Row> resultDf3 = spark.read()
+ .format("iceberg")
+ .option(SparkReadOptions.SNAPSHOT_ID, snapshot1.snapshotId())
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords3 = resultDf3.orderBy("id")
+ .collectAsList();
+
+ Assert.assertEquals("Records should match", originalRecords,
actualRecords3);
+ Assert.assertEquals("Schemas should match", originalSparkSchema,
resultDf3.schema());
+ }
+
+ @Test
+ public synchronized void testSnapshotReadAfterDropColumn() {
+ TableIdentifier tableIdentifier = TableIdentifier.of("db", "table");
+ Table table = createTable(tableIdentifier, SCHEMA2,
PartitionSpec.unpartitioned());
+
+ List<Row> originalRecords = Lists.newArrayList(
+ RowFactory.create(1, "x", "A"),
+ RowFactory.create(2, "y", "A"),
+ RowFactory.create(3, "z", "B"));
+
+ StructType originalSparkSchema = SparkSchemaUtil.convert(SCHEMA2);
+ Dataset<Row> inputDf = spark.createDataFrame(originalRecords,
originalSparkSchema);
+ inputDf.select("id", "data", "category").write()
+ .format("iceberg")
+ .mode(SaveMode.Append)
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+
+ Dataset<Row> resultDf = spark.read()
+ .format("iceberg")
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords = resultDf.orderBy("id")
+ .collectAsList();
+
+ Assert.assertEquals("Records should match", originalRecords,
actualRecords);
+
+ long tsBeforeDropColumn = waitUntilAfter(System.currentTimeMillis());
+ table.updateSchema().deleteColumn("data").commit();
+ long tsAfterDropColumn = waitUntilAfter(System.currentTimeMillis());
+
+ List<Row> newRecords = Lists.newArrayList(
+ RowFactory.create(4, "B"),
+ RowFactory.create(5, "C"));
+
+ StructType newSparkSchema = SparkSchemaUtil.convert(SCHEMA3);
+ Dataset<Row> inputDf2 = spark.createDataFrame(newRecords, newSparkSchema);
+ inputDf2.select("id", "category").write()
+ .format("iceberg")
+ .mode(SaveMode.Append)
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+
+ Dataset<Row> resultDf2 = spark.read()
+ .format("iceberg")
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords2 = resultDf2.orderBy("id")
+ .collectAsList();
+
+ List<Row> updatedRecords = Lists.newArrayList(
+ RowFactory.create(1, "A"),
+ RowFactory.create(2, "A"),
+ RowFactory.create(3, "B"),
+ RowFactory.create(4, "B"),
+ RowFactory.create(5, "C"));
+ Assert.assertEquals("Records should match", updatedRecords,
actualRecords2);
+
+ Dataset<Row> resultDf3 = spark.read()
+ .format("iceberg")
+ .option(SparkReadOptions.AS_OF_TIMESTAMP, tsBeforeDropColumn)
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords3 = resultDf3.orderBy("id")
+ .collectAsList();
+
+ Assert.assertEquals("Records should match", originalRecords,
actualRecords3);
+ Assert.assertEquals("Schemas should match", originalSparkSchema,
resultDf3.schema());
+
+ Dataset<Row> resultDf4 = spark.read()
+ .format("iceberg")
+ .option(SparkReadOptions.AS_OF_TIMESTAMP, tsAfterDropColumn)
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords4 = resultDf4.orderBy("id")
+ .collectAsList();
+
+ // At tsAfterDropColumn, there has been a schema change, but no new
snapshot,
+ // so the snapshot as of tsAfterDropColumn is the same as that as of
tsBeforeDropColumn.
Review comment:
This makes me think that we should only time travel with schema if
`snapshot-id` is set and not `as-of-timestamp` because this behavior is
inaccurate. If you read the table at time `tsAfterDropColumn`, then the result
would be different. That said, I think this is a consequence of a very
reasonable implementation. `as-of-timestamp` selects a snapshot and we read
with the snapshot's schema.
@RussellSpitzer and @yyanyy, what do you think?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
