zachdisc commented on code in PR #12840:
URL: https://github.com/apache/iceberg/pull/12840#discussion_r2279820485
##########
spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java:
##########
@@ -531,6 +534,115 @@ public void testRewriteLargeManifestsPartitionedTable()
throws IOException {
assertThat(newManifests).hasSizeGreaterThanOrEqualTo(2);
}
+ @TestTemplate
+ public void testRewriteManifestsPartitionedTableWithInvalidSortingColumns()
throws IOException {
+ PartitionSpec spec =
PartitionSpec.builderFor(SCHEMA).identity("c1").bucket("c3", 10).build();
+ Map<String, String> options = Maps.newHashMap();
+ options.put(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion));
+ options.put(TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED,
snapshotIdInheritanceEnabled);
+ Table table = TABLES.create(SCHEMA, spec, options, tableLocation);
+
+ SparkActions actions = org.apache.iceberg.spark.actions.SparkActions.get();
+
+ List<String> nonexistentFields = ImmutableList.of("c1", "c2");
+ assertThatThrownBy(
+ () ->
+ actions
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .sortBy(nonexistentFields)
+ .execute())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot set manifest sorting because specified field(s) [c2] were
not found in "
+ + "current partition spec [\n"
+ + " 1000: c1: identity(1)\n"
+ + " 1001: c3_bucket: bucket[10](3)\n"
+ + "]. Spec ID 0");
+
+ // c3_bucket is the correct internal partition name to use, c3 is the
untransformed column name,
+ // sortBy() expects the hidden partition column names
+ List<String> hasIncorrectPartitionFieldNames = ImmutableList.of("c1",
"c3");
+ assertThatThrownBy(
+ () ->
+ actions
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .sortBy(hasIncorrectPartitionFieldNames)
+ .execute())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot set manifest sorting because specified field(s) [c3] were
not found in "
+ + "current partition spec [\n"
+ + " 1000: c1: identity(1)\n"
+ + " 1001: c3_bucket: bucket[10](3)\n"
+ + "]. Spec ID 0");
+ }
+
+ @TestTemplate
+ public void testRewriteManifestsPartitionedTableWithCustomSorting() throws
IOException {
+ Random random = new Random(4141912);
+
+ PartitionSpec spec =
+ PartitionSpec.builderFor(SCHEMA).identity("c1").truncate("c2",
3).bucket("c3", 10).build();
+ Table table = TABLES.create(SCHEMA, spec, tableLocation);
+
+ // write a large number of random records so the rewrite will split into
multiple manifests
+ List<DataFile> dataFiles = Lists.newArrayList();
+ for (int i = 0; i < 1000; i++) {
+ dataFiles.add(
+ newDataFile(
+ table,
+ TestHelpers.Row.of(i, String.valueOf(random.nextInt() * 100),
random.nextInt(10))));
+ }
+ ManifestFile appendManifest = writeManifest(table, dataFiles);
+ table.newFastAppend().appendManifest(appendManifest).commit();
+
+ // force manifest splitting
+ table
+ .updateProperties()
+ .set(
+ TableProperties.MANIFEST_TARGET_SIZE_BYTES,
String.valueOf(appendManifest.length() / 2))
+ .commit();
+
+ List<String> clusterKeys = ImmutableList.of("c3_bucket", "c2_trunc", "c1");
+ SparkActions actions = SparkActions.get();
+ actions
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .sortBy(clusterKeys)
+ .option(RewriteManifestsSparkAction.USE_CACHING, useCaching)
+ .execute();
+
+ table.refresh();
+
+ // Read the manifests metadata table. The partition_summaries column
contains
+ // an array of structs with lower_bound and upper_bound as strings
+ Dataset<Row> manifestsDf =
spark.read().format("iceberg").load(tableLocation + "#manifests");
+
+ Dataset<Row> entries = spark.read().format("iceberg").load(tableLocation
+ "#entries");
Review Comment:
Good call. Will remove.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
