danielcweeks commented on code in PR #17434:
URL: https://github.com/apache/iceberg/pull/17434#discussion_r3686222044
##########
core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java:
##########
@@ -723,38 +734,192 @@ public void unknownManifestFormatThrows() throws
IOException {
InputFile badFile =
fileIO.newInputFile(tempDir.resolve("manifest-" + System.nanoTime() +
".txt").toString());
- try (V4ManifestReader reader = V4ManifestReader.builder(badFile,
UNPARTITIONED_SPECS).build()) {
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(badFile, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
assertThatThrownBy(reader::iterator)
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("Cannot determine format of manifest");
}
}
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void resolvesRelativeDataFileLocation(FileFormat format) throws
IOException {
+ TrackedFile file = dataFile("data/00000-0.parquet", EMPTY_PARTITION_DATA);
+
+ InputFile manifest = writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(file));
+
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
+ TrackedFile actual = Iterables.getOnlyElement(reader);
+ assertThat(actual.location()).isEqualTo(TABLE_LOCATION +
"/data/00000-0.parquet");
+ }
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void absoluteDataFileLocationIsUnchanged(FileFormat format) throws
IOException {
+ TrackedFile file = dataFile("hdfs://wh/db/table/data/00000-0.parquet",
EMPTY_PARTITION_DATA);
+
+ InputFile manifest = writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(file));
+
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
+ TrackedFile actual = Iterables.getOnlyElement(reader);
+
assertThat(actual.location()).isEqualTo("hdfs://wh/db/table/data/00000-0.parquet");
+ }
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void resolvesRelativeDeletionVectorLocation(FileFormat format) throws
IOException {
+ TrackedFile file = dataFile("data/00000-0.parquet", EMPTY_PARTITION_DATA,
dv("data/dv.puffin"));
+
+ InputFile manifest = writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(file));
+
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
+ TrackedFile actual = Iterables.getOnlyElement(reader);
+ assertThat(actual.location()).isEqualTo(TABLE_LOCATION +
"/data/00000-0.parquet");
+ assertThat(actual.deletionVector().location()).isEqualTo(TABLE_LOCATION
+ "/data/dv.puffin");
+ }
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void resolvesLeafManifestLocation(FileFormat format) throws
IOException {
+ TrackedFile leaf = manifestRef(FileContent.DATA_MANIFEST,
"metadata/leaf.avro");
+
+ InputFile manifest = writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(leaf));
+
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
+ TrackedFile actual = Iterables.getOnlyElement(reader);
+ assertThat(actual.location()).isEqualTo(TABLE_LOCATION +
"/metadata/leaf.avro");
+ }
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void resolvesDataFileAndDvSchemesIndependently(FileFormat format)
throws IOException {
+ // absolute data file paired with a relative DV, and relative data file
paired with an absolute
+ // DV: each location's scheme is evaluated on its own
+ TrackedFile absoluteFileRelativeDv =
+ dataFile("s3://other/abs.parquet", EMPTY_PARTITION_DATA,
dv("data/dv.puffin"));
+ TrackedFile relativeFileAbsoluteDv =
+ dataFile("data/rel.parquet", EMPTY_PARTITION_DATA,
dv("s3://other/abs-dv.puffin"));
+
+ InputFile manifest =
+ writeManifest(
+ format,
+ EMPTY_PARTITION,
+ ImmutableList.of(absoluteFileRelativeDv, relativeFileAbsoluteDv));
+
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
+ List<TrackedFile> actual = Lists.newArrayList(reader);
+ assertThat(actual.get(0).location()).isEqualTo("s3://other/abs.parquet");
+ assertThat(actual.get(0).deletionVector().location())
+ .isEqualTo(TABLE_LOCATION + "/data/dv.puffin");
+ assertThat(actual.get(1).location()).isEqualTo(TABLE_LOCATION +
"/data/rel.parquet");
+
assertThat(actual.get(1).deletionVector().location()).isEqualTo("s3://other/abs-dv.puffin");
+ }
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void stripsTrailingSlashFromTableLocation(FileFormat format) throws
IOException {
Review Comment:
It would be good to add a test that shows preserving non-standard paths
(e.g. datafile with `/` or `//` at the beginning).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
