anoopj commented on code in PR #17434:
URL: https://github.com/apache/iceberg/pull/17434#discussion_r3686383190
##########
core/src/main/java/org/apache/iceberg/V4ManifestReader.java:
##########
@@ -146,6 +151,22 @@ private TrackedFile prepare(TrackedFile trackedFile) {
return trackedFile;
}
+ // resolves stored locations against the table location
+ private TrackedFile copyResolved(TrackedFile trackedFile) {
+ TrackedFileStruct copy = (TrackedFileStruct) trackedFile.copy();
+ if (copy.location() != null) {
+ copy.setLocation(LocationUtil.resolveLocation(tableLocation,
copy.location()));
Review Comment:
We are already doing a copy, and this PR doesn't make anything worse. Yes,
we can always change this in future to make this lazy/zero copy.
##########
core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java:
##########
@@ -723,38 +734,192 @@ public void unknownManifestFormatThrows() throws
IOException {
InputFile badFile =
fileIO.newInputFile(tempDir.resolve("manifest-" + System.nanoTime() +
".txt").toString());
- try (V4ManifestReader reader = V4ManifestReader.builder(badFile,
UNPARTITIONED_SPECS).build()) {
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(badFile, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
assertThatThrownBy(reader::iterator)
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("Cannot determine format of manifest");
}
}
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void resolvesRelativeDataFileLocation(FileFormat format) throws
IOException {
+ TrackedFile file = dataFile("data/00000-0.parquet", EMPTY_PARTITION_DATA);
+
+ InputFile manifest = writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(file));
+
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
+ TrackedFile actual = Iterables.getOnlyElement(reader);
+ assertThat(actual.location()).isEqualTo(TABLE_LOCATION +
"/data/00000-0.parquet");
+ }
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void absoluteDataFileLocationIsUnchanged(FileFormat format) throws
IOException {
+ TrackedFile file = dataFile("hdfs://wh/db/table/data/00000-0.parquet",
EMPTY_PARTITION_DATA);
+
+ InputFile manifest = writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(file));
+
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
+ TrackedFile actual = Iterables.getOnlyElement(reader);
+
assertThat(actual.location()).isEqualTo("hdfs://wh/db/table/data/00000-0.parquet");
+ }
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void resolvesRelativeDeletionVectorLocation(FileFormat format) throws
IOException {
+ TrackedFile file = dataFile("data/00000-0.parquet", EMPTY_PARTITION_DATA,
dv("data/dv.puffin"));
+
+ InputFile manifest = writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(file));
+
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
+ TrackedFile actual = Iterables.getOnlyElement(reader);
+ assertThat(actual.location()).isEqualTo(TABLE_LOCATION +
"/data/00000-0.parquet");
+ assertThat(actual.deletionVector().location()).isEqualTo(TABLE_LOCATION
+ "/data/dv.puffin");
+ }
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void resolvesLeafManifestLocation(FileFormat format) throws
IOException {
+ TrackedFile leaf = manifestRef(FileContent.DATA_MANIFEST,
"metadata/leaf.avro");
+
+ InputFile manifest = writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(leaf));
+
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
+ TrackedFile actual = Iterables.getOnlyElement(reader);
+ assertThat(actual.location()).isEqualTo(TABLE_LOCATION +
"/metadata/leaf.avro");
+ }
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void resolvesDataFileAndDvSchemesIndependently(FileFormat format)
throws IOException {
+ // absolute data file paired with a relative DV, and relative data file
paired with an absolute
+ // DV: each location's scheme is evaluated on its own
+ TrackedFile absoluteFileRelativeDv =
+ dataFile("s3://other/abs.parquet", EMPTY_PARTITION_DATA,
dv("data/dv.puffin"));
+ TrackedFile relativeFileAbsoluteDv =
+ dataFile("data/rel.parquet", EMPTY_PARTITION_DATA,
dv("s3://other/abs-dv.puffin"));
+
+ InputFile manifest =
+ writeManifest(
+ format,
+ EMPTY_PARTITION,
+ ImmutableList.of(absoluteFileRelativeDv, relativeFileAbsoluteDv));
+
+ try (V4ManifestReader reader =
+ V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
+ List<TrackedFile> actual = Lists.newArrayList(reader);
+ assertThat(actual.get(0).location()).isEqualTo("s3://other/abs.parquet");
+ assertThat(actual.get(0).deletionVector().location())
+ .isEqualTo(TABLE_LOCATION + "/data/dv.puffin");
+ assertThat(actual.get(1).location()).isEqualTo(TABLE_LOCATION +
"/data/rel.parquet");
+
assertThat(actual.get(1).deletionVector().location()).isEqualTo("s3://other/abs-dv.puffin");
+ }
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void stripsTrailingSlashFromTableLocation(FileFormat format) throws
IOException {
Review Comment:
Added `preservesNonStandardDataFileLocation()` to test this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
