This is an automated email from the ASF dual-hosted git repository.
vinish pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
The following commit(s) were added to refs/heads/main by this push:
new 5ec6b8d0 Add latest metadata path field in InternalTable (#710)
5ec6b8d0 is described below
commit 5ec6b8d0443ca43945303b2dca40eb14d1439513
Author: Rahil C <[email protected]>
AuthorDate: Fri May 9 13:26:30 2025 -0700
Add latest metadata path field in InternalTable (#710)
* Add latest metadata path field
* ensure field is asserted
---
.../src/main/java/org/apache/xtable/model/InternalTable.java | 2 ++
.../main/java/org/apache/xtable/delta/DeltaTableExtractor.java | 1 +
.../src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java | 1 +
.../java/org/apache/xtable/iceberg/IcebergConversionSource.java | 4 ++++
xtable-core/src/test/java/org/apache/xtable/GenericTable.java | 2 ++
.../src/test/java/org/apache/xtable/TestAbstractHudiTable.java | 4 ++++
xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java | 8 ++++++++
.../src/test/java/org/apache/xtable/TestSparkDeltaTable.java | 5 +++++
.../java/org/apache/xtable/delta/ITDeltaConversionSource.java | 3 +++
.../test/java/org/apache/xtable/hudi/ITHudiConversionSource.java | 1 +
.../java/org/apache/xtable/iceberg/ITIcebergConversionSource.java | 1 +
.../src/test/java/org/apache/xtable/testutil/ITTestUtils.java | 2 ++
12 files changed, 34 insertions(+)
diff --git
a/xtable-api/src/main/java/org/apache/xtable/model/InternalTable.java
b/xtable-api/src/main/java/org/apache/xtable/model/InternalTable.java
index f7678750..8575ee81 100644
--- a/xtable-api/src/main/java/org/apache/xtable/model/InternalTable.java
+++ b/xtable-api/src/main/java/org/apache/xtable/model/InternalTable.java
@@ -50,4 +50,6 @@ public class InternalTable {
List<InternalPartitionField> partitioningFields;
// latest commit(write) on the table.
Instant latestCommitTime;
+ // Path to latest metadata
+ String latestMetdataPath;
}
diff --git
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java
index 4bedf1e4..1929974e 100644
--- a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java
+++ b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java
@@ -62,6 +62,7 @@ public class DeltaTableExtractor {
.partitioningFields(partitionFields)
.readSchema(schema)
.latestCommitTime(Instant.ofEpochMilli(snapshot.timestamp()))
+ .latestMetdataPath(snapshot.deltaLog().logPath().toString())
.build();
}
}
diff --git
a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java
b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java
index a3380b34..dd5996a7 100644
--- a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java
+++ b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java
@@ -87,6 +87,7 @@ public class HudiTableExtractor {
.partitioningFields(partitionFields)
.readSchema(canonicalSchema)
.latestCommitTime(HudiInstantUtils.parseFromInstantTime(commit.getTimestamp()))
+ .latestMetdataPath(metaClient.getMetaPath().toString())
.build();
}
diff --git
a/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergConversionSource.java
b/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergConversionSource.java
index d4a506e7..fe28be0d 100644
---
a/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergConversionSource.java
+++
b/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergConversionSource.java
@@ -35,12 +35,14 @@ import lombok.extern.log4j.Log4j2;
import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseTable;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.Table;
+import org.apache.iceberg.TableOperations;
import org.apache.iceberg.TableScan;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
@@ -107,6 +109,7 @@ public class IcebergConversionSource implements
ConversionSource<Snapshot> {
public InternalTable getTable(Snapshot snapshot) {
Table iceTable = getSourceTable();
Schema iceSchema = iceTable.schemas().get(snapshot.schemaId());
+ TableOperations iceOps = ((BaseTable) iceTable).operations();
IcebergSchemaExtractor schemaExtractor =
IcebergSchemaExtractor.getInstance();
InternalSchema irSchema = schemaExtractor.fromIceberg(iceSchema);
@@ -128,6 +131,7 @@ public class IcebergConversionSource implements
ConversionSource<Snapshot> {
.latestCommitTime(Instant.ofEpochMilli(snapshot.timestampMillis()))
.readSchema(irSchema)
.layoutStrategy(dataLayoutStrategy)
+ .latestMetdataPath(iceOps.current().metadataFileLocation())
.build();
}
diff --git a/xtable-core/src/test/java/org/apache/xtable/GenericTable.java
b/xtable-core/src/test/java/org/apache/xtable/GenericTable.java
index dce0f21a..db7e776b 100644
--- a/xtable-core/src/test/java/org/apache/xtable/GenericTable.java
+++ b/xtable-core/src/test/java/org/apache/xtable/GenericTable.java
@@ -52,6 +52,8 @@ public interface GenericTable<T, Q> extends AutoCloseable {
String getBasePath();
+ String getMetadataPath();
+
default String getDataPath() {
return getBasePath();
}
diff --git
a/xtable-core/src/test/java/org/apache/xtable/TestAbstractHudiTable.java
b/xtable-core/src/test/java/org/apache/xtable/TestAbstractHudiTable.java
index 3e9a133a..252b5b26 100644
--- a/xtable-core/src/test/java/org/apache/xtable/TestAbstractHudiTable.java
+++ b/xtable-core/src/test/java/org/apache/xtable/TestAbstractHudiTable.java
@@ -189,6 +189,10 @@ public abstract class TestAbstractHudiTable
return basePath;
}
+ public String getMetadataPath() {
+ return metaClient.getMetaPath().toString();
+ }
+
protected HoodieRecord<HoodieAvroPayload> getRecord(
Schema schema,
String key,
diff --git a/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java
b/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java
index a912241d..b7b86578 100644
--- a/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java
+++ b/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java
@@ -40,6 +40,7 @@ import lombok.SneakyThrows;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.AppendFiles;
+import org.apache.iceberg.BaseTable;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.OverwriteFiles;
@@ -49,6 +50,7 @@ import org.apache.iceberg.Schema;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.StructLike;
import org.apache.iceberg.Table;
+import org.apache.iceberg.TableOperations;
import org.apache.iceberg.UpdateSchema;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.data.Record;
@@ -249,6 +251,12 @@ public class TestIcebergTable implements
GenericTable<Record, String> {
return removeSlash(basePath) + "/" + tableName;
}
+ @Override
+ public String getMetadataPath() {
+ TableOperations iceOps = ((BaseTable) icebergTable).operations();
+ return iceOps.current().metadataFileLocation();
+ }
+
public String getDataPath() {
return getBasePath() + "/data";
}
diff --git
a/xtable-core/src/test/java/org/apache/xtable/TestSparkDeltaTable.java
b/xtable-core/src/test/java/org/apache/xtable/TestSparkDeltaTable.java
index ee5b1ccd..028eca1b 100644
--- a/xtable-core/src/test/java/org/apache/xtable/TestSparkDeltaTable.java
+++ b/xtable-core/src/test/java/org/apache/xtable/TestSparkDeltaTable.java
@@ -236,6 +236,11 @@ public class TestSparkDeltaTable implements
GenericTable<Row, Object>, Closeable
return basePath;
}
+ @Override
+ public String getMetadataPath() {
+ return deltaLog.logPath().toString();
+ }
+
@Override
public void close() {
// no-op as spark session lifecycle is managed by the caller
diff --git
a/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionSource.java
b/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionSource.java
index b29c7261..0685e919 100644
---
a/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionSource.java
+++
b/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionSource.java
@@ -188,6 +188,7 @@ public class ITDeltaConversionSource {
.build(),
DataLayoutStrategy.FLAT,
"file:" + basePath,
+ snapshot.getTable().getLatestMetdataPath(),
Collections.emptyList());
// Validate data files
List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT,
COL2_COLUMN_STAT);
@@ -244,6 +245,7 @@ public class ITDeltaConversionSource {
.build(),
DataLayoutStrategy.FLAT,
"file:" + basePath,
+ internalTable.getLatestMetdataPath(),
Collections.emptyList());
}
@@ -295,6 +297,7 @@ public class ITDeltaConversionSource {
.build(),
DataLayoutStrategy.HIVE_STYLE_PARTITION,
"file:" + basePath,
+ snapshot.getTable().getLatestMetdataPath(),
Collections.singletonList(
InternalPartitionField.builder()
.sourceField(partCol)
diff --git
a/xtable-core/src/test/java/org/apache/xtable/hudi/ITHudiConversionSource.java
b/xtable-core/src/test/java/org/apache/xtable/hudi/ITHudiConversionSource.java
index 376cceda..6b6349cc 100644
---
a/xtable-core/src/test/java/org/apache/xtable/hudi/ITHudiConversionSource.java
+++
b/xtable-core/src/test/java/org/apache/xtable/hudi/ITHudiConversionSource.java
@@ -219,6 +219,7 @@ public class ITHudiConversionSource {
internalSchema,
DataLayoutStrategy.FLAT,
"file:" + basePath + "_v1",
+ internalTable.getLatestMetdataPath(),
Collections.emptyList());
}
}
diff --git
a/xtable-core/src/test/java/org/apache/xtable/iceberg/ITIcebergConversionSource.java
b/xtable-core/src/test/java/org/apache/xtable/iceberg/ITIcebergConversionSource.java
index 20026cb5..76edf306 100644
---
a/xtable-core/src/test/java/org/apache/xtable/iceberg/ITIcebergConversionSource.java
+++
b/xtable-core/src/test/java/org/apache/xtable/iceberg/ITIcebergConversionSource.java
@@ -131,6 +131,7 @@ public class ITIcebergConversionSource {
internalSchema,
DataLayoutStrategy.FLAT,
testIcebergTable.getBasePath(),
+ testIcebergTable.getMetadataPath(),
Collections.emptyList());
}
}
diff --git
a/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java
b/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java
index 75c3833a..4b1dac84 100644
--- a/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java
+++ b/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java
@@ -46,12 +46,14 @@ public class ITTestUtils {
InternalSchema readSchema,
DataLayoutStrategy dataLayoutStrategy,
String basePath,
+ String latestMetadataPath,
List<InternalPartitionField> partitioningFields) {
Assertions.assertEquals(tableName, internalTable.getName());
Assertions.assertEquals(tableFormat, internalTable.getTableFormat());
Assertions.assertEquals(readSchema, internalTable.getReadSchema());
Assertions.assertEquals(dataLayoutStrategy,
internalTable.getLayoutStrategy());
Assertions.assertEquals(basePath, internalTable.getBasePath());
+ Assertions.assertEquals(latestMetadataPath,
internalTable.getLatestMetdataPath());
Assertions.assertEquals(partitioningFields,
internalTable.getPartitioningFields());
}
