chamikaramj commented on code in PR #39600: URL: https://github.com/apache/beam/pull/39600#discussion_r3718246411
########## sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/ResolveChanges.java: ########## @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.beam.sdk.io.iceberg.IcebergScanConfig; +import org.apache.beam.sdk.io.iceberg.IcebergUtils; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.join.CoGbkResult; +import org.apache.beam.sdk.util.RowFilter; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.iceberg.types.Types; +import org.joda.time.Instant; + +/** + * Receives a {@link CoGbkResult} containing inserts and deletes sharing the same snapshot sequence + * number and Primary Key, and uses {@link CdcResolver} to identify logical updates. + */ +class ResolveChanges extends DoFn<KV<CdcRowDescriptor, CoGbkResult>, Row> { + static final TupleTag<Row> DELETES = new TupleTag<>() {}; + static final TupleTag<Row> INSERTS = new TupleTag<>() {}; + private final IcebergScanConfig scanConfig; + private final RowFilter rowFilter; + private final Schema outputSchema; + + ResolveChanges(IcebergScanConfig scanConfig) { + this.scanConfig = scanConfig; + this.rowFilter = + new RowFilter( + CdcOutputUtils.readBeamSchemaWithRowMetadata( + scanConfig.getMetadataColumns(), scanConfig.getSchema())) + .keep( + CdcOutputUtils.readSchemaWithRowMetadata( + scanConfig.getMetadataColumns(), scanConfig.getProjectedSchema()) + .columns().stream() + .map(Types.NestedField::name) + .collect(Collectors.toList())); + this.outputSchema = + CdcOutputUtils.outputSchema( + scanConfig, + IcebergUtils.icebergSchemaToBeamSchema( + scanConfig.getProjectedSchema(), scanConfig.getUpdateCompatibilityVersion())); + } + + @ProcessElement + public void processElement( + @Element KV<CdcRowDescriptor, CoGbkResult> element, + @Timestamp Instant timestamp, + OutputReceiver<Row> out) { + CdcRowDescriptor descriptor = element.getKey(); + + Set<String> pkFields = new HashSet<>(descriptor.getPrimaryKey().getSchema().getFieldNames()); + CoGbkResult result = element.getValue(); + + // should be okay to materialize these lists. a PK collision will likely be a handful of records + // at most + List<Row> deletes = Lists.newArrayList(result.getAll(DELETES)); + List<Row> inserts = Lists.newArrayList(result.getAll(INSERTS)); + + new RowResolver(pkFields, scanConfig.getMetadataColumns()) + .resolve( + deletes, + inserts, + (kind, row) -> { + Row projectedRow = rowFilter.filter(row); + out.builder( + CdcOutputUtils.outputRow( + scanConfig.getMetadataColumns(), + outputSchema, + descriptor.getCommitSnapshotId(), + descriptor.getSnapshotSequenceNumber(), + kind, + projectedRow)) + .setValueKind(kind) + .setTimestamp(timestamp) + .output(); + }); + } + + private static final class RowResolver extends CdcResolver<Row> { + private final Set<String> pkFields; + private final List<String> metadataColumns; + + RowResolver(Set<String> pkFields, List<String> metadataColumns) { + this.pkFields = pkFields; + this.metadataColumns = metadataColumns; + } + + @Override + protected int nonPkHash(Row element) { Review Comment: How expensive is this ? Seems like we are hashing pretty much all fields here of all elements ? (not a blocker) ########## sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/IncrementalChangelogSource.java: ########## @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc; + +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.LARGE_BIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.SMALL_BIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.UNIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ResolveChanges.DELETES; +import static org.apache.beam.sdk.io.iceberg.cdc.ResolveChanges.INSERTS; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + +import java.util.List; +import java.util.stream.Collectors; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.io.iceberg.IcebergScanConfig; +import org.apache.beam.sdk.io.iceberg.IcebergUtils; +import org.apache.beam.sdk.io.iceberg.ReadUtils; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.Flatten; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.Redistribute; +import org.apache.beam.sdk.transforms.join.CoGroupByKey; +import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple; +import org.apache.beam.sdk.transforms.windowing.AfterWatermark; +import org.apache.beam.sdk.transforms.windowing.DefaultTrigger; +import org.apache.beam.sdk.transforms.windowing.GlobalWindows; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PBegin; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionList; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.iceberg.Table; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.joda.time.Instant; + +/** + * An Iceberg source that incrementally reads a table's changelogs, processing one snapshot at a + * time. + * + * <p>Each snapshot is resolved independently. For a given primary key, this source emits the <b>net + * change</b> the snapshot produces, and not its intermediate states. + * + * <p>Implications: if a writer batches several transitions for the same PK into one snapshot (for + * example {@code A → B, then B → C}), only the endpoints survive. The intermediate {@code B} is + * dropped. A round-trip within a single snapshot (for example {@code A → B → A}) is also dropped. + * + * <p>The streaming path uses {@link WatchForSnapshotsSdf} for proper per-snapshot watermarks. The + * bounded path creates the snapshot range up front. + */ +public class IncrementalChangelogSource extends PTransform<PBegin, PCollection<Row>> { + private final IcebergScanConfig scanConfig; + + public IncrementalChangelogSource(IcebergScanConfig scanConfig) { + this.scanConfig = scanConfig; + } + + @Override + public PCollection<Row> expand(PBegin input) { + // emit one SnapshotInfo per element, with element timestamp -> snapshot commit time. + PCollection<Long> snapshots = + MoreObjects.firstNonNull(scanConfig.getStreaming(), false) + ? unboundedSnapshots(input) + : boundedSnapshots(input); + + // process one snapshot at a time and produce batches of changelog scan tasks. + // tasks are emitted to three outputs: + // 1. unidirectional tasks: we know these won't have any updates Review Comment: How about deletes ? Did you mean "these only contain inserts" ? ########## sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/IncrementalChangelogSource.java: ########## @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc; + +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.LARGE_BIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.SMALL_BIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.UNIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ResolveChanges.DELETES; +import static org.apache.beam.sdk.io.iceberg.cdc.ResolveChanges.INSERTS; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + +import java.util.List; +import java.util.stream.Collectors; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.io.iceberg.IcebergScanConfig; +import org.apache.beam.sdk.io.iceberg.IcebergUtils; +import org.apache.beam.sdk.io.iceberg.ReadUtils; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.Flatten; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.Redistribute; +import org.apache.beam.sdk.transforms.join.CoGroupByKey; +import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple; +import org.apache.beam.sdk.transforms.windowing.AfterWatermark; +import org.apache.beam.sdk.transforms.windowing.DefaultTrigger; +import org.apache.beam.sdk.transforms.windowing.GlobalWindows; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PBegin; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionList; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.iceberg.Table; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.joda.time.Instant; + +/** + * An Iceberg source that incrementally reads a table's changelogs, processing one snapshot at a + * time. + * + * <p>Each snapshot is resolved independently. For a given primary key, this source emits the <b>net + * change</b> the snapshot produces, and not its intermediate states. + * + * <p>Implications: if a writer batches several transitions for the same PK into one snapshot (for + * example {@code A → B, then B → C}), only the endpoints survive. The intermediate {@code B} is + * dropped. A round-trip within a single snapshot (for example {@code A → B → A}) is also dropped. + * + * <p>The streaming path uses {@link WatchForSnapshotsSdf} for proper per-snapshot watermarks. The + * bounded path creates the snapshot range up front. + */ +public class IncrementalChangelogSource extends PTransform<PBegin, PCollection<Row>> { + private final IcebergScanConfig scanConfig; + + public IncrementalChangelogSource(IcebergScanConfig scanConfig) { + this.scanConfig = scanConfig; + } + + @Override + public PCollection<Row> expand(PBegin input) { + // emit one SnapshotInfo per element, with element timestamp -> snapshot commit time. + PCollection<Long> snapshots = + MoreObjects.firstNonNull(scanConfig.getStreaming(), false) + ? unboundedSnapshots(input) + : boundedSnapshots(input); + + // process one snapshot at a time and produce batches of changelog scan tasks. + // tasks are emitted to three outputs: + // 1. unidirectional tasks: we know these won't have any updates + // 2. small bidirectional tasks: these may contain an update, but the batch is small enough to + // resolve in-memory + // 2. large bidirectional tasks: may contain an update, but are too large for in-memory + // resolution. will + // need to run these output rows through a CoGBK + PCollectionTuple changelogTasks = + snapshots.apply( + "Create Changelog Tasks", + ParDo.of(new ChangelogScanner(scanConfig)) + .withOutputTags( + UNIDIRECTIONAL_TASKS, + TupleTagList.of(LARGE_BIDIRECTIONAL_TASKS).and(SMALL_BIDIRECTIONAL_TASKS))); + KvCoder<ChangelogDescriptor, List<SerializableChangelogTask>> tasksCoder = + ChangelogScanner.coder(scanConfig.rowIdBeamSchema()); + changelogTasks.get(UNIDIRECTIONAL_TASKS).setCoder(tasksCoder); + changelogTasks.get(SMALL_BIDIRECTIONAL_TASKS).setCoder(tasksCoder); + changelogTasks.get(LARGE_BIDIRECTIONAL_TASKS).setCoder(tasksCoder); + + Schema projectedRowSchema = + IcebergUtils.icebergSchemaToBeamSchema( + scanConfig.getProjectedSchema(), scanConfig.getUpdateCompatibilityVersion()); + Schema outputRowSchema = CdcOutputUtils.outputSchema(scanConfig, projectedRowSchema); + + // reads UNIDIRECTIONAL and BIDIRECTIONAL tags and produces rows. + ReadFromChangelogs.Output outputRows = changelogTasks.apply(new ReadFromChangelogs(scanConfig)); + + // Small overlapping groups get resolved entirely in memory with no shuffle. + PCollection<Row> smallBidirectionalCdcRows = + changelogTasks + .get(SMALL_BIDIRECTIONAL_TASKS) + .apply("Redistribute Small Bidirectional Changes", Redistribute.arbitrarily()) Review Comment: Seems like we have a shuffle here ? ########## sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/IncrementalChangelogSource.java: ########## @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc; + +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.LARGE_BIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.SMALL_BIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.UNIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ResolveChanges.DELETES; +import static org.apache.beam.sdk.io.iceberg.cdc.ResolveChanges.INSERTS; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + +import java.util.List; +import java.util.stream.Collectors; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.io.iceberg.IcebergScanConfig; +import org.apache.beam.sdk.io.iceberg.IcebergUtils; +import org.apache.beam.sdk.io.iceberg.ReadUtils; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.Flatten; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.Redistribute; +import org.apache.beam.sdk.transforms.join.CoGroupByKey; +import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple; +import org.apache.beam.sdk.transforms.windowing.AfterWatermark; +import org.apache.beam.sdk.transforms.windowing.DefaultTrigger; +import org.apache.beam.sdk.transforms.windowing.GlobalWindows; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PBegin; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionList; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.iceberg.Table; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.joda.time.Instant; + +/** + * An Iceberg source that incrementally reads a table's changelogs, processing one snapshot at a + * time. + * + * <p>Each snapshot is resolved independently. For a given primary key, this source emits the <b>net + * change</b> the snapshot produces, and not its intermediate states. + * + * <p>Implications: if a writer batches several transitions for the same PK into one snapshot (for + * example {@code A → B, then B → C}), only the endpoints survive. The intermediate {@code B} is + * dropped. A round-trip within a single snapshot (for example {@code A → B → A}) is also dropped. + * + * <p>The streaming path uses {@link WatchForSnapshotsSdf} for proper per-snapshot watermarks. The + * bounded path creates the snapshot range up front. + */ +public class IncrementalChangelogSource extends PTransform<PBegin, PCollection<Row>> { + private final IcebergScanConfig scanConfig; + + public IncrementalChangelogSource(IcebergScanConfig scanConfig) { + this.scanConfig = scanConfig; + } + + @Override + public PCollection<Row> expand(PBegin input) { + // emit one SnapshotInfo per element, with element timestamp -> snapshot commit time. + PCollection<Long> snapshots = + MoreObjects.firstNonNull(scanConfig.getStreaming(), false) + ? unboundedSnapshots(input) + : boundedSnapshots(input); + + // process one snapshot at a time and produce batches of changelog scan tasks. + // tasks are emitted to three outputs: + // 1. unidirectional tasks: we know these won't have any updates + // 2. small bidirectional tasks: these may contain an update, but the batch is small enough to + // resolve in-memory + // 2. large bidirectional tasks: may contain an update, but are too large for in-memory + // resolution. will + // need to run these output rows through a CoGBK + PCollectionTuple changelogTasks = + snapshots.apply( + "Create Changelog Tasks", + ParDo.of(new ChangelogScanner(scanConfig)) + .withOutputTags( + UNIDIRECTIONAL_TASKS, + TupleTagList.of(LARGE_BIDIRECTIONAL_TASKS).and(SMALL_BIDIRECTIONAL_TASKS))); + KvCoder<ChangelogDescriptor, List<SerializableChangelogTask>> tasksCoder = + ChangelogScanner.coder(scanConfig.rowIdBeamSchema()); + changelogTasks.get(UNIDIRECTIONAL_TASKS).setCoder(tasksCoder); + changelogTasks.get(SMALL_BIDIRECTIONAL_TASKS).setCoder(tasksCoder); + changelogTasks.get(LARGE_BIDIRECTIONAL_TASKS).setCoder(tasksCoder); + + Schema projectedRowSchema = + IcebergUtils.icebergSchemaToBeamSchema( + scanConfig.getProjectedSchema(), scanConfig.getUpdateCompatibilityVersion()); + Schema outputRowSchema = CdcOutputUtils.outputSchema(scanConfig, projectedRowSchema); + + // reads UNIDIRECTIONAL and BIDIRECTIONAL tags and produces rows. + ReadFromChangelogs.Output outputRows = changelogTasks.apply(new ReadFromChangelogs(scanConfig)); + + // Small overlapping groups get resolved entirely in memory with no shuffle. + PCollection<Row> smallBidirectionalCdcRows = + changelogTasks + .get(SMALL_BIDIRECTIONAL_TASKS) + .apply("Redistribute Small Bidirectional Changes", Redistribute.arbitrarily()) + .apply("Resolve Locally", ParDo.of(new LocalResolveDoFn(scanConfig))) + .setRowSchema(outputRowSchema); + + // BIDIRECTIONAL records go through a CoGBK and ResolveChanges + // We window locally using a custom WindowFn based on the snapsot's commit time. Each snapshot + // exists in its own window. + // We re-window the resolved output back to GlobalWindows before the final Flatten + // to align with the other branches. + Window<KV<CdcRowDescriptor, Row>> keyedWindowing = + Window.<KV<CdcRowDescriptor, Row>>into(new SnapshotWindowFn()) + .triggering(AfterWatermark.pastEndOfWindow()) + .withAllowedLateness(Duration.ZERO) + .discardingFiredPanes(); + PCollection<KV<CdcRowDescriptor, Row>> keyedInserts = + outputRows.biDirectionalInserts().apply("Window Inserts", keyedWindowing); + PCollection<KV<CdcRowDescriptor, Row>> keyedDeletes = + outputRows.biDirectionalDeletes().apply("Window Deletes", keyedWindowing); + PCollection<Row> biDirectionalCdcRows = + KeyedPCollectionTuple.of(INSERTS, keyedInserts) + .and(DELETES, keyedDeletes) + .apply("CoGroupBy Primary Key", CoGroupByKey.create()) + .apply("Resolve Delete-Insert Pairs", ParDo.of(new ResolveChanges(scanConfig))) Review Comment: I suspect this is something we can do at the framework level later right ? (Without having to do this kind of optimizations per source) ########## sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java: ########## @@ -693,6 +712,35 @@ public ReadRows withFilter(@Nullable String filter) { return toBuilder().setFilter(filter).build(); } + public ReadRows withWatermarkColumn(@Nullable String watermarkColumn) { + return toBuilder().setWatermarkColumn(watermarkColumn).build(); + } + + public ReadRows withWatermarkColumnTimeUnit(@Nullable String timeUnit) { + return toBuilder().setWatermarkColumnTimeUnit(timeUnit).build(); + } + + public ReadRows withMaxSnapshotDiscoveryDelay(@Nullable Duration delay) { + return toBuilder().setMaxSnapshotDiscoveryDelay(delay).build(); + } + + /** + * Appends top-level metadata columns to CDC output rows. + * + * <p>Supported values are {@code _change_type}, {@code _commit_snapshot_id}, {@code + * _commit_snapshot_sequence_number}, {@code _row_id}, and {@code + * _last_updated_sequence_number}. The row metadata columns are read from Iceberg data files and + * require a row-lineage table. The changelog metadata columns come from the emitted change kind + * and snapshot context and are appended when final Beam rows are emitted. + * + * <p>This option is only valid {@link #withCdc()}. Review Comment: Let's fail if the incorrect combination is set. ########## sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/IncrementalChangelogSource.java: ########## @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc; + +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.LARGE_BIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.SMALL_BIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ChangelogScanner.UNIDIRECTIONAL_TASKS; +import static org.apache.beam.sdk.io.iceberg.cdc.ResolveChanges.DELETES; +import static org.apache.beam.sdk.io.iceberg.cdc.ResolveChanges.INSERTS; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + +import java.util.List; +import java.util.stream.Collectors; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.io.iceberg.IcebergScanConfig; +import org.apache.beam.sdk.io.iceberg.IcebergUtils; +import org.apache.beam.sdk.io.iceberg.ReadUtils; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.Flatten; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.Redistribute; +import org.apache.beam.sdk.transforms.join.CoGroupByKey; +import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple; +import org.apache.beam.sdk.transforms.windowing.AfterWatermark; +import org.apache.beam.sdk.transforms.windowing.DefaultTrigger; +import org.apache.beam.sdk.transforms.windowing.GlobalWindows; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PBegin; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionList; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.iceberg.Table; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.joda.time.Instant; + +/** + * An Iceberg source that incrementally reads a table's changelogs, processing one snapshot at a + * time. + * + * <p>Each snapshot is resolved independently. For a given primary key, this source emits the <b>net + * change</b> the snapshot produces, and not its intermediate states. + * + * <p>Implications: if a writer batches several transitions for the same PK into one snapshot (for + * example {@code A → B, then B → C}), only the endpoints survive. The intermediate {@code B} is + * dropped. A round-trip within a single snapshot (for example {@code A → B → A}) is also dropped. + * + * <p>The streaming path uses {@link WatchForSnapshotsSdf} for proper per-snapshot watermarks. The + * bounded path creates the snapshot range up front. + */ +public class IncrementalChangelogSource extends PTransform<PBegin, PCollection<Row>> { + private final IcebergScanConfig scanConfig; + + public IncrementalChangelogSource(IcebergScanConfig scanConfig) { + this.scanConfig = scanConfig; + } + + @Override + public PCollection<Row> expand(PBegin input) { + // emit one SnapshotInfo per element, with element timestamp -> snapshot commit time. + PCollection<Long> snapshots = + MoreObjects.firstNonNull(scanConfig.getStreaming(), false) + ? unboundedSnapshots(input) + : boundedSnapshots(input); + + // process one snapshot at a time and produce batches of changelog scan tasks. + // tasks are emitted to three outputs: + // 1. unidirectional tasks: we know these won't have any updates + // 2. small bidirectional tasks: these may contain an update, but the batch is small enough to + // resolve in-memory + // 2. large bidirectional tasks: may contain an update, but are too large for in-memory + // resolution. will + // need to run these output rows through a CoGBK + PCollectionTuple changelogTasks = + snapshots.apply( + "Create Changelog Tasks", + ParDo.of(new ChangelogScanner(scanConfig)) + .withOutputTags( + UNIDIRECTIONAL_TASKS, + TupleTagList.of(LARGE_BIDIRECTIONAL_TASKS).and(SMALL_BIDIRECTIONAL_TASKS))); + KvCoder<ChangelogDescriptor, List<SerializableChangelogTask>> tasksCoder = + ChangelogScanner.coder(scanConfig.rowIdBeamSchema()); + changelogTasks.get(UNIDIRECTIONAL_TASKS).setCoder(tasksCoder); + changelogTasks.get(SMALL_BIDIRECTIONAL_TASKS).setCoder(tasksCoder); + changelogTasks.get(LARGE_BIDIRECTIONAL_TASKS).setCoder(tasksCoder); + + Schema projectedRowSchema = + IcebergUtils.icebergSchemaToBeamSchema( + scanConfig.getProjectedSchema(), scanConfig.getUpdateCompatibilityVersion()); + Schema outputRowSchema = CdcOutputUtils.outputSchema(scanConfig, projectedRowSchema); + + // reads UNIDIRECTIONAL and BIDIRECTIONAL tags and produces rows. + ReadFromChangelogs.Output outputRows = changelogTasks.apply(new ReadFromChangelogs(scanConfig)); + + // Small overlapping groups get resolved entirely in memory with no shuffle. + PCollection<Row> smallBidirectionalCdcRows = + changelogTasks + .get(SMALL_BIDIRECTIONAL_TASKS) + .apply("Redistribute Small Bidirectional Changes", Redistribute.arbitrarily()) + .apply("Resolve Locally", ParDo.of(new LocalResolveDoFn(scanConfig))) + .setRowSchema(outputRowSchema); + + // BIDIRECTIONAL records go through a CoGBK and ResolveChanges + // We window locally using a custom WindowFn based on the snapsot's commit time. Each snapshot + // exists in its own window. + // We re-window the resolved output back to GlobalWindows before the final Flatten + // to align with the other branches. + Window<KV<CdcRowDescriptor, Row>> keyedWindowing = + Window.<KV<CdcRowDescriptor, Row>>into(new SnapshotWindowFn()) + .triggering(AfterWatermark.pastEndOfWindow()) + .withAllowedLateness(Duration.ZERO) Review Comment: Is dropping late data the correct approach here ? ########## sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/ResolveChanges.java: ########## @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.beam.sdk.io.iceberg.IcebergScanConfig; +import org.apache.beam.sdk.io.iceberg.IcebergUtils; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.join.CoGbkResult; +import org.apache.beam.sdk.util.RowFilter; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.iceberg.types.Types; +import org.joda.time.Instant; + +/** + * Receives a {@link CoGbkResult} containing inserts and deletes sharing the same snapshot sequence + * number and Primary Key, and uses {@link CdcResolver} to identify logical updates. + */ +class ResolveChanges extends DoFn<KV<CdcRowDescriptor, CoGbkResult>, Row> { + static final TupleTag<Row> DELETES = new TupleTag<>() {}; + static final TupleTag<Row> INSERTS = new TupleTag<>() {}; + private final IcebergScanConfig scanConfig; + private final RowFilter rowFilter; + private final Schema outputSchema; + + ResolveChanges(IcebergScanConfig scanConfig) { + this.scanConfig = scanConfig; + this.rowFilter = + new RowFilter( + CdcOutputUtils.readBeamSchemaWithRowMetadata( + scanConfig.getMetadataColumns(), scanConfig.getSchema())) + .keep( + CdcOutputUtils.readSchemaWithRowMetadata( + scanConfig.getMetadataColumns(), scanConfig.getProjectedSchema()) + .columns().stream() + .map(Types.NestedField::name) + .collect(Collectors.toList())); + this.outputSchema = + CdcOutputUtils.outputSchema( + scanConfig, + IcebergUtils.icebergSchemaToBeamSchema( + scanConfig.getProjectedSchema(), scanConfig.getUpdateCompatibilityVersion())); + } + + @ProcessElement + public void processElement( + @Element KV<CdcRowDescriptor, CoGbkResult> element, + @Timestamp Instant timestamp, + OutputReceiver<Row> out) { + CdcRowDescriptor descriptor = element.getKey(); + + Set<String> pkFields = new HashSet<>(descriptor.getPrimaryKey().getSchema().getFieldNames()); + CoGbkResult result = element.getValue(); + + // should be okay to materialize these lists. a PK collision will likely be a handful of records + // at most + List<Row> deletes = Lists.newArrayList(result.getAll(DELETES)); + List<Row> inserts = Lists.newArrayList(result.getAll(INSERTS)); + + new RowResolver(pkFields, scanConfig.getMetadataColumns()) + .resolve( + deletes, + inserts, + (kind, row) -> { + Row projectedRow = rowFilter.filter(row); + out.builder( + CdcOutputUtils.outputRow( + scanConfig.getMetadataColumns(), + outputSchema, + descriptor.getCommitSnapshotId(), + descriptor.getSnapshotSequenceNumber(), + kind, + projectedRow)) + .setValueKind(kind) + .setTimestamp(timestamp) + .output(); + }); + } + + private static final class RowResolver extends CdcResolver<Row> { + private final Set<String> pkFields; + private final List<String> metadataColumns; + + RowResolver(Set<String> pkFields, List<String> metadataColumns) { + this.pkFields = pkFields; + this.metadataColumns = metadataColumns; + } + + @Override + protected int nonPkHash(Row element) { + int hash = 1; + for (String field : element.getSchema().getFieldNames()) { + if (pkFields.contains(field) + || (IcebergCdcMetadataColumns.isSupportedColumn(field) + && metadataColumns.contains(field))) { + continue; + } + hash = + 31 * hash + + Row.Equals.deepHashCode( + element.getValue(field), element.getSchema().getField(field).getType()); + } + return hash; + } + + @Override + protected boolean nonPkEquals(Row delete, Row insert) { Review Comment: Ditto (might be good to do a perf analysis after the PR is submitted) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
