This is an automated email from the ASF dual-hosted git repository. damondouglas pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 957a7cd22b6 created CsvIOReadFiles class (#31738) 957a7cd22b6 is described below commit 957a7cd22b6974a4c565846ad32b9fb9fc9c8122 Author: lahariguduru <108150650+laharigud...@users.noreply.github.com> AuthorDate: Mon Jul 1 20:20:15 2024 +0000 created CsvIOReadFiles class (#31738) Co-authored-by: Lahari Guduru <laharigud...@google.com> --- .../org/apache/beam/sdk/io/csv/CsvIOReadFiles.java | 54 ++++++++++++++++++++++ .../apache/beam/sdk/io/csv/CsvIOReadFilesTest.java | 32 +++++++++++++ 2 files changed, 86 insertions(+) diff --git a/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIOReadFiles.java b/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIOReadFiles.java new file mode 100644 index 00000000000..0f6267c6b34 --- /dev/null +++ b/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIOReadFiles.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.csv; + +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.FileIO; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.TupleTag; + +/** + * Skeleton for error handling in CsvIO that transforms a {@link FileIO.ReadableFile} into the + * result of parsing. + */ +// TODO(https://github.com/apache/beam/issues/31736): Plan completion in future PR after +// dependencies are completed. +class CsvIOReadFiles<T> extends PTransform<PCollection<FileIO.ReadableFile>, CsvIOParseResult<T>> { + /** Stores required parameters for parsing. */ + private final CsvIOParseConfiguration.Builder configBuilder; + + CsvIOReadFiles(CsvIOParseConfiguration.Builder configBuilder) { + this.configBuilder = configBuilder; + } + + /** {@link PTransform} that parses and relays the filename associated with each error. */ + // TODO: complete expand method to unsure parsing from FileIO.ReadableFile to CsvIOParseResult. + @Override + public CsvIOParseResult<T> expand(PCollection<FileIO.ReadableFile> input) { + // TODO(https://github.com/apache/beam/issues/31736): Needed to prevent check errors, will + // remove with future PR. + configBuilder.build(); + TupleTag<T> outputTag = new TupleTag<>(); + TupleTag<CsvIOParseError> errorTag = new TupleTag<>(); + Pipeline p = input.getPipeline(); + PCollectionTuple tuple = PCollectionTuple.empty(p); + return CsvIOParseResult.of(outputTag, errorTag, tuple); + } +} diff --git a/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOReadFilesTest.java b/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOReadFilesTest.java new file mode 100644 index 00000000000..c4a62f84eae --- /dev/null +++ b/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOReadFilesTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.csv; + +import org.apache.beam.sdk.util.SerializableUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link CsvIOReadFiles}. */ +@RunWith(JUnit4.class) +public class CsvIOReadFilesTest { + @Test + public void isSerializable() { + SerializableUtils.ensureSerializable(CsvIOReadFiles.class); + } +}