This is an automated email from the ASF dual-hosted git repository. echauchot pushed a commit to branch spark-runner_structured-streaming in repository https://gitbox.apache.org/repos/asf/beam.git
commit 1fe87911591940f49d2db95d3cda5a906a04fd96 Author: Etienne Chauchot <echauc...@apache.org> AuthorDate: Tue Jan 15 13:35:52 2019 +0100 Comment schema choices --- .../structuredstreaming/translation/batch/DatasetSourceBatch.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java index c4cfeaf..2a13d98 100644 --- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java +++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java @@ -96,6 +96,9 @@ public class DatasetSourceBatch implements DataSourceV2, ReadSupport { @Override public StructType readSchema() { // TODO: find a way to extend schema with a WindowedValue schema + // we use a binary schema for now because: + // using a empty schema raises a indexOutOfBoundsException + // using a NullType schema stores null in the elements StructField[] array = new StructField[1]; StructField binaryStructField = StructField .apply("binaryStructField", DataTypes.BinaryType, true, Metadata.empty());