This closes #2050
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/61e31e62 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/61e31e62 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/61e31e62 Branch: refs/heads/master Commit: 61e31e622ec5b1f3c4cd1417c859810689f2683c Parents: a41afdc 2c70743 Author: Sela <ans...@paypal.com> Authored: Wed Mar 1 00:18:45 2017 +0200 Committer: Sela <ans...@paypal.com> Committed: Wed Mar 1 00:18:45 2017 +0200 ---------------------------------------------------------------------- .../apache/beam/runners/core/LateDataUtils.java | 88 ++++ .../core/UnsupportedSideInputReader.java | 52 +++ runners/spark/pom.xml | 5 +- .../runners/spark/SparkPipelineOptions.java | 6 +- .../beam/runners/spark/SparkPipelineResult.java | 4 + .../apache/beam/runners/spark/SparkRunner.java | 8 +- .../runners/spark/TestSparkPipelineOptions.java | 35 ++ .../beam/runners/spark/TestSparkRunner.java | 88 ++-- .../aggregators/AggregatorsAccumulator.java | 2 +- .../beam/runners/spark/coders/CoderHelpers.java | 23 + .../apache/beam/runners/spark/io/ConsoleIO.java | 4 +- .../beam/runners/spark/io/CreateStream.java | 198 +++++++-- .../spark/metrics/MetricsAccumulator.java | 4 - .../spark/metrics/SparkMetricsContainer.java | 5 +- .../SparkGroupAlsoByWindowViaWindowSet.java | 431 +++++++++++++++++++ .../spark/stateful/SparkStateInternals.java | 418 ++++++++++++++++++ .../spark/stateful/SparkTimerInternals.java | 185 ++++++++ .../spark/translation/EvaluationContext.java | 6 - .../translation/GroupCombineFunctions.java | 237 ++++------ .../translation/SparkGroupAlsoByWindowFn.java | 198 --------- ...SparkGroupAlsoByWindowViaOutputBufferFn.java | 179 ++++++++ .../spark/translation/TransformTranslator.java | 238 ++++++---- .../spark/translation/TranslationUtils.java | 24 +- .../streaming/StreamingTransformTranslator.java | 277 +++++------- .../translation/streaming/UnboundedDataset.java | 59 +-- .../beam/runners/spark/ForceStreamingTest.java | 17 +- .../spark/GlobalWatermarkHolderTest.java | 151 +++++++ .../apache/beam/runners/spark/PipelineRule.java | 120 ++++++ .../runners/spark/ProvidedSparkContextTest.java | 2 +- .../beam/runners/spark/ReuseSparkContext.java | 46 -- .../runners/spark/ReuseSparkContextRule.java | 46 ++ .../runners/spark/SparkPipelineStateTest.java | 33 +- .../beam/runners/spark/WatermarkTest.java | 231 ---------- .../metrics/sink/NamedAggregatorsTest.java | 11 +- .../beam/runners/spark/io/AvroPipelineTest.java | 6 +- .../beam/runners/spark/io/NumShardsTest.java | 6 +- .../io/hadoop/HadoopFileFormatPipelineTest.java | 6 +- .../spark/translation/StorageLevelTest.java | 8 +- .../translation/streaming/CreateStreamTest.java | 369 ++++++++++++++++ .../streaming/EmptyStreamAssertionTest.java | 87 ---- .../streaming/FlattenStreamingTest.java | 82 ---- .../streaming/KafkaStreamingTest.java | 270 ------------ .../ResumeFromCheckpointStreamingTest.java | 292 +++++++++---- .../streaming/SimpleStreamingWordCountTest.java | 84 ---- .../streaming/TrackStreamingSourcesTest.java | 45 +- .../utils/KafkaWriteOnBatchCompleted.java | 105 ----- .../streaming/utils/PAssertStreaming.java | 121 ------ .../utils/SparkTestPipelineOptions.java | 42 -- .../SparkTestPipelineOptionsForStreaming.java | 37 -- .../windowing/TimestampTransform.java | 3 +- 50 files changed, 2971 insertions(+), 2023 deletions(-) ----------------------------------------------------------------------
