This closes #1143
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/dc61a001 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/dc61a001 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/dc61a001 Branch: refs/heads/python-sdk Commit: dc61a001c1b9389450ade3f49b98a9a22c10f794 Parents: c94b8ea b72e7e3 Author: Sela <ans...@paypal.com> Authored: Wed Oct 26 20:52:17 2016 +0300 Committer: Sela <ans...@paypal.com> Committed: Wed Oct 26 20:52:17 2016 +0300 ---------------------------------------------------------------------- runners/spark/pom.xml | 44 ++-- .../runners/spark/SparkPipelineOptions.java | 11 + .../runners/spark/io/EmptyCheckpointMark.java | 52 ++++ .../apache/beam/runners/spark/io/KafkaIO.java | 131 ---------- .../beam/runners/spark/io/MicrobatchSource.java | 262 +++++++++++++++++++ .../beam/runners/spark/io/SourceDStream.java | 156 +++++++++++ .../apache/beam/runners/spark/io/SourceRDD.java | 75 +++++- .../runners/spark/io/SparkUnboundedSource.java | 167 ++++++++++++ .../spark/stateful/StateSpecFunctions.java | 167 ++++++++++++ .../runners/spark/stateful/package-info.java | 22 ++ .../spark/translation/TranslationUtils.java | 17 ++ .../SparkRunnerStreamingContextFactory.java | 7 + .../streaming/StreamingTransformTranslator.java | 37 +-- .../streaming/KafkaStreamingTest.java | 150 ++++++++--- .../ResumeFromCheckpointStreamingTest.java | 34 ++- .../streaming/utils/EmbeddedKafkaCluster.java | 51 ++-- .../utils/KafkaWriteOnBatchCompleted.java | 105 ++++++++ 17 files changed, 1224 insertions(+), 264 deletions(-) ----------------------------------------------------------------------