This closes #1507
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/88de0cb2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/88de0cb2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/88de0cb2 Branch: refs/heads/gearpump-runner Commit: 88de0cb2367327f0fe41282d64f02d761c3ce25e Parents: 2812405 2afc0cd Author: Kenneth Knowles <k...@google.com> Authored: Tue Dec 6 08:36:54 2016 -0800 Committer: Kenneth Knowles <k...@google.com> Committed: Tue Dec 6 08:36:54 2016 -0800 ---------------------------------------------------------------------- .gitignore | 4 + .jenkins/common_job_properties.groovy | 166 ++++++ ...job_beam_PostCommit_Java_MavenInstall.groovy | 42 ++ ...ommit_Java_RunnableOnService_Dataflow.groovy | 39 ++ ...stCommit_Java_RunnableOnService_Flink.groovy | 38 ++ ...ommit_Java_RunnableOnService_Gearpump.groovy | 41 ++ ...stCommit_Java_RunnableOnService_Spark.groovy | 38 ++ .../job_beam_PostCommit_Python_Verify.groovy | 37 ++ .../job_beam_PreCommit_Java_MavenInstall.groovy | 42 ++ .../job_beam_Release_NightlySnapshot.groovy | 46 ++ .jenkins/job_seed.groovy | 47 ++ .travis.yml | 2 +- .travis/README.md | 2 +- .../org/apache/beam/examples/complete/README.md | 14 +- .../beam/examples/cookbook/DeDupExample.java | 96 --- .../beam/examples/cookbook/DistinctExample.java | 96 +++ .../org/apache/beam/examples/cookbook/README.md | 16 +- .../org/apache/beam/examples/WordCountIT.java | 2 +- pom.xml | 10 +- runners/apex/pom.xml | 11 +- .../translation/ApexPipelineTranslator.java | 12 +- .../apex/translation/TranslationContext.java | 6 +- .../operators/ApexGroupByKeyOperator.java | 14 +- .../core/ElementAndRestrictionCoder.java | 8 + .../runners/core/GBKIntoKeyedWorkItems.java | 55 -- .../GroupAlsoByWindowsViaOutputBufferDoFn.java | 6 +- .../core/GroupByKeyViaGroupByKeyOnly.java | 22 +- .../runners/core/PerKeyCombineFnRunners.java | 263 +++++++++ .../core/PushbackSideInputDoFnRunner.java | 20 +- .../beam/runners/core/SimpleDoFnRunner.java | 76 ++- .../beam/runners/core/SimpleOldDoFnRunner.java | 16 - .../beam/runners/core/SplittableParDo.java | 378 +++++++++--- .../core/GroupAlsoByWindowsProperties.java | 590 +++++++++++-------- .../core/PushbackSideInputDoFnRunnerTest.java | 18 +- .../beam/runners/core/ReduceFnRunnerTest.java | 5 +- .../beam/runners/core/SplittableParDoTest.java | 132 ++++- .../beam/runners/core/WindowMatchers.java | 204 +++++++ .../beam/runners/core/WindowMatchersTest.java | 82 +++ .../direct/AbstractModelEnforcement.java | 2 +- .../direct/BoundedReadEvaluatorFactory.java | 8 +- .../beam/runners/direct/CommittedResult.java | 2 +- .../beam/runners/direct/CompletionCallback.java | 2 +- .../direct/ConsumerTrackingPipelineVisitor.java | 173 ------ ...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 41 +- .../apache/beam/runners/direct/DirectGraph.java | 89 +++ .../beam/runners/direct/DirectGraphVisitor.java | 145 +++++ .../beam/runners/direct/DirectGroupByKey.java | 2 +- .../beam/runners/direct/DirectRunner.java | 48 +- .../runners/direct/DirectTimerInternals.java | 4 +- .../runners/direct/DisplayDataValidator.java | 6 +- .../runners/direct/DoFnLifecycleManager.java | 4 +- ...ecycleManagerRemovingTransformEvaluator.java | 2 +- .../runners/direct/EmptyTransformEvaluator.java | 4 +- .../beam/runners/direct/EvaluationContext.java | 87 ++- .../direct/ExecutorServiceParallelExecutor.java | 17 +- .../runners/direct/FlattenEvaluatorFactory.java | 10 +- .../GroupAlsoByWindowEvaluatorFactory.java | 5 +- .../direct/GroupByKeyOnlyEvaluatorFactory.java | 2 +- .../ImmutabilityCheckingBundleFactory.java | 21 +- .../direct/ImmutabilityEnforcementFactory.java | 2 +- .../direct/KeyedPValueTrackingVisitor.java | 12 +- .../beam/runners/direct/ModelEnforcement.java | 2 +- .../beam/runners/direct/ParDoEvaluator.java | 33 +- .../runners/direct/ParDoEvaluatorFactory.java | 94 ++- .../direct/ParDoMultiOverrideFactory.java | 76 ++- .../ParDoSingleViaMultiOverrideFactory.java | 6 +- .../direct/PassthroughTransformEvaluator.java | 4 +- ...littableProcessElementsEvaluatorFactory.java | 144 +++++ .../direct/StatefulParDoEvaluatorFactory.java | 256 ++++++++ .../runners/direct/StepTransformResult.java | 38 +- .../direct/TestStreamEvaluatorFactory.java | 2 +- .../beam/runners/direct/TransformEvaluator.java | 2 +- .../direct/TransformEvaluatorRegistry.java | 7 + .../beam/runners/direct/TransformExecutor.java | 4 +- .../beam/runners/direct/TransformResult.java | 16 +- .../direct/UnboundedReadEvaluatorFactory.java | 3 +- .../runners/direct/ViewEvaluatorFactory.java | 2 +- .../direct/WatermarkCallbackExecutor.java | 34 ++ .../beam/runners/direct/WatermarkManager.java | 50 +- .../runners/direct/WindowEvaluatorFactory.java | 6 +- .../direct/BoundedReadEvaluatorFactoryTest.java | 16 +- .../ConsumerTrackingPipelineVisitorTest.java | 287 --------- .../runners/direct/DirectGraphVisitorTest.java | 239 ++++++++ ...leManagerRemovingTransformEvaluatorTest.java | 4 +- .../runners/direct/EvaluationContextTest.java | 49 +- .../direct/FlattenEvaluatorFactoryTest.java | 6 +- .../ImmutabilityCheckingBundleFactoryTest.java | 6 +- .../ImmutabilityEnforcementFactoryTest.java | 6 +- .../beam/runners/direct/ParDoEvaluatorTest.java | 2 +- .../beam/runners/direct/SplittableDoFnTest.java | 231 -------- .../StatefulParDoEvaluatorFactoryTest.java | 300 ++++++++++ .../runners/direct/StepTransformResultTest.java | 25 +- .../direct/TestStreamEvaluatorFactoryTest.java | 10 +- .../runners/direct/TransformExecutorTest.java | 223 +++---- .../UnboundedReadEvaluatorFactoryTest.java | 20 +- .../runners/direct/WatermarkManagerTest.java | 23 +- .../direct/WindowEvaluatorFactoryTest.java | 16 +- runners/flink/runner/pom.xml | 9 +- .../apache/beam/runners/flink/FlinkRunner.java | 12 +- .../FlinkBatchPipelineTranslator.java | 14 +- .../FlinkStreamingPipelineTranslator.java | 16 +- .../FlinkStreamingTransformTranslators.java | 63 +- .../PipelineTranslationOptimizer.java | 10 +- .../FlinkMergingNonShuffleReduceFunction.java | 2 +- .../FlinkMergingPartialReduceFunction.java | 2 +- .../functions/FlinkMergingReduceFunction.java | 2 +- .../functions/FlinkPartialReduceFunction.java | 2 +- .../functions/FlinkProcessContextBase.java | 8 - .../functions/FlinkReduceFunction.java | 2 +- runners/gearpump/pom.xml | 9 +- .../gearpump/GearpumpPipelineTranslator.java | 12 +- .../translators/TranslationContext.java | 4 +- .../gearpump/translators/io/ValuesSource.java | 71 ++- runners/google-cloud-dataflow-java/pom.xml | 4 + .../dataflow/DataflowPipelineTranslator.java | 33 +- .../beam/runners/dataflow/DataflowRunner.java | 53 +- .../DataflowPipelineTranslatorTest.java | 2 +- .../runners/dataflow/DataflowRunnerTest.java | 4 +- .../dataflow/RecordingPipelineVisitor.java | 6 +- runners/spark/pom.xml | 7 +- .../runners/spark/SparkPipelineOptions.java | 5 + .../apache/beam/runners/spark/SparkRunner.java | 80 ++- .../spark/aggregators/AccumulatorSingleton.java | 2 +- .../beam/runners/spark/io/MicrobatchSource.java | 20 +- .../beam/runners/spark/io/SourceDStream.java | 3 +- .../spark/stateful/StateSpecFunctions.java | 2 +- .../spark/translation/BoundedDataset.java | 5 +- .../beam/runners/spark/translation/Dataset.java | 2 +- .../spark/translation/EvaluationContext.java | 10 +- .../spark/translation/SparkProcessContext.java | 9 - .../translation/StorageLevelPTransform.java | 43 ++ .../spark/translation/TransformTranslator.java | 27 + .../SparkRunnerStreamingContextFactory.java | 2 +- .../streaming/StreamingTransformTranslator.java | 2 +- .../translation/streaming/UnboundedDataset.java | 13 +- .../spark/translation/StorageLevelTest.java | 56 ++ .../streaming/EmptyStreamAssertionTest.java | 2 + .../streaming/FlattenStreamingTest.java | 2 + .../streaming/SimpleStreamingWordCountTest.java | 1 + .../SparkTestPipelineOptionsForStreaming.java | 6 - sdks/java/core/pom.xml | 29 +- .../beam/sdk/AggregatorPipelineExtractor.java | 6 +- .../main/java/org/apache/beam/sdk/Pipeline.java | 151 ++--- .../org/apache/beam/sdk/io/FileBasedSink.java | 22 +- .../java/org/apache/beam/sdk/io/PubsubIO.java | 176 +++++- .../apache/beam/sdk/io/PubsubUnboundedSink.java | 23 +- .../beam/sdk/io/PubsubUnboundedSource.java | 40 +- .../java/org/apache/beam/sdk/io/TextIO.java | 28 +- .../java/org/apache/beam/sdk/io/XmlSink.java | 4 +- .../beam/sdk/options/PipelineOptions.java | 7 + .../sdk/options/PipelineOptionsFactory.java | 1 + .../sdk/options/ProxyInvocationHandler.java | 26 + .../apache/beam/sdk/options/ValueProvider.java | 2 +- .../beam/sdk/runners/TransformHierarchy.java | 340 +++++++++-- .../beam/sdk/runners/TransformTreeNode.java | 257 -------- .../beam/sdk/testing/FileChecksumMatcher.java | 168 +++++- .../apache/beam/sdk/testing/GatherAllPanes.java | 88 +++ .../org/apache/beam/sdk/testing/PAssert.java | 77 +-- .../apache/beam/sdk/testing/PaneExtractors.java | 55 +- .../beam/sdk/testing/UsesSplittableParDo.java | 25 + .../beam/sdk/testing/ValueInSingleWindow.java | 134 +++++ .../org/apache/beam/sdk/transforms/Combine.java | 37 +- .../org/apache/beam/sdk/transforms/DoFn.java | 16 +- .../beam/sdk/transforms/DoFnAdapters.java | 2 + .../apache/beam/sdk/transforms/DoFnTester.java | 350 ++++++----- .../org/apache/beam/sdk/transforms/OldDoFn.java | 8 +- .../org/apache/beam/sdk/transforms/ParDo.java | 10 - .../reflect/ByteBuddyDoFnInvokerFactory.java | 28 +- .../reflect/ByteBuddyOnTimerInvokerFactory.java | 10 +- .../sdk/transforms/reflect/DoFnInvoker.java | 3 + .../sdk/transforms/reflect/DoFnInvokers.java | 16 +- .../sdk/transforms/reflect/DoFnSignature.java | 8 + .../apache/beam/sdk/util/GatherAllPanes.java | 86 --- .../apache/beam/sdk/util/IdentityWindowFn.java | 2 +- .../beam/sdk/util/PerKeyCombineFnRunners.java | 258 -------- .../apache/beam/sdk/util/TimerInternals.java | 107 ++-- .../beam/sdk/util/WindowingInternals.java | 10 - .../beam/sdk/util/common/ReflectHelpers.java | 3 +- .../sdk/util/state/InMemoryTimerInternals.java | 8 +- .../sdk/util/state/TimerInternalsFactory.java | 36 ++ .../sdk/AggregatorPipelineExtractorTest.java | 20 +- .../org/apache/beam/sdk/WindowMatchers.java | 204 ------- .../org/apache/beam/sdk/WindowMatchersTest.java | 82 --- .../org/apache/beam/sdk/io/PubsubIOTest.java | 43 +- .../beam/sdk/io/PubsubUnboundedSinkTest.java | 20 +- .../beam/sdk/io/PubsubUnboundedSourceTest.java | 14 +- .../org/apache/beam/sdk/io/XmlSinkTest.java | 6 +- .../beam/sdk/options/PipelineOptionsTest.java | 24 + .../beam/sdk/options/ValueProviderTest.java | 18 + .../sdk/runners/TransformHierarchyTest.java | 230 ++++++++ .../beam/sdk/runners/TransformTreeTest.java | 12 +- .../sdk/testing/FileChecksumMatcherTest.java | 131 +++- .../beam/sdk/testing/GatherAllPanesTest.java | 140 +++++ .../beam/sdk/testing/PaneExtractorsTest.java | 133 ++--- .../testing/ValueInSingleWindowCoderTest.java | 51 ++ .../beam/sdk/transforms/DoFnTesterTest.java | 38 +- .../apache/beam/sdk/transforms/ParDoTest.java | 177 +++++- .../beam/sdk/transforms/SplittableDoFnTest.java | 401 +++++++++++++ .../display/DisplayDataEvaluator.java | 8 +- .../beam/sdk/util/GatherAllPanesTest.java | 143 ----- .../apache/beam/sdk/util/ReleaseInfoTest.java | 45 ++ .../beam/sdk/util/TimerInternalsTest.java | 4 +- .../sorter/BufferedExternalSorter.java | 6 +- .../sdk/extensions/sorter/ExternalSorter.java | 6 +- .../sorter/BufferedExternalSorterTest.java | 58 +- .../extensions/sorter/ExternalSorterTest.java | 53 +- .../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 52 +- .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 26 +- .../apache/beam/sdk/io/mongodb/MongoDbIO.java | 6 +- .../examples-java8/generate-sources.sh | 82 +++ .../maven-archetypes/examples-java8/pom.xml | 177 ++++++ .../META-INF/maven/archetype-metadata.xml | 39 ++ .../main/resources/archetype-resources/pom.xml | 209 +++++++ .../projects/basic/archetype.properties | 21 + .../src/test/resources/projects/basic/goal.txt | 1 + .../examples/generate-sources.sh | 68 +++ sdks/java/maven-archetypes/examples/pom.xml | 27 +- .../main/resources/archetype-resources/pom.xml | 123 +++- .../src/main/java/DebuggingWordCount.java | 164 ------ .../src/main/java/MinimalWordCount.java | 118 ---- .../src/main/java/WindowedWordCount.java | 229 ------- .../src/main/java/WordCount.java | 186 ------ .../common/ExampleBigQueryTableOptions.java | 55 -- .../src/main/java/common/ExampleOptions.java | 37 -- ...xamplePubsubTopicAndSubscriptionOptions.java | 45 -- .../java/common/ExamplePubsubTopicOptions.java | 45 -- .../src/main/java/common/ExampleUtils.java | 352 ----------- .../src/test/java/DebuggingWordCountTest.java | 52 -- .../src/test/java/WordCountTest.java | 85 --- sdks/java/maven-archetypes/pom.xml | 38 +- .../main/resources/archetype-resources/pom.xml | 8 + .../resources/projects/basic/reference/pom.xml | 8 + .../update-examples-archetype.sh | 59 -- 233 files changed, 7991 insertions(+), 5310 deletions(-) ----------------------------------------------------------------------