Closes #1193
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/3933b557 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/3933b557 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/3933b557 Branch: refs/heads/gearpump-runner Commit: 3933b5577da1808bdb8535e406f26682f62f57fb Parents: 8f4334c 3f06382 Author: Dan Halperin <dhalp...@google.com> Authored: Wed Oct 26 09:42:56 2016 -0700 Committer: Dan Halperin <dhalp...@google.com> Committed: Wed Oct 26 09:42:56 2016 -0700 ---------------------------------------------------------------------- .gitignore | 8 + .travis.yml | 10 +- examples/java/pom.xml | 154 ++- .../beam/examples/DebuggingWordCount.java | 2 +- .../apache/beam/examples/MinimalWordCount.java | 3 +- .../apache/beam/examples/WindowedWordCount.java | 2 +- .../org/apache/beam/examples/WordCount.java | 4 +- .../common/ExampleBigQueryTableOptions.java | 2 +- ...xamplePubsubTopicAndSubscriptionOptions.java | 2 +- .../common/ExamplePubsubTopicOptions.java | 2 +- .../beam/examples/common/ExampleUtils.java | 2 +- .../examples/common/PubsubFileInjector.java | 153 --- .../beam/examples/complete/AutoComplete.java | 2 +- .../apache/beam/examples/complete/TfIdf.java | 2 +- .../examples/complete/TopWikipediaSessions.java | 2 +- .../examples/cookbook/BigQueryTornadoes.java | 2 +- .../cookbook/CombinePerKeyExamples.java | 2 +- .../examples/cookbook/DatastoreWordCount.java | 261 ----- .../beam/examples/cookbook/DeDupExample.java | 4 +- .../beam/examples/cookbook/FilterExamples.java | 2 +- .../beam/examples/cookbook/JoinExamples.java | 2 +- .../examples/cookbook/MaxPerKeyExamples.java | 2 +- .../beam/examples/cookbook/TriggerExample.java | 28 +- .../beam/examples/WindowedWordCountIT.java | 75 ++ .../org/apache/beam/examples/WordCountIT.java | 6 - .../examples/cookbook/BigQueryTornadoesIT.java | 14 +- examples/java8/pom.xml | 2 +- .../beam/examples/complete/game/GameStats.java | 10 +- .../examples/complete/game/HourlyTeamScore.java | 8 +- .../examples/complete/game/LeaderBoard.java | 12 +- .../beam/examples/complete/game/UserScore.java | 10 +- .../complete/game/injector/Injector.java | 10 +- .../examples/complete/game/LeaderBoardTest.java | 5 +- examples/pom.xml | 2 +- pom.xml | 134 ++- runners/core-java/pom.xml | 8 +- .../beam/runners/core/AggregatorFactory.java | 39 + .../beam/runners/core/BatchTimerInternals.java | 140 --- .../apache/beam/runners/core/DoFnRunner.java | 8 +- .../beam/runners/core/DoFnRunnerBase.java | 559 ----------- .../apache/beam/runners/core/DoFnRunners.java | 191 +++- .../runners/core/ElementAndRestriction.java | 42 + .../core/ElementAndRestrictionCoder.java | 67 ++ .../runners/core/ElementByteSizeObservable.java | 5 +- .../runners/core/GBKIntoKeyedWorkItems.java | 55 + .../core/GroupAlsoByWindowViaWindowSetDoFn.java | 5 + .../runners/core/GroupAlsoByWindowsDoFn.java | 19 - .../GroupAlsoByWindowsViaOutputBufferDoFn.java | 12 +- .../runners/core/ReduceFnContextFactory.java | 2 +- .../beam/runners/core/ReduceFnRunner.java | 42 +- .../beam/runners/core/SideInputHandler.java | 2 +- .../beam/runners/core/SimpleDoFnRunner.java | 554 ++++++++++- .../beam/runners/core/SimpleOldDoFnRunner.java | 521 ++++++++++ .../beam/runners/core/SplittableParDo.java | 469 +++++++++ .../apache/beam/runners/core/TriggerRunner.java | 247 ----- .../core/UnboundedReadFromBoundedSource.java | 2 +- .../core/triggers/AfterAllStateMachine.java | 109 ++ .../AfterDelayFromFirstElementStateMachine.java | 337 +++++++ .../core/triggers/AfterEachStateMachine.java | 130 +++ .../core/triggers/AfterFirstStateMachine.java | 112 +++ .../core/triggers/AfterPaneStateMachine.java | 139 +++ .../AfterProcessingTimeStateMachine.java | 93 ++ ...rSynchronizedProcessingTimeStateMachine.java | 63 ++ .../triggers/AfterWatermarkStateMachine.java | 325 ++++++ .../triggers/DefaultTriggerStateMachine.java | 81 ++ .../triggers/ExecutableTriggerStateMachine.java | 160 +++ .../runners/core/triggers/FinishedTriggers.java | 44 + .../core/triggers/FinishedTriggersBitSet.java | 67 ++ .../core/triggers/FinishedTriggersSet.java | 72 ++ .../core/triggers/NeverStateMachine.java | 60 ++ .../core/triggers/OrFinallyStateMachine.java | 85 ++ .../core/triggers/RepeatedlyStateMachine.java | 88 ++ .../triggers/ReshuffleTriggerStateMachine.java | 50 + .../core/triggers/TriggerStateMachine.java | 487 +++++++++ .../TriggerStateMachineContextFactory.java | 509 ++++++++++ .../triggers/TriggerStateMachineRunner.java | 234 +++++ .../core/triggers/TriggerStateMachines.java | 215 ++++ .../runners/core/triggers/package-info.java | 23 + .../runners/core/BatchTimerInternalsTest.java | 118 --- .../core/ElementAndRestrictionCoderTest.java | 127 +++ .../beam/runners/core/ReduceFnRunnerTest.java | 281 +++--- .../beam/runners/core/ReduceFnTester.java | 405 +++----- .../beam/runners/core/SimpleDoFnRunnerTest.java | 88 -- .../runners/core/SimpleOldDoFnRunnerTest.java | 88 ++ .../beam/runners/core/SplittableParDoTest.java | 467 +++++++++ .../core/triggers/AfterAllStateMachineTest.java | 140 +++ .../triggers/AfterEachStateMachineTest.java | 108 ++ .../triggers/AfterFirstStateMachineTest.java | 159 +++ .../triggers/AfterPaneStateMachineTest.java | 117 +++ .../AfterProcessingTimeStateMachineTest.java | 172 ++++ ...chronizedProcessingTimeStateMachineTest.java | 110 ++ .../AfterWatermarkStateMachineTest.java | 382 +++++++ .../DefaultTriggerStateMachineTest.java | 165 +++ .../ExecutableTriggerStateMachineTest.java | 108 ++ .../triggers/FinishedTriggersBitSetTest.java | 55 + .../triggers/FinishedTriggersProperties.java | 115 +++ .../core/triggers/FinishedTriggersSetTest.java | 60 ++ .../core/triggers/NeverStateMachineTest.java | 59 ++ .../triggers/OrFinallyStateMachineTest.java | 177 ++++ .../triggers/RepeatedlyStateMachineTest.java | 200 ++++ .../ReshuffleTriggerStateMachineTest.java | 68 ++ .../core/triggers/StubTriggerStateMachine.java | 60 ++ .../core/triggers/TriggerStateMachineTest.java | 98 ++ .../triggers/TriggerStateMachineTester.java | 431 ++++++++ .../core/triggers/TriggerStateMachinesTest.java | 199 ++++ runners/direct-java/pom.xml | 26 +- .../runners/direct/AggregatorContainer.java | 9 +- .../direct/BoundedReadEvaluatorFactory.java | 155 +-- .../beam/runners/direct/BundleFactory.java | 15 +- .../runners/direct/CloningBundleFactory.java | 98 ++ .../beam/runners/direct/CompletionCallback.java | 4 +- .../runners/direct/DirectExecutionContext.java | 2 +- ...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 66 ++ .../beam/runners/direct/DirectMetrics.java | 338 +++++++ .../beam/runners/direct/DirectOptions.java | 40 +- .../beam/runners/direct/DirectRunner.java | 103 +- .../runners/direct/DirectTimerInternals.java | 2 +- .../runners/direct/DoFnLifecycleManager.java | 86 +- .../beam/runners/direct/EmptyInputProvider.java | 45 + .../direct/EncodabilityEnforcementFactory.java | 50 +- .../beam/runners/direct/EvaluationContext.java | 27 +- .../direct/ExecutorServiceParallelExecutor.java | 124 ++- .../runners/direct/FlattenEvaluatorFactory.java | 26 +- .../runners/direct/ForwardingPTransform.java | 2 +- .../GroupAlsoByWindowEvaluatorFactory.java | 14 +- .../direct/GroupByKeyOnlyEvaluatorFactory.java | 21 +- .../ImmutabilityCheckingBundleFactory.java | 18 +- .../direct/ImmutableListBundleFactory.java | 76 +- .../beam/runners/direct/KeyedResourcePool.java | 47 - .../runners/direct/LockedKeyedResourcePool.java | 95 -- .../beam/runners/direct/ParDoEvaluator.java | 13 +- .../direct/ParDoMultiEvaluatorFactory.java | 29 +- .../runners/direct/ParDoOverrideFactory.java | 55 + .../direct/ParDoSingleEvaluatorFactory.java | 25 +- .../beam/runners/direct/PipelineExecutor.java | 2 +- .../beam/runners/direct/RootInputProvider.java | 46 + .../runners/direct/RootProviderRegistry.java | 66 ++ .../runners/direct/StepTransformResult.java | 49 +- .../beam/runners/direct/StructuralKey.java | 88 +- .../direct/TestStreamEvaluatorFactory.java | 152 +-- .../beam/runners/direct/TransformEvaluator.java | 2 +- .../direct/TransformEvaluatorFactory.java | 22 +- .../direct/TransformEvaluatorRegistry.java | 44 +- .../beam/runners/direct/TransformExecutor.java | 53 +- .../beam/runners/direct/TransformResult.java | 16 +- .../direct/UnboundedReadEvaluatorFactory.java | 318 +++--- .../direct/UncommittedBundleOutputManager.java | 4 +- .../runners/direct/ViewEvaluatorFactory.java | 16 +- .../beam/runners/direct/WatermarkManager.java | 53 +- .../runners/direct/WindowEvaluatorFactory.java | 18 +- .../direct/WriteWithShardingFactory.java | 6 +- .../direct/BoundedReadEvaluatorFactoryTest.java | 181 ++-- .../direct/CloningBundleFactoryTest.java | 177 ++++ .../runners/direct/CommittedResultTest.java | 10 +- .../ConsumerTrackingPipelineVisitorTest.java | 32 +- .../beam/runners/direct/DirectMetricsTest.java | 133 +++ .../beam/runners/direct/DirectRunnerTest.java | 180 +++- ...leManagerRemovingTransformEvaluatorTest.java | 16 +- .../direct/DoFnLifecycleManagerTest.java | 86 +- .../direct/DoFnLifecycleManagersTest.java | 48 +- .../EncodabilityEnforcementFactoryTest.java | 132 ++- .../runners/direct/EvaluationContextTest.java | 31 +- .../direct/FlattenEvaluatorFactoryTest.java | 36 +- .../direct/ForwardingPTransformTest.java | 7 +- .../direct/GroupByKeyEvaluatorFactoryTest.java | 27 +- .../GroupByKeyOnlyEvaluatorFactoryTest.java | 35 +- .../ImmutabilityCheckingBundleFactoryTest.java | 67 +- .../ImmutabilityEnforcementFactoryTest.java | 14 +- .../direct/ImmutableListBundleFactoryTest.java | 52 +- .../direct/KeyedPValueTrackingVisitorTest.java | 8 +- .../direct/LockedKeyedResourcePoolTest.java | 163 --- .../beam/runners/direct/ParDoEvaluatorTest.java | 15 +- .../direct/ParDoMultiEvaluatorFactoryTest.java | 162 +-- .../direct/ParDoSingleEvaluatorFactoryTest.java | 145 ++- .../beam/runners/direct/SplittableDoFnTest.java | 231 +++++ .../runners/direct/StepTransformResultTest.java | 4 +- .../beam/runners/direct/StructuralKeyTest.java | 9 + .../direct/TestStreamEvaluatorFactoryTest.java | 223 ++--- .../runners/direct/TransformExecutorTest.java | 115 +-- .../UnboundedReadEvaluatorFactoryTest.java | 321 ++++-- .../direct/ViewEvaluatorFactoryTest.java | 6 +- .../runners/direct/WatermarkManagerTest.java | 84 +- .../direct/WindowEvaluatorFactoryTest.java | 12 +- runners/flink/README.md | 10 +- runners/flink/examples/pom.xml | 2 +- .../beam/runners/flink/examples/TFIDF.java | 12 +- .../beam/runners/flink/examples/WordCount.java | 4 +- .../flink/examples/streaming/AutoComplete.java | 4 +- runners/flink/pom.xml | 4 +- runners/flink/runner/pom.xml | 10 +- .../flink/FlinkDetachedRunnerResult.java | 76 ++ .../FlinkPipelineExecutionEnvironment.java | 7 + .../runners/flink/FlinkPipelineOptions.java | 19 +- .../apache/beam/runners/flink/FlinkRunner.java | 29 +- .../runners/flink/FlinkRunnerRegistrar.java | 4 +- .../beam/runners/flink/FlinkRunnerResult.java | 17 +- .../beam/runners/flink/TestFlinkRunner.java | 9 +- .../FlinkBatchPipelineTranslator.java | 2 +- .../FlinkStreamingPipelineTranslator.java | 2 +- .../functions/FlinkDoFnFunction.java | 2 +- .../functions/FlinkMultiOutputDoFnFunction.java | 2 +- .../functions/FlinkProcessContext.java | 10 +- .../wrappers/streaming/DoFnOperator.java | 31 +- .../wrappers/streaming/FlinkStateInternals.java | 8 +- .../wrappers/streaming/WindowDoFnOperator.java | 177 +++- .../streaming/io/BoundedSourceWrapper.java | 9 +- .../streaming/io/UnboundedSourceWrapper.java | 187 +++- .../beam/runners/flink/FlinkTestPipeline.java | 6 +- .../apache/beam/runners/flink/package-info.java | 22 - .../flink/streaming/DoFnOperatorTest.java | 9 +- .../streaming/UnboundedSourceWrapperTest.java | 123 +-- runners/gearpump/pom.xml | 4 +- .../gearpump/GearpumpPipelineResult.java | 6 + .../gearpump/GearpumpPipelineRunner.java | 193 ---- .../GearpumpPipelineRunnerRegistrar.java | 62 -- .../beam/runners/gearpump/GearpumpRunner.java | 191 ++++ .../gearpump/GearpumpRunnerRegistrar.java | 62 ++ .../runners/gearpump/TestGearpumpRunner.java | 4 +- .../gearpump/examples/StreamingWordCount.java | 4 +- .../translators/utils/GearpumpDoFnRunner.java | 16 +- runners/google-cloud-dataflow-java/pom.xml | 22 +- .../runners/dataflow/DataflowPipelineJob.java | 52 +- .../dataflow/DataflowPipelineTranslator.java | 137 ++- .../beam/runners/dataflow/DataflowRunner.java | 57 +- .../dataflow/internal/AssignWindows.java | 6 +- .../DataflowUnboundedReadFromBoundedSource.java | 18 +- .../runners/dataflow/internal/IsmFormat.java | 13 +- .../dataflow/internal/ReadTranslator.java | 2 +- .../options/DataflowPipelineDebugOptions.java | 7 +- .../options/DataflowPipelineOptions.java | 14 +- .../DataflowPipelineWorkerPoolOptions.java | 11 +- .../options/DataflowProfilingOptions.java | 2 +- .../options/DataflowWorkerLoggingOptions.java | 4 +- .../dataflow/testing/TestDataflowRunner.java | 144 ++- .../beam/runners/dataflow/util/DoFnInfo.java | 30 +- .../runners/dataflow/util/MonitoringUtil.java | 2 +- .../runners/dataflow/util/RandomAccessData.java | 8 +- .../beam/runners/dataflow/util/Stager.java | 2 +- .../dataflow/DataflowPipelineJobTest.java | 120 +++ .../DataflowPipelineTranslatorTest.java | 40 +- .../runners/dataflow/DataflowRunnerTest.java | 13 +- ...aflowUnboundedReadFromBoundedSourceTest.java | 83 ++ .../testing/TestDataflowRunnerTest.java | 287 +++++- .../dataflow/util/MonitoringUtilTest.java | 4 +- runners/pom.xml | 3 +- runners/spark/pom.xml | 94 +- .../beam/runners/spark/EvaluationResult.java | 4 +- .../runners/spark/SparkPipelineOptions.java | 48 +- .../apache/beam/runners/spark/SparkRunner.java | 176 ++-- .../beam/runners/spark/TestSparkRunner.java | 17 +- .../spark/aggregators/AccumulatorSingleton.java | 53 + .../runners/spark/coders/WritableCoder.java | 2 +- .../beam/runners/spark/examples/WordCount.java | 2 +- .../apache/beam/runners/spark/io/SourceRDD.java | 200 ++++ .../spark/io/hadoop/ShardNameTemplateAware.java | 2 +- .../runners/spark/translation/DoFnFunction.java | 75 +- .../spark/translation/EvaluationContext.java | 32 +- .../translation/GroupCombineFunctions.java | 313 ++++++ .../spark/translation/MultiDoFnFunction.java | 77 +- .../translation/SparkAbstractCombineFn.java | 134 +++ .../spark/translation/SparkContextFactory.java | 50 +- .../spark/translation/SparkGlobalCombineFn.java | 260 +++++ .../spark/translation/SparkKeyedCombineFn.java | 273 +++++ .../translation/SparkPipelineEvaluator.java | 57 -- .../translation/SparkPipelineTranslator.java | 5 +- .../spark/translation/SparkProcessContext.java | 168 +++- .../spark/translation/SparkRuntimeContext.java | 44 +- .../spark/translation/TransformTranslator.java | 592 +++-------- .../spark/translation/TranslationUtils.java | 197 ++++ .../SparkRunnerStreamingContextFactory.java | 106 ++ .../streaming/StreamingEvaluationContext.java | 81 +- .../streaming/StreamingTransformTranslator.java | 561 +++++++---- .../spark/util/SparkSideInputReader.java | 95 ++ .../runners/spark/ClearAggregatorsRule.java | 33 + .../apache/beam/runners/spark/DeDupTest.java | 59 -- .../beam/runners/spark/EmptyInputTest.java | 75 -- .../beam/runners/spark/SimpleWordCountTest.java | 105 -- .../apache/beam/runners/spark/TfIdfTest.java | 260 ----- .../beam/runners/spark/io/AvroPipelineTest.java | 4 +- .../beam/runners/spark/io/NumShardsTest.java | 4 +- .../io/hadoop/HadoopFileFormatPipelineTest.java | 4 +- .../spark/translation/CombineGloballyTest.java | 102 -- .../spark/translation/CombinePerKeyTest.java | 79 -- .../spark/translation/DoFnOutputTest.java | 67 -- .../translation/MultiOutputWordCountTest.java | 176 ---- .../spark/translation/SerializationTest.java | 201 ---- .../spark/translation/SideEffectsTest.java | 21 +- .../translation/SparkPipelineOptionsTest.java | 42 - .../translation/TransformTranslatorTest.java | 104 -- .../translation/WindowedWordCountTest.java | 120 --- .../streaming/EmptyStreamAssertionTest.java | 80 ++ .../streaming/FlattenStreamingTest.java | 57 +- .../streaming/KafkaStreamingTest.java | 32 +- .../ResumeFromCheckpointStreamingTest.java | 182 ++++ .../streaming/SimpleStreamingWordCountTest.java | 67 +- .../streaming/utils/PAssertStreaming.java | 87 +- .../utils/TestOptionsForStreaming.java | 55 + .../spark/src/test/resources/metrics.properties | 61 +- sdks/java/build-tools/pom.xml | 2 +- .../src/main/resources/beam/checkstyle.xml | 28 +- .../src/main/resources/beam/findbugs-filter.xml | 2 +- .../src/main/resources/beam/suppressions.xml | 11 +- sdks/java/core/pom.xml | 43 +- .../main/java/org/apache/beam/sdk/Pipeline.java | 14 +- .../org/apache/beam/sdk/PipelineResult.java | 15 +- .../beam/sdk/annotations/Experimental.java | 15 +- .../sdk/coders/CannotProvideCoderException.java | 2 +- .../java/org/apache/beam/sdk/coders/Coder.java | 28 +- .../apache/beam/sdk/coders/CoderFactory.java | 4 +- .../apache/beam/sdk/coders/CoderProvider.java | 2 +- .../apache/beam/sdk/coders/DelegateCoder.java | 4 +- .../beam/sdk/coders/IterableLikeCoder.java | 22 +- .../apache/beam/sdk/coders/NullableCoder.java | 10 +- .../apache/beam/sdk/coders/package-info.java | 2 +- .../beam/sdk/coders/protobuf/ProtoCoder.java | 4 +- .../java/org/apache/beam/sdk/io/AvroIO.java | 162 ++- .../java/org/apache/beam/sdk/io/AvroSource.java | 98 +- .../sdk/io/BoundedReadFromUnboundedSource.java | 11 +- .../org/apache/beam/sdk/io/BoundedSource.java | 19 +- .../apache/beam/sdk/io/CompressedSource.java | 18 +- .../org/apache/beam/sdk/io/FileBasedSink.java | 145 ++- .../apache/beam/sdk/io/OffsetBasedSource.java | 6 +- .../java/org/apache/beam/sdk/io/PubsubIO.java | 24 +- .../beam/sdk/io/PubsubUnboundedSource.java | 2 +- .../main/java/org/apache/beam/sdk/io/Read.java | 4 +- .../sdk/io/SerializableAvroCodecFactory.java | 112 +++ .../main/java/org/apache/beam/sdk/io/Sink.java | 5 +- .../java/org/apache/beam/sdk/io/TextIO.java | 236 +++-- .../main/java/org/apache/beam/sdk/io/Write.java | 8 +- .../java/org/apache/beam/sdk/io/XmlSource.java | 13 +- .../apache/beam/sdk/io/range/ByteKeyRange.java | 6 +- .../apache/beam/sdk/io/range/RangeTracker.java | 1 + .../org/apache/beam/sdk/metrics/Counter.java | 40 + .../apache/beam/sdk/metrics/CounterCell.java | 76 ++ .../org/apache/beam/sdk/metrics/DirtyState.java | 98 ++ .../apache/beam/sdk/metrics/Distribution.java | 30 + .../beam/sdk/metrics/DistributionCell.java | 58 ++ .../beam/sdk/metrics/DistributionData.java | 59 ++ .../beam/sdk/metrics/DistributionResult.java | 42 + .../org/apache/beam/sdk/metrics/Metric.java | 24 + .../org/apache/beam/sdk/metrics/MetricCell.java | 47 + .../org/apache/beam/sdk/metrics/MetricKey.java | 40 + .../org/apache/beam/sdk/metrics/MetricName.java | 46 + .../beam/sdk/metrics/MetricNameFilter.java | 60 ++ .../beam/sdk/metrics/MetricQueryResults.java | 33 + .../apache/beam/sdk/metrics/MetricResult.java | 45 + .../apache/beam/sdk/metrics/MetricResults.java | 34 + .../apache/beam/sdk/metrics/MetricUpdates.java | 72 ++ .../org/apache/beam/sdk/metrics/Metrics.java | 110 ++ .../beam/sdk/metrics/MetricsContainer.java | 150 +++ .../beam/sdk/metrics/MetricsEnvironment.java | 85 ++ .../apache/beam/sdk/metrics/MetricsFilter.java | 86 ++ .../org/apache/beam/sdk/metrics/MetricsMap.java | 86 ++ .../apache/beam/sdk/metrics/package-info.java | 28 + .../options/CloudResourceManagerOptions.java | 40 + .../org/apache/beam/sdk/options/GcpOptions.java | 8 +- .../org/apache/beam/sdk/options/GcsOptions.java | 4 +- .../beam/sdk/options/GoogleApiDebugOptions.java | 2 +- .../beam/sdk/options/PipelineOptions.java | 52 +- .../sdk/options/PipelineOptionsFactory.java | 269 +++-- .../sdk/options/ProxyInvocationHandler.java | 264 +++-- .../apache/beam/sdk/options/ValueProvider.java | 239 +++++ .../java/org/apache/beam/sdk/package-info.java | 2 +- .../apache/beam/sdk/runners/PipelineRunner.java | 1 - .../sdk/runners/PipelineRunnerRegistrar.java | 2 +- .../apache/beam/sdk/runners/package-info.java | 17 +- .../beam/sdk/testing/BigqueryMatcher.java | 239 +++++ .../org/apache/beam/sdk/testing/PAssert.java | 4 +- .../beam/sdk/testing/SerializableMatchers.java | 4 +- .../beam/sdk/testing/SourceTestUtils.java | 3 +- .../apache/beam/sdk/testing/StreamingIT.java | 35 + .../apache/beam/sdk/testing/TestPipeline.java | 2 - .../org/apache/beam/sdk/testing/TestStream.java | 2 +- .../apache/beam/sdk/testing/package-info.java | 4 +- .../apache/beam/sdk/transforms/Aggregator.java | 11 +- .../sdk/transforms/ApproximateQuantiles.java | 5 +- .../org/apache/beam/sdk/transforms/Combine.java | 95 +- .../beam/sdk/transforms/CombineFnBase.java | 17 +- .../apache/beam/sdk/transforms/CombineFns.java | 41 +- .../beam/sdk/transforms/CombineWithContext.java | 7 +- .../sdk/transforms/DelegatingAggregator.java | 125 +++ .../org/apache/beam/sdk/transforms/DoFn.java | 431 +++++++- .../beam/sdk/transforms/DoFnAdapters.java | 184 +++- .../apache/beam/sdk/transforms/DoFnTester.java | 257 +++-- .../beam/sdk/transforms/FlatMapElements.java | 6 +- .../apache/beam/sdk/transforms/GroupByKey.java | 4 +- .../transforms/IntraBundleParallelization.java | 361 ------- .../org/apache/beam/sdk/transforms/Latest.java | 12 +- .../apache/beam/sdk/transforms/MapElements.java | 8 +- .../org/apache/beam/sdk/transforms/OldDoFn.java | 318 ++++-- .../apache/beam/sdk/transforms/PTransform.java | 10 +- .../org/apache/beam/sdk/transforms/ParDo.java | 343 +++++-- .../apache/beam/sdk/transforms/Partition.java | 4 +- .../beam/sdk/transforms/RemoveDuplicates.java | 5 +- .../sdk/transforms/SerializableFunction.java | 2 +- .../org/apache/beam/sdk/transforms/ViewFn.java | 2 +- .../apache/beam/sdk/transforms/WithKeys.java | 2 +- .../sdk/transforms/display/DisplayData.java | 530 ++++++---- .../sdk/transforms/reflect/DoFnInvoker.java | 48 +- .../sdk/transforms/reflect/DoFnInvokers.java | 658 ++++++++---- .../sdk/transforms/reflect/DoFnSignature.java | 466 ++++++++- .../sdk/transforms/reflect/DoFnSignatures.java | 946 +++++++++++++++--- .../splittabledofn/RestrictionTracker.java | 42 + .../transforms/splittabledofn/package-info.java | 22 + .../beam/sdk/transforms/windowing/AfterAll.java | 51 +- .../windowing/AfterDelayFromFirstElement.java | 110 +- .../sdk/transforms/windowing/AfterEach.java | 63 +- .../sdk/transforms/windowing/AfterFirst.java | 52 +- .../sdk/transforms/windowing/AfterPane.java | 59 +- .../windowing/AfterProcessingTime.java | 7 - .../AfterSynchronizedProcessingTime.java | 13 +- .../transforms/windowing/AfterWatermark.java | 174 +--- .../transforms/windowing/DefaultTrigger.java | 37 +- .../beam/sdk/transforms/windowing/Never.java | 28 +- .../transforms/windowing/OrFinallyTrigger.java | 55 +- .../beam/sdk/transforms/windowing/PaneInfo.java | 6 +- .../sdk/transforms/windowing/Repeatedly.java | 36 +- .../transforms/windowing/SlidingWindows.java | 3 +- .../beam/sdk/transforms/windowing/Trigger.java | 421 ++------ .../beam/sdk/transforms/windowing/Window.java | 20 +- .../beam/sdk/transforms/windowing/WindowFn.java | 2 +- ...AttemptAndTimeBoundedExponentialBackOff.java | 173 ++++ .../util/AttemptBoundedExponentialBackOff.java | 86 ++ .../beam/sdk/util/BaseExecutionContext.java | 4 +- .../apache/beam/sdk/util/CredentialFactory.java | 2 +- .../apache/beam/sdk/util/ExecutableTrigger.java | 40 +- .../sdk/util/ExposedByteArrayOutputStream.java | 1 + .../apache/beam/sdk/util/FinishedTriggers.java | 44 - .../beam/sdk/util/FinishedTriggersBitSet.java | 67 -- .../beam/sdk/util/FinishedTriggersSet.java | 72 -- .../apache/beam/sdk/util/GatherAllPanes.java | 10 +- .../apache/beam/sdk/util/GcpProjectUtil.java | 106 ++ .../apache/beam/sdk/util/GcsPathValidator.java | 2 +- .../java/org/apache/beam/sdk/util/GcsUtil.java | 94 +- .../apache/beam/sdk/util/IOChannelFactory.java | 2 +- .../beam/sdk/util/MergingActiveWindowSet.java | 12 +- .../apache/beam/sdk/util/PCollectionViews.java | 9 +- .../org/apache/beam/sdk/util/PathValidator.java | 6 +- .../beam/sdk/util/PerKeyCombineFnRunner.java | 26 +- .../beam/sdk/util/PerKeyCombineFnRunners.java | 4 +- .../org/apache/beam/sdk/util/PropertyNames.java | 1 + .../org/apache/beam/sdk/util/PubsubClient.java | 5 +- .../apache/beam/sdk/util/PubsubGrpcClient.java | 34 +- .../apache/beam/sdk/util/PubsubTestClient.java | 2 +- .../sdk/util/ReifyTimestampAndWindowsDoFn.java | 16 +- .../org/apache/beam/sdk/util/ReleaseInfo.java | 6 +- .../apache/beam/sdk/util/ReshuffleTrigger.java | 16 +- .../org/apache/beam/sdk/util/StringUtils.java | 2 +- .../java/org/apache/beam/sdk/util/Timer.java | 56 ++ .../apache/beam/sdk/util/TimerInternals.java | 6 +- .../org/apache/beam/sdk/util/TimerSpec.java | 30 + .../org/apache/beam/sdk/util/TimerSpecs.java | 41 + .../java/org/apache/beam/sdk/util/Timers.java | 10 +- .../org/apache/beam/sdk/util/Transport.java | 17 + .../beam/sdk/util/TriggerContextFactory.java | 507 ---------- .../apache/beam/sdk/util/ValueWithRecordId.java | 8 +- .../beam/sdk/util/common/ReflectHelpers.java | 22 - .../apache/beam/sdk/util/common/Reiterable.java | 2 +- .../apache/beam/sdk/util/common/Reiterator.java | 2 +- .../CopyOnAccessInMemoryStateInternals.java | 2 +- .../sdk/util/state/InMemoryTimerInternals.java | 235 +++++ .../beam/sdk/util/state/ReadableState.java | 10 +- .../apache/beam/sdk/util/state/StateBinder.java | 67 ++ .../beam/sdk/util/state/StateContext.java | 6 +- .../apache/beam/sdk/util/state/StateSpec.java | 39 + .../apache/beam/sdk/util/state/StateSpecs.java | 452 +++++++++ .../apache/beam/sdk/util/state/StateTag.java | 82 +- .../apache/beam/sdk/util/state/StateTags.java | 386 +------- .../util/state/TestInMemoryStateInternals.java | 61 ++ .../beam/sdk/util/state/TimerCallback.java | 35 + .../apache/beam/sdk/values/PCollectionView.java | 15 +- .../java/org/apache/beam/sdk/values/PInput.java | 8 +- .../org/apache/beam/sdk/values/POutput.java | 8 +- .../java/org/apache/beam/sdk/values/PValue.java | 4 +- .../apache/beam/sdk/values/TypeDescriptors.java | 40 +- .../apache/beam/sdk/values/package-info.java | 4 +- .../dataflow/util/GcsPathValidatorTest.java | 103 -- .../org/apache/beam/sdk/DataflowMatchers.java | 64 -- .../java/org/apache/beam/sdk/PipelineTest.java | 2 +- .../apache/beam/sdk/coders/AvroCoderTest.java | 4 +- .../beam/sdk/coders/CoderRegistryTest.java | 1 + .../org/apache/beam/sdk/coders/KvCoderTest.java | 99 +- .../beam/sdk/coders/NullableCoderTest.java | 60 +- .../java/org/apache/beam/sdk/io/AvroIOTest.java | 107 +- .../org/apache/beam/sdk/io/AvroSourceTest.java | 43 + .../io/BoundedReadFromUnboundedSourceTest.java | 14 +- .../beam/sdk/io/CompressedSourceTest.java | 4 +- .../sdk/io/DrunkWritableByteChannelFactory.java | 80 ++ .../apache/beam/sdk/io/FileBasedSinkTest.java | 108 ++ .../beam/sdk/io/OffsetBasedSourceTest.java | 30 +- .../java/org/apache/beam/sdk/io/ReadTest.java | 62 +- .../io/SerializableAvroCodecFactoryTest.java | 100 ++ .../java/org/apache/beam/sdk/io/TextIOTest.java | 601 ++++++----- .../java/org/apache/beam/sdk/io/WriteTest.java | 6 +- .../beam/sdk/metrics/CounterCellTest.java | 55 + .../apache/beam/sdk/metrics/DirtyStateTest.java | 56 ++ .../beam/sdk/metrics/DistributionCellTest.java | 53 + .../apache/beam/sdk/metrics/MetricMatchers.java | 99 ++ .../beam/sdk/metrics/MetricsContainerTest.java | 129 +++ .../sdk/metrics/MetricsEnvironmentTest.java | 63 ++ .../apache/beam/sdk/metrics/MetricsMapTest.java | 103 ++ .../apache/beam/sdk/metrics/MetricsTest.java | 98 ++ .../sdk/options/PipelineOptionsFactoryTest.java | 184 +++- .../beam/sdk/options/PipelineOptionsTest.java | 49 +- .../options/PipelineOptionsValidatorTest.java | 18 +- .../sdk/options/ProxyInvocationHandlerTest.java | 111 ++- .../beam/sdk/options/ValueProviderTest.java | 220 ++++ .../beam/sdk/testing/BigqueryMatcherTest.java | 176 ++++ .../sdk/testing/PCollectionViewTesting.java | 7 - .../beam/sdk/testing/SystemNanoTimeSleeper.java | 2 +- .../beam/sdk/testing/TestPipelineTest.java | 4 +- .../beam/sdk/transforms/CombineFnsTest.java | 7 +- .../apache/beam/sdk/transforms/CombineTest.java | 81 +- .../DoFnDelegatingAggregatorTest.java | 5 +- .../beam/sdk/transforms/DoFnTesterTest.java | 458 +++++---- .../apache/beam/sdk/transforms/FlattenTest.java | 20 + .../IntraBundleParallelizationTest.java | 280 ------ .../beam/sdk/transforms/LatestFnTest.java | 233 +++++ .../beam/sdk/transforms/LatestFnTests.java | 233 ----- .../apache/beam/sdk/transforms/ParDoTest.java | 101 +- .../display/DisplayDataEvaluator.java | 13 +- .../transforms/display/DisplayDataMatchers.java | 141 ++- .../display/DisplayDataMatchersTest.java | 67 +- .../sdk/transforms/display/DisplayDataTest.java | 367 ++++--- .../transforms/reflect/DoFnInvokersTest.java | 710 +++++++------ .../reflect/DoFnInvokersTestHelper.java | 116 --- .../DoFnSignaturesProcessElementTest.java | 213 ++++ .../DoFnSignaturesSplittableDoFnTest.java | 543 ++++++++++ .../transforms/reflect/DoFnSignaturesTest.java | 991 ++++++++++++++----- .../reflect/DoFnSignaturesTestUtils.java | 67 ++ .../testhelper/DoFnInvokersTestHelper.java | 124 +++ .../sdk/transforms/windowing/AfterAllTest.java | 98 -- .../sdk/transforms/windowing/AfterEachTest.java | 64 -- .../transforms/windowing/AfterFirstTest.java | 120 --- .../sdk/transforms/windowing/AfterPaneTest.java | 77 -- .../windowing/AfterProcessingTimeTest.java | 94 -- .../AfterSynchronizedProcessingTimeTest.java | 75 -- .../windowing/AfterWatermarkTest.java | 308 ------ .../windowing/DefaultTriggerTest.java | 130 --- .../sdk/transforms/windowing/NeverTest.java | 34 +- .../windowing/OrFinallyTriggerTest.java | 136 --- .../transforms/windowing/RepeatedlyTest.java | 161 +-- .../sdk/transforms/windowing/StubTrigger.java | 17 - .../sdk/transforms/windowing/TriggerTest.java | 28 - .../sdk/transforms/windowing/WindowTest.java | 4 +- .../apache/beam/sdk/util/ApiSurfaceTest.java | 28 +- ...mptAndTimeBoundedExponentialBackOffTest.java | 213 ++++ .../AttemptBoundedExponentialBackOffTest.java | 85 ++ .../beam/sdk/util/ExecutableTriggerTest.java | 18 - .../sdk/util/FinishedTriggersBitSetTest.java | 55 - .../sdk/util/FinishedTriggersProperties.java | 110 -- .../beam/sdk/util/FinishedTriggersSetTest.java | 60 -- .../beam/sdk/util/GcpProjectUtilTest.java | 76 ++ .../beam/sdk/util/GcsPathValidatorTest.java | 100 ++ .../org/apache/beam/sdk/util/GcsUtilTest.java | 112 ++- .../beam/sdk/util/PubsubGrpcClientTest.java | 108 +- .../beam/sdk/util/PubsubJsonClientTest.java | 16 +- .../beam/sdk/util/ReshuffleTriggerTest.java | 23 - .../org/apache/beam/sdk/util/TriggerTester.java | 592 ----------- .../util/state/InMemoryTimerInternalsTest.java | 116 +++ sdks/java/extensions/join-library/pom.xml | 2 +- sdks/java/extensions/pom.xml | 2 +- sdks/java/io/google-cloud-platform/pom.xml | 16 +- .../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 79 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 239 +++-- .../sdk/io/gcp/bigquery/BigQueryServices.java | 15 +- .../io/gcp/bigquery/BigQueryServicesImpl.java | 106 +- .../gcp/bigquery/BigQueryTableRowIterator.java | 130 ++- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 56 +- .../beam/sdk/io/gcp/datastore/DatastoreIO.java | 2 +- .../beam/sdk/io/gcp/datastore/DatastoreV1.java | 99 +- .../io/gcp/bigquery/BigQueryAvroUtilsTest.java | 149 ++- .../sdk/io/gcp/bigquery/BigQueryIOTest.java | 570 ++++++++++- .../gcp/bigquery/BigQueryServicesImplTest.java | 149 ++- .../bigquery/BigQueryTableRowIteratorTest.java | 169 +++- .../sdk/io/gcp/bigtable/BigtableIOTest.java | 104 +- .../sdk/io/gcp/datastore/SplitQueryFnIT.java | 4 +- .../beam/sdk/io/gcp/datastore/V1ReadIT.java | 2 +- .../beam/sdk/io/gcp/datastore/V1WriteIT.java | 2 +- sdks/java/io/hdfs/pom.xml | 2 +- .../beam/sdk/io/hdfs/AvroHDFSFileSource.java | 2 +- .../beam/sdk/io/hdfs/AvroWrapperCoder.java | 2 +- .../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 10 +- .../apache/beam/sdk/io/hdfs/WritableCoder.java | 2 +- .../SimpleAuthAvroHDFSFileSource.java | 2 +- .../hdfs/simpleauth/SimpleAuthHDFSFileSink.java | 2 +- .../simpleauth/SimpleAuthHDFSFileSource.java | 7 +- sdks/java/io/jdbc/pom.xml | 138 +++ .../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 443 +++++++++ .../apache/beam/sdk/io/jdbc/package-info.java | 22 + .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 271 +++++ sdks/java/io/jms/pom.xml | 2 +- .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 11 +- sdks/java/io/kafka/pom.xml | 2 +- .../beam/sdk/io/kafka/KafkaCheckpointMark.java | 8 +- .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 90 +- .../apache/beam/sdk/io/kafka/KafkaIOTest.java | 166 +++- sdks/java/io/kinesis/pom.xml | 3 +- .../beam/sdk/io/kinesis/CustomOptional.java | 7 +- .../sdk/io/kinesis/GetKinesisRecordsResult.java | 2 +- .../sdk/io/kinesis/KinesisClientProvider.java | 2 +- .../apache/beam/sdk/io/kinesis/KinesisIO.java | 18 +- .../beam/sdk/io/kinesis/KinesisReader.java | 10 +- .../sdk/io/kinesis/KinesisReaderCheckpoint.java | 4 +- .../beam/sdk/io/kinesis/KinesisRecordCoder.java | 2 +- .../beam/sdk/io/kinesis/KinesisSource.java | 6 +- .../beam/sdk/io/kinesis/RecordFilter.java | 6 +- .../apache/beam/sdk/io/kinesis/RoundRobin.java | 2 +- .../beam/sdk/io/kinesis/ShardCheckpoint.java | 6 +- .../sdk/io/kinesis/ShardRecordsIterator.java | 4 +- .../sdk/io/kinesis/SimplifiedKinesisClient.java | 8 +- .../beam/sdk/io/kinesis/StartingPoint.java | 2 +- .../beam/sdk/io/kinesis/KinesisTestOptions.java | 2 +- .../beam/sdk/io/kinesis/KinesisUploader.java | 5 +- .../beam/sdk/io/kinesis/package-info.java | 22 - sdks/java/io/mongodb/pom.xml | 14 +- .../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 449 +++++++++ .../apache/beam/sdk/io/mongodb/MongoDbIO.java | 310 +++--- .../sdk/io/mongodb/MongoDBGridFSIOTest.java | 276 ++++++ .../beam/sdk/io/mongodb/MongoDbIOTest.java | 19 +- .../beam/sdk/io/mongodb/package-info.java | 22 - sdks/java/io/pom.xml | 3 +- sdks/java/java8tests/pom.xml | 2 +- .../PipelineOptionsFactoryJava8Test.java | 8 +- sdks/java/maven-archetypes/examples/pom.xml | 27 +- .../main/resources/archetype-resources/pom.xml | 26 +- .../src/main/java/DebuggingWordCount.java | 34 +- .../src/main/java/MinimalWordCount.java | 50 +- .../src/main/java/WindowedWordCount.java | 139 +-- .../src/main/java/WordCount.java | 79 +- .../java/common/DataflowExampleOptions.java | 32 - .../main/java/common/DataflowExampleUtils.java | 391 -------- .../common/ExampleBigQueryTableOptions.java | 11 +- .../src/main/java/common/ExampleOptions.java | 32 + ...xamplePubsubTopicAndSubscriptionOptions.java | 45 + .../java/common/ExamplePubsubTopicOptions.java | 17 +- .../src/main/java/common/ExampleUtils.java | 353 +++++++ .../main/java/common/PubsubFileInjector.java | 153 --- .../src/test/java/WordCountTest.java | 9 +- sdks/java/maven-archetypes/pom.xml | 2 +- sdks/java/maven-archetypes/starter/pom.xml | 10 +- .../main/resources/archetype-resources/pom.xml | 8 +- .../resources/projects/basic/reference/pom.xml | 8 +- sdks/java/microbenchmarks/pom.xml | 2 +- .../transforms/DoFnInvokersBenchmark.java | 7 + sdks/java/pom.xml | 2 +- sdks/pom.xml | 2 +- 647 files changed, 37183 insertions(+), 17519 deletions(-) ----------------------------------------------------------------------
