Closes #1861
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/27cf68ee Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/27cf68ee Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/27cf68ee Branch: refs/heads/master Commit: 27cf68ee72bd58475c170712f7afe20102601606 Parents: 1bc6859 f1b8679 Author: Dan Halperin <dhalp...@google.com> Authored: Sun Jan 29 08:21:18 2017 -0800 Committer: Dan Halperin <dhalp...@google.com> Committed: Sun Jan 29 08:21:18 2017 -0800 ---------------------------------------------------------------------- .jenkins/common_job_properties.groovy | 9 +- ...job_beam_PostCommit_Java_MavenInstall.groovy | 2 +- .../job_beam_PreCommit_Java_MavenInstall.groovy | 2 +- .../job_beam_Release_NightlySnapshot.groovy | 2 +- .jenkins/job_seed.groovy | 2 +- .travis/README.md | 2 +- DISCLAIMER | 10 - NOTICE | 4 +- README.md | 46 +- examples/java/README.md | 16 +- examples/java/pom.xml | 21 +- .../beam/examples/DebuggingWordCount.java | 4 +- .../org/apache/beam/examples/WordCount.java | 6 +- .../beam/examples/complete/AutoComplete.java | 2 +- .../org/apache/beam/examples/complete/README.md | 14 +- .../apache/beam/examples/complete/TfIdf.java | 2 +- .../examples/complete/TopWikipediaSessions.java | 2 +- .../examples/complete/TrafficMaxLaneFlow.java | 2 +- .../beam/examples/complete/TrafficRoutes.java | 2 +- .../examples/cookbook/BigQueryTornadoes.java | 2 +- .../cookbook/CombinePerKeyExamples.java | 2 +- .../org/apache/beam/examples/cookbook/README.md | 14 +- .../beam/examples/cookbook/TriggerExample.java | 4 +- .../beam/examples/WindowedWordCountIT.java | 16 +- examples/java8/pom.xml | 2 +- .../beam/examples/complete/game/GameStats.java | 7 +- .../examples/complete/game/LeaderBoard.java | 5 +- .../beam/examples/complete/game/UserScore.java | 2 +- examples/pom.xml | 16 +- pom.xml | 41 +- runners/apex/README.md | 4 +- runners/apex/pom.xml | 3 +- .../beam/runners/apex/ApexPipelineOptions.java | 7 +- .../apache/beam/runners/apex/ApexRunner.java | 43 +- .../beam/runners/apex/ApexYarnLauncher.java | 23 +- .../translation/CreateValuesTranslator.java | 18 +- .../FlattenPCollectionTranslator.java | 28 +- .../apex/translation/GroupByKeyTranslator.java | 2 +- .../translation/ParDoBoundMultiTranslator.java | 27 +- .../apex/translation/ParDoBoundTranslator.java | 4 +- .../apex/translation/TranslationContext.java | 27 +- .../apex/translation/WindowBoundTranslator.java | 8 +- .../operators/ApexGroupByKeyOperator.java | 4 +- .../operators/ApexParDoOperator.java | 6 +- .../ApexReadUnboundedInputOperator.java | 17 +- .../beam/runners/apex/ApexRunnerTest.java | 75 ++ .../beam/runners/apex/ApexYarnLauncherTest.java | 9 +- .../runners/apex/examples/WordCountTest.java | 2 +- .../translation/ParDoBoundTranslatorTest.java | 6 +- .../translation/ReadUnboundTranslatorTest.java | 8 +- .../utils/ApexStateInternalsTest.java | 2 +- .../test/resources/beam-runners-apex.properties | 20 + runners/core-java/pom.xml | 2 +- .../beam/runners/core/AssignWindowsDoFn.java | 3 +- .../apache/beam/runners/core/DoFnAdapters.java | 343 ++++++ .../apache/beam/runners/core/DoFnRunner.java | 21 - .../apache/beam/runners/core/DoFnRunners.java | 138 +-- .../core/GroupAlsoByWindowViaWindowSetDoFn.java | 10 +- .../runners/core/GroupAlsoByWindowsDoFn.java | 5 +- .../beam/runners/core/KeyedWorkItemCoder.java | 4 +- .../core/LateDataDroppingDoFnRunner.java | 1 - .../apache/beam/runners/core/NonEmptyPanes.java | 2 +- .../org/apache/beam/runners/core/OldDoFn.java | 472 ++++++++ .../runners/core/PerKeyCombineFnRunner.java | 70 -- .../runners/core/PerKeyCombineFnRunners.java | 101 -- .../beam/runners/core/SimpleDoFnRunner.java | 63 - .../beam/runners/core/SimpleOldDoFnRunner.java | 7 +- .../beam/runners/core/SplittableParDo.java | 7 - .../core/UnboundedReadFromBoundedSource.java | 14 +- .../AfterDelayFromFirstElementStateMachine.java | 2 +- .../core/triggers/AfterPaneStateMachine.java | 2 +- .../core/DoFnDelegatingAggregatorTest.java | 144 +++ .../core/GroupAlsoByWindowsProperties.java | 2 +- .../runners/core/KeyedWorkItemCoderTest.java | 6 + .../core/LateDataDroppingDoFnRunnerTest.java | 2 +- .../apache/beam/runners/core/NoOpOldDoFn.java | 72 ++ .../beam/runners/core/OldDoFnContextTest.java | 72 ++ .../apache/beam/runners/core/OldDoFnTest.java | 192 +++ .../beam/runners/core/ReduceFnRunnerTest.java | 12 +- .../beam/runners/core/ReduceFnTester.java | 2 +- .../runners/core/SimpleOldDoFnRunnerTest.java | 2 +- .../UnboundedReadFromBoundedSourceTest.java | 12 +- runners/direct-java/pom.xml | 3 +- .../direct/BoundedReadEvaluatorFactory.java | 10 +- ...ecycleManagerRemovingTransformEvaluator.java | 19 +- .../beam/runners/direct/EvaluationContext.java | 2 +- .../direct/ExecutorServiceParallelExecutor.java | 4 +- .../runners/direct/FlattenEvaluatorFactory.java | 4 +- .../GroupAlsoByWindowEvaluatorFactory.java | 9 +- .../direct/GroupByKeyOnlyEvaluatorFactory.java | 10 +- .../beam/runners/direct/ParDoEvaluator.java | 16 +- .../runners/direct/ParDoEvaluatorFactory.java | 19 +- .../direct/ParDoMultiOverrideFactory.java | 13 +- .../direct/StatefulParDoEvaluatorFactory.java | 27 +- .../direct/TestStreamEvaluatorFactory.java | 5 +- .../direct/UnboundedReadEvaluatorFactory.java | 22 +- .../runners/direct/ViewEvaluatorFactory.java | 8 +- .../beam/runners/direct/WatermarkManager.java | 6 +- .../runners/direct/WindowEvaluatorFactory.java | 3 +- .../runners/direct/AggregatorContainerTest.java | 16 +- .../direct/BoundedReadEvaluatorFactoryTest.java | 5 - .../CopyOnAccessInMemoryStateInternalsTest.java | 4 +- .../runners/direct/DirectGraphVisitorTest.java | 16 +- .../beam/runners/direct/DirectRunnerTest.java | 5 - ...leManagerRemovingTransformEvaluatorTest.java | 103 +- .../runners/direct/EvaluationContextTest.java | 6 +- .../beam/runners/direct/ParDoEvaluatorTest.java | 3 +- .../StatefulParDoEvaluatorFactoryTest.java | 4 +- .../UnboundedReadEvaluatorFactoryTest.java | 13 +- runners/flink/README.md | 6 +- runners/flink/examples/pom.xml | 2 +- .../beam/runners/flink/examples/WordCount.java | 2 +- .../flink/examples/streaming/AutoComplete.java | 2 +- .../examples/streaming/KafkaIOExamples.java | 4 +- .../KafkaWindowedWordCountExample.java | 2 +- .../examples/streaming/WindowedWordCount.java | 2 +- runners/flink/pom.xml | 2 +- runners/flink/runner/pom.xml | 4 +- .../runners/flink/FlinkPipelineOptions.java | 6 +- .../runners/flink/OldPerKeyCombineFnRunner.java | 62 + .../flink/OldPerKeyCombineFnRunners.java | 155 +++ .../FlinkBatchTransformTranslators.java | 40 +- .../FlinkBatchTranslationContext.java | 21 +- .../FlinkStreamingTransformTranslators.java | 46 +- .../FlinkStreamingTranslationContext.java | 20 +- .../functions/FlinkDoFnFunction.java | 4 +- .../FlinkMergingNonShuffleReduceFunction.java | 10 +- .../FlinkMergingPartialReduceFunction.java | 8 +- .../functions/FlinkMergingReduceFunction.java | 8 +- .../functions/FlinkMultiOutputDoFnFunction.java | 4 +- .../FlinkMultiOutputProcessContext.java | 2 +- .../functions/FlinkNoElementAssignContext.java | 2 +- .../functions/FlinkPartialReduceFunction.java | 10 +- .../functions/FlinkProcessContextBase.java | 4 +- .../functions/FlinkReduceFunction.java | 10 +- .../FlinkSingleOutputProcessContext.java | 2 +- .../wrappers/streaming/DoFnOperator.java | 6 +- .../streaming/SingletonKeyedWorkItemCoder.java | 10 +- .../wrappers/streaming/WindowDoFnOperator.java | 2 +- .../streaming/io/BoundedSourceWrapper.java | 2 +- .../streaming/io/UnboundedFlinkSink.java | 6 + .../streaming/io/UnboundedSourceWrapper.java | 2 +- .../beam/runners/flink/PipelineOptionsTest.java | 13 + .../streaming/FlinkStateInternalsTest.java | 2 +- .../streaming/UnboundedSourceWrapperTest.java | 464 +++---- runners/google-cloud-dataflow-java/pom.xml | 15 +- .../beam/runners/dataflow/AssignWindows.java | 89 ++ .../dataflow/DataflowAggregatorTransforms.java | 79 ++ .../dataflow/DataflowMetricUpdateExtractor.java | 109 ++ .../runners/dataflow/DataflowPipelineJob.java | 2 - .../dataflow/DataflowPipelineTranslator.java | 510 +++----- .../beam/runners/dataflow/DataflowRunner.java | 109 +- .../DataflowUnboundedReadFromBoundedSource.java | 547 +++++++++ .../beam/runners/dataflow/ReadTranslator.java | 102 ++ .../runners/dataflow/TransformTranslator.java | 120 ++ .../dataflow/internal/AssignWindows.java | 89 -- .../dataflow/internal/CustomSources.java | 5 - .../internal/DataflowAggregatorTransforms.java | 79 -- .../internal/DataflowMetricUpdateExtractor.java | 109 -- .../DataflowUnboundedReadFromBoundedSource.java | 556 --------- .../runners/dataflow/internal/IsmFormat.java | 20 +- .../dataflow/internal/ReadTranslator.java | 107 -- .../DataflowPipelineWorkerPoolOptions.java | 16 +- .../beam/runners/dataflow/util/DoFnInfo.java | 66 +- .../beam/runners/dataflow/util/GcsStager.java | 18 +- .../beam/runners/dataflow/util/PackageUtil.java | 352 ++++-- .../beam/runners/dataflow/dataflow.properties | 6 +- .../dataflow/DataflowPipelineJobTest.java | 38 +- .../DataflowPipelineTranslatorTest.java | 3 +- .../runners/dataflow/DataflowRunnerTest.java | 8 +- ...aflowUnboundedReadFromBoundedSourceTest.java | 79 ++ ...aflowUnboundedReadFromBoundedSourceTest.java | 83 -- .../DataflowPipelineDebugOptionsTest.java | 2 +- .../options/DataflowPipelineOptionsTest.java | 4 +- .../options/DataflowProfilingOptionsTest.java | 4 +- .../runners/dataflow/util/PackageUtilTest.java | 69 +- runners/pom.xml | 16 +- runners/spark/README.md | 8 +- runners/spark/pom.xml | 29 +- .../spark/aggregators/NamedAggregators.java | 4 +- .../coders/BeamSparkRunnerRegistrator.java | 48 +- .../spark/coders/StatelessJavaSerializer.java | 97 ++ .../runners/spark/coders/WritableCoder.java | 4 +- .../beam/runners/spark/examples/WordCount.java | 2 +- .../beam/runners/spark/io/MicrobatchSource.java | 9 +- .../runners/spark/io/SparkUnboundedSource.java | 127 +- .../spark/stateful/StateSpecFunctions.java | 37 +- .../runners/spark/translation/DoFnFunction.java | 15 +- .../spark/translation/EvaluationContext.java | 83 +- .../translation/GroupCombineFunctions.java | 8 +- .../spark/translation/MultiDoFnFunction.java | 14 +- .../translation/SparkAbstractCombineFn.java | 12 +- .../spark/translation/SparkGlobalCombineFn.java | 13 +- .../translation/SparkGroupAlsoByWindowFn.java | 2 +- .../spark/translation/SparkKeyedCombineFn.java | 13 +- .../spark/translation/SparkPCollectionView.java | 99 ++ .../spark/translation/SparkRuntimeContext.java | 63 +- .../spark/translation/TransformTranslator.java | 67 +- .../spark/translation/TranslationUtils.java | 37 +- .../streaming/StreamingTransformTranslator.java | 114 +- .../runners/spark/util/BroadcastHelper.java | 127 -- .../runners/spark/util/SideInputBroadcast.java | 77 ++ .../spark/util/SparkSideInputReader.java | 8 +- .../coders/BeamSparkRunnerRegistratorTest.java | 57 - .../streaming/KafkaStreamingTest.java | 57 +- .../ResumeFromCheckpointStreamingTest.java | 20 +- .../streaming/utils/PAssertStreaming.java | 4 +- sdks/java/build-tools/pom.xml | 2 +- .../src/main/resources/beam/findbugs-filter.xml | 26 - sdks/java/core/pom.xml | 2 +- .../beam/sdk/annotations/Experimental.java | 5 +- .../org/apache/beam/sdk/coders/AtomicCoder.java | 2 +- .../org/apache/beam/sdk/coders/AvroCoder.java | 30 +- .../apache/beam/sdk/coders/BigDecimalCoder.java | 6 +- .../beam/sdk/coders/BigEndianIntegerCoder.java | 7 + .../beam/sdk/coders/BigEndianLongCoder.java | 7 + .../apache/beam/sdk/coders/ByteArrayCoder.java | 7 + .../org/apache/beam/sdk/coders/ByteCoder.java | 7 + .../apache/beam/sdk/coders/ByteStringCoder.java | 8 + .../java/org/apache/beam/sdk/coders/Coder.java | 7 + .../apache/beam/sdk/coders/CollectionCoder.java | 12 +- .../org/apache/beam/sdk/coders/CustomCoder.java | 18 +- .../apache/beam/sdk/coders/DelegateCoder.java | 29 +- .../org/apache/beam/sdk/coders/DoubleCoder.java | 7 + .../apache/beam/sdk/coders/DurationCoder.java | 8 + .../apache/beam/sdk/coders/InstantCoder.java | 7 + .../apache/beam/sdk/coders/IterableCoder.java | 12 +- .../org/apache/beam/sdk/coders/JAXBCoder.java | 48 +- .../org/apache/beam/sdk/coders/KvCoder.java | 35 +- .../beam/sdk/coders/LengthPrefixCoder.java | 145 +++ .../org/apache/beam/sdk/coders/ListCoder.java | 7 + .../org/apache/beam/sdk/coders/MapCoder.java | 62 +- .../apache/beam/sdk/coders/NullableCoder.java | 6 + .../beam/sdk/coders/SerializableCoder.java | 17 +- .../org/apache/beam/sdk/coders/SetCoder.java | 12 +- .../apache/beam/sdk/coders/StandardCoder.java | 39 +- .../beam/sdk/coders/StringDelegateCoder.java | 16 +- .../apache/beam/sdk/coders/StringUtf8Coder.java | 7 + .../beam/sdk/coders/TableRowJsonCoder.java | 7 + .../beam/sdk/coders/TextualIntegerCoder.java | 8 + .../org/apache/beam/sdk/coders/VarIntCoder.java | 10 +- .../apache/beam/sdk/coders/VarLongCoder.java | 7 + .../org/apache/beam/sdk/coders/VoidCoder.java | 7 + .../beam/sdk/coders/protobuf/ProtoCoder.java | 8 +- .../java/org/apache/beam/sdk/io/AvroSource.java | 5 - .../sdk/io/BoundedReadFromUnboundedSource.java | 79 +- .../org/apache/beam/sdk/io/BoundedSource.java | 8 - .../apache/beam/sdk/io/CompressedSource.java | 8 - .../org/apache/beam/sdk/io/CountingSource.java | 5 - .../org/apache/beam/sdk/io/FileSystems.java | 32 +- .../java/org/apache/beam/sdk/io/PubsubIO.java | 1142 +++++++++--------- .../apache/beam/sdk/io/PubsubUnboundedSink.java | 88 +- .../beam/sdk/io/PubsubUnboundedSource.java | 104 +- .../main/java/org/apache/beam/sdk/io/Read.java | 7 +- .../java/org/apache/beam/sdk/io/TextIO.java | 5 - .../java/org/apache/beam/sdk/io/XmlSource.java | 5 - .../org/apache/beam/sdk/options/GcpOptions.java | 36 +- .../org/apache/beam/sdk/options/GcsOptions.java | 4 +- .../beam/sdk/options/PipelineOptions.java | 2 +- .../sdk/options/PipelineOptionsFactory.java | 10 +- .../apache/beam/sdk/options/ValueProvider.java | 6 +- .../beam/sdk/runners/TransformHierarchy.java | 33 +- .../testing/FlattenWithHeterogeneousCoders.java | 29 + .../org/apache/beam/sdk/testing/PAssert.java | 12 +- .../beam/sdk/testing/RunnableOnService.java | 14 +- .../beam/sdk/testing/SourceTestUtils.java | 5 - .../org/apache/beam/sdk/testing/TestStream.java | 8 + .../sdk/testing/UsesUnboundedPCollections.java | 23 + .../beam/sdk/testing/ValueInSingleWindow.java | 6 +- .../sdk/transforms/AggregatorRetriever.java | 13 +- .../beam/sdk/transforms/AppliedPTransform.java | 11 +- .../org/apache/beam/sdk/transforms/Combine.java | 197 +-- .../apache/beam/sdk/transforms/CombineFns.java | 14 +- .../org/apache/beam/sdk/transforms/Count.java | 4 +- .../org/apache/beam/sdk/transforms/Create.java | 5 - .../sdk/transforms/DelegatingAggregator.java | 2 +- .../beam/sdk/transforms/DoFnAdapters.java | 504 -------- .../apache/beam/sdk/transforms/DoFnTester.java | 7 - .../apache/beam/sdk/transforms/GroupByKey.java | 2 +- .../org/apache/beam/sdk/transforms/Max.java | 124 +- .../org/apache/beam/sdk/transforms/Mean.java | 27 +- .../org/apache/beam/sdk/transforms/Min.java | 122 +- .../org/apache/beam/sdk/transforms/OldDoFn.java | 758 ------------ .../apache/beam/sdk/transforms/PTransform.java | 9 +- .../org/apache/beam/sdk/transforms/ParDo.java | 46 +- .../org/apache/beam/sdk/transforms/Regex.java | 589 ++++++++- .../org/apache/beam/sdk/transforms/Sum.java | 57 +- .../apache/beam/sdk/transforms/ToString.java | 198 +++ .../org/apache/beam/sdk/transforms/Top.java | 27 +- .../beam/sdk/transforms/join/CoGbkResult.java | 35 +- .../sdk/transforms/reflect/DoFnInvoker.java | 20 - .../sdk/transforms/reflect/DoFnInvokers.java | 142 +-- .../sdk/transforms/reflect/DoFnSignature.java | 15 +- .../windowing/AfterDelayFromFirstElement.java | 2 +- .../sdk/transforms/windowing/AfterPane.java | 2 +- .../sdk/transforms/windowing/GlobalWindow.java | 6 + .../transforms/windowing/IntervalWindow.java | 4 +- .../beam/sdk/transforms/windowing/Window.java | 3 +- .../org/apache/beam/sdk/util/CoderUtils.java | 28 +- .../beam/sdk/util/CombineContextFactory.java | 18 - .../org/apache/beam/sdk/util/DefaultBucket.java | 105 ++ .../util/EmptyOnDeserializationThreadLocal.java | 39 + .../apache/beam/sdk/util/GcpProjectUtil.java | 2 +- .../java/org/apache/beam/sdk/util/GcsUtil.java | 36 +- .../org/apache/beam/sdk/util/NameUtils.java | 162 +++ .../org/apache/beam/sdk/util/PropertyNames.java | 1 + .../org/apache/beam/sdk/util/PubsubClient.java | 28 +- .../apache/beam/sdk/util/PubsubGrpcClient.java | 6 +- .../apache/beam/sdk/util/PubsubJsonClient.java | 4 +- .../apache/beam/sdk/util/PubsubTestClient.java | 6 +- .../org/apache/beam/sdk/util/StringUtils.java | 100 -- .../apache/beam/sdk/util/TimerInternals.java | 4 +- .../org/apache/beam/sdk/util/WindowedValue.java | 23 +- .../beam/sdk/util/state/StateContexts.java | 4 +- .../org/apache/beam/sdk/values/PValueBase.java | 4 +- .../beam/sdk/values/TimestampedValue.java | 10 +- .../sdk/AggregatorPipelineExtractorTest.java | 16 +- .../apache/beam/sdk/coders/AvroCoderTest.java | 7 + .../beam/sdk/coders/BigDecimalCoderTest.java | 46 +- .../sdk/coders/BigEndianIntegerCoderTest.java | 9 + .../beam/sdk/coders/BigEndianLongCoderTest.java | 9 + .../beam/sdk/coders/ByteArrayCoderTest.java | 6 + .../apache/beam/sdk/coders/ByteCoderTest.java | 9 + .../beam/sdk/coders/ByteStringCoderTest.java | 8 + .../beam/sdk/coders/CoderRegistryTest.java | 6 + .../org/apache/beam/sdk/coders/CoderTest.java | 8 + .../beam/sdk/coders/CollectionCoderTest.java | 16 + .../beam/sdk/coders/DefaultCoderTest.java | 4 +- .../beam/sdk/coders/DelegateCoderTest.java | 35 +- .../apache/beam/sdk/coders/DoubleCoderTest.java | 9 + .../beam/sdk/coders/DurationCoderTest.java | 10 + .../beam/sdk/coders/InstantCoderTest.java | 9 + .../beam/sdk/coders/IterableCoderTest.java | 27 +- .../apache/beam/sdk/coders/JAXBCoderTest.java | 26 +- .../org/apache/beam/sdk/coders/KvCoderTest.java | 29 + .../beam/sdk/coders/LengthPrefixCoderTest.java | 129 ++ .../apache/beam/sdk/coders/ListCoderTest.java | 16 +- .../apache/beam/sdk/coders/MapCoderTest.java | 21 +- .../beam/sdk/coders/NullableCoderTest.java | 12 + .../beam/sdk/coders/SerializableCoderTest.java | 9 + .../apache/beam/sdk/coders/SetCoderTest.java | 16 + .../beam/sdk/coders/StandardCoderTest.java | 40 + .../sdk/coders/StringDelegateCoderTest.java | 11 + .../beam/sdk/coders/StringUtf8CoderTest.java | 9 + .../beam/sdk/coders/TableRowJsonCoderTest.java | 9 + .../sdk/coders/TextualIntegerCoderTest.java | 9 + .../apache/beam/sdk/coders/VarIntCoderTest.java | 9 + .../beam/sdk/coders/VarLongCoderTest.java | 9 + .../apache/beam/sdk/coders/VoidCoderTest.java | 40 + .../beam/sdk/io/AvroIOGeneratedClassTest.java | 285 ----- .../apache/beam/sdk/io/AvroIOTransformTest.java | 324 +++++ .../beam/sdk/io/CompressedSourceTest.java | 5 - .../apache/beam/sdk/io/FileBasedSourceTest.java | 5 - .../org/apache/beam/sdk/io/FileSystemsTest.java | 33 +- .../beam/sdk/io/OffsetBasedSourceTest.java | 5 - .../org/apache/beam/sdk/io/PubsubIOTest.java | 86 +- .../beam/sdk/io/PubsubUnboundedSinkTest.java | 41 +- .../beam/sdk/io/PubsubUnboundedSourceTest.java | 10 +- .../java/org/apache/beam/sdk/io/ReadTest.java | 5 - .../java/org/apache/beam/sdk/io/WriteTest.java | 10 +- .../apache/beam/sdk/options/GcpOptionsTest.java | 4 +- .../sdk/options/PipelineOptionsFactoryTest.java | 6 +- .../beam/sdk/options/PipelineOptionsTest.java | 3 +- .../beam/sdk/options/ValueProviderTest.java | 36 +- .../sdk/options/ValueProviderUtilsTest.java | 2 +- .../sdk/runners/TransformHierarchyTest.java | 30 +- .../apache/beam/sdk/testing/TestStreamTest.java | 5 + .../testing/ValueInSingleWindowCoderTest.java | 7 + .../sdk/transforms/ApproximateUniqueTest.java | 483 ++++---- .../beam/sdk/transforms/CombineFnsTest.java | 20 +- .../apache/beam/sdk/transforms/CombineTest.java | 99 +- .../apache/beam/sdk/transforms/CountTest.java | 2 +- .../apache/beam/sdk/transforms/CreateTest.java | 8 - .../DoFnDelegatingAggregatorTest.java | 142 --- .../apache/beam/sdk/transforms/DoFnTest.java | 15 +- .../beam/sdk/transforms/DoFnTesterTest.java | 6 +- .../apache/beam/sdk/transforms/FlattenTest.java | 27 + .../apache/beam/sdk/transforms/KvSwapTest.java | 13 +- .../org/apache/beam/sdk/transforms/MaxTest.java | 20 +- .../apache/beam/sdk/transforms/MeanTest.java | 7 +- .../org/apache/beam/sdk/transforms/MinTest.java | 21 +- .../apache/beam/sdk/transforms/NoOpOldDoFn.java | 71 -- .../beam/sdk/transforms/OldDoFnContextTest.java | 69 -- .../apache/beam/sdk/transforms/OldDoFnTest.java | 188 --- .../apache/beam/sdk/transforms/ParDoTest.java | 74 +- .../apache/beam/sdk/transforms/RegexTest.java | 127 +- .../apache/beam/sdk/transforms/SampleTest.java | 405 ++++--- .../beam/sdk/transforms/SimpleStatsFnsTest.java | 36 +- .../org/apache/beam/sdk/transforms/SumTest.java | 24 +- .../beam/sdk/transforms/ToStringTest.java | 125 ++ .../org/apache/beam/sdk/transforms/TopTest.java | 13 +- .../apache/beam/sdk/transforms/ViewTest.java | 2 +- .../transforms/join/CoGbkResultCoderTest.java | 10 +- .../sdk/transforms/join/UnionCoderTest.java | 24 +- .../transforms/reflect/DoFnInvokersTest.java | 42 - .../transforms/windowing/GlobalWindowTest.java | 64 + .../apache/beam/sdk/util/CombineFnUtilTest.java | 8 +- .../apache/beam/sdk/util/DefaultBucketTest.java | 112 ++ .../org/apache/beam/sdk/util/GcsUtilTest.java | 56 + .../org/apache/beam/sdk/util/NameUtilsTest.java | 177 +++ .../beam/sdk/util/PubsubGrpcClientTest.java | 8 +- .../beam/sdk/util/PubsubJsonClientTest.java | 3 +- .../beam/sdk/util/PubsubTestClientTest.java | 4 +- .../beam/sdk/util/SerializableUtilsTest.java | 4 +- .../apache/beam/sdk/util/StringUtilsTest.java | 100 -- .../beam/sdk/util/TimerInternalsTest.java | 5 + .../beam/sdk/util/ValueWithRecordIdTest.java | 34 + .../apache/beam/sdk/util/WindowedValueTest.java | 23 + .../util/state/InMemoryStateInternalsTest.java | 2 +- .../beam/sdk/util/state/StateTagTest.java | 11 +- .../beam/sdk/values/TimestampedValueTest.java | 19 +- sdks/java/extensions/join-library/README.md | 10 - sdks/java/extensions/join-library/pom.xml | 2 +- sdks/java/extensions/pom.xml | 2 +- sdks/java/extensions/sorter/pom.xml | 2 +- sdks/java/io/elasticsearch/pom.xml | 175 +++ .../sdk/io/elasticsearch/ElasticsearchIO.java | 819 +++++++++++++ .../beam/sdk/io/elasticsearch/package-info.java | 20 + .../elasticsearch/ElasticSearchIOTestUtils.java | 129 ++ .../io/elasticsearch/ElasticsearchIOTest.java | 358 ++++++ sdks/java/io/google-cloud-platform/pom.xml | 2 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 308 +++-- .../sdk/io/gcp/bigquery/BigQueryServices.java | 16 +- .../io/gcp/bigquery/BigQueryServicesImpl.java | 76 +- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 8 - .../io/gcp/bigtable/BigtableTestOptions.java | 37 - .../sdk/io/gcp/bigquery/BigQueryIOTest.java | 205 +++- .../gcp/bigquery/BigQueryServicesImplTest.java | 141 +++ .../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 3 +- .../sdk/io/gcp/bigtable/BigtableIOTest.java | 5 +- .../io/gcp/bigtable/BigtableTestOptions.java | 37 + sdks/java/io/hdfs/pom.xml | 2 +- .../beam/sdk/io/hdfs/AvroWrapperCoder.java | 4 +- .../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 5 - .../apache/beam/sdk/io/hdfs/WritableCoder.java | 4 +- .../beam/sdk/io/hdfs/AvroWrapperCoderTest.java | 1 - sdks/java/io/jdbc/pom.xml | 2 +- sdks/java/io/jms/pom.xml | 2 +- sdks/java/io/kafka/pom.xml | 2 +- .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 7 +- .../beam/sdk/io/kafka/KafkaRecordCoder.java | 4 +- .../apache/beam/sdk/io/kafka/KafkaIOTest.java | 9 +- .../beam/sdk/io/kafka/KafkaRecordCoderTest.java | 34 + sdks/java/io/kinesis/pom.xml | 2 +- .../beam/sdk/io/kinesis/KinesisRecordCoder.java | 4 +- .../beam/sdk/io/kinesis/package-info.java | 2 +- sdks/java/io/mongodb/pom.xml | 2 +- .../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 5 - .../apache/beam/sdk/io/mongodb/MongoDbIO.java | 5 - sdks/java/io/mqtt/pom.xml | 152 +++ .../org/apache/beam/sdk/io/mqtt/MqttIO.java | 588 +++++++++ .../apache/beam/sdk/io/mqtt/package-info.java | 22 + .../org/apache/beam/sdk/io/mqtt/MqttIOTest.java | 197 +++ sdks/java/io/pom.xml | 4 +- sdks/java/java8tests/pom.xml | 2 +- .../maven-archetypes/examples-java8/pom.xml | 2 +- .../main/resources/archetype-resources/pom.xml | 19 +- sdks/java/maven-archetypes/examples/pom.xml | 2 +- .../main/resources/archetype-resources/pom.xml | 19 +- sdks/java/maven-archetypes/pom.xml | 2 +- sdks/java/maven-archetypes/starter/pom.xml | 2 +- .../main/resources/archetype-resources/pom.xml | 4 +- .../resources/projects/basic/reference/pom.xml | 4 +- sdks/java/pom.xml | 2 +- sdks/pom.xml | 15 +- sdks/python/pom.xml | 2 +- 466 files changed, 13883 insertions(+), 8047 deletions(-) ----------------------------------------------------------------------