Merge branch 'master' upto commit 686b774ceda8bee32032cb421651e8350ca5bf3d into jstorm-runner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f1e170a5 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f1e170a5 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f1e170a5 Branch: refs/heads/jstorm-runner Commit: f1e170a5fa9dc4d462af42f9f382afd0ecd798b6 Parents: f6a89b0 686b774 Author: Pei He <[email protected]> Authored: Tue Apr 25 17:37:52 2017 +0800 Committer: Pei He <[email protected]> Committed: Tue Apr 25 18:19:41 2017 +0800 ---------------------------------------------------------------------- .gitignore | 3 + .jenkins/common_job_properties.groovy | 261 ---- .../job_beam_PerformanceTests_Dataflow.groovy | 43 - .jenkins/job_beam_PerformanceTests_JDBC.groovy | 60 - .jenkins/job_beam_PerformanceTests_Spark.groovy | 44 - ...job_beam_PostCommit_Java_MavenInstall.groovy | 42 - ..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 - ...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 - ...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 - ...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 - ...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 - .../job_beam_PostCommit_Python_Verify.groovy | 55 - .../job_beam_PreCommit_Java_MavenInstall.groovy | 42 - .../job_beam_PreCommit_Website_Stage.groovy | 80 - .jenkins/job_beam_PreCommit_Website_Test.groovy | 65 - .../job_beam_Release_NightlySnapshot.groovy | 45 - .jenkins/job_seed.groovy | 53 - .../jenkins/common_job_properties.groovy | 261 ++++ .../job_beam_PerformanceTests_Dataflow.groovy | 43 + .../job_beam_PerformanceTests_JDBC.groovy | 60 + .../job_beam_PerformanceTests_Spark.groovy | 44 + ...job_beam_PostCommit_Java_MavenInstall.groovy | 42 + ..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 + ...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 + ...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 + ...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 + ...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 + .../job_beam_PostCommit_Python_Verify.groovy | 55 + .../job_beam_PreCommit_Java_MavenInstall.groovy | 42 + .../job_beam_PreCommit_Website_Stage.groovy | 80 + .../job_beam_PreCommit_Website_Test.groovy | 65 + .../job_beam_Release_NightlySnapshot.groovy | 45 + .test-infra/jenkins/job_seed.groovy | 53 + .../cassandra-service-for-local-dev.yaml | 28 + .../cassandra-svc-statefulset.yaml | 114 ++ .../LargeITCluster/cassandra-svc-temp.yaml | 74 + .../cassandra/LargeITCluster/data-load.sh | 122 ++ .../cassandra/LargeITCluster/show_health.sh | 47 + .../cassandra/LargeITCluster/start-up.sh | 22 + .../cassandra/LargeITCluster/teardown.sh | 25 + .../cassandra-service-for-local-dev.yaml | 30 + .../SmallITCluster/cassandra-svc-rc.yaml | 74 + .../cassandra/SmallITCluster/data-load.sh | 86 + .../cassandra/SmallITCluster/show_health.sh | 47 + .../cassandra/SmallITCluster/start-up.sh | 23 + .../cassandra/SmallITCluster/teardown.sh | 22 + .../kubernetes/cassandra/data-load-setup.sh | 29 + .../elasticsearch-service-for-local-dev.yaml | 33 + .../es-services-deployments.yaml | 258 +++ .../LargeProductionCluster/start-up.sh | 22 + .../LargeProductionCluster/teardown.sh | 21 + .../elasticsearch-service-for-local-dev.yaml | 34 + .../SmallITCluster/elasticsearch-svc-rc.yaml | 96 ++ .../elasticsearch/SmallITCluster/start-up.sh | 23 + .../elasticsearch/SmallITCluster/teardown.sh | 21 + .../kubernetes/elasticsearch/data-load-setup.sh | 26 + .../kubernetes/elasticsearch/data-load.sh | 33 + .../kubernetes/elasticsearch/es_test_data.py | 299 ++++ .../kubernetes/elasticsearch/show-health.sh | 33 + .../postgres/postgres-service-for-local-dev.yml | 28 + .test-infra/kubernetes/postgres/postgres.yml | 56 + .test-infra/travis/README.md | 23 + .test-infra/travis/settings.xml | 33 + .test-infra/travis/test_wordcount.sh | 125 ++ .travis.yml | 4 +- .travis/README.md | 23 - .travis/settings.xml | 33 - .travis/test_wordcount.sh | 125 -- README.md | 1 + examples/java8/pom.xml | 34 + .../beam/examples/MinimalWordCountJava8.java | 9 +- .../beam/examples/complete/game/GameStats.java | 8 +- .../examples/complete/game/LeaderBoard.java | 2 +- .../beam/examples/complete/game/UserScore.java | 5 +- .../examples/MinimalWordCountJava8Test.java | 9 +- .../complete/game/HourlyTeamScoreTest.java | 5 +- .../examples/complete/game/UserScoreTest.java | 6 +- .../apache/beam/runners/apex/ApexRunner.java | 34 +- .../beam/runners/apex/ApexYarnLauncher.java | 111 +- .../translation/ApexPipelineTranslator.java | 2 +- .../FlattenPCollectionTranslator.java | 13 +- .../apex/translation/GroupByKeyTranslator.java | 4 +- .../apex/translation/ParDoTranslator.java | 30 +- .../apex/translation/TranslationContext.java | 22 +- .../translation/WindowAssignTranslator.java | 58 +- .../operators/ApexGroupByKeyOperator.java | 273 +--- .../operators/ApexParDoOperator.java | 235 ++- .../operators/ApexProcessFnOperator.java | 184 +++ .../translation/utils/ApexStateInternals.java | 73 +- .../apex/translation/utils/NoOpStepContext.java | 2 +- .../utils/SerializablePipelineOptions.java | 13 +- .../translation/utils/StateInternalsProxy.java | 67 + .../translation/ApexGroupByKeyOperatorTest.java | 2 +- .../apex/translation/ParDoTranslatorTest.java | 20 +- .../utils/ApexStateInternalsTest.java | 25 +- runners/core-construction-java/pom.xml | 36 + .../beam/runners/core/construction/Coders.java | 174 +++ .../DeduplicatedFlattenFactory.java | 79 +- .../EmptyFlattenAsCreateFactory.java | 25 +- .../runners/core/construction/PCollections.java | 97 ++ .../core/construction/PTransformMatchers.java | 7 +- .../construction/PTransformReplacements.java | 69 + .../core/construction/PrimitiveCreate.java | 18 +- .../core/construction/ReplacementOutputs.java | 63 +- .../core/construction/SdkComponents.java | 159 ++ .../SingleInputOutputOverrideFactory.java | 14 +- .../runners/core/construction/Triggers.java | 336 ++++ .../UnboundedReadFromBoundedSource.java | 542 +++++++ .../UnsupportedOverrideFactory.java | 20 +- .../core/construction/WindowingStrategies.java | 245 +++ .../runners/core/construction/CodersTest.java | 163 ++ .../DeduplicatedFlattenFactoryTest.java | 24 +- .../EmptyFlattenAsCreateFactoryTest.java | 42 +- .../core/construction/PCollectionsTest.java | 188 +++ .../construction/PTransformMatchersTest.java | 132 +- .../PTransformReplacementsTest.java | 131 ++ .../construction/ReplacementOutputsTest.java | 109 +- .../core/construction/SdkComponentsTest.java | 157 ++ .../SingleInputOutputOverrideFactoryTest.java | 37 +- .../runners/core/construction/TriggersTest.java | 111 ++ .../UnboundedReadFromBoundedSourceTest.java | 373 +++++ .../UnsupportedOverrideFactoryTest.java | 16 +- .../construction/WindowingStrategiesTest.java | 110 ++ runners/core-java/pom.xml | 10 +- .../beam/runners/core/AssignWindowsDoFn.java | 78 - .../beam/runners/core/BaseExecutionContext.java | 13 +- .../apache/beam/runners/core/DoFnAdapters.java | 323 ---- .../apache/beam/runners/core/DoFnRunners.java | 10 +- .../beam/runners/core/ExecutionContext.java | 13 +- .../GroupAlsoByWindowViaOutputBufferDoFn.java | 19 +- .../core/GroupAlsoByWindowViaWindowSetDoFn.java | 9 +- .../GroupAlsoByWindowViaWindowSetNewDoFn.java | 19 +- .../core/GroupAlsoByWindowsAggregators.java | 28 + .../runners/core/GroupAlsoByWindowsDoFn.java | 46 - .../core/LateDataDroppingDoFnRunner.java | 3 +- .../org/apache/beam/runners/core/OldDoFn.java | 41 +- ...eBoundedSplittableProcessElementInvoker.java | 133 +- .../beam/runners/core/OutputWindowedValue.java | 10 +- .../beam/runners/core/SimpleDoFnRunner.java | 59 +- .../beam/runners/core/SimpleOldDoFnRunner.java | 63 +- .../beam/runners/core/SplittableParDo.java | 36 +- .../core/SplittableProcessElementInvoker.java | 22 +- .../core/UnboundedReadFromBoundedSource.java | 542 ------- .../beam/runners/core/WindowingInternals.java | 8 +- .../core/WindowingInternalsAdapters.java | 8 +- .../triggers/AfterWatermarkStateMachine.java | 14 +- ...roupAlsoByWindowViaOutputBufferDoFnTest.java | 4 +- .../core/GroupAlsoByWindowsProperties.java | 37 +- .../apache/beam/runners/core/NoOpOldDoFn.java | 4 +- .../apache/beam/runners/core/OldDoFnTest.java | 4 +- ...ndedSplittableProcessElementInvokerTest.java | 27 +- .../beam/runners/core/ReduceFnTester.java | 13 +- .../runners/core/SimpleOldDoFnRunnerTest.java | 4 +- .../beam/runners/core/SplittableParDoTest.java | 238 +-- .../UnboundedReadFromBoundedSourceTest.java | 373 ----- .../direct/BoundedReadEvaluatorFactory.java | 2 +- ...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 16 +- .../beam/runners/direct/DirectGraphVisitor.java | 5 +- .../direct/DirectGroupByKeyOverrideFactory.java | 14 +- .../direct/ExecutorServiceParallelExecutor.java | 4 +- .../runners/direct/FlattenEvaluatorFactory.java | 2 +- .../GroupAlsoByWindowEvaluatorFactory.java | 18 +- .../direct/GroupByKeyOnlyEvaluatorFactory.java | 4 +- .../direct/KeyedPValueTrackingVisitor.java | 14 +- .../beam/runners/direct/ModelEnforcement.java | 13 +- .../beam/runners/direct/ParDoEvaluator.java | 4 +- .../runners/direct/ParDoEvaluatorFactory.java | 20 +- .../direct/ParDoMultiOverrideFactory.java | 29 +- ...littableProcessElementsEvaluatorFactory.java | 8 +- .../direct/StatefulParDoEvaluatorFactory.java | 10 +- .../direct/TestStreamEvaluatorFactory.java | 20 +- .../direct/TransformEvaluatorFactory.java | 10 +- .../direct/TransformExecutorServices.java | 37 +- .../direct/UnboundedReadEvaluatorFactory.java | 4 +- .../runners/direct/ViewEvaluatorFactory.java | 4 +- .../runners/direct/ViewOverrideFactory.java | 23 +- .../beam/runners/direct/WatermarkManager.java | 19 +- .../runners/direct/WindowEvaluatorFactory.java | 2 +- .../direct/WriteWithShardingFactory.java | 22 +- .../runners/direct/DirectGraphVisitorTest.java | 7 +- .../DirectGroupByKeyOverrideFactoryTest.java | 12 +- .../beam/runners/direct/DirectRunnerTest.java | 5 +- .../beam/runners/direct/ParDoEvaluatorTest.java | 8 +- .../direct/ParDoMultiOverrideFactoryTest.java | 45 - .../StatefulParDoEvaluatorFactoryTest.java | 2 +- .../direct/TestStreamEvaluatorFactoryTest.java | 11 - .../direct/TransformExecutorServicesTest.java | 48 + .../runners/direct/ViewOverrideFactoryTest.java | 44 +- .../direct/WriteWithShardingFactoryTest.java | 23 +- .../examples/streaming/KafkaIOExamples.java | 338 ---- .../KafkaWindowedWordCountExample.java | 164 -- runners/flink/pom.xml | 45 +- runners/flink/runner/pom.xml | 109 +- .../flink/FlinkBatchTransformTranslators.java | 36 +- .../flink/FlinkBatchTranslationContext.java | 11 +- .../flink/FlinkDetachedRunnerResult.java | 3 +- .../flink/FlinkStreamingPipelineTranslator.java | 61 +- .../FlinkStreamingTransformTranslators.java | 139 +- .../flink/FlinkStreamingTranslationContext.java | 12 +- .../functions/FlinkDoFnFunction.java | 4 +- .../functions/FlinkNoOpStepContext.java | 2 +- .../functions/FlinkStatefulDoFnFunction.java | 4 +- .../types/EncodedValueTypeInformation.java | 9 - .../flink/translation/types/FlinkCoder.java | 63 - .../utils/SerializedPipelineOptions.java | 2 + .../wrappers/streaming/DoFnOperator.java | 14 +- .../streaming/SingletonKeyedWorkItem.java | 2 - .../streaming/SingletonKeyedWorkItemCoder.java | 4 +- .../streaming/SplittableDoFnOperator.java | 10 +- .../wrappers/streaming/WindowDoFnOperator.java | 7 +- .../streaming/io/UnboundedFlinkSink.java | 200 --- .../streaming/io/UnboundedFlinkSource.java | 120 -- .../beam/runners/flink/PipelineOptionsTest.java | 2 +- .../flink/streaming/DoFnOperatorTest.java | 34 +- runners/google-cloud-dataflow-java/pom.xml | 25 +- .../dataflow/BatchStatefulParDoOverrides.java | 51 +- .../runners/dataflow/BatchViewOverrides.java | 23 +- .../dataflow/DataflowPipelineTranslator.java | 37 +- .../beam/runners/dataflow/DataflowRunner.java | 146 +- .../DataflowUnboundedReadFromBoundedSource.java | 547 ------- .../dataflow/PrimitiveParDoSingleFactory.java | 15 +- .../dataflow/ReshuffleOverrideFactory.java | 12 +- .../dataflow/StreamingViewOverrides.java | 14 +- .../runners/dataflow/TransformTranslator.java | 6 +- .../dataflow/testing/TestDataflowRunner.java | 34 +- .../dataflow/BatchViewOverridesTest.java | 4 +- .../dataflow/DataflowPipelineJobTest.java | 7 +- ...aflowUnboundedReadFromBoundedSourceTest.java | 79 - .../PrimitiveParDoSingleFactoryTest.java | 59 +- .../testing/TestDataflowRunnerTest.java | 3 +- .../apache/beam/runners/spark/SparkRunner.java | 20 +- .../beam/runners/spark/TestSparkRunner.java | 21 +- .../beam/runners/spark/io/MicrobatchSource.java | 113 +- .../beam/runners/spark/io/SourceDStream.java | 11 +- .../SparkGroupAlsoByWindowViaWindowSet.java | 22 +- .../spark/stateful/StateSpecFunctions.java | 6 +- .../runners/spark/translation/DoFnFunction.java | 130 -- .../spark/translation/EvaluationContext.java | 11 +- .../spark/translation/MultiDoFnFunction.java | 4 +- .../spark/translation/SparkAssignWindowFn.java | 3 +- ...SparkGroupAlsoByWindowViaOutputBufferFn.java | 18 +- .../spark/translation/SparkProcessContext.java | 2 +- .../spark/translation/SparkRuntimeContext.java | 2 + .../spark/translation/TransformTranslator.java | 93 +- .../streaming/StreamingTransformTranslator.java | 96 +- .../spark/util/SparkSideInputReader.java | 3 +- .../ResumeFromCheckpointStreamingTest.java | 14 +- sdks/common/fn-api/pom.xml | 5 - .../fn-api/src/main/proto/beam_fn_api.proto | 174 +-- .../src/main/proto/beam_runner_api.proto | 14 +- sdks/java/core/pom.xml | 44 - .../main/java/org/apache/beam/sdk/Pipeline.java | 39 +- .../java/org/apache/beam/sdk/io/AvroIO.java | 13 +- .../java/org/apache/beam/sdk/io/PubsubIO.java | 1195 -------------- .../apache/beam/sdk/io/PubsubUnboundedSink.java | 494 ------ .../beam/sdk/io/PubsubUnboundedSource.java | 1463 ------------------ .../main/java/org/apache/beam/sdk/io/Sink.java | 2 +- .../org/apache/beam/sdk/io/UnboundedSource.java | 13 + .../beam/sdk/options/PipelineOptions.java | 12 +- .../sdk/runners/PTransformOverrideFactory.java | 31 +- .../apache/beam/sdk/runners/PipelineRunner.java | 7 +- .../beam/sdk/runners/TransformHierarchy.java | 80 +- .../apache/beam/sdk/testing/TestPipeline.java | 2 + .../beam/sdk/transforms/AppliedPTransform.java | 24 +- .../org/apache/beam/sdk/transforms/Combine.java | 4 +- .../org/apache/beam/sdk/transforms/Create.java | 7 +- .../org/apache/beam/sdk/transforms/DoFn.java | 96 +- .../apache/beam/sdk/transforms/DoFnTester.java | 46 +- .../beam/sdk/transforms/FlatMapElements.java | 113 +- .../apache/beam/sdk/transforms/GroupByKey.java | 4 +- .../apache/beam/sdk/transforms/MapElements.java | 99 +- .../org/apache/beam/sdk/transforms/ParDo.java | 82 +- .../apache/beam/sdk/transforms/Partition.java | 2 +- .../org/apache/beam/sdk/transforms/Sample.java | 4 + .../transforms/join/KeyedPCollectionTuple.java | 12 +- .../reflect/ByteBuddyDoFnInvokerFactory.java | 47 +- .../sdk/transforms/reflect/DoFnInvoker.java | 4 +- .../sdk/transforms/reflect/DoFnSignature.java | 10 +- .../sdk/transforms/reflect/DoFnSignatures.java | 96 +- .../splittabledofn/HasDefaultTracker.java | 30 + .../transforms/splittabledofn/OffsetRange.java | 8 +- .../splittabledofn/OffsetRangeTracker.java | 33 +- .../splittabledofn/RestrictionTracker.java | 8 + .../transforms/windowing/AfterWatermark.java | 14 +- .../beam/sdk/transforms/windowing/Triggers.java | 320 ---- .../beam/sdk/transforms/windowing/WindowFn.java | 13 - .../org/apache/beam/sdk/util/PubsubClient.java | 544 ------- .../apache/beam/sdk/util/PubsubGrpcClient.java | 424 ----- .../apache/beam/sdk/util/PubsubJsonClient.java | 317 ---- .../apache/beam/sdk/util/PubsubTestClient.java | 436 ------ .../org/apache/beam/sdk/util/Transport.java | 3 +- .../beam/sdk/util/WindowingStrategies.java | 267 ---- .../java/org/apache/beam/sdk/values/PBegin.java | 6 +- .../apache/beam/sdk/values/PCollectionList.java | 27 +- .../beam/sdk/values/PCollectionTuple.java | 13 +- .../java/org/apache/beam/sdk/values/PDone.java | 6 +- .../java/org/apache/beam/sdk/values/PInput.java | 4 +- .../org/apache/beam/sdk/values/POutput.java | 4 +- .../java/org/apache/beam/sdk/values/PValue.java | 4 +- .../org/apache/beam/sdk/values/PValueBase.java | 6 +- .../apache/beam/sdk/values/TaggedPValue.java | 5 + .../org/apache/beam/sdk/values/TupleTag.java | 26 +- .../apache/beam/sdk/values/TupleTagList.java | 2 +- .../apache/beam/sdk/values/TypeDescriptors.java | 25 +- .../org/apache/beam/sdk/values/TypedPValue.java | 4 +- .../java/org/apache/beam/sdk/PipelineTest.java | 108 +- .../org/apache/beam/sdk/io/PubsubIOTest.java | 197 --- .../beam/sdk/io/PubsubUnboundedSinkTest.java | 190 --- .../beam/sdk/io/PubsubUnboundedSourceTest.java | 411 ----- .../apache/beam/sdk/metrics/MetricsTest.java | 2 +- .../sdk/runners/TransformHierarchyTest.java | 72 +- .../beam/sdk/testing/StaticWindowsTest.java | 10 +- .../apache/beam/sdk/transforms/CreateTest.java | 8 +- .../beam/sdk/transforms/MapElementsTest.java | 25 +- .../apache/beam/sdk/transforms/ParDoTest.java | 293 ++-- .../beam/sdk/transforms/SplittableDoFnTest.java | 58 +- .../transforms/reflect/DoFnInvokersTest.java | 153 +- .../DoFnSignaturesProcessElementTest.java | 2 +- .../DoFnSignaturesSplittableDoFnTest.java | 117 +- .../splittabledofn/OffsetRangeTrackerTest.java | 49 +- .../sdk/transforms/windowing/TriggersTest.java | 100 -- .../apache/beam/sdk/util/PubsubClientTest.java | 189 --- .../beam/sdk/util/PubsubGrpcClientTest.java | 207 --- .../beam/sdk/util/PubsubJsonClientTest.java | 140 -- .../beam/sdk/util/PubsubTestClientTest.java | 114 -- .../beam/sdk/util/WindowingStrategiesTest.java | 91 -- .../beam/sdk/values/PCollectionListTest.java | 70 +- .../beam/sdk/values/PCollectionTupleTest.java | 13 +- .../apache/beam/sdk/values/TypedPValueTest.java | 46 +- .../beam/fn/harness/fake/FakeStepContext.java | 2 +- .../control/ProcessBundleHandlerTest.java | 30 +- sdks/java/io/common/pom.xml | 4 + .../apache/beam/sdk/io/common/HashingFn.java | 109 ++ .../sdk/io/elasticsearch/ElasticsearchIO.java | 2 + sdks/java/io/google-cloud-platform/pom.xml | 75 +- .../sdk/io/gcp/bigquery/BatchLoadBigQuery.java | 7 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 59 +- .../io/gcp/bigquery/BigQueryTableSource.java | 30 +- .../sdk/io/gcp/bigquery/WritePartition.java | 6 +- .../beam/sdk/io/gcp/bigquery/WriteResult.java | 11 +- .../beam/sdk/io/gcp/pubsub/PubsubClient.java | 544 +++++++ .../sdk/io/gcp/pubsub/PubsubGrpcClient.java | 424 +++++ .../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 1014 ++++++++++++ .../sdk/io/gcp/pubsub/PubsubJsonClient.java | 319 ++++ .../sdk/io/gcp/pubsub/PubsubTestClient.java | 436 ++++++ .../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 490 ++++++ .../io/gcp/pubsub/PubsubUnboundedSource.java | 1463 ++++++++++++++++++ .../beam/sdk/io/gcp/pubsub/package-info.java | 24 + .../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 5 +- .../sdk/io/gcp/bigquery/BigQueryIOTest.java | 4 +- .../sdk/io/gcp/pubsub/PubsubClientTest.java | 189 +++ .../sdk/io/gcp/pubsub/PubsubGrpcClientTest.java | 208 +++ .../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 189 +++ .../sdk/io/gcp/pubsub/PubsubJsonClientTest.java | 139 ++ .../sdk/io/gcp/pubsub/PubsubTestClientTest.java | 114 ++ .../io/gcp/pubsub/PubsubUnboundedSinkTest.java | 188 +++ .../gcp/pubsub/PubsubUnboundedSourceTest.java | 409 +++++ .../hadoop/inputformat/HadoopInputFormatIO.java | 36 +- .../inputformat/HadoopInputFormatIOTest.java | 99 +- sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 46 +- .../inputformat/HIFIOWithElasticTest.java | 6 +- .../HIFIOWithEmbeddedCassandraTest.java | 215 +++ .../hadoop/inputformat/hashing/HashingFn.java | 109 -- .../integration/tests/HIFIOCassandraIT.java | 8 +- .../integration/tests/HIFIOElasticIT.java | 6 +- .../SmallITCluster/cassandra-svc-rc.yaml | 88 -- .../cassandra/SmallITCluster/start-up.sh | 21 - .../cassandra/SmallITCluster/teardown.sh | 21 - .../kubernetes/cassandra/data-load-setup.sh | 29 - .../resources/kubernetes/cassandra/data-load.sh | 67 - .../LargeProductionCluster/es-services.yaml | 277 ---- .../LargeProductionCluster/start-up.sh | 21 - .../LargeProductionCluster/teardown.sh | 20 - .../SmallITCluster/elasticsearch-svc-rc.yaml | 84 - .../elasticsearch/SmallITCluster/start-up.sh | 22 - .../elasticsearch/SmallITCluster/teardown.sh | 20 - .../kubernetes/elasticsearch/data-load-setup.sh | 26 - .../kubernetes/elasticsearch/data-load.sh | 33 - .../kubernetes/elasticsearch/es_test_data.py | 299 ---- .../kubernetes/elasticsearch/show-health.sh | 25 - sdks/java/io/hadoop/pom.xml | 12 +- .../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 2 + .../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 2 + .../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 61 +- .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 10 +- .../kubernetes/postgres-pod-no-vol.yml | 32 - .../kubernetes/postgres-service-public.yml | 28 - .../jdbc/src/test/resources/kubernetes/setup.sh | 20 - .../src/test/resources/kubernetes/teardown.sh | 20 - .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 2 + .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 2 + .../apache/beam/sdk/io/kinesis/KinesisIO.java | 2 + .../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 9 +- .../apache/beam/sdk/io/mongodb/MongoDbIO.java | 2 + .../org/apache/beam/sdk/io/mqtt/MqttIO.java | 2 + .../transforms/FlatMapElementsJava8Test.java | 10 +- .../sdk/transforms/MapElementsJava8Test.java | 10 +- sdks/python/.pylintrc | 6 +- sdks/python/apache_beam/coders/coder_impl.py | 50 +- sdks/python/apache_beam/coders/coders.py | 6 +- .../examples/complete/game/user_score.py | 8 +- .../examples/complete/top_wikipedia_sessions.py | 8 - .../examples/cookbook/group_with_coder.py | 6 +- .../examples/snippets/snippets_test.py | 3 +- sdks/python/apache_beam/examples/wordcount.py | 48 +- .../apache_beam/examples/wordcount_debugging.py | 4 - sdks/python/apache_beam/internal/gcp/auth.py | 80 +- .../apache_beam/internal/gcp/auth_test.py | 44 - .../apache_beam/internal/gcp/json_value.py | 6 - sdks/python/apache_beam/internal/pickler.py | 20 +- sdks/python/apache_beam/io/avroio_test.py | 14 +- sdks/python/apache_beam/io/concat_source.py | 74 +- .../python/apache_beam/io/concat_source_test.py | 12 +- sdks/python/apache_beam/io/filebasedsource.py | 54 +- .../apache_beam/io/filebasedsource_test.py | 26 +- sdks/python/apache_beam/io/fileio.py | 72 +- sdks/python/apache_beam/io/fileio_test.py | 45 +- sdks/python/apache_beam/io/filesystem.py | 3 +- sdks/python/apache_beam/io/filesystem_test.py | 242 ++- sdks/python/apache_beam/io/filesystems_util.py | 10 +- sdks/python/apache_beam/io/gcp/bigquery.py | 38 +- .../io/gcp/datastore/v1/datastoreio.py | 24 +- .../io/gcp/datastore/v1/datastoreio_test.py | 4 +- .../apache_beam/io/gcp/datastore/v1/helper.py | 16 +- .../io/gcp/datastore/v1/query_splitter.py | 2 +- sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 7 +- .../io/gcp/tests/bigquery_matcher.py | 3 +- sdks/python/apache_beam/io/iobase.py | 7 +- sdks/python/apache_beam/io/localfilesystem.py | 3 +- sdks/python/apache_beam/io/range_trackers.py | 19 +- sdks/python/apache_beam/io/source_test_utils.py | 79 +- .../apache_beam/io/source_test_utils_test.py | 20 +- sdks/python/apache_beam/io/textio.py | 16 +- sdks/python/apache_beam/io/textio_test.py | 18 +- sdks/python/apache_beam/io/tfrecordio.py | 12 +- sdks/python/apache_beam/metrics/cells.py | 28 +- sdks/python/apache_beam/metrics/execution.py | 3 +- sdks/python/apache_beam/metrics/metric.py | 9 +- sdks/python/apache_beam/pipeline.py | 4 +- sdks/python/apache_beam/pipeline_test.py | 13 +- sdks/python/apache_beam/pvalue.py | 16 +- sdks/python/apache_beam/runners/common.py | 9 +- .../runners/dataflow/dataflow_metrics_test.py | 3 +- .../runners/dataflow/dataflow_runner.py | 39 +- .../runners/dataflow/dataflow_runner_test.py | 2 +- .../runners/dataflow/internal/apiclient.py | 7 +- .../runners/dataflow/internal/dependency.py | 6 +- .../runners/dataflow/native_io/iobase_test.py | 2 +- .../runners/dataflow/test_dataflow_runner.py | 14 +- .../runners/direct/bundle_factory.py | 14 +- .../consumer_tracking_pipeline_visitor_test.py | 22 +- .../apache_beam/runners/direct/direct_runner.py | 27 +- .../runners/direct/evaluation_context.py | 10 +- .../apache_beam/runners/direct/executor.py | 9 +- .../runners/direct/transform_evaluator.py | 38 - sdks/python/apache_beam/runners/runner.py | 80 +- sdks/python/apache_beam/runners/runner_test.py | 41 + .../apache_beam/tests/pipeline_verifiers.py | 7 +- sdks/python/apache_beam/transforms/combiners.py | 62 +- .../apache_beam/transforms/combiners_test.py | 4 +- sdks/python/apache_beam/transforms/core.py | 192 ++- .../apache_beam/transforms/create_test.py | 121 ++ sdks/python/apache_beam/transforms/display.py | 4 +- .../apache_beam/transforms/display_test.py | 36 - .../python/apache_beam/transforms/ptransform.py | 38 +- .../apache_beam/transforms/ptransform_test.py | 17 +- .../python/apache_beam/transforms/sideinputs.py | 11 +- .../apache_beam/transforms/sideinputs_test.py | 6 +- sdks/python/apache_beam/transforms/trigger.py | 26 +- .../apache_beam/transforms/trigger_test.py | 6 +- sdks/python/apache_beam/typehints/decorators.py | 26 +- .../apache_beam/typehints/trivial_inference.py | 26 +- .../typehints/trivial_inference_test.py | 3 +- sdks/python/apache_beam/typehints/typecheck.py | 7 +- sdks/python/apache_beam/typehints/typehints.py | 66 +- .../apache_beam/typehints/typehints_test.py | 7 +- .../apache_beam/utils/annotations_test.py | 2 +- sdks/python/apache_beam/utils/path.py | 3 +- .../apache_beam/utils/pipeline_options.py | 109 +- .../apache_beam/utils/pipeline_options_test.py | 52 +- sdks/python/apache_beam/utils/proto_utils.py | 15 +- sdks/python/apache_beam/utils/retry.py | 14 +- sdks/python/apache_beam/utils/timestamp.py | 6 +- sdks/python/apache_beam/utils/value_provider.py | 110 -- .../apache_beam/utils/value_provider_test.py | 165 -- sdks/python/apache_beam/utils/windowed_value.py | 17 +- sdks/python/run_postcommit.sh | 6 +- sdks/python/run_pylint.sh | 2 +- sdks/python/setup.py | 8 +- sdks/python/tox.ini | 18 +- 490 files changed, 17307 insertions(+), 17743 deletions(-) ----------------------------------------------------------------------
