Merge branch 'master' of https://github.com/apache/beam into sync-master
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/fed98c84 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/fed98c84 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/fed98c84 Branch: refs/heads/master Commit: fed98c8402f0f8c2c6b5bad544256d11cd16f540 Parents: 559e3c3 0cabdf6 Author: manuzhang <owenzhang1...@gmail.com> Authored: Sat Jun 17 10:52:41 2017 +0800 Committer: manuzhang <owenzhang1...@gmail.com> Committed: Sat Jun 17 10:52:41 2017 +0800 ---------------------------------------------------------------------- .gitignore | 2 +- pom.xml | 7 + runners/apex/pom.xml | 7 + .../utils/ApexStateInternalsTest.java | 411 +++---------- .../core/InMemoryStateInternalsTest.java | 569 ++--------------- .../beam/runners/core/StateInternalsTest.java | 613 +++++++++++++++++++ .../beam/runners/direct/CommittedResult.java | 12 +- .../beam/runners/direct/EvaluationContext.java | 26 +- .../direct/ExecutorServiceParallelExecutor.java | 9 +- .../direct/ParDoMultiOverrideFactory.java | 90 ++- .../direct/StatefulParDoEvaluatorFactory.java | 11 +- .../beam/runners/direct/WatermarkManager.java | 4 +- .../runners/direct/CommittedResultTest.java | 17 +- .../StatefulParDoEvaluatorFactoryTest.java | 65 +- .../runners/direct/TransformExecutorTest.java | 11 +- .../runners/direct/WatermarkManagerTest.java | 15 +- runners/flink/pom.xml | 9 +- .../streaming/state/FlinkStateInternals.java | 425 ++++++------- .../FlinkBroadcastStateInternalsTest.java | 242 ++------ .../FlinkKeyGroupStateInternalsTest.java | 359 +++++------ .../streaming/FlinkSplitStateInternalsTest.java | 132 ++-- .../streaming/FlinkStateInternalsTest.java | 343 +---------- runners/spark/pom.xml | 7 + .../apache/beam/runners/spark/SparkRunner.java | 7 +- .../spark/stateful/SparkStateInternalsTest.java | 66 ++ .../fn-api/src/main/proto/beam_fn_api.proto | 63 +- .../src/main/resources/beam/findbugs-filter.xml | 9 + .../sdk/options/ProxyInvocationHandler.java | 15 +- .../sdk/options/ProxyInvocationHandlerTest.java | 12 + .../apache/beam/sdk/transforms/CombineTest.java | 225 ++++--- .../beam/sdk/transforms/SplittableDoFnTest.java | 145 ++++- sdks/java/harness/pom.xml | 6 + .../harness/control/ProcessBundleHandler.java | 293 +++------ .../beam/runners/core/BeamFnDataReadRunner.java | 70 ++- .../runners/core/BeamFnDataWriteRunner.java | 67 +- .../beam/runners/core/BoundedSourceRunner.java | 74 ++- .../beam/runners/core/DoFnRunnerFactory.java | 182 ++++++ .../runners/core/PTransformRunnerFactory.java | 81 +++ .../control/ProcessBundleHandlerTest.java | 521 +++------------- .../runners/core/BeamFnDataReadRunnerTest.java | 112 +++- .../runners/core/BeamFnDataWriteRunnerTest.java | 120 +++- .../runners/core/BoundedSourceRunnerTest.java | 124 +++- .../runners/core/DoFnRunnerFactoryTest.java | 209 +++++++ .../beam/sdk/io/gcp/spanner/MutationGroup.java | 67 ++ .../io/gcp/spanner/MutationSizeEstimator.java | 9 + .../beam/sdk/io/gcp/spanner/SpannerIO.java | 53 +- .../gcp/spanner/MutationSizeEstimatorTest.java | 12 + .../beam/sdk/io/gcp/spanner/SpannerIOTest.java | 76 ++- sdks/java/io/hbase/pom.xml | 2 +- sdks/python/apache_beam/coders/coder_impl.py | 4 + sdks/python/apache_beam/coders/coders.py | 2 +- .../apache_beam/coders/coders_test_common.py | 8 + sdks/python/apache_beam/io/filesystem.py | 14 +- .../apache_beam/options/pipeline_options.py | 26 +- .../options/pipeline_options_test.py | 39 +- .../apache_beam/options/value_provider_test.py | 93 +-- sdks/python/apache_beam/pipeline.py | 218 ++++++- sdks/python/apache_beam/pipeline_test.py | 35 ++ .../apache_beam/portability/api/__init__.py | 21 + .../apache_beam/portability/runners/__init__.py | 18 - .../portability/runners/api/__init__.py | 21 - sdks/python/apache_beam/pvalue.py | 2 +- .../runners/dataflow/dataflow_runner.py | 14 +- .../runners/dataflow/internal/apiclient_test.py | 24 + .../runners/dataflow/native_io/iobase_test.py | 39 +- .../runners/direct/bundle_factory.py | 2 +- .../apache_beam/runners/direct/direct_runner.py | 39 ++ .../runners/direct/evaluation_context.py | 56 +- .../apache_beam/runners/direct/executor.py | 64 +- .../runners/direct/transform_evaluator.py | 157 ++++- .../runners/direct/transform_result.py | 3 +- .../apache_beam/runners/pipeline_context.py | 2 +- .../runners/portability/fn_api_runner.py | 18 +- .../runners/portability/fn_api_runner_test.py | 4 +- .../apache_beam/runners/worker/data_plane.py | 2 +- .../runners/worker/data_plane_test.py | 2 +- .../apache_beam/runners/worker/log_handler.py | 2 +- .../runners/worker/log_handler_test.py | 2 +- .../apache_beam/runners/worker/operations.py | 1 + .../apache_beam/runners/worker/sdk_worker.py | 4 +- .../runners/worker/sdk_worker_main.py | 2 +- .../runners/worker/sdk_worker_test.py | 2 +- sdks/python/apache_beam/transforms/core.py | 91 +-- .../python/apache_beam/transforms/ptransform.py | 2 +- sdks/python/apache_beam/transforms/trigger.py | 2 +- sdks/python/apache_beam/transforms/window.py | 4 +- sdks/python/apache_beam/utils/plugin.py | 42 ++ sdks/python/apache_beam/utils/urns.py | 2 +- sdks/python/gen_protos.py | 2 +- sdks/python/run_pylint.sh | 2 +- 90 files changed, 4049 insertions(+), 3044 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/fed98c84/pom.xml ----------------------------------------------------------------------
