This closes #3515: Sync gearpump-runner with master and upgrade to gearpump 0.8.4
Fix ParDoTest#testPipelineOptionsParameter Upgrade to gearpump 0.8.4 Fix javadoc generation for AmqpIO, CassandraIO and HCatalogIO Simplified ByteBuddyOnTimerInvokerFactory Fix bad merge Made DataflowRunner TransformTranslator public Process timer firings for a window together Ignore processing time timers in expired windows Add timeout to initialization of partition in KafkaIO [BEAM-2534] Handle offset gaps in Kafka messages. Fix PValue input in _PubSubReadEvaluator Update SDK dependencies Disallow Combiner Lifting for multi-window WindowFns [BEAM-2553] Update Maven exec plugin to 1.6.0 to incorporate messaging improvements Website Mergebot Job Update Python SDK version [maven-release-plugin] prepare for next development iteration [maven-release-plugin] prepare branch release-2.1.0 For GCS operations use an http client with a default timeout value. [BEAM-2530] Fix compilation of modules with Java 9 that depend on jdk.tools Make modules that depend on Hadoop and Spark use the same version property Fix DoFn javadoc: StateSpec does not require a key Add support for PipelineOptions parameters Properly convert milliseconds whether there's less than 3/more than 9 digits. TimeUtil did not properly convert (and returned null) when the number of digits for fractions of seconds was less than 3 digits or more than 9 digits. The solution is to pad with zeros when there is less than 3 digits and to truncate when there is more than 3. Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1ce60b48 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1ce60b48 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1ce60b48 Branch: refs/heads/master Commit: 1ce60b488e991562712937aaaf13564e3a909117 Parents: 627ae0b 725f547 Author: Kenneth Knowles <k...@google.com> Authored: Sun Jul 9 21:47:12 2017 -0700 Committer: Kenneth Knowles <k...@google.com> Committed: Sun Jul 9 21:47:12 2017 -0700 ---------------------------------------------------------------------- .../jenkins/common_job_properties.groovy | 5 +- .../job_beam_PreCommit_Website_Merge.groovy | 59 +++++++++ examples/java/pom.xml | 20 ++- examples/java8/pom.xml | 20 ++- .../examples/complete/game/LeaderBoardTest.java | 2 + examples/pom.xml | 2 +- pom.xml | 87 ++++++++++++- runners/apex/pom.xml | 11 +- runners/core-construction-java/pom.xml | 2 +- runners/core-java/pom.xml | 2 +- ...eBoundedSplittableProcessElementInvoker.java | 5 + .../beam/runners/core/ReduceFnRunner.java | 106 ++++++++++----- .../beam/runners/core/SimpleDoFnRunner.java | 20 +++ .../beam/runners/core/ReduceFnRunnerTest.java | 81 +++++++++++- runners/direct-java/pom.xml | 4 +- runners/flink/pom.xml | 2 +- runners/gearpump/pom.xml | 4 +- .../gearpump/translators/io/GearpumpSource.java | 12 +- .../translators/utils/DoFnRunnerFactory.java | 5 +- .../translators/utils/TranslatorUtils.java | 19 +++ runners/google-cloud-dataflow-java/pom.xml | 2 +- .../dataflow/DataflowPipelineTranslator.java | 1 + .../runners/dataflow/TransformTranslator.java | 4 +- .../beam/runners/dataflow/util/TimeUtil.java | 24 ++-- .../runners/dataflow/util/TimeUtilTest.java | 6 + runners/pom.xml | 2 +- runners/spark/pom.xml | 16 +-- sdks/common/fn-api/pom.xml | 2 +- sdks/common/pom.xml | 2 +- sdks/common/runner-api/pom.xml | 2 +- sdks/java/build-tools/pom.xml | 2 +- sdks/java/core/pom.xml | 2 +- .../org/apache/beam/sdk/transforms/DoFn.java | 2 +- .../apache/beam/sdk/transforms/DoFnTester.java | 5 + .../reflect/ByteBuddyDoFnInvokerFactory.java | 6 + .../reflect/ByteBuddyOnTimerInvokerFactory.java | 73 ++++------- .../sdk/transforms/reflect/DoFnInvoker.java | 13 +- .../sdk/transforms/reflect/DoFnSignature.java | 23 ++++ .../sdk/transforms/reflect/DoFnSignatures.java | 22 +++- .../reflect/OnTimerMethodSpecifier.java | 37 ++++++ .../apache/beam/sdk/transforms/ParDoTest.java | 63 +++++++++ .../transforms/reflect/DoFnSignaturesTest.java | 14 ++ .../google-cloud-platform-core/pom.xml | 2 +- sdks/java/extensions/jackson/pom.xml | 2 +- sdks/java/extensions/join-library/pom.xml | 2 +- sdks/java/extensions/pom.xml | 2 +- sdks/java/extensions/protobuf/pom.xml | 2 +- sdks/java/extensions/sorter/pom.xml | 8 +- sdks/java/harness/pom.xml | 2 +- sdks/java/io/amqp/pom.xml | 2 +- sdks/java/io/cassandra/pom.xml | 2 +- sdks/java/io/common/pom.xml | 2 +- sdks/java/io/elasticsearch/pom.xml | 2 +- sdks/java/io/google-cloud-platform/pom.xml | 2 +- sdks/java/io/hadoop-common/pom.xml | 2 +- sdks/java/io/hadoop-file-system/pom.xml | 33 +---- sdks/java/io/hadoop/input-format/pom.xml | 2 +- sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 4 +- sdks/java/io/hadoop/pom.xml | 2 +- sdks/java/io/hbase/pom.xml | 18 ++- sdks/java/io/hcatalog/pom.xml | 20 ++- sdks/java/io/jdbc/pom.xml | 4 +- sdks/java/io/jms/pom.xml | 2 +- sdks/java/io/kafka/pom.xml | 2 +- .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 130 +++++++++++++------ .../apache/beam/sdk/io/kafka/KafkaIOTest.java | 30 +++++ sdks/java/io/kinesis/pom.xml | 2 +- sdks/java/io/mongodb/pom.xml | 2 +- sdks/java/io/mqtt/pom.xml | 2 +- sdks/java/io/pom.xml | 33 +---- sdks/java/io/xml/pom.xml | 2 +- sdks/java/java8tests/pom.xml | 2 +- sdks/java/javadoc/pom.xml | 19 ++- .../maven-archetypes/examples-java8/pom.xml | 2 +- .../main/resources/archetype-resources/pom.xml | 1 - sdks/java/maven-archetypes/examples/pom.xml | 2 +- .../main/resources/archetype-resources/pom.xml | 1 - sdks/java/maven-archetypes/pom.xml | 2 +- sdks/java/maven-archetypes/starter/pom.xml | 2 +- .../resources/projects/basic/reference/pom.xml | 2 +- sdks/java/pom.xml | 2 +- sdks/pom.xml | 2 +- sdks/python/apache_beam/io/gcp/gcsio.py | 10 +- .../runners/direct/transform_evaluator.py | 5 +- sdks/python/apache_beam/version.py | 2 +- sdks/python/pom.xml | 2 +- sdks/python/setup.py | 4 +- 87 files changed, 854 insertions(+), 317 deletions(-) ----------------------------------------------------------------------
