This is an automated email from the ASF dual-hosted git repository. chamikara pushed a commit to branch go-sdk in repository https://gitbox.apache.org/repos/asf/beam.git
commit 5e6db92862521b0c54e4b3e532ae98883c7f5098 Merge: e2d7408 ed922cb Author: Chamikara Jayalath <[email protected]> AuthorDate: Tue Feb 13 23:28:34 2018 -0800 Merge pull request #4682: [BEAM-3684] Integrate master to go-sdk and fixup coder urns .github/PULL_REQUEST_TEMPLATE.md | 13 +- .gitignore | 1 + .test-infra/jenkins/common_job_properties.groovy | 4 + ...am_PostCommit_Java_ValidatesRunner_Flink.groovy | 2 +- ...am_PostCommit_Java_ValidatesRunner_Spark.groovy | 2 +- ...mmit_Python_ValidatesContainer_Dataflow.groovy} | 37 +- ...vy => job_beam_PreCommit_Go_GradleBuild.groovy} | 19 +- .../job_beam_PreCommit_Java_GradleBuild.groovy | 4 +- build.gradle | 12 +- build_rules.gradle | 102 +- examples/java/build.gradle | 12 +- .../apache/beam/examples/WindowedWordCountIT.java | 5 - .../complete/game/StatefulTeamScoreTest.java | 2 - gradle/wrapper/gradle-wrapper.jar | Bin 54712 -> 54333 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- model/fn-execution/build.gradle | 7 - .../fn-execution/src/main/proto/beam_fn_api.proto | 9 +- model/job-management/build.gradle | 7 - model/pipeline/build.gradle | 7 - .../resources/org/apache/beam/model/common_urns.md | 134 ++ pom.xml | 60 +- runners/apex/build.gradle | 7 - .../runners/apex/translation/ParDoTranslator.java | 2 +- .../translation/operators/ApexParDoOperator.java | 2 +- .../apex/translation/ParDoTranslatorTest.java | 2 + runners/core-construction-java/build.gradle | 7 - .../core/construction/ArtifactServiceStager.java | 32 +- .../runners/core/construction/Environments.java | 2 +- .../core/construction/ImpulseTranslation.java | 65 + .../core/construction/ModelCoderRegistrar.java | 18 +- .../core/construction/PTransformTranslation.java | 27 +- .../beam/runners/core/construction/UrnUtils.java | 63 + .../core/construction/WindowIntoTranslation.java | 9 +- .../construction/WindowingStrategyTranslation.java | 7 + .../core/construction/graph/ExecutableStage.java | 84 ++ .../graph/GreedilyFusedExecutableStage.java | 195 +++ .../graph/GreedyPCollectionFusers.java | 257 ++++ .../core/construction/graph/PipelineNode.java | 55 + .../core/construction/graph/QueryablePipeline.java | 281 ++++ .../core/construction/graph}/package-info.java | 12 +- .../core/construction/EnvironmentsTest.java | 4 +- .../core/construction/PTransformMatchersTest.java | 2 +- .../core/construction/ParDoTranslationTest.java | 4 +- .../core/construction/SplittableParDoTest.java | 7 +- .../runners/core/construction/UrnUtilsTest.java | 42 +- .../graph/GreedilyFusedExecutableStageTest.java | 826 +++++++++++ .../construction/graph/QueryablePipelineTest.java | 389 ++++++ .../construction/metrics/MetricFilteringTest.java | 2 - runners/core-java/build.gradle | 7 - .../beam/runners/core/InMemoryStateInternals.java | 77 +- ...TimeBoundedSplittableProcessElementInvoker.java | 100 +- .../apache/beam/runners/core/SimpleDoFnRunner.java | 8 +- .../core/SplittableParDoViaKeyedWorkItems.java | 4 +- .../core/SplittableProcessElementInvoker.java | 2 +- ...BoundedSplittableProcessElementInvokerTest.java | 111 +- .../runners/core/SplittableParDoProcessFnTest.java | 34 +- .../core/metrics/MetricsContainerStepMapTest.java | 3 +- runners/direct-java/build.gradle | 7 - .../direct/CopyOnAccessInMemoryStateInternals.java | 10 +- .../apache/beam/runners/direct/DirectOptions.java | 8 + .../apache/beam/runners/direct/DirectRunner.java | 15 +- .../SplittableProcessElementsEvaluatorFactory.java | 6 +- .../runners/direct/TransformEvaluatorRegistry.java | 4 +- .../direct/UnboundedReadEvaluatorFactory.java | 3 - .../beam/runners/direct/DirectMetricsTest.java | 2 +- .../direct/DirectTransformExecutorTest.java | 1 - .../direct/FlattenEvaluatorFactoryTest.java | 22 +- runners/flink/build.gradle | 7 - .../flink/FlinkBatchTransformTranslators.java | 3 +- .../flink/FlinkPipelineExecutionEnvironment.java | 5 +- .../flink/FlinkStreamingTransformTranslators.java | 8 +- .../flink/PipelineTranslationOptimizer.java | 16 +- .../wrappers/streaming/DoFnOperator.java | 30 +- .../wrappers/streaming/SplittableDoFnOperator.java | 2 +- .../streaming/state/FlinkStateInternals.java | 2 +- .../FlinkPipelineExecutionEnvironmentTest.java | 68 + .../runners/flink/streaming/DoFnOperatorTest.java | 105 ++ .../translation/types/CoderTypeSerializerTest.java | 1 - runners/gcp/gcemd/build.gradle | 14 + runners/gcp/gcsproxy/build.gradle | 14 + runners/gearpump/build.gradle | 7 - runners/gearpump/pom.xml | 11 - .../runners/gearpump/GearpumpPipelineOptions.java | 3 - .../beam/runners/gearpump/GearpumpRunner.java | 4 - .../beam/runners/gearpump/TestGearpumpRunner.java | 1 - .../CreateGearpumpPCollectionViewTranslator.java | 1 - .../translators/GearpumpPipelineTranslator.java | 3 - .../translators/ParDoMultiOutputTranslator.java | 2 - .../translators/ReadUnboundedTranslator.java | 1 - .../gearpump/translators/TransformTranslator.java | 1 - .../gearpump/translators/TranslationContext.java | 3 - .../translators/WindowAssignTranslator.java | 2 - .../translators/functions/DoFnFunction.java | 4 - .../translators/io/BoundedSourceWrapper.java | 1 - .../gearpump/translators/io/GearpumpSource.java | 2 - .../translators/io/UnboundedSourceWrapper.java | 1 - .../translators/utils/DoFnRunnerFactory.java | 1 - .../translators/utils/NoOpStepContext.java | 1 - .../translators/utils/TranslatorUtils.java | 3 - .../beam/runners/gearpump/PipelineOptionsTest.java | 3 +- .../FlattenPCollectionsTranslatorTest.java | 1 - .../translators/GroupByKeyTranslatorTest.java | 2 - .../translators/io/GearpumpSourceTest.java | 2 - .../gearpump/translators/io/ValueSoureTest.java | 2 - .../translators/utils/TranslatorUtilsTest.java | 2 - runners/google-cloud-dataflow-java/build.gradle | 7 - runners/google-cloud-dataflow-java/pom.xml | 2 +- .../beam/runners/dataflow/BatchViewOverrides.java | 614 +++++---- .../beam/runners/dataflow/DataflowMetrics.java | 2 +- .../dataflow/DataflowPipelineTranslator.java | 7 + .../beam/runners/dataflow/TestDataflowRunner.java | 10 +- .../beam/runners/dataflow/util/PackageUtil.java | 64 +- .../beam/runners/dataflow/util/PropertyNames.java | 1 + .../beam/runners/dataflow/DataflowMetricsTest.java | 2 - runners/java-fn-execution/build.gradle | 7 - runners/java-fn-execution/pom.xml | 11 - .../artifact/ArtifactRetrievalService.java | 12 +- .../fnexecution/artifact}/package-info.java | 6 +- .../fnexecution/control/FnApiControlClient.java | 41 +- .../control/FnApiControlClientPoolService.java | 5 +- .../fnexecution/control/SdkHarnessClient.java | 37 +- .../control/SdkHarnessClientControlService.java | 75 + .../graph/LengthPrefixUnknownCoders.java | 19 +- .../control/FnApiControlClientPoolServiceTest.java | 9 +- .../control/FnApiControlClientTest.java | 30 +- .../fnexecution/control/SdkHarnessClientTest.java | 17 +- runners/local-artifact-service-java/build.gradle | 9 +- runners/local-artifact-service-java/pom.xml | 11 + .../local/LocalArtifactStagingLocation.java | 117 ++ .../LocalFileSystemArtifactRetrievalService.java | 120 ++ .../LocalFileSystemArtifactStagerService.java | 34 +- .../local/LocalArtifactStagingLocationTest.java | 125 ++ ...ocalFileSystemArtifactRetrievalServiceTest.java | 240 ++++ .../LocalFileSystemArtifactStagerServiceTest.java | 4 +- runners/local-java/build.gradle | 7 - runners/reference/java/build.gradle | 7 - runners/reference/job-server/build.gradle | 7 - .../reference/job/ReferenceRunnerJobServer.java | 3 - runners/spark/build.gradle | 7 - .../spark/translation/EvaluationContext.java | 12 +- sdks/go/gogradle.lock | 697 ++++++++++ sdks/go/pkg/beam/core/runtime/graphx/coder.go | 24 +- sdks/java/build-tools/build.gradle | 7 - .../src/main/resources/beam/checkstyle.xml | 22 +- .../src/main/resources/beam/findbugs-filter.xml | 11 +- .../src/main/resources/beam/suppressions.xml | 12 +- sdks/java/container/build.gradle | 12 +- sdks/java/core/build.gradle | 6 - sdks/java/core/pom.xml | 9 - .../coders/{BeamRecordCoder.java => RowCoder.java} | 50 +- .../org/apache/beam/sdk/coders/SnappyCoder.java | 65 + .../main/java/org/apache/beam/sdk/io/AvroSink.java | 2 - .../main/java/org/apache/beam/sdk/io/FileIO.java | 2 +- .../beam/sdk/metrics/DistributionResult.java | 4 +- .../beam/sdk/runners/TransformHierarchy.java | 36 +- .../java/org/apache/beam/sdk/state/StateSpecs.java | 42 +- .../org/apache/beam/sdk/testing/TestStream.java | 2 +- .../beam/sdk/transforms/ApproximateQuantiles.java | 10 +- .../org/apache/beam/sdk/transforms/Distinct.java | 138 +- .../java/org/apache/beam/sdk/transforms/DoFn.java | 56 +- .../org/apache/beam/sdk/transforms/DoFnTester.java | 219 ++- .../org/apache/beam/sdk/transforms/Impulse.java | 59 + .../java/org/apache/beam/sdk/transforms/ParDo.java | 5 + .../java/org/apache/beam/sdk/transforms/Wait.java | 120 ++ .../java/org/apache/beam/sdk/transforms/Watch.java | 167 ++- .../beam/sdk/transforms/reflect/DoFnInvoker.java | 6 +- .../sdk/transforms/reflect/DoFnSignatures.java | 6 +- .../splittabledofn/HasDefaultTracker.java | 2 +- .../splittabledofn/OffsetRangeTracker.java | 12 +- .../splittabledofn/RestrictionTracker.java | 76 +- .../util/BufferedElementCountingOutputStream.java | 15 +- .../java/org/apache/beam/sdk/util/CoderUtils.java | 2 - .../java/org/apache/beam/sdk/util/MoreFutures.java | 251 ++++ .../apache/beam/sdk/util/ThrowingRunnable.java} | 11 +- .../apache/beam/sdk/util/ThrowingSupplier.java} | 14 +- .../org/apache/beam/sdk/util/WindowedValue.java | 283 ++-- .../java/org/apache/beam/sdk/util/ZipFiles.java | 10 +- .../org/apache/beam/sdk/values/BeamRecordType.java | 96 -- .../beam/sdk/values/{BeamRecord.java => Row.java} | 194 +-- .../java/org/apache/beam/sdk/values/RowType.java | 140 ++ .../beam/sdk/values/reflect/ByteBuddyUtils.java | 109 ++ .../sdk/values/reflect/DefaultRowTypeFactory.java | 100 ++ .../beam/sdk/values/reflect/FieldValueGetter.java | 20 +- .../sdk/values/reflect/GeneratedGetterFactory.java | 106 ++ .../beam/sdk/values/reflect/GetterFactory.java | 18 +- .../beam/sdk/values/reflect/ReflectionGetter.java | 59 + .../values/reflect/ReflectionGetterFactory.java | 55 + .../beam/sdk/values/reflect/ReflectionUtils.java | 73 + .../apache/beam/sdk/values/reflect/RowFactory.java | 124 ++ .../beam/sdk/values/reflect/RowTypeFactory.java} | 35 +- .../beam/sdk/values/reflect/RowTypeGetters.java} | 38 +- .../beam/sdk/values/reflect}/package-info.java | 5 +- .../apache/beam/sdk/coders/DelegateCoderTest.java | 3 - .../sdk/io/BoundedReadFromUnboundedSourceTest.java | 257 +++- .../org/apache/beam/sdk/io/FileBasedSinkTest.java | 4 +- .../org/apache/beam/sdk/io/WriteFilesTest.java | 5 - .../sdk/runners/dataflow/TestCountingSource.java | 238 ---- .../runners/dataflow/TestCountingSourceTest.java | 72 - .../apache/beam/sdk/transforms/DistinctTest.java | 208 +-- .../beam/sdk/transforms/SplittableDoFnTest.java | 6 +- .../org/apache/beam/sdk/transforms/WaitTest.java | 304 +++++ .../org/apache/beam/sdk/transforms/WatchTest.java | 126 +- .../sdk/transforms/display/DisplayDataTest.java | 4 +- .../beam/sdk/transforms/join/CoGroupByKeyTest.java | 42 +- .../sdk/transforms/reflect/DoFnInvokersTest.java | 14 +- .../reflect/DoFnSignaturesProcessElementTest.java | 2 +- .../reflect/DoFnSignaturesSplittableDoFnTest.java | 11 +- .../splittabledofn/OffsetRangeTrackerTest.java | 73 +- .../BufferedElementCountingOutputStreamTest.java | 36 + .../apache/beam/sdk/util/MovingFunctionTest.java | 6 +- .../beam/sdk/util/SerializableUtilsTest.java | 1 - .../apache/beam/sdk/util/WindowedValueTest.java | 33 +- .../java/org/apache/beam/sdk/values/RowTest.java | 113 ++ .../org/apache/beam/sdk/values/RowTypeTest.java | 85 ++ .../values/reflect/DefaultRowTypeFactoryTest.java | 125 ++ .../values/reflect/GeneratedGetterFactoryTest.java | 107 ++ .../reflect/ReflectionGetterFactoryTest.java | 107 ++ .../sdk/values/reflect/ReflectionGetterTest.java | 95 ++ .../beam/sdk/values/reflect/RowFactoryTest.java | 142 ++ .../sdk/values/reflect/RowTypeGettersTest.java} | 26 +- .../google-cloud-platform-core/build.gradle | 7 - .../java/org/apache/beam/sdk/util/GcsUtil.java | 21 +- sdks/java/extensions/jackson/build.gradle | 7 - sdks/java/extensions/join-library/build.gradle | 7 - .../extensions/joinlibrary/OuterFullJoinTest.java | 1 - sdks/java/extensions/protobuf/build.gradle | 7 - .../{join-library => sketching}/build.gradle | 22 +- sdks/java/extensions/sketching/pom.xml | 5 - .../extensions/sketching/SketchFrequencies.java | 2 - .../sketching/ApproximateDistinctTest.java | 4 - .../sketching/SketchFrequenciesTest.java | 1 - sdks/java/extensions/sorter/build.gradle | 7 - sdks/java/extensions/sql/build.gradle | 24 +- sdks/java/extensions/sql/pom.xml | 19 +- .../sql/src/main/codegen/data/Parser.tdd | 3 +- .../sql/src/main/codegen/includes/parserImpls.ftl | 17 + .../beam/sdk/extensions/sql/BeamRecordSqlType.java | 248 ---- .../apache/beam/sdk/extensions/sql/BeamSql.java | 19 +- .../apache/beam/sdk/extensions/sql/BeamSqlCli.java | 16 +- .../sdk/extensions/sql/BeamSqlSeekableTable.java | 6 +- .../beam/sdk/extensions/sql/BeamSqlTable.java | 9 +- .../{BeamSqlRecordHelper.java => RowHelper.java} | 51 +- .../apache/beam/sdk/extensions/sql/RowSqlType.java | 113 ++ .../beam/sdk/extensions/sql/SqlRowTypeFactory.java | 90 ++ .../beam/sdk/extensions/sql/SqlTypeCoder.java | 156 +++ .../beam/sdk/extensions/sql/SqlTypeCoders.java | 67 + .../sdk/extensions/sql/example/BeamSqlExample.java | 46 +- .../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 40 +- .../interpreter/BeamSqlExpressionExecutor.java | 6 +- .../sql/impl/interpreter/BeamSqlFnExecutor.java | 7 +- .../operator/BeamSqlCaseExpression.java | 4 +- .../operator/BeamSqlCastExpression.java | 4 +- .../interpreter/operator/BeamSqlExpression.java | 8 +- .../operator/BeamSqlInputRefExpression.java | 6 +- .../interpreter/operator/BeamSqlPrimitive.java | 7 +- .../interpreter/operator/BeamSqlUdfExpression.java | 4 +- .../operator/BeamSqlWindowEndExpression.java | 4 +- .../operator/BeamSqlWindowExpression.java | 4 +- .../operator/BeamSqlWindowStartExpression.java | 4 +- .../arithmetic/BeamSqlArithmeticExpression.java | 4 +- .../comparison/BeamSqlCompareExpression.java | 4 +- .../comparison/BeamSqlIsNotNullExpression.java | 4 +- .../comparison/BeamSqlIsNullExpression.java | 4 +- .../date/BeamSqlCurrentDateExpression.java | 4 +- .../date/BeamSqlCurrentTimeExpression.java | 4 +- .../date/BeamSqlCurrentTimestampExpression.java | 4 +- .../operator/date/BeamSqlDateCeilExpression.java | 4 +- .../operator/date/BeamSqlDateFloorExpression.java | 4 +- .../date/BeamSqlDatetimeMinusExpression.java | 6 +- .../date/BeamSqlDatetimePlusExpression.java | 10 +- .../operator/date/BeamSqlExtractExpression.java | 4 +- .../date/BeamSqlIntervalMultiplyExpression.java | 6 +- .../BeamSqlTimestampMinusIntervalExpression.java | 5 +- .../BeamSqlTimestampMinusTimestampExpression.java | 5 +- .../interpreter/operator/date/TimeUnitUtils.java | 1 - .../operator/logical/BeamSqlAndExpression.java | 4 +- .../operator/logical/BeamSqlNotExpression.java | 4 +- .../operator/logical/BeamSqlOrExpression.java | 4 +- .../operator/math/BeamSqlMathBinaryExpression.java | 4 +- .../operator/math/BeamSqlMathUnaryExpression.java | 4 +- .../operator/math/BeamSqlPiExpression.java | 4 +- .../operator/math/BeamSqlRandExpression.java | 6 +- .../math/BeamSqlRandIntegerExpression.java | 8 +- .../reinterpret/BeamSqlReinterpretExpression.java | 5 +- .../reinterpret/ReinterpretConversion.java | 2 - .../string/BeamSqlCharLengthExpression.java | 4 +- .../operator/string/BeamSqlConcatExpression.java | 4 +- .../operator/string/BeamSqlInitCapExpression.java | 4 +- .../operator/string/BeamSqlLowerExpression.java | 4 +- .../operator/string/BeamSqlOverlayExpression.java | 4 +- .../operator/string/BeamSqlPositionExpression.java | 4 +- .../string/BeamSqlSubstringExpression.java | 4 +- .../operator/string/BeamSqlTrimExpression.java | 4 +- .../operator/string/BeamSqlUpperExpression.java | 4 +- .../extensions/sql/impl/parser/ParserUtils.java | 4 +- .../extensions/sql/impl/parser/SqlDropTable.java | 79 ++ .../sql/impl/planner/BeamQueryPlanner.java | 4 +- .../sql/impl/rel/BeamAggregationRel.java | 177 ++- .../sdk/extensions/sql/impl/rel/BeamFilterRel.java | 14 +- .../sdk/extensions/sql/impl/rel/BeamIOSinkRel.java | 6 +- .../extensions/sql/impl/rel/BeamIOSourceRel.java | 12 +- .../extensions/sql/impl/rel/BeamIntersectRel.java | 4 +- .../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 100 +- .../sdk/extensions/sql/impl/rel/BeamMinusRel.java | 4 +- .../extensions/sql/impl/rel/BeamProjectRel.java | 10 +- .../sdk/extensions/sql/impl/rel/BeamRelNode.java | 4 +- .../sql/impl/rel/BeamSetOperatorRelBase.java | 16 +- .../sdk/extensions/sql/impl/rel/BeamSortRel.java | 28 +- .../sdk/extensions/sql/impl/rel/BeamUnionRel.java | 9 +- .../sdk/extensions/sql/impl/rel/BeamValuesRel.java | 49 +- .../sdk/extensions/sql/impl/rel/package-info.java | 4 + .../sql/impl/rule/AggregateWindowFactory.java | 158 +++ .../rule/AggregateWindowField.java} | 39 +- .../sql/impl/rule/BeamAggregationRule.java | 94 +- .../sdk/extensions/sql/impl/rule/package-info.java | 4 + .../extensions/sql/impl/schema/BaseBeamTable.java | 12 +- .../sql/impl/schema/BeamPCollectionTable.java | 20 +- .../extensions/sql/impl/schema/BeamTableUtils.java | 45 +- .../impl/transform/BeamAggregationTransforms.java | 213 ++- .../sql/impl/transform/BeamJoinTransforms.java | 179 +-- .../impl/transform/BeamSetOperatorsTransforms.java | 26 +- .../sql/impl/transform/BeamSqlFilterFn.java | 6 +- .../impl/transform/BeamSqlOutputToConsoleFn.java | 6 +- .../sql/impl/transform/BeamSqlProjectFn.java | 46 +- .../impl/transform/agg/CovarianceAccumulator.java | 151 ++ .../sql/impl/transform/agg/CovarianceFn.java | 120 ++ .../agg => utils}/BigDecimalConverter.java | 33 +- .../extensions/sql/impl/utils/CalciteUtils.java | 102 +- .../extensions/sql/impl/utils/SqlTypeUtils.java | 2 - .../beam/sdk/extensions/sql/meta/Column.java | 5 +- .../extensions/sql/meta/provider/MetaUtils.java | 25 +- .../sql/meta/provider/TableProvider.java | 7 + .../sql/meta/provider/kafka/BeamKafkaCSVTable.java | 58 +- .../sql/meta/provider/kafka/BeamKafkaTable.java | 30 +- .../meta/provider/kafka/KafkaTableProvider.java | 12 +- .../sql/meta/provider/text/BeamTextCSVTable.java | 26 +- .../provider/text/BeamTextCSVTableIOReader.java | 22 +- .../provider/text/BeamTextCSVTableIOWriter.java | 29 +- .../sql/meta/provider/text/BeamTextTable.java | 6 +- .../sql/meta/provider/text/TextTableProvider.java | 12 +- .../sql/meta/store/InMemoryMetaStore.java | 10 + .../sdk/extensions/sql/meta/store/MetaStore.java | 6 +- .../sdk/extensions/sql/BeamRecordSqlTypeTest.java | 115 -- .../sdk/extensions/sql/BeamSqlApiSurfaceTest.java | 4 +- .../beam/sdk/extensions/sql/BeamSqlCliTest.java | 45 + .../sql/BeamSqlDslAggregationCovarianceTest.java | 116 ++ .../extensions/sql/BeamSqlDslAggregationTest.java | 452 ++++-- .../sql/BeamSqlDslAggregationVarianceTest.java | 25 +- .../beam/sdk/extensions/sql/BeamSqlDslBase.java | 116 +- .../sdk/extensions/sql/BeamSqlDslFilterTest.java | 24 +- .../sdk/extensions/sql/BeamSqlDslJoinTest.java | 86 +- .../sdk/extensions/sql/BeamSqlDslProjectTest.java | 117 +- .../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java | 37 +- .../sdk/extensions/sql/BeamSqlNonAsciiTest.java | 10 +- .../sdk/extensions/sql/SqlRowTypeFactoryTest.java | 133 ++ .../apache/beam/sdk/extensions/sql/TestUtils.java | 158 ++- .../interpreter/BeamSqlFnExecutorTestBase.java | 31 +- .../operator/BeamNullExperssionTest.java | 8 +- .../operator/BeamSqlAndOrExpressionTest.java | 8 +- .../operator/BeamSqlCaseExpressionTest.java | 6 +- .../operator/BeamSqlCastExpressionTest.java | 20 +- .../operator/BeamSqlCompareExpressionTest.java | 24 +- .../operator/BeamSqlInputRefExpressionTest.java | 12 +- .../interpreter/operator/BeamSqlPrimitiveTest.java | 10 +- .../operator/BeamSqlReinterpretExpressionTest.java | 5 +- .../operator/BeamSqlUdfExpressionTest.java | 2 +- .../BeamSqlArithmeticExpressionTest.java | 50 +- .../date/BeamSqlCurrentDateExpressionTest.java | 2 +- .../date/BeamSqlCurrentTimeExpressionTest.java | 2 +- .../BeamSqlCurrentTimestampExpressionTest.java | 2 +- .../date/BeamSqlDateCeilExpressionTest.java | 4 +- .../date/BeamSqlDateExpressionTestBase.java | 1 - .../date/BeamSqlDateFloorExpressionTest.java | 4 +- .../date/BeamSqlDatetimeMinusExpressionTest.java | 8 +- .../date/BeamSqlDatetimePlusExpressionTest.java | 5 +- .../date/BeamSqlExtractExpressionTest.java | 14 +- .../BeamSqlIntervalMultiplyExpressionTest.java | 5 +- ...eamSqlTimestampMinusIntervalExpressionTest.java | 5 +- ...amSqlTimestampMinusTimestampExpressionTest.java | 5 +- .../operator/date/TimeUnitUtilsTest.java | 19 +- .../operator/logical/BeamSqlNotExpressionTest.java | 6 +- .../math/BeamSqlMathBinaryExpressionTest.java | 46 +- .../math/BeamSqlMathUnaryExpressionTest.java | 78 +- .../DatetimeReinterpretConversionsTest.java | 1 - .../reinterpret/ReinterpretConversionTest.java | 2 - .../operator/reinterpret/ReinterpreterTest.java | 1 - .../string/BeamSqlCharLengthExpressionTest.java | 2 +- .../string/BeamSqlConcatExpressionTest.java | 2 +- .../string/BeamSqlInitCapExpressionTest.java | 6 +- .../string/BeamSqlLowerExpressionTest.java | 2 +- .../string/BeamSqlOverlayExpressionTest.java | 8 +- .../string/BeamSqlPositionExpressionTest.java | 6 +- .../string/BeamSqlSubstringExpressionTest.java | 14 +- .../operator/string/BeamSqlTrimExpressionTest.java | 8 +- .../string/BeamSqlUpperExpressionTest.java | 2 +- .../sql/impl/parser/BeamSqlParserTest.java | 19 +- .../sdk/extensions/sql/impl/rel/BaseRelTest.java | 4 +- .../sql/impl/rel/BeamIntersectRelTest.java | 32 +- .../impl/rel/BeamJoinRelBoundedVsBoundedTest.java | 72 +- .../rel/BeamJoinRelUnboundedVsBoundedTest.java | 78 +- .../rel/BeamJoinRelUnboundedVsUnboundedTest.java | 52 +- .../extensions/sql/impl/rel/BeamMinusRelTest.java | 32 +- .../sql/impl/rel/BeamSetOperatorRelBaseTest.java | 20 +- .../extensions/sql/impl/rel/BeamSortRelTest.java | 92 +- .../extensions/sql/impl/rel/BeamUnionRelTest.java | 26 +- .../extensions/sql/impl/rel/BeamValuesRelTest.java | 30 +- .../sdk/extensions/sql/impl/rel/CheckSize.java | 8 +- .../sql/impl/schema/BeamSqlRowCoderTest.java | 63 +- .../transform/BeamAggregationTransformTest.java | 235 ++-- .../schema/transform/BeamTransformBaseTest.java | 165 +-- .../agg => utils}/BigDecimalConverterTest.java | 21 +- .../BeamSqlArithmeticOperatorsIntegrationTest.java | 1 - ...BeamSqlBuiltinFunctionsIntegrationTestBase.java | 64 +- .../BeamSqlComparisonOperatorsIntegrationTest.java | 64 +- .../BeamSqlDateFunctionsIntegrationTest.java | 13 +- .../meta/provider/kafka/BeamKafkaCSVTableTest.java | 50 +- .../provider/kafka/KafkaTableProviderTest.java | 7 +- .../meta/provider/text/BeamTextCSVTableTest.java | 72 +- .../meta/provider/text/TextTableProviderTest.java | 7 +- .../sql/meta/store/InMemoryMetaStoreTest.java | 18 +- .../extensions/sql/mock/MockedBoundedTable.java | 26 +- .../beam/sdk/extensions/sql/mock/MockedTable.java | 10 +- .../extensions/sql/mock/MockedUnboundedTable.java | 18 +- .../{BeamRecordAsserts.java => RowAsserts.java} | 26 +- sdks/java/fn-execution/build.gradle | 7 - .../sdk/fn/stream/SynchronizedStreamObserver.java | 64 + sdks/java/harness/build.gradle | 7 - .../apache/beam/fn/harness/FnApiDoFnRunner.java | 6 +- .../org/apache/beam/fn/harness/MapFnRunner.java | 120 ++ .../beam/fn/harness/WindowMappingFnRunner.java | 67 + .../data/BeamFnDataBufferingOutboundObserver.java | 4 +- .../apache/beam/fn/harness/MapFnRunnerTest.java | 97 ++ .../beam/fn/harness/WindowMappingFnRunnerTest.java | 68 + sdks/java/io/amazon-web-services/build.gradle | 7 - .../apache/beam/sdk/io/aws/s3/S3FileSystem.java | 10 +- sdks/java/io/amqp/build.gradle | 7 - .../apache/beam/sdk/io/amqp/AmqpMessageCoder.java | 2 - .../io/amqp/AmqpMessageCoderProviderRegistrar.java | 2 - .../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 2 - sdks/java/io/cassandra/build.gradle | 7 - .../apache/beam/sdk/io/cassandra/CassandraIO.java | 4 - .../beam/sdk/io/cassandra/CassandraService.java | 1 - .../beam/sdk/io/cassandra/CassandraIOTest.java | 4 - .../sdk/io/cassandra/CassandraServiceImplTest.java | 2 - .../sdk/io/cassandra/CassandraTestDataSet.java | 1 - sdks/java/io/common/build.gradle | 7 - .../elasticsearch-tests-2/build.gradle | 7 - .../elasticsearch-tests-5/build.gradle | 7 - .../elasticsearch-tests-common/build.gradle | 7 - sdks/java/io/elasticsearch/build.gradle | 7 - .../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 15 +- sdks/java/io/file-based-io-tests/build.gradle | 7 - sdks/java/io/google-cloud-platform/build.gradle | 10 - .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 44 +- .../sdk/io/gcp/bigquery/BigQueryQuerySource.java | 18 +- .../beam/sdk/io/gcp/bigquery/StreamingWriteFn.java | 6 - .../beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java | 5 - .../sdk/io/gcp/bigquery/WriteBundlesToFiles.java | 3 - .../gcp/bigquery/WriteGroupedRecordsToFiles.java | 3 - .../beam/sdk/io/gcp/bigquery/WriteTables.java | 3 - .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 34 +- .../beam/sdk/io/gcp/bigtable/BigtableService.java | 4 +- .../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 24 +- .../beam/sdk/io/gcp/spanner/MutationGroup.java | 1 - .../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 8 +- .../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 127 +- .../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 16 +- .../io/gcp/bigtable/BigtableServiceImplTest.java | 10 +- sdks/java/io/hadoop-common/build.gradle | 7 - sdks/java/io/hadoop-file-system/build.gradle | 7 - sdks/java/io/hadoop-input-format/build.gradle | 7 - sdks/java/io/hadoop-input-format/pom.xml | 10 +- .../io/hadoop/inputformat/HadoopInputFormatIO.java | 8 +- sdks/java/io/hbase/build.gradle | 7 - .../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 79 +- .../beam/sdk/io/hbase/SerializableScanTest.java | 1 - sdks/java/io/hcatalog/build.gradle | 7 - sdks/java/io/jdbc/build.gradle | 8 +- sdks/java/io/jdbc/pom.xml | 24 +- .../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 138 +- .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 118 +- sdks/java/io/jms/build.gradle | 7 - sdks/java/io/jms/pom.xml | 10 - .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 5 +- .../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 1 - sdks/java/io/kafka/build.gradle | 7 - sdks/java/io/kafka/pom.xml | 1 + .../org/apache/beam/sdk/io/kafka/ConsumerSpEL.java | 65 +- .../beam/sdk/io/kafka/KafkaCheckpointMark.java | 7 +- .../beam/sdk/io/kafka/KafkaExactlyOnceSink.java | 643 +++++++++ .../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 1440 +------------------- .../apache/beam/sdk/io/kafka/KafkaRecordCoder.java | 40 +- .../beam/sdk/io/kafka/KafkaUnboundedReader.java | 663 +++++++++ .../beam/sdk/io/kafka/KafkaUnboundedSource.java | 165 +++ .../org/apache/beam/sdk/io/kafka/KafkaWriter.java | 132 ++ .../org/apache/beam/sdk/io/kafka/ProducerSpEL.java | 13 +- .../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 71 +- sdks/java/io/kinesis/build.gradle | 8 +- sdks/java/io/kinesis/pom.xml | 34 +- .../beam/sdk/io/kinesis/AWSClientsProvider.java | 10 +- .../beam/sdk/io/kinesis/BasicKinesisProvider.java | 90 ++ .../sdk/io/kinesis/DynamicCheckpointGenerator.java | 2 - .../org/apache/beam/sdk/io/kinesis/KinesisIO.java | 446 +++++- .../beam/sdk/io/kinesis/KinesisPartitioner.java} | 15 +- .../apache/beam/sdk/io/kinesis/KinesisReader.java | 1 - .../sdk/io/kinesis/KinesisReaderCheckpoint.java | 2 - .../apache/beam/sdk/io/kinesis/KinesisRecord.java | 2 - .../beam/sdk/io/kinesis/KinesisRecordCoder.java | 1 - .../apache/beam/sdk/io/kinesis/KinesisSource.java | 1 - .../beam/sdk/io/kinesis/ShardCheckpoint.java | 2 - .../beam/sdk/io/kinesis/ShardReadersPool.java | 2 - .../beam/sdk/io/kinesis/ShardRecordsIterator.java | 2 - .../apache/beam/sdk/io/kinesis/StartingPoint.java | 2 - .../sdk/io/kinesis/StartingPointShardsFinder.java | 2 - .../beam/sdk/io/kinesis/AmazonKinesisMock.java | 11 + .../beam/sdk/io/kinesis/CustomOptionalTest.java | 2 - .../io/kinesis/DynamicCheckpointGeneratorTest.java | 2 - .../apache/beam/sdk/io/kinesis/KinesisIOIT.java | 126 ++ .../beam/sdk/io/kinesis/KinesisMockReadTest.java | 4 +- .../beam/sdk/io/kinesis/KinesisMockWriteTest.java | 289 ++++ .../beam/sdk/io/kinesis/KinesisProducerMock.java | 120 ++ .../io/kinesis/KinesisReaderCheckpointTest.java | 2 - .../beam/sdk/io/kinesis/KinesisReaderIT.java | 121 -- .../beam/sdk/io/kinesis/KinesisReaderTest.java | 2 + .../sdk/io/kinesis/KinesisRecordCoderTest.java | 1 - .../beam/sdk/io/kinesis/KinesisServiceMock.java | 88 ++ .../beam/sdk/io/kinesis/RecordFilterTest.java | 2 - .../beam/sdk/io/kinesis/ShardCheckpointTest.java | 2 - .../beam/sdk/io/kinesis/ShardReadersPoolTest.java | 6 +- .../io/kinesis/SimplifiedKinesisClientTest.java | 2 - .../io/kinesis/StartingPointShardsFinderTest.java | 2 - sdks/java/io/mongodb/build.gradle | 7 - sdks/java/io/mqtt/build.gradle | 7 - .../java/org/apache/beam/sdk/io/mqtt/MqttIO.java | 1 - sdks/java/io/redis/build.gradle | 7 - .../sdk/io/redis/RedisConnectionConfiguration.java | 4 - sdks/java/io/solr/build.gradle | 7 - sdks/java/io/solr/pom.xml | 7 + .../beam/sdk/io/solr/AuthorizedSolrClient.java | 1 - .../beam/sdk/io/solr/JavaBinCodecCoderTest.java | 1 - sdks/java/io/tika/build.gradle | 7 - sdks/java/io/tika/pom.xml | 12 - sdks/java/io/xml/build.gradle | 7 - sdks/java/java8tests/build.gradle | 7 - sdks/java/javadoc/pom.xml | 43 +- sdks/java/maven-archetypes/examples/build.gradle | 7 - .../src/main/resources/archetype-resources/pom.xml | 19 + sdks/java/maven-archetypes/starter/build.gradle | 7 - sdks/java/nexmark/build.gradle | 7 - sdks/java/nexmark/pom.xml | 4 + .../apache/beam/sdk/nexmark/NexmarkOptions.java | 1 - .../org/apache/beam/sdk/nexmark/NexmarkUtils.java | 5 - .../sql/{BeamRecordSize.java => RowSize.java} | 79 +- .../model/sql/{ToBeamRecord.java => ToRow.java} | 25 +- .../model/sql/adapter/ModelAdaptersMapping.java | 8 +- .../model/sql/adapter/ModelFieldsAdapter.java | 16 +- .../nexmark/model/sql/adapter/package-info.java | 2 +- .../sdk/nexmark/queries/AbstractSimulator.java | 1 - .../beam/sdk/nexmark/queries/AuctionOrBid.java | 19 +- .../beam/sdk/nexmark/queries/Query0Model.java | 1 - .../apache/beam/sdk/nexmark/queries/Query10.java | 1 - .../beam/sdk/nexmark/queries/Query1Model.java | 1 - .../beam/sdk/nexmark/queries/Query2Model.java | 1 - .../beam/sdk/nexmark/queries/Query3Model.java | 2 - .../apache/beam/sdk/nexmark/queries/Query4.java | 19 +- .../apache/beam/sdk/nexmark/queries/Query6.java | 6 +- .../beam/sdk/nexmark/queries/Query7Model.java | 1 - .../apache/beam/sdk/nexmark/queries/Query9.java | 9 +- .../beam/sdk/nexmark/queries/Query9Model.java | 1 - .../beam/sdk/nexmark/queries/WinningBids.java | 92 +- .../sdk/nexmark/queries/WinningBidsSimulator.java | 1 - .../sdk/nexmark/queries/sql/NexmarkSqlQuery.java | 14 +- .../beam/sdk/nexmark/queries/sql/SqlQuery0.java | 36 +- .../beam/sdk/nexmark/queries/sql/SqlQuery1.java | 22 +- .../beam/sdk/nexmark/queries/sql/SqlQuery2.java | 22 +- .../beam/sdk/nexmark/queries/sql/SqlQuery3.java | 49 +- .../sdk/nexmark/sources/UnboundedEventSource.java | 2 - .../sources/generator/GeneratorCheckpoint.java | 1 - .../nexmark/sources/generator/GeneratorConfig.java | 1 - .../{BeamRecordSizeTest.java => RowSizeTest.java} | 69 +- .../sql/{ToBeamRecordTest.java => ToRowTest.java} | 45 +- .../sql/adapter/ModelAdaptersMappingTest.java | 24 +- .../sdk/nexmark/queries/sql/SqlQuery0Test.java | 18 +- .../sdk/nexmark/queries/sql/SqlQuery1Test.java | 18 +- .../sdk/nexmark/queries/sql/SqlQuery2Test.java | 31 +- .../sdk/nexmark/queries/sql/SqlQuery3Test.java | 46 +- .../nexmark/sources/UnboundedEventSourceTest.java | 1 - sdks/python/apache_beam/coders/coders.py | 33 +- sdks/python/apache_beam/coders/typecoders.py | 8 +- .../apache_beam/examples/snippets/snippets.py | 208 +-- .../apache_beam/examples/snippets/snippets_test.py | 16 +- .../apache_beam/examples/wordcount_it_test.py | 2 +- sdks/python/apache_beam/io/filebasedsource.py | 27 +- sdks/python/apache_beam/io/filesystemio.py | 267 ++++ sdks/python/apache_beam/io/filesystemio_test.py | 185 +++ sdks/python/apache_beam/io/filesystems.py | 2 +- sdks/python/apache_beam/io/gcp/gcsio.py | 480 +------ sdks/python/apache_beam/io/gcp/gcsio_test.py | 48 +- sdks/python/apache_beam/io/gcp/pubsub_test.py | 58 +- sdks/python/apache_beam/io/hadoopfilesystem.py | 126 +- .../python/apache_beam/io/hadoopfilesystem_test.py | 153 ++- sdks/python/apache_beam/io/iobase.py | 8 +- sdks/python/apache_beam/io/tfrecordio.py | 47 +- sdks/python/apache_beam/io/tfrecordio_test.py | 482 ++++--- sdks/python/apache_beam/metrics/execution.py | 15 +- sdks/python/apache_beam/metrics/metricbase.py | 10 + .../python/apache_beam/options/pipeline_options.py | 28 + sdks/python/apache_beam/options/value_provider.py | 26 +- .../apache_beam/options/value_provider_test.py | 34 + sdks/python/apache_beam/pipeline.py | 99 +- sdks/python/apache_beam/pipeline_test.py | 71 +- .../python_urns.py} | 37 +- sdks/python/apache_beam/pvalue.py | 20 +- .../runners/dataflow/dataflow_runner.py | 24 +- .../runners/dataflow/dataflow_runner_test.py | 4 +- .../runners/dataflow/native_io/iobase_test.py | 3 +- .../runners/dataflow/ptransform_overrides.py | 6 +- .../apache_beam/runners/direct/direct_runner.py | 318 +++-- .../runners/direct/direct_runner_test.py | 18 +- .../runners/direct/helper_transforms.py | 8 +- .../runners/direct/sdf_direct_runner.py | 9 +- .../runners/direct/sdf_direct_runner_test.py | 6 +- .../runners/direct/transform_evaluator.py | 31 +- .../python/apache_beam/runners/pipeline_context.py | 3 + .../runners/portability/fn_api_runner.py | 75 +- .../runners/portability/fn_api_runner_test.py | 4 +- sdks/python/apache_beam/runners/runner.py | 12 +- sdks/python/apache_beam/runners/runner_test.py | 14 + sdks/python/apache_beam/runners/sdf_common.py | 15 +- .../apache_beam/runners/worker/bundle_processor.py | 45 +- .../apache_beam/runners/worker/data_plane.py | 2 +- .../apache_beam/runners/worker/operations.py | 7 +- .../apache_beam/runners/worker/sideinputs.py | 2 +- .../runners/worker/statesampler_test.py | 2 + sdks/python/apache_beam/transforms/combiners.py | 41 +- sdks/python/apache_beam/transforms/core.py | 75 +- sdks/python/apache_beam/transforms/cy_combiners.py | 1 + sdks/python/apache_beam/transforms/ptransform.py | 6 +- .../apache_beam/transforms/ptransform_test.py | 41 +- sdks/python/apache_beam/transforms/trigger.py | 1 + sdks/python/apache_beam/transforms/window.py | 22 +- sdks/python/apache_beam/typehints/typecheck.py | 75 +- sdks/python/apache_beam/typehints/typehints.py | 5 + sdks/python/apache_beam/utils/urns.py | 42 - sdks/python/container/build.gradle | 10 +- sdks/python/container/run_validatescontainer.sh | 120 ++ sdks/python/generate_pydoc.sh | 1 - sdks/python/setup.py | 32 +- sdks/python/tox.ini | 10 +- settings.gradle | 1 + 650 files changed, 19048 insertions(+), 8966 deletions(-) -- To stop receiving notification emails like this one, please contact [email protected].
