This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to tag nightly-master in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! *** from fd3075f (commit) to 194fe17 (commit) from fd3075f Merge pull request #13995 from [BEAM-11322] Apache Beam Example to tokenize sensitive data add 6eb3c24 Update dependencies add eb8ceeb Update dependencies add 460e093 Merge pull request #14379: Update python dependencies add 3d8798e Moving from vendored gRPC 1.26 to 1.36 add 76c49ac Merge pull request #14295: [BEAM-11227] Moving from vendored gRPC 1.26 to 1.36 add ab509b3 [BEAM-9185] Publish pre-release python artifacts (RCs) to PyPI (#14325) add 93a6927 Update httplib2 to version 1.19.1 on Python container add 288d379 Merge pull request #14386 from iemejia/update-python-deps add 50031c6 Update website build dependencies add ec9e413 Merge pull request #14387: Update website build dependencies add dcd8778 Merge pull request #14113 from [BEAM-11277] Respect schemaUpdateOptions during BigQuery load with temporary tables add 1499d16 Reenable type checking for iobase (#14281) add 12a2d18 Better dataframe support for beam notebooks. add a582fd6 Merge pull request #14356 Better dataframe support for beam notebooks. add e1571c3 Documenting temp_dataset parameter in ReadFromBigQuery add 196d55f Merge pull request #14376 from apache/pabloem-patch-2 add 194fe17 [BEAM-12062] Python-Katas Doc Fixes and Lesson Enhancements (#14353) No new revisions were added by this update. Summary of changes: .github/workflows/build_wheels.yml | 93 ++++++ .../jenkins/metrics_report/requirements.txt | 2 +- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 10 +- .../apache/beam/gradle/GrpcVendoring_1_26_0.groovy | 222 ------------- examples/java/build.gradle | 2 +- .../transforms/DataProtectors.java | 6 +- .../kafkatopubsub/kafka/consumer/Utils.java | 4 +- .../Common Transforms/Aggregation/Largest/task.md | 2 +- .../Common Transforms/Aggregation/Largest/task.py | 2 +- .../Common Transforms/Aggregation/Largest/tests.py | 4 +- .../Common Transforms/WithKeys/WithKeys/task.md | 2 +- .../Windowing/Adding Timestamp/ParDo/task.md | 2 +- release/src/main/scripts/choose_rc_commit.sh | 0 .../main/scripts/deploy_release_candidate_pypi.sh | 172 ++++++++++ .../scripts/download_github_actions_artifacts.py | 30 +- runners/core-construction-java/build.gradle | 2 +- .../beam/runners/core/construction/BeamUrns.java | 2 +- .../core/construction/CoderTranslation.java | 2 +- .../core/construction/CoderTranslators.java | 2 +- .../core/construction/CombineTranslation.java | 2 +- .../CreatePCollectionViewTranslation.java | 2 +- .../DefaultExpansionServiceClientFactory.java | 2 +- .../core/construction/DisplayDataTranslation.java | 2 +- .../runners/core/construction/Environments.java | 4 +- .../beam/runners/core/construction/External.java | 6 +- .../runners/core/construction/ModelCoders.java | 2 +- .../construction/PCollectionViewTranslation.java | 2 +- .../core/construction/ParDoTranslation.java | 4 +- .../construction/PipelineOptionsTranslation.java | 6 +- .../runners/core/construction/ReadTranslation.java | 4 +- .../core/construction/TestStreamTranslation.java | 2 +- .../core/construction/WindowIntoTranslation.java | 2 +- .../construction/WindowingStrategyTranslation.java | 8 +- .../core/construction/WriteFilesTranslation.java | 2 +- .../graph/GreedyPCollectionFusers.java | 2 +- .../core/construction/graph/QueryablePipeline.java | 2 +- .../runners/core/construction/CommonCoderTest.java | 2 +- .../PipelineOptionsTranslationTest.java | 6 +- .../core/construction/ValidateRunnerXlangTest.java | 8 +- .../construction/WindowIntoTranslationTest.java | 2 +- .../construction/graph/ProtoOverridesTest.java | 2 +- runners/core-java/build.gradle | 2 +- .../core/metrics/MetricsContainerStepMap.java | 4 +- .../core/metrics/MonitoringInfoEncodings.java | 2 +- .../core/metrics/MonitoringInfoEncodingsTest.java | 2 +- runners/direct-java/build.gradle | 4 +- runners/flink/flink_runner.gradle | 2 +- .../FlinkBatchPortablePipelineTranslator.java | 2 +- .../apache/beam/runners/flink/FlinkJobInvoker.java | 2 +- .../beam/runners/flink/FlinkPipelineRunner.java | 2 +- .../FlinkStreamingPortablePipelineTranslator.java | 2 +- .../utils/FlinkPortableRunnerUtils.java | 2 +- .../streaming/ExecutableStageDoFnOperator.java | 4 +- .../wrappers/streaming/FlinkKeyUtils.java | 2 +- .../FlinkExecutableStageFunctionTest.java | 2 +- .../streaming/ExecutableStageDoFnOperatorTest.java | 4 +- .../wrappers/streaming/FlinkKeyUtilsTest.java | 2 +- runners/google-cloud-dataflow-java/build.gradle | 2 +- .../dataflow/DataflowPipelineTranslator.java | 2 +- .../beam/runners/dataflow/DataflowRunner.java | 4 +- .../beam/runners/dataflow/DataflowRunnerTest.java | 2 +- .../google-cloud-dataflow-java/worker/build.gradle | 2 +- .../worker/legacy-worker/build.gradle | 4 +- .../runners/dataflow/worker/ByteStringCoder.java | 2 +- .../dataflow/worker/DataflowRunnerHarness.java | 2 +- .../worker/DataflowWorkerHarnessHelper.java | 2 +- .../worker/GroupAlsoByWindowParDoFnFactory.java | 2 +- .../beam/runners/dataflow/worker/PubsubSink.java | 2 +- .../beam/runners/dataflow/worker/StateFetcher.java | 2 +- .../dataflow/worker/StreamingDataflowWorker.java | 4 +- .../worker/StreamingModeExecutionContext.java | 2 +- .../dataflow/worker/StreamingSideInputFetcher.java | 4 +- .../dataflow/worker/WindmillComputationKey.java | 4 +- .../dataflow/worker/WindmillNamespacePrefix.java | 2 +- .../beam/runners/dataflow/worker/WindmillSink.java | 2 +- .../dataflow/worker/WindmillStateCache.java | 2 +- .../dataflow/worker/WindmillStateInternals.java | 2 +- .../dataflow/worker/WindmillStateReader.java | 2 +- .../dataflow/worker/WindmillTimerInternals.java | 2 +- .../dataflow/worker/WorkerCustomSources.java | 2 +- .../dataflow/worker/fn/BeamFnControlService.java | 6 +- .../control/RegisterAndProcessBundleOperation.java | 4 +- .../worker/fn/data/BeamFnDataGrpcService.java | 4 +- .../worker/fn/logging/BeamFnLoggingService.java | 4 +- .../fn/stream/ServerStreamObserverFactory.java | 6 +- .../graph/CreateExecutableStageNodeFunction.java | 4 +- ...nsertFetchAndFilterStreamingSideInputNodes.java | 2 +- .../worker/graph/RegisterNodeFunction.java | 4 +- .../worker/windmill/DirectStreamObserver.java | 4 +- .../windmill/ForwardingClientResponseObserver.java | 6 +- .../worker/windmill/GrpcWindmillServer.java | 30 +- .../worker/windmill/StreamObserverFactory.java | 4 +- .../runners/dataflow/harness/test/TestStreams.java | 4 +- .../runners/dataflow/worker/DataflowMatchers.java | 2 +- .../worker/DataflowWorkerHarnessHelperTest.java | 2 +- .../runners/dataflow/worker/PubsubReaderTest.java | 2 +- .../runners/dataflow/worker/PubsubSinkTest.java | 2 +- .../runners/dataflow/worker/ReaderCacheTest.java | 2 +- .../runners/dataflow/worker/StateFetcherTest.java | 2 +- .../worker/StreamingDataflowWorkerTest.java | 6 +- .../worker/StreamingGroupAlsoByWindowFnsTest.java | 2 +- ...reamingGroupAlsoByWindowsReshuffleDoFnTest.java | 2 +- .../worker/StreamingModeExecutionContextTest.java | 2 +- .../worker/StreamingSideInputDoFnRunnerTest.java | 2 +- .../worker/StreamingSideInputFetcherTest.java | 2 +- .../dataflow/worker/WindmillKeyedWorkItemTest.java | 2 +- .../worker/WindmillReaderIteratorBaseTest.java | 2 +- .../dataflow/worker/WindmillStateCacheTest.java | 2 +- .../worker/WindmillStateInternalsTest.java | 2 +- .../dataflow/worker/WindmillStateReaderTest.java | 4 +- .../dataflow/worker/WorkerCustomSourcesTest.java | 2 +- .../worker/fn/BeamFnControlServiceTest.java | 6 +- .../RegisterAndProcessBundleOperationTest.java | 2 +- .../worker/fn/data/BeamFnDataGrpcServiceTest.java | 32 +- .../fn/logging/BeamFnLoggingServiceTest.java | 12 +- .../fn/stream/ServerStreamObserverFactoryTest.java | 4 +- ...tFetchAndFilterStreamingSideInputNodesTest.java | 2 +- .../logging/DataflowWorkerLoggingHandlerTest.java | 2 +- .../worker/windmill/GrpcWindmillServerTest.java | 14 +- runners/java-fn-execution/build.gradle | 2 +- .../apache/beam/runners/fnexecution/FnService.java | 6 +- .../GrpcContextHeaderAccessorProvider.java | 16 +- .../beam/runners/fnexecution/GrpcFnServer.java | 2 +- .../fnexecution/InProcessServerFactory.java | 8 +- .../beam/runners/fnexecution/ServerFactory.java | 20 +- .../artifact/ArtifactRetrievalService.java | 8 +- .../artifact/ArtifactStagingService.java | 10 +- .../control/BundleCheckpointHandlers.java | 2 +- .../fnexecution/control/FnApiControlClient.java | 6 +- .../control/FnApiControlClientPoolService.java | 6 +- .../control/ProcessBundleDescriptors.java | 2 +- .../runners/fnexecution/data/GrpcDataService.java | 2 +- .../environment/ExternalEnvironmentFactory.java | 2 +- .../fnexecution/logging/GrpcLoggingService.java | 2 +- .../runners/fnexecution/provisioning/JobInfo.java | 2 +- .../provisioning/StaticGrpcProvisionService.java | 2 +- .../fnexecution/state/GrpcStateService.java | 4 +- .../fnexecution/state/StateRequestHandlers.java | 2 +- .../status/BeamWorkerStatusGrpcService.java | 2 +- .../fnexecution/status/WorkerStatusClient.java | 2 +- .../translation/PipelineTranslatorUtils.java | 2 +- .../runners/fnexecution/wire/ByteStringCoder.java | 2 +- .../GrpcContextHeaderAccessorProviderTest.java | 20 +- .../runners/fnexecution/ServerFactoryTest.java | 12 +- .../artifact/ArtifactRetrievalServiceTest.java | 10 +- .../artifact/ArtifactStagingServiceTest.java | 12 +- .../control/DefaultJobBundleFactoryTest.java | 4 +- .../control/FnApiControlClientPoolServiceTest.java | 8 +- .../control/FnApiControlClientTest.java | 2 +- .../fnexecution/control/RemoteExecutionTest.java | 2 +- .../fnexecution/control/SdkHarnessClientTest.java | 2 +- .../fnexecution/data/GrpcDataServiceTest.java | 8 +- .../logging/GrpcLoggingServiceTest.java | 6 +- .../StaticGrpcProvisionServiceTest.java | 10 +- .../fnexecution/state/GrpcStateServiceTest.java | 4 +- .../state/StateRequestHandlersTest.java | 2 +- .../status/BeamWorkerStatusGrpcServiceTest.java | 8 +- .../fnexecution/status/WorkerStatusClientTest.java | 2 +- .../fnexecution/wire/ByteStringCoderTest.java | 2 +- runners/java-job-service/build.gradle | 2 +- .../runners/jobsubmission/InMemoryJobService.java | 10 +- .../beam/runners/jobsubmission/JobInvocation.java | 2 +- .../beam/runners/jobsubmission/JobInvoker.java | 2 +- .../beam/runners/jobsubmission/JobPreparation.java | 2 +- .../jobsubmission/PortablePipelineJarCreator.java | 4 +- .../jobsubmission/PortablePipelineJarUtils.java | 6 +- .../jobsubmission/InMemoryJobServiceTest.java | 6 +- .../runners/jobsubmission/JobInvocationTest.java | 2 +- .../PortablePipelineJarCreatorTest.java | 2 +- runners/portability/java/build.gradle | 2 +- .../runners/portability/ExternalWorkerService.java | 2 +- .../portability/JobServicePipelineResult.java | 2 +- .../beam/runners/portability/PortableRunner.java | 4 +- .../portability/testing/TestJobService.java | 2 +- .../runners/portability/PortableRunnerTest.java | 8 +- runners/samza/build.gradle | 2 +- .../beam/runners/samza/SamzaJobServerDriver.java | 2 +- .../samza/translation/WindowAssignTranslator.java | 2 +- .../samza/util/SamzaPipelineTranslatorUtils.java | 2 +- runners/spark/spark_runner.gradle | 2 +- .../apache/beam/runners/spark/SparkJobInvoker.java | 2 +- .../beam/runners/spark/SparkPipelineRunner.java | 2 +- runners/twister2/build.gradle | 2 +- .../functions/AssignWindowsFunction.java | 2 +- .../translators/functions/DoFnFunction.java | 2 +- .../functions/GroupByWindowFunction.java | 2 +- sdks/java/core/build.gradle | 2 +- .../java/org/apache/beam/sdk/io/TextSource.java | 2 +- .../java/org/apache/beam/sdk/io/range/ByteKey.java | 4 +- .../apache/beam/sdk/schemas/SchemaTranslation.java | 2 +- sdks/java/expansion-service/build.gradle | 2 +- .../sdk/expansion/service/ExpansionServer.java | 4 +- .../sdk/expansion/service/ExpansionService.java | 6 +- .../expansion/service/ExpansionServiceTest.java | 2 +- .../beam/sdk/expansion/service/ExternalTest.java | 4 +- .../schemaio-expansion-service/build.gradle | 2 +- .../ExternalSchemaIOTransformRegistrar.java | 2 +- sdks/java/fn-execution/build.gradle | 2 +- .../beam/sdk/fn/channel/ManagedChannelFactory.java | 18 +- .../beam/sdk/fn/channel/SocketAddressFactory.java | 2 +- .../data/BeamFnDataBufferingOutboundObserver.java | 2 +- .../sdk/fn/data/BeamFnDataGrpcMultiplexer.java | 6 +- .../sdk/fn/data/BeamFnDataInboundObserver.java | 2 +- ...amFnDataSizeBasedBufferingOutboundObserver.java | 4 +- ...amFnDataTimeBasedBufferingOutboundObserver.java | 2 +- .../beam/sdk/fn/data/DecodingFnDataReceiver.java | 2 +- .../beam/sdk/fn/data/RemoteGrpcPortRead.java | 2 +- .../beam/sdk/fn/data/RemoteGrpcPortWrite.java | 2 +- .../sdk/fn/stream/BufferingStreamObserver.java | 4 +- .../org/apache/beam/sdk/fn/stream/DataStreams.java | 2 +- .../beam/sdk/fn/stream/DirectStreamObserver.java | 4 +- .../stream/ForwardingClientResponseObserver.java | 6 +- .../sdk/fn/stream/OutboundObserverFactory.java | 4 +- .../sdk/fn/stream/SynchronizedStreamObserver.java | 2 +- .../fn/test/InProcessManagedChannelFactory.java | 4 +- .../org/apache/beam/sdk/fn/test/TestStreams.java | 4 +- .../sdk/fn/windowing/EncodedBoundedWindow.java | 2 +- .../sdk/fn/channel/ManagedChannelFactoryTest.java | 6 +- .../sdk/fn/channel/SocketAddressFactoryTest.java | 2 +- .../sdk/fn/data/BeamFnDataGrpcMultiplexerTest.java | 2 +- ...DataSizeBasedBufferingOutboundObserverTest.java | 2 +- ...DataTimeBasedBufferingOutboundObserverTest.java | 2 +- .../beam/sdk/fn/data/RemoteGrpcPortReadTest.java | 2 +- .../beam/sdk/fn/data/RemoteGrpcPortWriteTest.java | 2 +- .../apache/beam/sdk/fn/stream/DataStreamsTest.java | 2 +- .../ForwardingClientResponseObserverTest.java | 6 +- .../sdk/fn/stream/OutboundObserverFactoryTest.java | 4 +- .../sdk/fn/windowing/EncodedBoundedWindowTest.java | 2 +- sdks/java/harness/build.gradle | 2 +- .../apache/beam/fn/harness/FnApiDoFnRunner.java | 16 +- .../java/org/apache/beam/fn/harness/FnHarness.java | 4 +- .../harness/control/AddHarnessIdInterceptor.java | 8 +- .../fn/harness/control/BeamFnControlClient.java | 4 +- .../fn/harness/control/ProcessBundleHandler.java | 4 +- .../beam/fn/harness/data/BeamFnDataClient.java | 2 +- .../beam/fn/harness/data/BeamFnDataGrpcClient.java | 4 +- .../fn/harness/data/QueueingBeamFnDataClient.java | 2 +- .../fn/harness/logging/BeamFnLoggingClient.java | 12 +- .../apache/beam/fn/harness/state/BagUserState.java | 2 +- .../harness/state/BeamFnStateGrpcClientCache.java | 4 +- .../beam/fn/harness/state/FnApiStateAccessor.java | 2 +- .../beam/fn/harness/state/IterableSideInput.java | 2 +- .../beam/fn/harness/state/MultimapSideInput.java | 2 +- .../beam/fn/harness/state/StateBackedIterable.java | 2 +- .../fn/harness/state/StateFetchingIterators.java | 2 +- .../stream/HarnessStreamObserverFactories.java | 2 +- .../beam/fn/harness/FnApiDoFnRunnerTest.java | 38 +-- .../org/apache/beam/fn/harness/FnHarnessTest.java | 8 +- .../harness/control/BeamFnControlClientTest.java | 8 +- .../harness/control/ProcessBundleHandlerTest.java | 4 +- .../fn/harness/data/BeamFnDataGrpcClientTest.java | 14 +- .../data/BeamFnDataInboundObserverTest.java | 2 +- .../harness/data/QueueingBeamFnDataClientTest.java | 14 +- .../harness/logging/BeamFnLoggingClientTest.java | 16 +- .../beam/fn/harness/state/BagUserStateTest.java | 2 +- .../state/BeamFnStateGrpcClientCacheTest.java | 16 +- .../fn/harness/state/FakeBeamFnStateClient.java | 2 +- .../fn/harness/state/MultimapSideInputTest.java | 2 +- .../fn/harness/state/StateBackedIterableTest.java | 2 +- .../harness/state/StateFetchingIteratorsTest.java | 2 +- .../stream/HarnessStreamObserverFactoriesTest.java | 4 +- sdks/java/io/contextualtextio/build.gradle | 2 +- .../contextualtextio/ContextualTextIOSource.java | 2 +- sdks/java/io/debezium/build.gradle | 2 +- .../apache/beam/io/debezium/SourceRecordJson.java | 4 +- sdks/java/io/google-cloud-platform/build.gradle | 2 +- .../io/gcp/spanner/SpannerTransformRegistrar.java | 2 +- .../sdk/io/gcp/pubsub/PubsubIOExternalTest.java | 4 +- .../beam/sdk/io/kafka/KafkaIOExternalTest.java | 4 +- sdks/python/apache_beam/io/gcp/bigquery.py | 4 + .../apache_beam/io/gcp/bigquery_file_loads.py | 180 ++++++++++- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 54 +++- .../apache_beam/io/gcp/bigquery_tools_test.py | 29 ++ .../apache_beam/io/gcp/bigquery_write_it_test.py | 45 +++ sdks/python/apache_beam/io/gcp/pubsub.py | 9 +- sdks/python/apache_beam/io/iobase.py | 89 ++++-- .../runners/interactive/interactive_beam.py | 10 +- .../runners/interactive/interactive_runner_test.py | 24 ++ .../runners/interactive/recording_manager.py | 8 + .../apache_beam/runners/interactive/utils.py | 13 +- sdks/python/container/base_image_requirements.txt | 4 +- sdks/python/mypy.ini | 3 + sdks/python/setup.py | 2 +- settings.gradle.kts | 1 - vendor/grpc-1_26_0/build.gradle | 37 --- .../site/content/en/contribute/release-guide.md | 31 +- website/www/yarn.lock | 355 ++++++++++----------- 287 files changed, 1449 insertions(+), 1056 deletions(-) delete mode 100644 buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_26_0.groovy mode change 100644 => 100755 release/src/main/scripts/choose_rc_commit.sh create mode 100755 release/src/main/scripts/deploy_release_candidate_pypi.sh delete mode 100644 vendor/grpc-1_26_0/build.gradle