This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch nightly-refs/heads/master in repository https://gitbox.apache.org/repos/asf/beam.git.
from bee56a6 [BEAM-13288] improve logging for no rows present error (#16096) add 45955da Merge pull request #16117 from [BEAM-13368][Playground][Bugfix] Fix CI and failed unit tests on master add 0830a02 Run python SpannerIO IT with python 3.7 only to avoid overload spanner instance (#16112) add 862ece1 Merge pull request #15378 from [RFC] Define and document per-key ordering semantics for runners add 3422774 Bump python containers to beam-master-20211202 (#16129) add 29213ce [BEAM-13354, BEAM-13015, BEAM-12802, BEAM-12588] Support prefetch for multimap and set state making loading keys and values truly lazy (#16092) add 09bbb48 [BEAM-11936] Fix errorprone UnusedVariable in IO (#16036) add 3c948fc Fix failing RecommendationAICatalogItemIT add 89e2b23 Merge pull request #16132 from y1chi/fix_post add 91950d6 Clarify instructions on how to get contributor list. add dfb5726 Merge pull request #16133 from tvalentyn/tags_command add 6b9e374 Revert "Revert "Allow wildcards for java class lookup transform providers."" add 8faf018 Move stand-alone expansion service jar into its own project. add bfff840 Merge pull request #16044 Restore "Allow wildcards for java class lookup transform providers." No new revisions were added by this update. Summary of changes: ...Java_PortableValidatesRunner_Flink_Batch.groovy | 2 +- build.gradle.kts | 3 + playground/backend/build.gradle.kts | 2 +- playground/backend/cmd/server/controller_test.go | 21 +- runners/flink/job-server/flink_job_server.gradle | 2 + runners/spark/job-server/spark_job_server.gradle | 1 + ...edMetrics.java => UsesPerKeyOrderInBundle.java} | 10 +- ...leParDo.java => UsesPerKeyOrderedDelivery.java} | 9 +- .../org/apache/beam/sdk/transforms/ParDoTest.java | 3 +- .../beam/sdk/transforms/PerKeyOrderingTest.java | 328 +++++++++++++++ .../app}/build.gradle | 21 +- sdks/java/expansion-service/build.gradle | 5 +- .../expansion/service/ExpansionServiceOptions.java | 10 +- .../service/JavaClassLookupTransformProvider.java | 83 +++- .../JavaClassLookupTransformProviderTest.java | 42 ++ .../ml/RecommendationAICatalogItemIT.java | 4 +- .../beam/fn/harness/state/FnApiStateAccessor.java | 19 +- .../beam/fn/harness/state/MultimapUserState.java | 165 +++++--- .../fn/harness/state/MultimapUserStateTest.java | 449 +++++++++++++++------ sdks/java/io/amazon-web-services/build.gradle | 1 - sdks/java/io/amazon-web-services2/build.gradle | 1 - .../apache/beam/sdk/io/aws2/kinesis/KinesisIO.java | 6 - .../beam/sdk/io/aws2/s3/S3FileSystemTest.java | 2 +- sdks/java/io/amqp/build.gradle | 2 +- .../java/org/apache/beam/sdk/io/amqp/AmqpIO.java | 8 +- .../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 2 +- sdks/java/io/azure/build.gradle | 1 - .../beam/sdk/io/azure/options/AzureModule.java | 6 - sdks/java/io/bigquery-io-perf-tests/build.gradle | 2 +- .../beam/sdk/bigqueryioperftests/BigQueryIOIT.java | 4 - sdks/java/io/cassandra/build.gradle | 1 - .../apache/beam/sdk/io/cassandra/CassandraIO.java | 1 - .../beam/sdk/io/cassandra/CassandraIOTest.java | 1 - sdks/java/io/clickhouse/build.gradle | 1 - sdks/java/io/common/build.gradle | 2 +- sdks/java/io/contextualtextio/build.gradle | 1 - .../contextualtextio/ContextualTextIOSource.java | 4 - sdks/java/io/debezium/build.gradle | 1 - .../io/debezium/expansion-service/build.gradle | 1 - .../io/debezium/DebeziumTransformRegistrar.java | 3 - .../apache/beam/io/debezium/SourceRecordJson.java | 4 +- .../apache/beam/io/debezium/DebeziumIOTest.java | 4 +- .../io/debezium/KafkaSourceConsumerFnTest.java | 23 +- .../elasticsearch-tests-5/build.gradle | 1 - .../elasticsearch-tests-6/build.gradle | 1 - .../elasticsearch-tests-7/build.gradle | 1 - .../elasticsearch-tests-common/build.gradle | 1 - sdks/java/io/elasticsearch/build.gradle | 2 +- sdks/java/io/expansion-service/build.gradle | 1 - sdks/java/io/file-based-io-tests/build.gradle | 2 +- .../java/org/apache/beam/sdk/io/avro/AvroIOIT.java | 4 - .../apache/beam/sdk/io/parquet/ParquetIOIT.java | 4 - .../java/org/apache/beam/sdk/io/text/TextIOIT.java | 7 - .../apache/beam/sdk/io/tfrecord/TFRecordIOIT.java | 4 - .../java/org/apache/beam/sdk/io/xml/XmlIOIT.java | 4 - sdks/java/io/google-cloud-platform/build.gradle | 1 - .../expansion-service/build.gradle | 1 - .../sdk/io/gcp/bigquery/BigQueryQuerySource.java | 4 - .../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 3 - .../gcp/bigquery/BigQueryStorageArrowReader.java | 5 - .../sdk/io/gcp/bigquery/BigQueryTableSource.java | 3 - .../beam/sdk/io/gcp/bigquery/CreateTables.java | 6 - .../beam/sdk/io/gcp/bigquery/StorageApiLoads.java | 3 - .../bigquery/StorageApiWritesShardedRecords.java | 5 +- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 1 - .../apache/beam/sdk/io/gcp/healthcare/FhirIO.java | 1 - .../io/gcp/healthcare/HttpHealthcareApiClient.java | 2 +- .../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 4 - .../beam/sdk/io/gcp/pubsub/TestPubsubSignal.java | 9 +- .../sdk/io/gcp/spanner/MutationKeyEncoder.java | 4 +- .../sdk/io/gcp/testing/FakeDatasetService.java | 1 - .../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 3 - .../io/gcp/bigquery/BigQueryNestedRecordsIT.java | 2 - .../io/gcp/bigquery/BigQueryServicesImplTest.java | 4 - .../sdk/io/gcp/bigquery/BigQueryToTableIT.java | 5 +- .../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 2 - .../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 3 - .../gcp/firestore/BaseFirestoreV1WriteFnTest.java | 5 +- .../beam/sdk/io/gcp/healthcare/FhirIOLROIT.java | 4 +- .../beam/sdk/io/gcp/healthcare/FhirIOTestUtil.java | 3 - .../beam/sdk/io/gcp/pubsub/PubsubReadIT.java | 3 - .../sdk/io/gcp/spanner/ReadSpannerSchemaTest.java | 2 - .../beam/sdk/io/gcp/spanner/SpannerIOReadTest.java | 30 +- .../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 1 - sdks/java/io/hadoop-common/build.gradle | 2 +- sdks/java/io/hadoop-file-system/build.gradle | 1 - sdks/java/io/hadoop-format/build.gradle | 1 - .../sdk/io/hadoop/format/HadoopFormatIOIT.java | 4 - .../io/hadoop/format/HadoopFormatIOReadTest.java | 3 - sdks/java/io/hbase/build.gradle | 2 +- .../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 2 - sdks/java/io/hcatalog/build.gradle | 2 +- sdks/java/io/influxdb/build.gradle | 3 +- .../apache/beam/sdk/io/influxdb/InfluxDbIO.java | 3 - sdks/java/io/jdbc/build.gradle | 1 - .../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 4 - .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 26 +- sdks/java/io/jms/build.gradle | 1 - sdks/java/io/kafka/build.gradle | 1 - .../beam/sdk/io/kafka/KafkaExactlyOnceSink.java | 4 +- .../sdk/io/kafka/WatchKafkaTopicPartitionDoFn.java | 5 +- .../beam/sdk/io/kafka/KafkaIOExternalTest.java | 3 +- sdks/java/io/kinesis/build.gradle | 2 +- .../java/io/kinesis/expansion-service/build.gradle | 1 - .../beam/sdk/io/kinesis/KinesisIOReadTest.java | 2 - .../beam/sdk/io/kinesis/serde/AwsModuleTest.java | 2 - sdks/java/io/kudu/build.gradle | 1 - .../java/org/apache/beam/sdk/io/kudu/KuduIO.java | 3 - .../java/org/apache/beam/sdk/io/kudu/KuduIOIT.java | 3 - .../org/apache/beam/sdk/io/kudu/KuduIOTest.java | 1 - sdks/java/io/mongodb/build.gradle | 2 +- .../org/apache/beam/sdk/io/mongodb/MongoDbIO.java | 1 - .../apache/beam/sdk/io/mongodb/MongoDBIOIT.java | 4 - sdks/java/io/mqtt/build.gradle | 2 +- sdks/java/io/parquet/build.gradle | 1 - .../apache/beam/sdk/io/parquet/ParquetIOTest.java | 1 - sdks/java/io/rabbitmq/build.gradle | 2 +- .../beam/sdk/io/rabbitmq/RabbitMqIOTest.java | 3 - sdks/java/io/redis/build.gradle | 2 +- sdks/java/io/snowflake/build.gradle | 1 - .../io/snowflake/expansion-service/build.gradle | 1 - .../apache/beam/sdk/io/snowflake/SnowflakeIO.java | 3 +- .../services/SnowflakeBatchServiceImpl.java | 3 - .../services/SnowflakeStreamingServiceImpl.java | 4 +- .../test/unit/DataSourceConfigurationTest.java | 4 - .../test/unit/write/SchemaDispositionTest.java | 1 - .../test/unit/write/StreamingWriteTest.java | 4 +- sdks/java/io/solr/build.gradle | 1 - sdks/java/io/splunk/build.gradle | 2 +- .../beam/sdk/io/splunk/SplunkEventWriter.java | 3 +- sdks/java/io/synthetic/build.gradle | 1 - sdks/java/io/thrift/build.gradle | 1 - .../apache/beam/sdk/io/thrift/ThriftIOTest.java | 6 +- sdks/java/io/tika/build.gradle | 2 +- sdks/java/io/xml/build.gradle | 2 +- .../io/gcp/experimental/spannerio_read_it_test.py | 20 +- .../io/gcp/experimental/spannerio_write_it_test.py | 10 +- .../apache_beam/runners/dataflow/internal/names.py | 4 +- sdks/python/apache_beam/transforms/external.py | 56 ++- .../python/apache_beam/transforms/external_test.py | 24 ++ sdks/python/apache_beam/utils/processes_test.py | 8 + sdks/python/apache_beam/utils/subprocess_server.py | 47 ++- .../apache_beam/utils/subprocess_server_test.py | 51 +++ sdks/python/test-suites/dataflow/common.gradle | 24 ++ sdks/python/test-suites/direct/common.gradle | 20 + sdks/python/test-suites/portable/common.gradle | 38 +- settings.gradle.kts | 1 + .../site/content/en/contribute/release-guide.md | 2 +- .../site/content/en/documentation/runtime/model.md | 50 ++- website/www/site/data/capability_matrix.yaml | 39 ++ .../documentation/capability-matrix-row.html | 2 + .../site/layouts/shortcodes/capability-matrix.html | 2 + .../documentation/capability-matrix-single.html | 6 +- 153 files changed, 1391 insertions(+), 566 deletions(-) copy sdks/java/core/src/main/java/org/apache/beam/sdk/testing/{UsesCommittedMetrics.java => UsesPerKeyOrderInBundle.java} (71%) copy sdks/java/core/src/main/java/org/apache/beam/sdk/testing/{UsesBoundedSplittableParDo.java => UsesPerKeyOrderedDelivery.java} (75%) create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/PerKeyOrderingTest.java copy sdks/java/{io/snowflake/expansion-service => expansion-service/app}/build.gradle (72%)