[beam] branch master updated: [BEAM-14035 ] Implement BigQuerySchema Read/Write TransformProvider (#17607)
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new ee5888dbf5f [BEAM-14035 ] Implement BigQuerySchema Read/Write
TransformProvider (#17607)
ee5888dbf5f is described below
commit ee5888dbf5f14ce7c427b41b5884e342f5e971c7
Author: Damon Douglas
AuthorDate: Mon May 16 18:02:39 2022 +
[BEAM-14035 ] Implement BigQuerySchema Read/Write TransformProvider (#17607)
---
.../transforms/TypedSchemaTransformProvider.java | 4 +-
.../TypedSchemaTransformProviderTest.java | 2 +-
.../BigQuerySchemaTransformReadConfiguration.java | 100
.../BigQuerySchemaTransformReadProvider.java | 177 +
.../BigQuerySchemaTransformWriteConfiguration.java | 86 +++
.../BigQuerySchemaTransformWriteProvider.java | 279 +
.../BigQuerySchemaTransformReadProviderTest.java | 252 +++
.../BigQuerySchemaTransformWriteProviderTest.java | 256 +++
8 files changed, 1153 insertions(+), 3 deletions(-)
diff --git
a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProvider.java
b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProvider.java
index c237d03e052..944f33a596d 100644
---
a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProvider.java
+++
b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProvider.java
@@ -42,13 +42,13 @@ import org.apache.beam.sdk.values.Row;
@Experimental(Kind.SCHEMAS)
public abstract class TypedSchemaTransformProvider implements
SchemaTransformProvider {
- abstract Class configurationClass();
+ protected abstract Class configurationClass();
/**
* Produce a SchemaTransform from ConfigT. Can throw a {@link
InvalidConfigurationException} or a
* {@link InvalidSchemaException}.
*/
- abstract SchemaTransform from(ConfigT configuration);
+ protected abstract SchemaTransform from(ConfigT configuration);
/**
* List the dependencies needed for this transform. Jars from classpath are
used by default when
diff --git
a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java
b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java
index e4018d3b6c1..744b4f3bf0b 100644
---
a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java
+++
b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java
@@ -63,7 +63,7 @@ public class TypedSchemaTransformProviderTest {
}
@Override
-Class configurationClass() {
+protected Class configurationClass() {
return Configuration.class;
}
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySchemaTransformReadConfiguration.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySchemaTransformReadConfiguration.java
new file mode 100644
index 000..f964a87be16
--- /dev/null
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySchemaTransformReadConfiguration.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.auto.value.AutoValue;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.schemas.AutoValueSchema;
+import org.apache.beam.sdk.schemas.annotations.DefaultSchema;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.TypeDescriptor;
+
+/**
+ * Configuration for reading from BigQuery.
+ *
+ * This class is meant to be used with {@link
BigQuerySchemaTransformReadProvider}.
+ *
+ * Internal only: This class is actively being worked on, and it
will likely ch
[beam] branch master updated: Better test assertion. (#17551)
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new dc20185f6af Better test assertion. (#17551)
dc20185f6af is described below
commit dc20185f6af2e862bd62f9df7b835593a466c5f1
Author: Robert Bradshaw
AuthorDate: Thu May 12 13:02:14 2022 -0700
Better test assertion. (#17551)
---
sdks/python/apache_beam/io/gcp/bigquery_json_it_test.py | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_json_it_test.py
b/sdks/python/apache_beam/io/gcp/bigquery_json_it_test.py
index 13bf63a46a1..25ce2b359f3 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_json_it_test.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_json_it_test.py
@@ -100,13 +100,13 @@ class BigQueryJsonIT(unittest.TestCase):
# Test country (JSON String)
country_actual = json.loads(row["country"])
country_expected = json.loads(expected["country"])
-self.assertTrue(country_expected == country_actual)
+self.assertEqual(country_expected, country_actual)
# Test stats (JSON String in BigQuery struct)
for stat, value in row["stats"].items():
stats_actual = json.loads(value)
stats_expected = json.loads(expected["stats"][stat])
- self.assertTrue(stats_expected == stats_actual)
+ self.assertEqual(stats_expected, stats_actual)
# Test cities (JSON String in BigQuery array of structs)
for city_row in row["cities"]:
@@ -115,7 +115,7 @@ class BigQueryJsonIT(unittest.TestCase):
city_actual = json.loads(city)
city_expected = json.loads(expected["cities"][city_name])
- self.assertTrue(city_expected == city_actual)
+ self.assertEqual(city_expected, city_actual)
# Test landmarks (JSON String in BigQuery array)
landmarks_actual = row["landmarks"]
@@ -123,7 +123,7 @@ class BigQueryJsonIT(unittest.TestCase):
for i in range(len(landmarks_actual)):
l_actual = json.loads(landmarks_actual[i])
l_expected = json.loads(landmarks_expected[i])
- self.assertTrue(l_expected == l_actual)
+ self.assertEqual(l_expected, l_actual)
parser = argparse.ArgumentParser()
parser.add_argument('--read_method')
[beam] branch master updated: Factors enable_prime flag in when checking use_unified_worker conditions (#17271)
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a88b2a109e1 Factors enable_prime flag in when checking
use_unified_worker conditions (#17271)
a88b2a109e1 is described below
commit a88b2a109e1647337a326d0d792024ee6661ba19
Author: Yichi Zhang
AuthorDate: Thu Apr 7 09:43:51 2022 -0700
Factors enable_prime flag in when checking use_unified_worker conditions
(#17271)
* Factors enable_prime flag in when checking use_unified_worker conditions
* Address comments
---
.../apache_beam/runners/dataflow/internal/apiclient.py | 11 +++
.../runners/dataflow/internal/apiclient_test.py | 17 +
2 files changed, 28 insertions(+)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index 771d98d7763..6ac8f778959 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -1174,11 +1174,22 @@ def _use_unified_worker(pipeline_options):
debug_options = pipeline_options.view_as(DebugOptions)
use_unified_worker_flag = 'use_unified_worker'
use_runner_v2_flag = 'use_runner_v2'
+ enable_prime_flag = 'enable_prime'
if (debug_options.lookup_experiment(use_runner_v2_flag) and
not debug_options.lookup_experiment(use_unified_worker_flag)):
debug_options.add_experiment(use_unified_worker_flag)
+ dataflow_service_options = pipeline_options.view_as(
+ GoogleCloudOptions).dataflow_service_options or []
+ if ((debug_options.lookup_experiment(enable_prime_flag) or
+ enable_prime_flag in dataflow_service_options) and
+ not any([debug_options.lookup_experiment('disable_prime_runner_v2'),
+ debug_options.lookup_experiment('disable_runner_v2')])):
+debug_options.add_experiment(use_runner_v2_flag)
+debug_options.add_experiment(use_unified_worker_flag)
+debug_options.add_experiment(enable_prime_flag)
+
return debug_options.lookup_experiment(use_unified_worker_flag)
diff --git
a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
index 0e0a9e02197..49180c77ad7 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
@@ -1003,6 +1003,23 @@ class UtilTest(unittest.TestCase):
['--experiments=use_runner_v2', '--experiments=beam_fn_api'])
self.assertTrue(apiclient._use_unified_worker(pipeline_options))
+pipeline_options = PipelineOptions(['--experiments=enable_prime'])
+self.assertTrue(apiclient._use_unified_worker(pipeline_options))
+
+pipeline_options = PipelineOptions(
+['--dataflow_service_options=enable_prime'])
+self.assertTrue(apiclient._use_unified_worker(pipeline_options))
+
+pipeline_options = PipelineOptions([
+'--dataflow_service_options=enable_prime',
+'--experiments=disable_prime_runner_v2'
+])
+self.assertFalse(apiclient._use_unified_worker(pipeline_options))
+
+pipeline_options = PipelineOptions(
+['--experiments=enable_prime',
'--experiments=disable_prime_runner_v2'])
+self.assertFalse(apiclient._use_unified_worker(pipeline_options))
+
pipeline_options = PipelineOptions([
'--experiments=use_unified_worker',
'--experiments=use_runner_v2',
[beam] branch master updated (6d60ac3 -> 837e505)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 6d60ac3 [BEAM-13738] Reenable ignored SQS test after bumping elasticmq for fixed version (#16914) add 837e505 fix build status link (#16907) No new revisions were added by this update. Summary of changes: .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[beam] branch master updated (c975031 -> 1c579f9)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from c975031 [BEAM-12572] Fix failing python examples tests in Dataflow runner (#16853) add 1c579f9 Remove build status from PR (#16902) No new revisions were added by this update. Summary of changes: .github/PULL_REQUEST_TEMPLATE.md | 363 + .../BUILD_STATUS.md| 25 +- 2 files changed, 17 insertions(+), 371 deletions(-) copy .github/PULL_REQUEST_TEMPLATE.md => .test-infra/BUILD_STATUS.md (95%)
[beam] branch master updated: fixing release date
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new dc04bfc fixing release date new bb4ba7c Merge pull request #15453 from angoenka/website-2.32 dc04bfc is described below commit dc04bfc6c900d05d6e50e30fa7707c0f0a3588d0 Author: Ankur AuthorDate: Thu Sep 2 13:57:58 2021 -0700 fixing release date --- website/www/site/content/en/get-started/downloads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/site/content/en/get-started/downloads.md b/website/www/site/content/en/get-started/downloads.md index b52b773..0a7c844 100644 --- a/website/www/site/content/en/get-started/downloads.md +++ b/website/www/site/content/en/get-started/downloads.md @@ -88,7 +88,7 @@ versions denoted `0.x.y`. ## Releases -### 2.32.0 (2021-08-11) +### 2.32.0 (2021-08-25) Official [source code download](https://www.apache.org/dyn/closer.cgi/beam/2.32.0/apache-beam-2.32.0-source-release.zip). [SHA-512](https://downloads.apache.org/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512). [signature](https://downloads.apache.org/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc).
[beam] branch master updated (637ad65 -> c3dd07d)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 637ad65 Merge pull request #15401 from angoenka/blogpost-2.32.0 new b0bc22b Docs are built in build_release_candidate.sh new 0c91b67 Add pypy to email new a56efee Update CHANGES.md along with website new c3dd07d Merge pull request #15143 from apilloud/guide The 32803 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../site/content/en/contribute/release-guide.md| 46 -- 1 file changed, 8 insertions(+), 38 deletions(-)
[beam] branch master updated: Fix Change log for Dataframe preview
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 64c934d Fix Change log for Dataframe preview new 637ad65 Merge pull request #15401 from angoenka/blogpost-2.32.0 64c934d is described below commit 64c934d566fbf7e2725d21c23c790791095447ee Author: Ankur Goenka AuthorDate: Thu Aug 26 12:18:22 2021 -0700 Fix Change log for Dataframe preview --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7f6bf2e..5e503cf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -154,7 +154,7 @@ ## Highlights * The [Beam DataFrame API](https://beam.apache.org/documentation/dsls/dataframes/overview/) is no - longer experimental! We've spent the time since the [2.32.0 preview + longer experimental! We've spent the time since the [2.26.0 preview announcement](https://beam.apache.org/blog/dataframe-api-preview-available/) implementing the most frequently used pandas operations ([BEAM-9547](https://issues.apache.org/jira/browse/BEAM-9547)), improving
[beam-site] branch release-docs updated: adding .asf.yaml
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch release-docs in repository https://gitbox.apache.org/repos/asf/beam-site.git The following commit(s) were added to refs/heads/release-docs by this push: new b1c3d32 adding .asf.yaml b1c3d32 is described below commit b1c3d322ee091527e52a935059f1e0e2f4f4a507 Author: Ankur AuthorDate: Mon Aug 30 11:32:48 2021 -0700 adding .asf.yaml --- .asf.yaml | 21 + asf.yaml | 3 --- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.asf.yaml b/.asf.yaml new file mode 100644 index 000..015046e --- /dev/null +++ b/.asf.yaml @@ -0,0 +1,21 @@ + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is required to publish the Beam Docs. + +publish: + whoami: release-docs + subdir: content/releases diff --git a/asf.yaml b/asf.yaml deleted file mode 100644 index d6adfb6..000 --- a/asf.yaml +++ /dev/null @@ -1,3 +0,0 @@ -publish: - whoami: release-docs - subdir: content/releases
[beam] branch master updated: Revert "Merge pull request #15271 Decreasing peak memory usage for beam.TupleCombineFn."
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 08ebb14 Revert "Merge pull request #15271 Decreasing peak memory
usage for beam.TupleCombineFn."
new 8c6ade2 Merge pull request #15406 from ibzib/combine-rollback
08ebb14 is described below
commit 08ebb14ba5be6a337eae4ecdb65811d53a049f6d
Author: Kyle Weaver
AuthorDate: Thu Aug 26 16:28:24 2021 -0700
Revert "Merge pull request #15271 Decreasing peak memory usage for
beam.TupleCombineFn."
This reverts commit 4559c75863d9d6c9dd9e48c2b4f12f2139410524, reversing
changes made to 7611831443399f31fc505bc3451f2b56f245d4e4.
---
sdks/python/apache_beam/transforms/combiners.py| 34 ---
.../apache_beam/transforms/combiners_test.py | 38 +-
2 files changed, 7 insertions(+), 65 deletions(-)
diff --git a/sdks/python/apache_beam/transforms/combiners.py
b/sdks/python/apache_beam/transforms/combiners.py
index 7e6b1f9..bcedd86 100644
--- a/sdks/python/apache_beam/transforms/combiners.py
+++ b/sdks/python/apache_beam/transforms/combiners.py
@@ -21,7 +21,6 @@
import copy
import heapq
-import itertools
import operator
import random
from typing import Any
@@ -598,24 +597,16 @@ class SampleCombineFn(core.CombineFn):
class _TupleCombineFnBase(core.CombineFn):
- def __init__(self, *combiners, merge_accumulators_batch_size=None):
+ def __init__(self, *combiners):
self._combiners = [core.CombineFn.maybe_from_callable(c) for c in
combiners]
self._named_combiners = combiners
-# If the `merge_accumulators_batch_size` value is not specified, we chose a
-# bounded default that is inversely proportional to the number of
-# accumulators in merged tuples.
-self._merge_accumulators_batch_size = (
-merge_accumulators_batch_size or max(10, 1000 // len(combiners)))
def display_data(self):
combiners = [
c.__name__ if hasattr(c, '__name__') else c.__class__.__name__
for c in self._named_combiners
]
-return {
-'combiners': str(combiners),
-'merge_accumulators_batch_size': self._merge_accumulators_batch_size
-}
+return {'combiners': str(combiners)}
def setup(self, *args, **kwargs):
for c in self._combiners:
@@ -625,23 +616,10 @@ class _TupleCombineFnBase(core.CombineFn):
return [c.create_accumulator(*args, **kwargs) for c in self._combiners]
def merge_accumulators(self, accumulators, *args, **kwargs):
-# Make sure that `accumulators` is an iterator (so that the position is
-# remembered).
-accumulators = iter(accumulators)
-result = next(accumulators)
-while True:
- # Load accumulators into memory and merge in batches to decrease peak
- # memory usage.
- accumulators_batch = list(
- itertools.islice(accumulators, self._merge_accumulators_batch_size))
- if not accumulators_batch:
-break
- accumulators_batch += [result]
- result = [
- c.merge_accumulators(a, *args, **kwargs) for c,
- a in zip(self._combiners, zip(*accumulators_batch))
- ]
-return result
+return [
+c.merge_accumulators(a, *args, **kwargs) for c,
+a in zip(self._combiners, zip(*accumulators))
+]
def compact(self, accumulator, *args, **kwargs):
return [
diff --git a/sdks/python/apache_beam/transforms/combiners_test.py
b/sdks/python/apache_beam/transforms/combiners_test.py
index 68b273e..d826287 100644
--- a/sdks/python/apache_beam/transforms/combiners_test.py
+++ b/sdks/python/apache_beam/transforms/combiners_test.py
@@ -249,8 +249,7 @@ class CombineTest(unittest.TestCase):
dd = DisplayData.create_from(transform)
expected_items = [
DisplayDataItemMatcher('combine_fn', combine.TupleCombineFn),
-DisplayDataItemMatcher('combiners', "['max', 'MeanCombineFn', 'sum']"),
-DisplayDataItemMatcher('merge_accumulators_batch_size', 333),
+DisplayDataItemMatcher('combiners', "['max', 'MeanCombineFn', 'sum']")
]
hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
@@ -359,41 +358,6 @@ class CombineTest(unittest.TestCase):
max).with_common_input()).without_defaults())
assert_that(result, equal_to([(1, 7.0 / 4, 3)]))
- def test_tuple_combine_fn_batched_merge(self):
-num_combine_fns = 10
-max_num_accumulators_in_memory = 30
-# Maximum number of accumulator tuples in memory - 1 for the merge result.
-merge_accumulators_batch_size = (
-max_num_accumulators_in_memory // num_combine_fns - 1)
-num_accumulator_tuples_to_merge = 20
-
-class CountedAccumulator:
- count = 0
- oom = False
-
- def __init__(self):
-if CountedAccumulato
[beam-site] branch release-docs updated: Adding asf.yaml
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch release-docs in repository https://gitbox.apache.org/repos/asf/beam-site.git The following commit(s) were added to refs/heads/release-docs by this push: new bbc3992 Adding asf.yaml bbc3992 is described below commit bbc3992cd28544a88d00e64841b342cb47cfd2db Author: Ankur AuthorDate: Mon Aug 30 09:58:26 2021 -0700 Adding asf.yaml Suggested in https://issues.apache.org/jira/browse/INFRA-22264 --- asf.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/asf.yaml b/asf.yaml new file mode 100644 index 000..d6adfb6 --- /dev/null +++ b/asf.yaml @@ -0,0 +1,3 @@ +publish: + whoami: release-docs + subdir: content/releases
[beam] branch master updated (3a5b52c -> 7cad244)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 3a5b52c Merge pull request #15317 from angoenka/website-2.32 new b1e4d7c Add a blogpost for Apache Beam 2.32.0 new dd79449 Updating date for beam 2.32.0 blog post new cf4bafe Fixing author name new 7cad244 Merge pull request #15324 from angoenka/blogpost-2.32.0 The 32772 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGES.md | 32 +-- website/www/site/content/en/blog/beam-2.32.0.md | 106 2 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 website/www/site/content/en/blog/beam-2.32.0.md
[beam] annotated tag v2.32.0 updated (a9775d7 -> 6c0dab7)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to annotated tag v2.32.0 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.32.0 was modified! *** from a9775d7 (commit) to 6c0dab7 (tag) tagging 0aba787937090feed43ca5de5c6ae8a4c9d0ee16 (tag) length 150 bytes by Ankur Goenka on Wed Aug 25 23:10:23 2021 -0700 - Log - Beam 2.32.0 tag -BEGIN PGP SIGNATURE- iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmEnMFkACgkQiJNKZpmY WUgmnw/+KreTYVdQcVQqw5o2sya5qeG8qXmBFlzajUra3rvd2RwCabUpWlEfkViY sTVQInaqbimjbA4UL5of1cl3JkLR+HUW7XV2Qvi0ZrbOaRSOtXHyak/76372ala5 /qn0w0y39tK0KutfXEajVuvSGVh8zXlHBm0GhreTCGynesxw9hGagjlNiBgbG/NS khuwy1aPywPpf5RDbRERl8vU0+tV4X2zrlk3u2O6DpnoQDfizP8aKDFcx2G7GVl+ 9izd0LfvQnhlTDeeD86+cChSliroNvYYrs0zDUOXGXs7uXKNaA7R37PPUmvtHz0o 6ukaNZ9A9BqhRX+LPG1BnYV+rZ83N/QSx4Tk08tN71HoVrugBPNCdNfkip+yP/Vv WtFlpUve4ldYAEEMwzGsjIkm+yp+Et5dGwxx/Vy4ic0Lf1tKu55ucLyAddfJXKwz aMiGfmmCBa2NqNAw0LU+4Cwu5uV5CfD0Wy1aYaFrMYpQCkxJIvvPOc7nubuwpUJL MYT1Qb5WfjMNWDkHmzDqQR3AHF4GAMu2xMdlYKmltZ2Wlykx2lDsB63fdKltQrRh vYlbDFfDwVVUMqOoT1Y2K5nwbxNuZH1MIO1IgFmedvHPDNwrnon0AvBWbYKv8F+3 Ue6gKjVEjcmF4zIpaEEa2Yex3xjMOOItgwadL4OsVYMPjERpBGs= =OAsO -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
[beam] branch master updated (f4e54f2 -> 3a5b52c)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from f4e54f2 Add 2.34.0 section to CHANGES.md new 37ac4f7 Update Beam website to release 2.32.0 new 0b1b36a Update website/www/site/content/en/get-started/downloads.md new 3a5b52c Merge pull request #15317 from angoenka/website-2.32 The 32768 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: website/www/site/config.toml | 2 +- website/www/site/content/en/get-started/downloads.md | 8 website/www/site/static/.htaccess| 2 +- 3 files changed, 10 insertions(+), 2 deletions(-)
svn commit: r49442 - in /dev/beam/2.32.0: apache-beam-2.32.0-source-release.zip apache-beam-2.32.0-source-release.zip.asc apache-beam-2.32.0-source-release.zip.sha512
Author: goenka Date: Wed Aug 11 07:30:22 2021 New Revision: 49442 Log: Staging Java artifacts for Apache Beam 2.32.0 RC1 Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip (with props) Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip -- svn:mime-type = application/octet-stream Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc (original) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc Wed Aug 11 07:30:22 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETdMEACgkQiJNKZpmY -WUhqZRAAkfwJ+x10ZqSIiTxSCTqtzifv7w7BF0VsYu0D1z7nhOU7RnN2ILKFCWPW -SKuCKMVQNyHUcwtA2B/inz4BQMj4zmhZ/eN3BzKBNis6wc5Qq6y2ytr4rVAyfPQ1 -OjdlmHgBoIlurmfNO3gbzehoClhjQrv1dg8qn/c2LCt+QyTZT/M7dnxzGyk5Sw8E -/JiqTEn0N1CunkU0ifQVYpnqs6EQcLuH7PTTiXdrkGkNF3FMho/kEC74h8plXsIa -OCI+TArimapjXZq39xcZpe58CLmkg+nLigTQWcJGXU84BB0ikfhRJKwAVG1F2l7/ -YCpgPcy+4WhgSoUgL7VflR7vDmhtZkxZawLlR6KwoJzl57z5fW02H/UgsbQ6ohyP -6h6h/uTghQ+rFZ1+QwM33PGn4PUuTn8PhE6bd2gYIKreJJpxBTQNmweFFXBILniy -ApoU8/1ig2bCVNTZoZiUDMhdWCwyI8N8UD7sGEVKeJisIMf1Mj2q8qB7SxCNUDBN -boPFarBI+akz0S357RDR9qWmJygbyRuojUJsX1X5mwV3ua+1cD3BMwhm5rADfjCw -gYX+suFRmVncXL4Ovx8dVfsb38KbZKCg8pJd3ATAEXUvklBAywAuFl8w+BrseB77 -MAjiMfoiE6i1T9T8RzI87dVNHtNeiBtONA5LUqXXPjhMdCVgmmw= -=QEdj +iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETfHAACgkQiJNKZpmY +WUi/Fg/+Mf4biDhSxAfYIDPbcfamROXV/6kQbBqInkYSHVcvitqFqK9TOYwaYORX +g5oZS6w1YlwLF/eQ9SFmz5whZhuGD+8EOTxhjzZfDOBrLFiqWLdXegZcERRy2LsQ +B0f0QrvaL/lH8N2Kkqxag2jLNswOALRIftXI+YTw1SR88PkNr5cr0H0srOoUmmpw +rf6Sle6on1+N6bgWvFWGB53bYOliBWUHy4BDmkf8s6rr0Ba3/enNCl4LRUIhN+oq +2JunHh0nszbb1IuH7r+ISXCNpKjMXiGevLT56So5L//E7tjPJs10bhp3DkS05cMV +AQNCOQChE9jpCi+xwDFnwbO2yMS9u8uFNzzpAnsoX4A4/UW5j/mYXbuZYqN8fssP +T8WlX+7gjTWeOxwAb/Ky9Md8N44F3ZORAIlXOj9fFgUJjFoFGJX36LPrx8JDLVFd +KpCc885I0qka+cVtj+oLZvoRtjxKI3rjFcQoPsYJ8dWSadwsQFfvHAB7BLtoiTEC +PPxDNGBti5nRpJvkx/76pCYeftq/JIVtb+pqMAMHMYGmk63QALn31vz0Q2r9nnai +/lFSHth9yrwxAsBlkkfWG1oviNNViyVZUqu4rmcAfBORQGrb1n3Gmzgi/zjxNiqM +i0XGwfbQPbkAaQwCJeVDQhE+jUeVwDNmcAhI/Ddczh1DcXHCaWI= +=Q714 -END PGP SIGNATURE- Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 (original) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Wed Aug 11 07:30:22 2021 @@ -1 +1 @@ -7f47574a6665076fa5d759675322bb821bc41068c4006e782e270cf101fa32856638c269580020de70badf71b923d17f6f8a751bf3dc76b85176143acffaffc6 apache-beam-2.32.0-source-release.zip +d32d44afe78f5375215d513c8f43565f8b1f1975c3cff3addb8314587020923a6f055b2db515aae429b95d963bada14982964977e6cc2743922c27c260b7c10c apache-beam-2.32.0-source-release.zip
svn commit: r49440 - in /dev/beam/2.32.0: apache-beam-2.32.0-source-release.zip apache-beam-2.32.0-source-release.zip.asc apache-beam-2.32.0-source-release.zip.sha512
Author: goenka Date: Wed Aug 11 06:57:25 2021 New Revision: 49440 Log: Staging Java artifacts for Apache Beam 2.32.0 RC1 Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip (with props) Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip -- svn:mime-type = application/octet-stream Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc (original) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc Wed Aug 11 06:57:25 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETb+wACgkQiJNKZpmY -WUjxXQ//SvIkkrhfCsDsIWR1qfWIAbzwfqJHLBp8Mv0aN2ePMm6kygtCrrOkUnhq -XPWPiXa/1AN2jnP9cqOlTsEwBDPNMq/92tuQhkb+E8Qh/DV0GYp4bBnbn5aP7MXi -RnqJJuG0aZ4nlPlW5iufRYOgyOah6kQRE42Ar4KN5l70Y3vLe8a0496ax3m4gNJI -2mz+ExUHSf/mVdXxCCQWVy7m5m3JUAcb+SSEqAIe8oDg9PngJg14bOqtIKp2SWP0 -s2qIZwJUCB5nZlMz1tC1dPH2yRUGuxLCYrZ1sGIbspb+9SmAbvGiuocshkng9RTx -Dt4QiLfAbxZdGJkXhQna2f/qzl4fLAmkXrxCKqPUfQw4aYel/ROqC/YJMhgUFa1L -3oKfD5Ujf+fh2idKPDIz6/iqtgkOTho8yjhfY53ccVpnsqh14r721L2XNXy/KfHy -Vgz8it1Djir6GQbq7S0tRNT+dHrnSbRXml7haoZX0pM+DdaUqWhaSxjspzcCFNky -GyWOZWQ+0dKVAb+ip6ITluZj8xdJcm+P6T45fzmaFkWxin267OZYj4cjNU4B9YMz -ASDPPND7B+xv+goidjUiNzFapbfFlD0l4DVgw2whndKCYzroO9FfpWXQklQAYuBV -lw5EQ0Yr/tHTFPkP/2jLBXcSqHspr9cTW298mMZZUzH3blzhEKA= -=iD45 +iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETdMEACgkQiJNKZpmY +WUhqZRAAkfwJ+x10ZqSIiTxSCTqtzifv7w7BF0VsYu0D1z7nhOU7RnN2ILKFCWPW +SKuCKMVQNyHUcwtA2B/inz4BQMj4zmhZ/eN3BzKBNis6wc5Qq6y2ytr4rVAyfPQ1 +OjdlmHgBoIlurmfNO3gbzehoClhjQrv1dg8qn/c2LCt+QyTZT/M7dnxzGyk5Sw8E +/JiqTEn0N1CunkU0ifQVYpnqs6EQcLuH7PTTiXdrkGkNF3FMho/kEC74h8plXsIa +OCI+TArimapjXZq39xcZpe58CLmkg+nLigTQWcJGXU84BB0ikfhRJKwAVG1F2l7/ +YCpgPcy+4WhgSoUgL7VflR7vDmhtZkxZawLlR6KwoJzl57z5fW02H/UgsbQ6ohyP +6h6h/uTghQ+rFZ1+QwM33PGn4PUuTn8PhE6bd2gYIKreJJpxBTQNmweFFXBILniy +ApoU8/1ig2bCVNTZoZiUDMhdWCwyI8N8UD7sGEVKeJisIMf1Mj2q8qB7SxCNUDBN +boPFarBI+akz0S357RDR9qWmJygbyRuojUJsX1X5mwV3ua+1cD3BMwhm5rADfjCw +gYX+suFRmVncXL4Ovx8dVfsb38KbZKCg8pJd3ATAEXUvklBAywAuFl8w+BrseB77 +MAjiMfoiE6i1T9T8RzI87dVNHtNeiBtONA5LUqXXPjhMdCVgmmw= +=QEdj -END PGP SIGNATURE- Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 (original) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Wed Aug 11 06:57:25 2021 @@ -1 +1 @@ -f55714d84a0b12ad13ff1d94995f812abf187d01f73ccf0a6516f7f97b1a26e4d35b2840c7b9dbde46b439e236c4b6ef2ef23dce49338f0a04ac6a94f0640f50 apache-beam-2.32.0-source-release.zip +7f47574a6665076fa5d759675322bb821bc41068c4006e782e270cf101fa32856638c269580020de70badf71b923d17f6f8a751bf3dc76b85176143acffaffc6 apache-beam-2.32.0-source-release.zip
svn commit: r49439 - /dev/beam/2.32.0/python/
Author: goenka Date: Wed Aug 11 06:39:17 2021 New Revision: 49439 Log: Staging Python artifacts for Apache Beam 2.32.0 RC1 Modified: dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2014_aarch64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win32.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win_amd64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2014_aarch64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win32.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win_amd64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux1_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux2010_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux2010_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux2014_aarch64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-win32.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-win_amd64.whl.asc Modified: dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc == --- dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc (original) +++ dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc Wed Aug 11 06:39:17 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETRrIACgkQiJNKZpmY -WUhm3A//QdPA1G++gCdMha4zg+1lH7am9py7lGNqusxHrAZ3H3XhMNwA7ubYjJUm -HTNvmvaJQbAW7i6Sbklqhy1UuHLGEqSvozlL2EdYOIxaf1ukUfwFwfJi0JTYPeCr -nGEoehXU3gC27x1YiVZCyKTei6cJ+O3Md9goi62KVpujj57qGSlnrmaaiEJuMjzE -HluWmoV5cFhDtcHt0Nj5wmQc6TigCzr1keAJ8Yw4JN7rZ15InGMjPZUX1l2J6Mqa -/30XxkZtFSgiJVdSYfdgkPZOMmxw/J754845zYYgwiwyzu2iYTbTS+jriw5lvEfo -oyniQ1HnQcZGXb/1jEcL8xKJ2NAtm6nTUgIR57w2p0OSp+eN//A26xaS1cCF6ib9 -dOmMo5Bi7OI0kOo3vvmDNcm4SBn8Ct2dgfUUfQzJH/him6e+EoV4+dQSVxoDBzvn -zalypRYS8zuNVrQFya52fHEmX44V2OsUmB/2hijdUhueEmNkTvfF1KsqJZ+DoqwI -kgav3HeDkycyUa6u2GEeYe8dW+fbh3xqm0w9Uy1e2DSdWMQPBaDJ/s9oMGimwJ0x -7xTbTUH0ot78Z0Aap5Kv36bDuK3IKYLGPA1qxkbcDlMKtnCxFSERi5k72QpdyHX6 -3J3OokwV/zrnhf8QlT3WHEQ7YuOA243fyA5CQVX3OLt6now0U1o= -=cvK8 +iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETcFQACgkQiJNKZpmY +WUjupRAApXYF7MA2HYv/Bc/q43i+yH2NRfIqHqkmSyDwtQroy6RjYCDf+6owKYo+ +iioaxaKHi30Jb33pREYBAJFyMSf18k2Yk6jcklK7pXIh4ulucjXSjH5zddjez8aX +h1fuCYg8Ekw89pCH5T+Zd1VwRR5lMuJtSd4OKVOLxZoHkpLoTzd5vnomVWgq2/kU +LC46ei5NkUg7E7jKTnvSJDsEb/SXgOCo9oMXAnEpdHD559G+Ah79Vn3E9fdsitr/ +A39KkmTug5qRJl7L0h+nO4SmBSxDGJoEANa+o+YNX3dPig8vQhdeGDJOCWQy7uc/ +lOTFQ211ypcKJDGpWltpAKjQX4Sj17ZTabxd7+Hy5TXYM9hpQK3uoyzXU68gxp8t +iMLa73nD9yLWhWPYGTEsoCMsyrrkgFmusI+XyHQobP2ouZ42CcDvGg52NCUnH2Ty +/7TcAQG17BSFuXo2ZpbHkYEST3dktq0xHfCZn8Ue7tXMPn8w7KV6yT/Y1KMWPJOh +JCwALFNComBifw9yErSfBM50+ZF7aEfPJSQqm/oCXRdd+0KePI/O2KHtHqWWDXv3 +IQjVYMfRHKcitCOMVEqFIHtKiaHqIVhsrIDDsgUbP4InSv14KHToPjfolvLrmow4 +6V0DU8m6z1Rlq+xQ0lZuWeG0iMT8l8OjSrICXc1Y416Mfa06DOQ= +=kS90 -END PGP SIGNATURE- Modified: dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc == --- dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc (original) +++ dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc Wed Aug 11 06:39:17 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETRrQACgkQiJNKZpmY -WUhswhAAkJH+G4u2P6j2Ie7WumIPYGfIviMHQokj1NNURQL+qoKxBX/srfGpoUBk -C9lBdMhEhV9Injf+RL1cENGJR9iUAVFm5KMsgHu+zATtzI540GtR2cHe4CNyMQ3a -hIENw29w8li9pyLppOZejBlIZps8lh/h3AAvuM1lXDvoQ6cpVkoJZwG8q/hAjqDn -vhGbLTufEw8olBq8iQoXtnrFoCT1qX3Y6JW7cWgTY68fp2n+LqRi6kGyBNPrzGfa -ahSHi23eQY8X9wfUliwQMJCM6j8UAP4KdV5pGcCSwk6AyCZZdtWe1i+Dk/mtEbNN -BPcusuYDNRn+TCKbPPsUDZQ0EWVxCmIKfsFLH/YQtb/qDPG1iB8pplJqgGzAApue
svn commit: r49438 - in /dev/beam/2.32.0: apache-beam-2.32.0-source-release.zip apache-beam-2.32.0-source-release.zip.asc apache-beam-2.32.0-source-release.zip.sha512
Author: goenka Date: Wed Aug 11 06:37:01 2021 New Revision: 49438 Log: Staging Java artifacts for Apache Beam 2.32.0 RC1 Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip (with props) Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip -- svn:mime-type = application/octet-stream Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc (original) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc Wed Aug 11 06:37:01 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETRiAACgkQiJNKZpmY -WUjZRhAAo5atsQ8fPrwVIGO2RK2oke5gOxraOpmKY039I4a3CgFEaGrowRzLnA2y -syb+0ydw4Az6Z/dWVw+qQJ3aG+QAiu80myizRA2WfkmV0bRdQCxxWlfMeT4R6m44 -j+mmyfcO/HFRtu/ZSZorL4Z792RReuM7kvNn4wxuc/gwK30CTaRPQK/g8H9m+cJZ -OhI5QiV2i0QGsN6y7QEAjwR/mmHR8rZsFiA+zO/pgy6iqrsY+pmRns4af8uXKfo0 -8olJVLA/p2NKyyK3u9q8Dw42QV5cuDwO52XV0nK1Acr+52gW1VALsuReB+FzRvfa -psxCJ062RyjBY3yV9YDzinWekNqoQuexnElNc+UIYxVR2guQZNpzT7mRvJd/lOvF -XcVElp+jl+R/N6gVbuuFLA26Z4EWdgHr23VKEBSe/gUQZfA3UMqBWU0yARe/Y/pH -9LsNv1YdX2zuNXmtlG2VKtRXUlcXSFsr+qu4J4kW9vhPpTNZKTRohOwN2abjKku7 -MJxGtT4yWOpaiArLERS2ovYkrTVhevEATDVCWDjsDIrCMTTrINJq3zSSDG5acptX -/O/eidWV+4Em4KLZU6MGfaVwBtf+IJP/cA8CVBvc0SdtzE+FwTSb4pWqi+s6akmx -CSRSvKc4IzHKGb3GpvCiHwI23YhGx6Y3fGDnsLpPps0DCzQNOqc= -=5FFq +iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETb+wACgkQiJNKZpmY +WUjxXQ//SvIkkrhfCsDsIWR1qfWIAbzwfqJHLBp8Mv0aN2ePMm6kygtCrrOkUnhq +XPWPiXa/1AN2jnP9cqOlTsEwBDPNMq/92tuQhkb+E8Qh/DV0GYp4bBnbn5aP7MXi +RnqJJuG0aZ4nlPlW5iufRYOgyOah6kQRE42Ar4KN5l70Y3vLe8a0496ax3m4gNJI +2mz+ExUHSf/mVdXxCCQWVy7m5m3JUAcb+SSEqAIe8oDg9PngJg14bOqtIKp2SWP0 +s2qIZwJUCB5nZlMz1tC1dPH2yRUGuxLCYrZ1sGIbspb+9SmAbvGiuocshkng9RTx +Dt4QiLfAbxZdGJkXhQna2f/qzl4fLAmkXrxCKqPUfQw4aYel/ROqC/YJMhgUFa1L +3oKfD5Ujf+fh2idKPDIz6/iqtgkOTho8yjhfY53ccVpnsqh14r721L2XNXy/KfHy +Vgz8it1Djir6GQbq7S0tRNT+dHrnSbRXml7haoZX0pM+DdaUqWhaSxjspzcCFNky +GyWOZWQ+0dKVAb+ip6ITluZj8xdJcm+P6T45fzmaFkWxin267OZYj4cjNU4B9YMz +ASDPPND7B+xv+goidjUiNzFapbfFlD0l4DVgw2whndKCYzroO9FfpWXQklQAYuBV +lw5EQ0Yr/tHTFPkP/2jLBXcSqHspr9cTW298mMZZUzH3blzhEKA= +=iD45 -END PGP SIGNATURE- Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 (original) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Wed Aug 11 06:37:01 2021 @@ -1 +1 @@ -d38dd2b3b167b86040de96607f3eab88dc1963e3a59ae95aae211b7e0f9b5da32dcc6d47722064f299243393d7c9d737bd1304500ce0b68a1d9cbd3451d30d6f apache-beam-2.32.0-source-release.zip +f55714d84a0b12ad13ff1d94995f812abf187d01f73ccf0a6516f7f97b1a26e4d35b2840c7b9dbde46b439e236c4b6ef2ef23dce49338f0a04ac6a94f0640f50 apache-beam-2.32.0-source-release.zip
svn commit: r49434 - /dev/beam/2.32.0/python/
Author: goenka Date: Wed Aug 11 03:42:35 2021 New Revision: 49434 Log: Staging Python artifacts for Apache Beam 2.32.0 RC1 Modified: dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2014_aarch64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win32.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win_amd64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2014_aarch64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win32.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win_amd64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux1_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux2010_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux2010_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux2014_aarch64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-win32.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-win_amd64.whl.asc Modified: dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc == --- dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc (original) +++ dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc Wed Aug 11 03:42:35 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETL4cACgkQiJNKZpmY -WUjk4Q//VkPLEvwfDHs5TXOshnUKJLoUkym4MhAGVLbZ7g6tYTdVF90/BuxZfTqn -aSDUfNf5r8WBMsXwOSP6HJG3475gfdLGHZVTLAyldUY69tOAMUvTAQXXiznvAAUJ -rbrK6RdGHfRwoaU5B+/Af/7zdvH5T0JrYhcZ0Tb3bO12PFBjFsxGaPo/QvRoOYDL -s7iaRj+7wjB2FMTwhH7BO5l8ieRjTM06tOfqnPTZcIbZ5JuYzYUNmGNcL1FGERXM -tcFy8iq5R7a0Zla1e+9wg6kcK3vtStmvHEFJlUj2wrpZFH6o07lhgO2oOThWKk2m -is3HFKEoO8tTPNkFRiwxC5Ww8eGC8CL0wvluHM466/b9rDs0jaTq1t/zPIPqO1N9 -6Au8NR2lYPz4H2EPnixT3fIF2NZVio2Ii/W/EWY/0jG0ClDr6mvwuLNz9BnxCHTp -iAr/2cpwSJq093ZfWT5RDDkELQ4sm+avqYNrDWhT2JqSOPQNVcCQKR/7jQSSctqz -1lCDYdFssWOgAzPeIOnHatko+1GqZfXqGtqniCefdWGOcTG2IzHMfy8TqsyY7xl0 -92VFVHu8RTE/mQohsBH3RANIxjZcg/VIaeiRKMA3pMyOR/KQTXRrDEKzxdforU6k -wz+PVeQZWsas4/FZezBau/kocHFY/th+u2nZbi7r9JMqt8JINhE= -=ZLq0 +iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETRrIACgkQiJNKZpmY +WUhm3A//QdPA1G++gCdMha4zg+1lH7am9py7lGNqusxHrAZ3H3XhMNwA7ubYjJUm +HTNvmvaJQbAW7i6Sbklqhy1UuHLGEqSvozlL2EdYOIxaf1ukUfwFwfJi0JTYPeCr +nGEoehXU3gC27x1YiVZCyKTei6cJ+O3Md9goi62KVpujj57qGSlnrmaaiEJuMjzE +HluWmoV5cFhDtcHt0Nj5wmQc6TigCzr1keAJ8Yw4JN7rZ15InGMjPZUX1l2J6Mqa +/30XxkZtFSgiJVdSYfdgkPZOMmxw/J754845zYYgwiwyzu2iYTbTS+jriw5lvEfo +oyniQ1HnQcZGXb/1jEcL8xKJ2NAtm6nTUgIR57w2p0OSp+eN//A26xaS1cCF6ib9 +dOmMo5Bi7OI0kOo3vvmDNcm4SBn8Ct2dgfUUfQzJH/him6e+EoV4+dQSVxoDBzvn +zalypRYS8zuNVrQFya52fHEmX44V2OsUmB/2hijdUhueEmNkTvfF1KsqJZ+DoqwI +kgav3HeDkycyUa6u2GEeYe8dW+fbh3xqm0w9Uy1e2DSdWMQPBaDJ/s9oMGimwJ0x +7xTbTUH0ot78Z0Aap5Kv36bDuK3IKYLGPA1qxkbcDlMKtnCxFSERi5k72QpdyHX6 +3J3OokwV/zrnhf8QlT3WHEQ7YuOA243fyA5CQVX3OLt6now0U1o= +=cvK8 -END PGP SIGNATURE- Modified: dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc == --- dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc (original) +++ dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc Wed Aug 11 03:42:35 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETL4kACgkQiJNKZpmY -WUg1IQ/+PPE2+yr7y1Jpn4GwVkTfgeVYNdHa0UHk0LMlxyqjftFkYCSGrTBg46NG -1Bt6S5CPz9nI4Hc/yajsE32Wottf6HMAWiKMhBO+8dbZrcCM+fUzSYqh2jZ8cU4U -pvaAnVCBmREbQmASgp2WzraoSV9McD4WaX4+MYc4kaO0sk2OaDfG7yOv8ffh/sKi -PhjvQhPAo6NlFAjoPcTC4MD6L+8LSr1F+rcWst5UBTL0r8MfwqjkIHMUiLhY5XVQ -NjAELpguLqfoYvJsODHA9clr9QOVZZvNtBWmT3VisbsYrIJic+CeHpJo9ImzwegY -wosJg3Fq7mVSUdcmDX9S55WQ2hQzqiFIsVcyanbfYmE9E2rUoIbnB1urxMwsZdkr
svn commit: r49433 - in /dev/beam/2.32.0: apache-beam-2.32.0-source-release.zip apache-beam-2.32.0-source-release.zip.asc apache-beam-2.32.0-source-release.zip.sha512
Author: goenka Date: Wed Aug 11 03:38:29 2021 New Revision: 49433 Log: Staging Java artifacts for Apache Beam 2.32.0 RC1 Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip (with props) Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip -- svn:mime-type = application/octet-stream Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc (original) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc Wed Aug 11 03:38:29 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETLp4ACgkQiJNKZpmY -WUiF8w//eKZ/v5qcIAhIvMxewKhXNcdpb1TqEVSnmdwwFX+oEz9OUBhNFYHdlMAn -yHew3MXseIh+rV0gJHXHdb+CLNifjrJDEcAyBcSd0WtXXyKJzVda2fj345DZsrw2 -c9zhdNnXPV/kEUigC4U4tjEA1BpmPWkOl9JAC8vGID4fZjNwLHmibtBls59ZB4nd -NyidURlkKFlJUMaNl94e9Pi1eenJ57/92ueOnxQaQ7sTsUKXJrhV+tgQoNyf0Xnr -7IU8KlH0PciJm8S4tIOzHhVJdZ7HNuEspv56CabhEXO3c+nyVVOElvUqNQ/0drha -rtq90Q65HJcm7Zt8PtPvj7C1fF3tJ7EkpyXQugMShvGL1cVEsoMctKTCoGr/7m+y -kZXdYjxqgdhXR/7PBHd2vyZoJcgF1yWgCHIzCVS3jA4rT8kAVZUVlC7kte7uheUw -+OZHQWgwsuuMKdGgevdJgKb3nC8Iim2/Dn9GO63RDDZ64EXW9LD7M7ZXBPvjpcQv -RSGAVhHjBzr97EyfcuFTPuw6DcbrNgHyOo4ANNYo3S2tAnaIzlDzEAayzY7dIO+6 -jWvulAGV6cPGsEkTAIIi0p13XpnM90PPPq6qiZlBWcwCADbY6qD/TgjAwEVC+t06 -aid0it4LhlCLOJjQVx7QIC3jMLln6mM6/ur7nVQNpEHk+I+08Os= -=Gsfx +iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETRiAACgkQiJNKZpmY +WUjZRhAAo5atsQ8fPrwVIGO2RK2oke5gOxraOpmKY039I4a3CgFEaGrowRzLnA2y +syb+0ydw4Az6Z/dWVw+qQJ3aG+QAiu80myizRA2WfkmV0bRdQCxxWlfMeT4R6m44 +j+mmyfcO/HFRtu/ZSZorL4Z792RReuM7kvNn4wxuc/gwK30CTaRPQK/g8H9m+cJZ +OhI5QiV2i0QGsN6y7QEAjwR/mmHR8rZsFiA+zO/pgy6iqrsY+pmRns4af8uXKfo0 +8olJVLA/p2NKyyK3u9q8Dw42QV5cuDwO52XV0nK1Acr+52gW1VALsuReB+FzRvfa +psxCJ062RyjBY3yV9YDzinWekNqoQuexnElNc+UIYxVR2guQZNpzT7mRvJd/lOvF +XcVElp+jl+R/N6gVbuuFLA26Z4EWdgHr23VKEBSe/gUQZfA3UMqBWU0yARe/Y/pH +9LsNv1YdX2zuNXmtlG2VKtRXUlcXSFsr+qu4J4kW9vhPpTNZKTRohOwN2abjKku7 +MJxGtT4yWOpaiArLERS2ovYkrTVhevEATDVCWDjsDIrCMTTrINJq3zSSDG5acptX +/O/eidWV+4Em4KLZU6MGfaVwBtf+IJP/cA8CVBvc0SdtzE+FwTSb4pWqi+s6akmx +CSRSvKc4IzHKGb3GpvCiHwI23YhGx6Y3fGDnsLpPps0DCzQNOqc= +=5FFq -END PGP SIGNATURE- Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 (original) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Wed Aug 11 03:38:29 2021 @@ -1 +1 @@ -e944bdd0c45ec8ce4027833bb79a67a2968034ee2629c78d40c563df5562e9bb8b090760e6b2f172bfb8ec1213ba6b9d783edf18a616c7747709e857034c8103 apache-beam-2.32.0-source-release.zip +d38dd2b3b167b86040de96607f3eab88dc1963e3a59ae95aae211b7e0f9b5da32dcc6d47722064f299243393d7c9d737bd1304500ce0b68a1d9cbd3451d30d6f apache-beam-2.32.0-source-release.zip
svn commit: r49429 - /dev/beam/2.32.0/python/
Author: goenka Date: Wed Aug 11 02:02:19 2021 New Revision: 49429 Log: Staging Python artifacts for Apache Beam 2.32.0 RC1 Modified: dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2014_aarch64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win32.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win_amd64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2014_aarch64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win32.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win_amd64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux1_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux2010_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux2010_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux2014_aarch64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-win32.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-win_amd64.whl.asc Modified: dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc == --- dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc (original) +++ dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc Wed Aug 11 02:02:19 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCAAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmESLQQACgkQiJNKZpmY -WUiKlg/+LkhSpt/OtEiPztoqsY1vZEAp8uSeHB5YztRTO2AACHPi2BhvWCRFGwWc -HzYiCzEXl3xE5xuYsbefWb2zMt/xD45nrwP2cVyxCYkNi4z3iB23ZUKh4uACtsjN -hX7OZ4Sdsz94UloZXWHX6mSac/lVX1UUgxk/FLaGvEAwYcDiR88oaLT43XkDw99d -30XY6l6EY0iNigQA2ODY3J2uD/yUlfDQtAkSE9zmuX19pyYFdcaQdatRl2gtVqfr -ckGGL7bfXvSdqUXT+0zKMq6H4ZkBZp4Gawtg8EtpuHMc3XRlE32pVyeMb3VV47uZ -2hQh00cRxceleaeC2m9FgT8KVPZ0EZNQyAKhHoPSEJg9PvsBlwwFkom6JIGXt594 -NSBOLNTfsEWaS7xuSs2xzAQH/H/Orr4+hgbWM8nZ0108b9+4dUJr8ORomowUgbro -LCFokKEWuExcbEt0MnnoFdTCmQ+haouOgo110tYLJAqoPr1HpFwtA/4Q+Cgm6B2q -WjfmS/bJDULbE3Qu8BYA81OoEElhwkJYcPlNP6UKU+QZm+laRlTme7Ty7QmNlM4V -VzC0fUoP77pynHMQnbn6B7u4qbm+0lcwpsr2rzw3bf4stYMif7M1K2aeF7moYATe -mZyddt8m2or9Nu7Vv3Wdq1EfiSms0Wft6BdtC86NzdLgkOtQBUo= -=7yAE +iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETL4cACgkQiJNKZpmY +WUjk4Q//VkPLEvwfDHs5TXOshnUKJLoUkym4MhAGVLbZ7g6tYTdVF90/BuxZfTqn +aSDUfNf5r8WBMsXwOSP6HJG3475gfdLGHZVTLAyldUY69tOAMUvTAQXXiznvAAUJ +rbrK6RdGHfRwoaU5B+/Af/7zdvH5T0JrYhcZ0Tb3bO12PFBjFsxGaPo/QvRoOYDL +s7iaRj+7wjB2FMTwhH7BO5l8ieRjTM06tOfqnPTZcIbZ5JuYzYUNmGNcL1FGERXM +tcFy8iq5R7a0Zla1e+9wg6kcK3vtStmvHEFJlUj2wrpZFH6o07lhgO2oOThWKk2m +is3HFKEoO8tTPNkFRiwxC5Ww8eGC8CL0wvluHM466/b9rDs0jaTq1t/zPIPqO1N9 +6Au8NR2lYPz4H2EPnixT3fIF2NZVio2Ii/W/EWY/0jG0ClDr6mvwuLNz9BnxCHTp +iAr/2cpwSJq093ZfWT5RDDkELQ4sm+avqYNrDWhT2JqSOPQNVcCQKR/7jQSSctqz +1lCDYdFssWOgAzPeIOnHatko+1GqZfXqGtqniCefdWGOcTG2IzHMfy8TqsyY7xl0 +92VFVHu8RTE/mQohsBH3RANIxjZcg/VIaeiRKMA3pMyOR/KQTXRrDEKzxdforU6k +wz+PVeQZWsas4/FZezBau/kocHFY/th+u2nZbi7r9JMqt8JINhE= +=ZLq0 -END PGP SIGNATURE- Modified: dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc == --- dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc (original) +++ dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc Wed Aug 11 02:02:19 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCAAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmESLQ4ACgkQiJNKZpmY -WUi+IQ/9EJ471Sgn9mKNzStj0/jrKr5pAcwuLyK1wzDEqxWYTIIJm+RQfKFV0TsG -/o5wdtLBG+E6BX6kKu98aCG+ySYKg0R+yiOTk4N2O6pMm8b/E9NCz5DG8glyTO6U -DHNTGI4VKqzf2u9nQ5WH2/YQXeD0lwZvCmFxccM4nv291ujlExWhNMOEUI3fzHhr -luh1Tq03Ij9OxLPsr65PT3WdB6xpvtc8LMuPJSMkRbbY3MgzhBHxBclhbejxLhNN -aVdsNYqSUEK0YeHRUjxwrrhLtTrIw72AqN4/QvT3cs2Nwn6acRSp6OcC5DbHzT3v -1NTNcRWRvvX1btiqBCOlsD5hsDpzPYv70SUoiHZ9x0VVrA+93KDrcn31e6fGQA+4 -kR
svn commit: r49428 - in /dev/beam/2.32.0: apache-beam-2.32.0-source-release.zip apache-beam-2.32.0-source-release.zip.asc apache-beam-2.32.0-source-release.zip.sha512
Author: goenka Date: Wed Aug 11 01:59:19 2021 New Revision: 49428 Log: Staging Java artifacts for Apache Beam 2.32.0 RC1 Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip (with props) Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip -- svn:mime-type = application/octet-stream Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc (original) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc Wed Aug 11 01:59:19 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmESEIYACgkQiJNKZpmY -WUhaow//UwdrZNt1ekim7BQ2X5+0l041PpkST/n1dL3zloo1K8OU7cYjwuYgh0GK -M6woXVLamRl0kRe/95xzsEWjKHcmILIgAmLUyFFsLvIfJEilAL6snjGaY1Pk30lc -urGcAJUlNZ02mHono0Af+bMncIY31OtdQHUQv8FNHKMveLAEZK9ncm5RWv9qdDeu -C4ykagcuSNoxsAm+UlOGMvNWec6x4WTJnYGn/XoiJZVfvk1lTQwBTM7Bx9YtCs3N -CfS1P7ZWp4Quqn0f0VUijKOgWPPu81mwoRuGK/WrC47v8OwwyBRaJZxDL8ztW+Zj -hNcqwUXP9kQnyyPjMlFzPYVGg0qVMpPOfREQXByCVXWUGY2xpGB5WNFIHHNZNX4/ -0e1EpfGVrtJAhvjopVqFeOxp3Hf852w+lg53InEEZESvxBgsPA9fOf9fjkJmI1hV -qA6949koGMsPQEqdyzaMTSR01BlF7KK/hLFB1dSAmpq4AICX1KUBaMYpJfsw9XcR -I8bxV2KbiiPPkyXUxn5p6jenUi7vIzEchg0jnGBO4pHzNTQ7rN3CuME5sz2l3Jee -vHCS772VVuAfQIV8Px464DKB3EHY7XMZ8wNrCOx3sJ9W0bvcSQ4Tde2rP297bTFb -0RCSGWsSsSBH3xNxiVLcKLPPqIbDlHRaNYVkCw1XfC2NHGxwBZo= -=/Kml +iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmETLp4ACgkQiJNKZpmY +WUiF8w//eKZ/v5qcIAhIvMxewKhXNcdpb1TqEVSnmdwwFX+oEz9OUBhNFYHdlMAn +yHew3MXseIh+rV0gJHXHdb+CLNifjrJDEcAyBcSd0WtXXyKJzVda2fj345DZsrw2 +c9zhdNnXPV/kEUigC4U4tjEA1BpmPWkOl9JAC8vGID4fZjNwLHmibtBls59ZB4nd +NyidURlkKFlJUMaNl94e9Pi1eenJ57/92ueOnxQaQ7sTsUKXJrhV+tgQoNyf0Xnr +7IU8KlH0PciJm8S4tIOzHhVJdZ7HNuEspv56CabhEXO3c+nyVVOElvUqNQ/0drha +rtq90Q65HJcm7Zt8PtPvj7C1fF3tJ7EkpyXQugMShvGL1cVEsoMctKTCoGr/7m+y +kZXdYjxqgdhXR/7PBHd2vyZoJcgF1yWgCHIzCVS3jA4rT8kAVZUVlC7kte7uheUw ++OZHQWgwsuuMKdGgevdJgKb3nC8Iim2/Dn9GO63RDDZ64EXW9LD7M7ZXBPvjpcQv +RSGAVhHjBzr97EyfcuFTPuw6DcbrNgHyOo4ANNYo3S2tAnaIzlDzEAayzY7dIO+6 +jWvulAGV6cPGsEkTAIIi0p13XpnM90PPPq6qiZlBWcwCADbY6qD/TgjAwEVC+t06 +aid0it4LhlCLOJjQVx7QIC3jMLln6mM6/ur7nVQNpEHk+I+08Os= +=Gsfx -END PGP SIGNATURE- Modified: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 (original) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Wed Aug 11 01:59:19 2021 @@ -1 +1 @@ -e40cd8d7e5c9b836191140be4237266cb6556be40356829b9da16e821e01c283726a3214a3fae067f12117a8b6d489e8f519aef6e0f985da2902a81128bc0b85 apache-beam-2.32.0-source-release.zip +e944bdd0c45ec8ce4027833bb79a67a2968034ee2629c78d40c563df5562e9bb8b090760e6b2f172bfb8ec1213ba6b9d783edf18a616c7747709e857034c8103 apache-beam-2.32.0-source-release.zip
svn commit: r49416 - /dev/beam/2.32.0/python/
Author: goenka Date: Tue Aug 10 07:53:04 2021 New Revision: 49416 Log: Staging Python artifacts for Apache Beam 2.32.0 RC1 Added: dev/beam/2.32.0/python/ dev/beam/2.32.0/python/apache-beam-2.32.0.zip (with props) dev/beam/2.32.0/python/apache-beam-2.32.0.zip.asc dev/beam/2.32.0/python/apache-beam-2.32.0.zip.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-macosx_10_9_x86_64.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_i686.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_i686.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_x86_64.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux1_x86_64.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_i686.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_i686.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_x86_64.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2010_x86_64.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2014_aarch64.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2014_aarch64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-manylinux2014_aarch64.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win32.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win32.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win32.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win_amd64.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win_amd64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp36-cp36m-win_amd64.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-macosx_10_9_x86_64.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-macosx_10_9_x86_64.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_i686.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_i686.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_x86_64.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux1_x86_64.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_i686.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_i686.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_x86_64.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2010_x86_64.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2014_aarch64.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2014_aarch64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-manylinux2014_aarch64.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win32.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win32.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win32.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win_amd64.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win_amd64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp37-cp37m-win_amd64.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-macosx_10_9_x86_64.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-macosx_10_9_x86_64.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-macosx_10_9_x86_64.whl.sha512 dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux1_i686.whl (with props) dev/beam/2.32.0/python/apache_beam-2.32.0-cp38-cp38-manylinux1_i686.whl.asc dev/beam/2.32.0/python/apache_beam-2.32.0-cp38
svn commit: r49411 - in /dev/beam/2.32.0: ./ apache-beam-2.32.0-source-release.zip apache-beam-2.32.0-source-release.zip.asc apache-beam-2.32.0-source-release.zip.sha512
Author: goenka Date: Tue Aug 10 05:39:05 2021 New Revision: 49411 Log: Staging Java artifacts for Apache Beam 2.32.0 RC1 Added: dev/beam/2.32.0/ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip (with props) dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip -- svn:mime-type = application/octet-stream Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc (added) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.asc Tue Aug 10 05:39:05 2021 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEY1bBqfCJsPo96HU2iJNKZpmYWUgFAmESEIYACgkQiJNKZpmY +WUhaow//UwdrZNt1ekim7BQ2X5+0l041PpkST/n1dL3zloo1K8OU7cYjwuYgh0GK +M6woXVLamRl0kRe/95xzsEWjKHcmILIgAmLUyFFsLvIfJEilAL6snjGaY1Pk30lc +urGcAJUlNZ02mHono0Af+bMncIY31OtdQHUQv8FNHKMveLAEZK9ncm5RWv9qdDeu +C4ykagcuSNoxsAm+UlOGMvNWec6x4WTJnYGn/XoiJZVfvk1lTQwBTM7Bx9YtCs3N +CfS1P7ZWp4Quqn0f0VUijKOgWPPu81mwoRuGK/WrC47v8OwwyBRaJZxDL8ztW+Zj +hNcqwUXP9kQnyyPjMlFzPYVGg0qVMpPOfREQXByCVXWUGY2xpGB5WNFIHHNZNX4/ +0e1EpfGVrtJAhvjopVqFeOxp3Hf852w+lg53InEEZESvxBgsPA9fOf9fjkJmI1hV +qA6949koGMsPQEqdyzaMTSR01BlF7KK/hLFB1dSAmpq4AICX1KUBaMYpJfsw9XcR +I8bxV2KbiiPPkyXUxn5p6jenUi7vIzEchg0jnGBO4pHzNTQ7rN3CuME5sz2l3Jee +vHCS772VVuAfQIV8Px464DKB3EHY7XMZ8wNrCOx3sJ9W0bvcSQ4Tde2rP297bTFb +0RCSGWsSsSBH3xNxiVLcKLPPqIbDlHRaNYVkCw1XfC2NHGxwBZo= +=/Kml +-END PGP SIGNATURE- Added: dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 == --- dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 (added) +++ dev/beam/2.32.0/apache-beam-2.32.0-source-release.zip.sha512 Tue Aug 10 05:39:05 2021 @@ -0,0 +1 @@ +e40cd8d7e5c9b836191140be4237266cb6556be40356829b9da16e821e01c283726a3214a3fae067f12117a8b6d489e8f519aef6e0f985da2902a81128bc0b85 apache-beam-2.32.0-source-release.zip
[beam] annotated tag v2.32.0-RC1 updated (a9775d7 -> 0aba787)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to annotated tag v2.32.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.32.0-RC1 was modified! *** from a9775d7 (commit) to 0aba787 (tag) tagging a9775d757ff766505702a2d769283d6a3030adf5 (commit) replaces jupyterlab-sidepanel-v1.0.0 by Ankur Goenka on Mon Aug 9 22:11:11 2021 -0700 - Log - v2.32.0-RC1 --- No new revisions were added by this update. Summary of changes:
[beam] branch release-2.32.0 updated: [BEAM-12678] Add dependency of java jars when running go VR on portable runners (#15276) (#15279)
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch release-2.32.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.32.0 by this push:
new c0cd295 [BEAM-12678] Add dependency of java jars when running go VR
on portable runners (#15276) (#15279)
c0cd295 is described below
commit c0cd29572c2ce1490d0519fd6d5946ef46cf28fe
Author: Ankur
AuthorDate: Wed Aug 4 18:07:49 2021 -0700
[BEAM-12678] Add dependency of java jars when running go VR on portable
runners (#15276) (#15279)
---
sdks/go/test/build.gradle | 7 +++
1 file changed, 7 insertions(+)
diff --git a/sdks/go/test/build.gradle b/sdks/go/test/build.gradle
index ebeb0ea..8e078fa 100644
--- a/sdks/go/test/build.gradle
+++ b/sdks/go/test/build.gradle
@@ -78,6 +78,8 @@ task dataflowValidatesRunner() {
task flinkValidatesRunner {
dependsOn ":sdks:go:test:goBuild"
dependsOn ":sdks:go:container:docker"
+ dependsOn ":sdks:java:container:java8:docker"
+ dependsOn ":sdks:java:container:java11:docker"
dependsOn
":runners:flink:${project.ext.latestFlinkVersion}:job-server:shadowJar"
dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar"
doLast {
@@ -101,6 +103,8 @@ task flinkValidatesRunner {
task samzaValidatesRunner {
dependsOn ":sdks:go:test:goBuild"
dependsOn ":sdks:go:container:docker"
+ dependsOn ":sdks:java:container:java8:docker"
+ dependsOn ":sdks:java:container:java11:docker"
dependsOn ":runners:samza:job-server:shadowJar"
dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar"
doLast {
@@ -123,6 +127,8 @@ task samzaValidatesRunner {
// with Spark to validate that the runner behaves as expected.
task sparkValidatesRunner {
dependsOn ":sdks:go:test:goBuild"
+ dependsOn ":sdks:java:container:java8:docker"
+ dependsOn ":sdks:java:container:java11:docker"
dependsOn ":runners:spark:2:job-server:shadowJar"
dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar"
doLast {
@@ -151,6 +157,7 @@ task sparkValidatesRunner {
task ulrValidatesRunner {
dependsOn ":sdks:go:test:goBuild"
dependsOn ":sdks:go:container:docker"
+ dependsOn ":sdks:java:container:java8:docker"
dependsOn ":sdks:java:container:java11:docker"
dependsOn "setupVirtualenv"
dependsOn ":sdks:python:buildPython"
[beam] branch master updated (3898642 -> dbb44bf)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 3898642 Merge pull request #15255: [BEAM-12715] Snowflake IO use getShards in batch write mode add dbb44bf [BEAM-12678] Add dependency of java jars when running go VR on portable runners (#15276) No new revisions were added by this update. Summary of changes: sdks/go/test/build.gradle | 7 +++ 1 file changed, 7 insertions(+)
[beam] branch revert-15226-cherrypick-15224 created (now 1d0afed)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch revert-15226-cherrypick-15224 in repository https://gitbox.apache.org/repos/asf/beam.git. at 1d0afed Revert "[BEAM-12661] Fix stuck GetData Windmill calls (#15224) (#15226)" No new revisions were added by this update.
[beam] branch release-2.32 created (now 92ea5f5)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch release-2.32 in repository https://gitbox.apache.org/repos/asf/beam.git. at 92ea5f5 [BEAM-11088] Add TestStream package to Go SDK testing capabilities (#15253) No new revisions were added by this update.
[beam] branch release-2.32.0 updated (edd9a43 -> 1175c09)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch release-2.32.0 in repository https://gitbox.apache.org/repos/asf/beam.git. from edd9a43 Merge pull request #15184: [BEAM-12589] - exclude testTwoTimersSettingEachOtherWithCreateAsInput… (#15240) add 1175c09 [BEAM-12625] Annotate testTwoTimersSettingEachOtherWithCreateAsInputUnbounded. (#15207) No new revisions were added by this update. Summary of changes: .../src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java| 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-)
[beam] branch release-2.32.0 updated (eaf5776 -> edd9a43)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch release-2.32.0 in repository https://gitbox.apache.org/repos/asf/beam.git. from eaf5776 [BEAM-12656] Fix go-licenses build (#15215) (#15222) add edd9a43 Merge pull request #15184: [BEAM-12589] - exclude testTwoTimersSettingEachOtherWithCreateAsInput… (#15240) No new revisions were added by this update. Summary of changes: runners/google-cloud-dataflow-java/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
[beam] branch release-2.32.0 updated (ca866af -> eaf5776)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch release-2.32.0 in repository https://gitbox.apache.org/repos/asf/beam.git. from ca866af [BEAM-12399] Move CPython license to own file. (#15201) (#15211) add eaf5776 [BEAM-12656] Fix go-licenses build (#15215) (#15222) No new revisions were added by this update. Summary of changes: release/go-licenses/Dockerfile | 2 +- release/go-licenses/get-licenses.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-)
[beam] branch release-2.32.0 updated (ad5d202 -> ca866af)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch release-2.32.0 in repository https://gitbox.apache.org/repos/asf/beam.git. from ad5d202 Revert "Default to Runner v2 for Python Streaming jobs. (#15140)" (#15210) (#15212) add ca866af [BEAM-12399] Move CPython license to own file. (#15201) (#15211) No new revisions were added by this update. Summary of changes: LICENSE| 260 + LICENSE.python | 258 build.gradle.kts | 3 + .../org/apache/beam/gradle/BeamModulePlugin.groovy | 1 + sdks/python/MANIFEST.in| 1 + sdks/python/container/Dockerfile | 1 + 6 files changed, 266 insertions(+), 258 deletions(-) create mode 100644 LICENSE.python
[beam] branch release-2.32.0 updated (8e32f3b -> ad5d202)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch release-2.32.0 in repository https://gitbox.apache.org/repos/asf/beam.git. from 8e32f3b [BEAM-12661] Fix stuck GetData Windmill calls (#15224) (#15226) add ad5d202 Revert "Default to Runner v2 for Python Streaming jobs. (#15140)" (#15210) (#15212) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 8 +--- .../apache_beam/runners/dataflow/dataflow_runner_test.py | 11 ++- 2 files changed, 3 insertions(+), 16 deletions(-)
[beam] branch release-2.32.0 updated: [BEAM-12661] Fix stuck GetData Windmill calls (#15224) (#15226)
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch release-2.32.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.32.0 by this push:
new 8e32f3b [BEAM-12661] Fix stuck GetData Windmill calls (#15224)
(#15226)
8e32f3b is described below
commit 8e32f3b9a67dc5df63069b6f0271c97158cc9639
Author: Ankur
AuthorDate: Wed Jul 28 13:01:53 2021 -0700
[BEAM-12661] Fix stuck GetData Windmill calls (#15224) (#15226)
[BEAM-12661] Fix stuck GetData Windmill calls
Co-authored-by: slavachernyak
---
.../beam/runners/dataflow/worker/windmill/GrpcWindmillServer.java | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/GrpcWindmillServer.java
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/GrpcWindmillServer.java
index 81ae092..3c06ee9 100644
---
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/GrpcWindmillServer.java
+++
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/GrpcWindmillServer.java
@@ -1525,8 +1525,8 @@ public class GrpcWindmillServer extends
WindmillServerStub {
try {
blockedStartMs.set(Instant.now().getMillis());
- current = queue.take();
- if (current != POISON_PILL) {
+ current = queue.poll(180, TimeUnit.SECONDS);
+ if (current != null && current != POISON_PILL) {
return true;
}
if (cancelled.get()) {
@@ -1535,7 +1535,8 @@ public class GrpcWindmillServer extends
WindmillServerStub {
if (complete.get()) {
return false;
}
- throw new IllegalStateException("Got poison pill but stream
is not done.");
+ throw new IllegalStateException(
+ "Got poison pill or timeout but stream is not done.");
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new CancellationException();
[beam] branch master updated (10e6025 -> 034ccdf)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 10e6025 [BEAM-12399] Move CPython license to own file. (#15201) add 034ccdf Revert "Default to Runner v2 for Python Streaming jobs. (#15140)" (#15210) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 8 +--- .../apache_beam/runners/dataflow/dataflow_runner_test.py | 11 ++- 2 files changed, 3 insertions(+), 16 deletions(-)
[beam] 01/01: Revert "Default to Runner v2 for Python Streaming jobs. (#15140)"
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch revert-15140-streaming-v2
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 8fba88e4f1bde48a58c8052d121d2e737fc635b1
Author: Ankur
AuthorDate: Fri Jul 23 10:40:07 2021 -0700
Revert "Default to Runner v2 for Python Streaming jobs. (#15140)"
This reverts commit 09d4fab4cba974edaecd6e9c603f1e2a2855e56b.
---
sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 8 +---
.../apache_beam/runners/dataflow/dataflow_runner_test.py | 11 ++-
2 files changed, 3 insertions(+), 16 deletions(-)
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index 0ac8c5f..c112bdd 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -594,15 +594,9 @@ class DataflowRunner(PipelineRunner):
return result
def _maybe_add_unified_worker_missing_options(self, options):
-debug_options = options.view_as(DebugOptions)
-# Streaming is always portable, default to runner v2.
-if options.view_as(StandardOptions).streaming:
- if not debug_options.lookup_experiment('disable_runner_v2'):
-debug_options.add_experiment('beam_fn_api')
-debug_options.add_experiment('use_runner_v2')
-debug_options.add_experiment('use_portable_job_submission')
# set default beam_fn_api experiment if use unified
# worker experiment flag exists, no-op otherwise.
+debug_options = options.view_as(DebugOptions)
from apache_beam.runners.dataflow.internal import apiclient
if apiclient._use_unified_worker(options):
if not debug_options.lookup_experiment('beam_fn_api'):
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
index 9c7546d..ec4edc9 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
@@ -256,7 +256,6 @@ class DataflowRunnerTest(unittest.TestCase,
ExtraAssertionsMixin):
def test_streaming_create_translation(self):
remote_runner = DataflowRunner()
self.default_properties.append("--streaming")
-self.default_properties.append("--experiments=disable_runner_v2")
with Pipeline(remote_runner, PipelineOptions(self.default_properties)) as
p:
p | ptransform.Create([1]) # pylint: disable=expression-not-assigned
job_dict = json.loads(str(remote_runner.job))
@@ -839,8 +838,7 @@ class DataflowRunnerTest(unittest.TestCase,
ExtraAssertionsMixin):
'Runner determined sharding not available in Dataflow for '
'GroupIntoBatches for jobs not using Runner V2'):
_ = self._run_group_into_batches_and_get_step_properties(
- True,
- ['--enable_streaming_engine', '--experiments=disable_runner_v2'])
+ True, ['--enable_streaming_engine'])
# JRH
with self.assertRaisesRegex(
@@ -848,12 +846,7 @@ class DataflowRunnerTest(unittest.TestCase,
ExtraAssertionsMixin):
'Runner determined sharding not available in Dataflow for '
'GroupIntoBatches for jobs not using Runner V2'):
_ = self._run_group_into_batches_and_get_step_properties(
- True,
- [
- '--enable_streaming_engine',
- '--experiments=beam_fn_api',
- '--experiments=disable_runner_v2'
- ])
+ True, ['--enable_streaming_engine', '--experiments=beam_fn_api'])
def test_pack_combiners(self):
class PackableCombines(beam.PTransform):
[beam] branch revert-15140-streaming-v2 created (now 8fba88e)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch revert-15140-streaming-v2 in repository https://gitbox.apache.org/repos/asf/beam.git. at 8fba88e Revert "Default to Runner v2 for Python Streaming jobs. (#15140)" This branch includes the following new commits: new 8fba88e Revert "Default to Runner v2 for Python Streaming jobs. (#15140)" The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[beam] branch release-2.32.0 updated: remove dev and snapshot from beam version
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch release-2.32.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.32.0 by this push: new 87d0050 remove dev and snapshot from beam version 87d0050 is described below commit 87d005061d577205cfac0ca025f2b425fb289a6d Author: Ankur Goenka AuthorDate: Wed Jul 21 21:14:09 2021 -0700 remove dev and snapshot from beam version --- gradle.properties | 4 ++-- sdks/go/pkg/beam/core/core.go | 2 +- sdks/python/apache_beam/version.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gradle.properties b/gradle.properties index 9661844..417baf0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -24,8 +24,8 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.32.0-SNAPSHOT -sdk_version=2.32.0.dev +version=2.32.0 +sdk_version=2.32.0 javaVersion=1.8 diff --git a/sdks/go/pkg/beam/core/core.go b/sdks/go/pkg/beam/core/core.go index 981570a..fdb331e 100644 --- a/sdks/go/pkg/beam/core/core.go +++ b/sdks/go/pkg/beam/core/core.go @@ -27,5 +27,5 @@ const ( // SdkName is the human readable name of the SDK for UserAgents. SdkName = "Apache Beam SDK for Go" // SdkVersion is the current version of the SDK. - SdkVersion = "2.32.0.dev" + SdkVersion = "2.32.0" ) diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py index a3eb9eb..2615b5f 100644 --- a/sdks/python/apache_beam/version.py +++ b/sdks/python/apache_beam/version.py @@ -17,4 +17,4 @@ """Apache Beam SDK version information and utilities.""" -__version__ = '2.32.0.dev' +__version__ = '2.32.0'
[beam] branch release-2.32.0 created (now 0d106ce)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch release-2.32.0 in repository https://gitbox.apache.org/repos/asf/beam.git. at 0d106ce Set Dataflow container to release version. This branch includes the following new commits: new 0d106ce Set Dataflow container to release version. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[beam] 01/01: Set Dataflow container to release version.
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch release-2.32.0
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 0d106ce3efc072bd14dc7483356087f94e059546
Author: Ankur Goenka
AuthorDate: Wed Jul 21 20:33:15 2021 -0700
Set Dataflow container to release version.
---
runners/google-cloud-dataflow-java/build.gradle | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index e01e441..d3bd73e 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -45,8 +45,8 @@ processResources {
filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
'dataflow.legacy_environment_major_version' : '8',
'dataflow.fnapi_environment_major_version' : '8',
-'dataflow.legacy_container_version' : 'beam-master-20210525',
-'dataflow.fnapi_container_version' : 'beam-master-20210524',
+'dataflow.legacy_container_version' : 'beam-2.32.0',
+'dataflow.fnapi_container_version' : 'beam-2.32.0',
'dataflow.container_base_repository' : 'gcr.io/cloud-dataflow/v1beta3',
]
}
[beam] branch master updated: Moving to 2.33.0-SNAPSHOT on master branch.
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 31c2b76 Moving to 2.33.0-SNAPSHOT on master branch.
31c2b76 is described below
commit 31c2b76710f863c54da79ac9f34e6a76ebaabbeb
Author: Ankur Goenka
AuthorDate: Wed Jul 21 20:30:14 2021 -0700
Moving to 2.33.0-SNAPSHOT on master branch.
---
.../src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy| 2 +-
gradle.properties | 4 ++--
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/python/apache_beam/version.py| 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 9b60019..a078fe9 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -361,7 +361,7 @@ class BeamModulePlugin implements Plugin {
// Automatically use the official release version if we are performing a
release
// otherwise append '-SNAPSHOT'
-project.version = '2.32.0'
+project.version = '2.33.0'
if (!isRelease(project)) {
project.version += '-SNAPSHOT'
}
diff --git a/gradle.properties b/gradle.properties
index 9661844..e71c2f6 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -24,8 +24,8 @@ offlineRepositoryRoot=offline-repository
signing.gnupg.executable=gpg
signing.gnupg.useLegacyGpg=true
-version=2.32.0-SNAPSHOT
-sdk_version=2.32.0.dev
+version=2.33.0-SNAPSHOT
+sdk_version=2.33.0.dev
javaVersion=1.8
diff --git a/sdks/go/pkg/beam/core/core.go b/sdks/go/pkg/beam/core/core.go
index 981570a..34fe071 100644
--- a/sdks/go/pkg/beam/core/core.go
+++ b/sdks/go/pkg/beam/core/core.go
@@ -27,5 +27,5 @@ const (
// SdkName is the human readable name of the SDK for UserAgents.
SdkName = "Apache Beam SDK for Go"
// SdkVersion is the current version of the SDK.
- SdkVersion = "2.32.0.dev"
+ SdkVersion = "2.33.0.dev"
)
diff --git a/sdks/python/apache_beam/version.py
b/sdks/python/apache_beam/version.py
index a3eb9eb..27d47f2 100644
--- a/sdks/python/apache_beam/version.py
+++ b/sdks/python/apache_beam/version.py
@@ -17,4 +17,4 @@
"""Apache Beam SDK version information and utilities."""
-__version__ = '2.32.0.dev'
+__version__ = '2.33.0.dev'
[beam] branch master updated: Default to Runner v2 for Python Streaming jobs. (#15140)
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 09d4fab Default to Runner v2 for Python Streaming jobs. (#15140)
09d4fab is described below
commit 09d4fab4cba974edaecd6e9c603f1e2a2855e56b
Author: Robert Bradshaw
AuthorDate: Wed Jul 21 20:01:12 2021 -0700
Default to Runner v2 for Python Streaming jobs. (#15140)
* Default to Runner v2 for Python Streaming jobs.
* Fix test expectations.
* yapf
---
sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 8 +++-
.../apache_beam/runners/dataflow/dataflow_runner_test.py | 11 +--
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index c112bdd..0ac8c5f 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -594,9 +594,15 @@ class DataflowRunner(PipelineRunner):
return result
def _maybe_add_unified_worker_missing_options(self, options):
+debug_options = options.view_as(DebugOptions)
+# Streaming is always portable, default to runner v2.
+if options.view_as(StandardOptions).streaming:
+ if not debug_options.lookup_experiment('disable_runner_v2'):
+debug_options.add_experiment('beam_fn_api')
+debug_options.add_experiment('use_runner_v2')
+debug_options.add_experiment('use_portable_job_submission')
# set default beam_fn_api experiment if use unified
# worker experiment flag exists, no-op otherwise.
-debug_options = options.view_as(DebugOptions)
from apache_beam.runners.dataflow.internal import apiclient
if apiclient._use_unified_worker(options):
if not debug_options.lookup_experiment('beam_fn_api'):
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
index ec4edc9..9c7546d 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
@@ -256,6 +256,7 @@ class DataflowRunnerTest(unittest.TestCase,
ExtraAssertionsMixin):
def test_streaming_create_translation(self):
remote_runner = DataflowRunner()
self.default_properties.append("--streaming")
+self.default_properties.append("--experiments=disable_runner_v2")
with Pipeline(remote_runner, PipelineOptions(self.default_properties)) as
p:
p | ptransform.Create([1]) # pylint: disable=expression-not-assigned
job_dict = json.loads(str(remote_runner.job))
@@ -838,7 +839,8 @@ class DataflowRunnerTest(unittest.TestCase,
ExtraAssertionsMixin):
'Runner determined sharding not available in Dataflow for '
'GroupIntoBatches for jobs not using Runner V2'):
_ = self._run_group_into_batches_and_get_step_properties(
- True, ['--enable_streaming_engine'])
+ True,
+ ['--enable_streaming_engine', '--experiments=disable_runner_v2'])
# JRH
with self.assertRaisesRegex(
@@ -846,7 +848,12 @@ class DataflowRunnerTest(unittest.TestCase,
ExtraAssertionsMixin):
'Runner determined sharding not available in Dataflow for '
'GroupIntoBatches for jobs not using Runner V2'):
_ = self._run_group_into_batches_and_get_step_properties(
- True, ['--enable_streaming_engine', '--experiments=beam_fn_api'])
+ True,
+ [
+ '--enable_streaming_engine',
+ '--experiments=beam_fn_api',
+ '--experiments=disable_runner_v2'
+ ])
def test_pack_combiners(self):
class PackableCombines(beam.PTransform):
[beam] branch master updated: [BEAM-12445] Handle none bq_client (#15194)
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 46842eb [BEAM-12445] Handle none bq_client (#15194)
46842eb is described below
commit 46842eb2878021962e619d0bc863c31da6dd474b
Author: Ankur
AuthorDate: Wed Jul 21 00:56:47 2021 -0700
[BEAM-12445] Handle none bq_client (#15194)
* [BEAM-12445] Handle none bq_client
* fix test case
---
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 13 -
sdks/python/apache_beam/io/gcp/bigquery_tools_test.py | 4 +---
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py
b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
index c9540ce..eefdb05 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
@@ -641,14 +641,17 @@ class BigQueryWrapper(object):
started_millis = int(time.time() * 1000)
try:
- table_ref = gcp_bigquery.DatasetReference(project_id,
-dataset_id).table(table_id)
+ table_ref_str = '%s.%s.%s' % (project_id, dataset_id, table_id)
errors = self.gcp_bq_client.insert_rows_json(
- table_ref, json_rows=rows, row_ids=insert_ids,
skip_invalid_rows=True)
+ table_ref_str,
+ json_rows=rows,
+ row_ids=insert_ids,
+ skip_invalid_rows=True)
if not errors:
service_call_metric.call('ok')
- for insert_error in errors:
-service_call_metric.call(insert_error['errors'][0])
+ else:
+for insert_error in errors:
+ service_call_metric.call(insert_error['errors'][0])
except HttpError as e:
service_call_metric.call(e)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
index 566e2b3..c9ed827 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
@@ -51,7 +51,6 @@ from apache_beam.options.value_provider import
StaticValueProvider
# pylint: disable=wrong-import-order, wrong-import-position
try:
from apitools.base.py.exceptions import HttpError, HttpForbiddenError
- from google.cloud import bigquery as gcp_bigquery
except ImportError:
HttpError = None
HttpForbiddenError = None
@@ -860,8 +859,7 @@ class TestBigQueryWriter(unittest.TestCase):
sample_row = {'i': 1, 'b': True, 's': 'abc', 'f': 3.14}
client.insert_rows_json.assert_called_with(
-gcp_bigquery.TableReference(
-gcp_bigquery.DatasetReference('project', 'dataset'), 'table'),
+'%s.%s.%s' % ('project', 'dataset', 'table'),
json_rows=[sample_row],
row_ids=['_1'],
skip_invalid_rows=True)
[beam] branch master updated (5b810ba -> 6fcd339)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 5b810ba Merge pull request #15193 from pcoet/runner-tabs add 6fcd339 Update Container beam-master-20210720 (#15197) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/runners/dataflow/internal/names.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[beam] branch master updated (437bdee -> 8122b33)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 437bdee Merge pull request #14541 Avoid sending zero msec counters. add 8122b33 [BEAM-366] Populate display data in portable job representation (#14470) No new revisions were added by this update. Summary of changes: .../pipeline/src/main/proto/beam_runner_api.proto | 15 +++-- .../core/construction/DisplayDataTranslation.java | 13 ++-- .../construction/DisplayDataTranslationTest.java | 12 ++-- sdks/python/apache_beam/io/gcp/bigquery.py | 3 +- sdks/python/apache_beam/pipeline.py| 4 +- sdks/python/apache_beam/pipeline_test.py | 74 ++ sdks/python/apache_beam/portability/common_urns.py | 3 + sdks/python/apache_beam/transforms/display.py | 42 8 files changed, 145 insertions(+), 21 deletions(-)
[beam] branch master updated: More flexible GBK tests.
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 26030bf More flexible GBK tests.
new e30c349 Merge pull request #14287 from robertwb/gbk-testing
26030bf is described below
commit 26030bfea03fd700560e771571094ff5394dc3bb
Author: Robert Bradshaw
AuthorDate: Fri Mar 19 16:49:18 2021 -0700
More flexible GBK tests.
---
sdks/python/apache_beam/transforms/ptransform_test.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py
b/sdks/python/apache_beam/transforms/ptransform_test.py
index 2f2bb98..6b35818 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -35,6 +35,7 @@ from builtins import map
from builtins import range
from builtins import zip
from functools import reduce
+from typing import Iterable
from typing import Optional
from unittest.mock import patch
@@ -2592,6 +2593,8 @@ def _sort_lists(result):
return tuple(_sort_lists(e) for e in result)
elif isinstance(result, dict):
return {k: _sort_lists(v) for k, v in result.items()}
+ elif isinstance(result, Iterable) and not isinstance(result, str):
+return sorted(result)
else:
return result
[beam] branch master updated: bump worker to 20210301
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 9c6af7b bump worker to 20210301 new 22aba9b Merge pull request #14131 from TheNeuralBit/py-worker-20210301 9c6af7b is described below commit 9c6af7b90c02410ea4dc5e92adeec3a9b0a17f98 Author: Brian Hulette AuthorDate: Tue Mar 2 12:39:07 2021 -0800 bump worker to 20210301 --- sdks/python/apache_beam/runners/dataflow/internal/names.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/names.py b/sdks/python/apache_beam/runners/dataflow/internal/names.py index 84dcd53..81d5fe3 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/names.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/names.py @@ -41,10 +41,10 @@ SERIALIZED_SOURCE_KEY = 'serialized_source' # Update this version to the next version whenever there is a change that will # require changes to legacy Dataflow worker execution environment. -BEAM_CONTAINER_VERSION = 'beam-master-20210216' +BEAM_CONTAINER_VERSION = 'beam-master-20210301' # Update this version to the next version whenever there is a change that # requires changes to SDK harness container or SDK harness launcher. -BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20210216' +BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20210301' DATAFLOW_CONTAINER_IMAGE_REPOSITORY = 'gcr.io/cloud-dataflow/v1beta3'
[beam] branch master updated (b24e159 -> 46bd665)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from b24e159 Merge pull request #14162 from [BEAM-11932] Add Dataflow ServiceOptions new f9eb9c0 upgrade errorprone version to 2.3.2 new dfeda2a upgrade to 2.3.4 due to https://github.com/google/error-prone/commit/3c3c7979c260861e93154a52476df7bc0eb26127 new c6963d6 suppress new (post 2.3.1) errorprone patterns new d5961df Use errorprone_version instead of hardcoding. new 46bd665 Merge pull request #14148 from sonam-vend/upgrade-errorprone The 30952 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/beam/gradle/BeamModulePlugin.groovy | 41 -- 1 file changed, 38 insertions(+), 3 deletions(-)
[beam] branch master updated: [BEAM-11634] Give JobInvoker threads unique names.
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9d19510 [BEAM-11634] Give JobInvoker threads unique names.
new a44768f Merge pull request #13746 from ibzib/BEAM-11634
9d19510 is described below
commit 9d195103c243ccb1cd8ead3f52f768d86a49cb73
Author: Kyle Weaver
AuthorDate: Tue Jan 12 19:59:50 2021 -0800
[BEAM-11634] Give JobInvoker threads unique names.
---
.../src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java| 2 +-
.../src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java| 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git
a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java
b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java
index e6bc7f7..0914015 100644
---
a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java
+++
b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java
@@ -49,7 +49,7 @@ public class FlinkJobInvoker extends JobInvoker {
private final FlinkJobServerDriver.FlinkServerConfiguration serverConfig;
protected FlinkJobInvoker(FlinkJobServerDriver.FlinkServerConfiguration
serverConfig) {
-super("flink-runner-job-invoker");
+super("flink-runner-job-invoker-%d");
this.serverConfig = serverConfig;
}
diff --git
a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java
b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java
index 79c0633..dd57a6c 100644
---
a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java
+++
b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java
@@ -49,7 +49,7 @@ public class SparkJobInvoker extends JobInvoker {
}
private SparkJobInvoker(SparkJobServerDriver.SparkServerConfiguration
configuration) {
-super("spark-runner-job-invoker");
+super("spark-runner-job-invoker-%d");
this.configuration = configuration;
}
[beam] branch master updated: [BEAM-11113] Switch default pickler compressor back to zlib for Coders in python sdk
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new b73fcc6 [BEAM-3] Switch default pickler compressor back to zlib
for Coders in python sdk
new 38feb03 Merge pull request #13183 from y1chi/BEAM-3
b73fcc6 is described below
commit b73fcc6502624a1b19db28ec108db4a1424df2d4
Author: Yichi Zhang
AuthorDate: Fri Oct 23 13:41:37 2020 -0700
[BEAM-3] Switch default pickler compressor back to zlib for Coders in
python sdk
---
sdks/python/apache_beam/coders/coders.py| 5 +++--
sdks/python/apache_beam/internal/pickler.py | 19 +++
2 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/sdks/python/apache_beam/coders/coders.py
b/sdks/python/apache_beam/coders/coders.py
index bc35779..1641ae1 100644
--- a/sdks/python/apache_beam/coders/coders.py
+++ b/sdks/python/apache_beam/coders/coders.py
@@ -107,12 +107,13 @@ ConstructorFn = Callable[[Optional[Any], List['Coder'],
'PipelineContext'], Any]
def serialize_coder(coder):
from apache_beam.internal import pickler
return b'%s$%s' % (
- coder.__class__.__name__.encode('utf-8'), pickler.dumps(coder))
+ coder.__class__.__name__.encode('utf-8'),
+ pickler.dumps(coder, use_zlib=True))
def deserialize_coder(serialized):
from apache_beam.internal import pickler
- return pickler.loads(serialized.split(b'$', 1)[1])
+ return pickler.loads(serialized.split(b'$', 1)[1], use_zlib=True)
# pylint: enable=wrong-import-order, wrong-import-position
diff --git a/sdks/python/apache_beam/internal/pickler.py
b/sdks/python/apache_beam/internal/pickler.py
index c4bfb44..395d511 100644
--- a/sdks/python/apache_beam/internal/pickler.py
+++ b/sdks/python/apache_beam/internal/pickler.py
@@ -39,6 +39,7 @@ import sys
import threading
import traceback
import types
+import zlib
from typing import Any
from typing import Dict
from typing import Tuple
@@ -241,7 +242,7 @@ if 'save_module' in dir(dill.dill):
logging.getLogger('dill').setLevel(logging.WARN)
-def dumps(o, enable_trace=True):
+def dumps(o, enable_trace=True, use_zlib=False):
# type: (...) -> bytes
"""For internal use only; no backwards-compatibility guarantees."""
@@ -260,18 +261,28 @@ def dumps(o, enable_trace=True):
# Compress as compactly as possible (compresslevel=9) to decrease peak memory
# usage (of multiple in-memory copies) and to avoid hitting protocol buffer
# limits.
- c = bz2.compress(s, compresslevel=9)
+ # WARNING: Be cautious about compressor change since it can lead to pipeline
+ # representation change, and can break streaming job update compatibility on
+ # runners such as Dataflow.
+ if use_zlib:
+c = zlib.compress(s, 9)
+ else:
+c = bz2.compress(s, compresslevel=9)
del s # Free up some possibly large and no-longer-needed memory.
return base64.b64encode(c)
-def loads(encoded, enable_trace=True):
+def loads(encoded, enable_trace=True, use_zlib=False):
"""For internal use only; no backwards-compatibility guarantees."""
c = base64.b64decode(encoded)
- s = bz2.decompress(c)
+ if use_zlib:
+s = zlib.decompress(c)
+ else:
+s = bz2.decompress(c)
+
del c # Free up some possibly large and no-longer-needed memory.
with _pickle_lock_unless_py2:
[beam] branch master updated: Update beam-2.24.0.md
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 9d76ee5 Update beam-2.24.0.md new 3f9f1aa Merge pull request #13182 from y1chi/patch-2 9d76ee5 is described below commit 9d76ee555f50549ee7f50214083985ce49721bed Author: Yichi Zhang AuthorDate: Fri Oct 23 13:39:37 2020 -0700 Update beam-2.24.0.md --- website/www/site/content/en/blog/beam-2.24.0.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/www/site/content/en/blog/beam-2.24.0.md b/website/www/site/content/en/blog/beam-2.24.0.md index 6ec264f..461514c 100644 --- a/website/www/site/content/en/blog/beam-2.24.0.md +++ b/website/www/site/content/en/blog/beam-2.24.0.md @@ -55,6 +55,9 @@ For more information on changes in 2.24.0, check out the --temp_location, or pass method="STREAMING_INSERTS" to WriteToBigQuery ([BEAM-6928](https://issues.apache.org/jira/browse/BEAM-6928)). * Python SDK now understands `typing.FrozenSet` type hints, which are not interchangeable with `typing.Set`. You may need to update your pipelines if type checking fails. ([BEAM-10197](https://issues.apache.org/jira/browse/BEAM-10197)) +## Known Issues + +* ([BEAM-3](https://issues.apache.org/jira/browse/BEAM-3)) Default compressor change breaks dataflow python streaming job update compatibility. ## List of Contributors
[beam] branch master updated (04f5e12 -> e6b3cf1)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 04f5e12 [BEAM-9681] Add textio.Read lesson to Go SDK katas (#12941) add e6b3cf1 [BEAM-10200] Respect profile_memory option and add memory profiler to… (#12562) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/utils/profiler.py | 203 +++-- sdks/python/apache_beam/utils/profiler_test.py | 64 2 files changed, 154 insertions(+), 113 deletions(-) create mode 100644 sdks/python/apache_beam/utils/profiler_test.py
[beam] branch master updated (04f5e12 -> e6b3cf1)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 04f5e12 [BEAM-9681] Add textio.Read lesson to Go SDK katas (#12941) add e6b3cf1 [BEAM-10200] Respect profile_memory option and add memory profiler to… (#12562) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/utils/profiler.py | 203 +++-- sdks/python/apache_beam/utils/profiler_test.py | 64 2 files changed, 154 insertions(+), 113 deletions(-) create mode 100644 sdks/python/apache_beam/utils/profiler_test.py
[beam] branch master updated (1a89550 -> de309d2)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 1a89550 Merge pull request #12798 [BEAM-10833] Fix type inference for BUILD_MAP. add de309d2 [BEAM-10876] Fix TypeError in dataflow_metrics when distribution sum … (#12817) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py | 12 1 file changed, 12 insertions(+)
[beam] branch master updated (f2959c3 -> 275b7c9)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from f2959c3 Merge pull request #12586: [BEAM-8019] Don't drop coders embedded in CombinePayload add 275b7c9 [BEAM-10200] Add optional experiment to enable heap dump through the … (#12537) No new revisions were added by this update. Summary of changes: .../runners/portability/portable_runner.py | 7 -- .../apache_beam/runners/worker/sdk_worker.py | 6 +++-- .../apache_beam/runners/worker/sdk_worker_main.py | 19 +++ .../apache_beam/runners/worker/worker_status.py| 28 +++--- .../runners/worker/worker_status_test.py | 9 +++ 5 files changed, 52 insertions(+), 17 deletions(-)
[beam] branch master updated: Bump up advanceProcessingTime duration in ParDoTest
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 37e914b Bump up advanceProcessingTime duration in ParDoTest
new ead6971 Merge pull request #12608 from y1chi/flaky_test
37e914b is described below
commit 37e914b88a276f440f98e73955d468b197a6
Author: Yichi Zhang
AuthorDate: Mon Aug 17 18:01:08 2020 -0700
Bump up advanceProcessingTime duration in ParDoTest
---
.../test/java/org/apache/beam/sdk/transforms/ParDoTest.java| 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git
a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index 6d365b9..e7cd5bb 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -4024,8 +4024,9 @@ public class ParDoTest implements Serializable {
TestStream.create(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()))
.addElements(KV.of("hello", 37))
.advanceProcessingTime(
- Duration.millis(DateTimeUtils.currentTimeMillis())
- .plus(Duration.standardSeconds(60)))
+ Duration.millis(
+ DateTimeUtils.currentTimeMillis() / 1000 * 1000) //
round to seconds
+ .plus(Duration.standardMinutes(2)))
.advanceWatermarkToInfinity();
PCollection output = pipeline.apply(stream).apply(ParDo.of(fn));
@@ -4193,8 +4194,9 @@ public class ParDoTest implements Serializable {
.addElements(KV.of("key", "input1"))
.addElements(KV.of("key", "input2"))
.advanceProcessingTime(
- Duration.millis(DateTimeUtils.currentTimeMillis())
- .plus(Duration.standardSeconds(60)))
+ Duration.millis(
+ DateTimeUtils.currentTimeMillis() / 1000 * 1000) //
round to seconds
+ .plus(Duration.standardMinutes(2)))
.advanceWatermarkToInfinity();
PCollection output = pipeline.apply(stream).apply(ParDo.of(fn));
[beam] branch master updated (7ae4648 -> eaa41cc)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 7ae4648 Merge pull request #12072 from damgad/tmp_cleanup add eaa41cc Change time granularity to seconds in ParDoTest TestStream timer test… (#11993) No new revisions were added by this update. Summary of changes: .../org/apache/beam/sdk/transforms/ParDoTest.java | 44 ++ 1 file changed, 29 insertions(+), 15 deletions(-)
[beam] branch master updated (4688bc6 -> 6c95955)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 4688bc6 Merge pull request #12078 from ibzib/BEAM-10315 add 6c95955 [BEAM-10189] Add ReadModifyWriteState user state to python sdk (#11916) No new revisions were added by this update. Summary of changes: .../apache_beam/runners/direct/direct_userstate.py | 51 +- .../runners/direct/sdf_direct_runner.py| 8 +-- .../runners/direct/transform_evaluator.py | 5 +- .../portability/fn_api_runner/fn_runner_test.py| 17 -- .../apache_beam/runners/worker/bundle_processor.py | 32 ++-- sdks/python/apache_beam/transforms/trigger.py | 16 +++--- sdks/python/apache_beam/transforms/userstate.py| 27 ++ .../apache_beam/transforms/userstate_test.py | 60 +- 8 files changed, 190 insertions(+), 26 deletions(-)
[beam] branch master updated: [BEAM-9707] Remove hardcoded unified harness image
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 8825995 [BEAM-9707] Remove hardcoded unified harness image
new 05f676e Merge pull request #12044 from angoenka/cleanup_uw_image
8825995 is described below
commit 8825995dffbe9ac5d9ea17a74e207a68c7ff25cc
Author: Ankur Goenka
AuthorDate: Fri Jun 19 18:08:30 2020 -0700
[BEAM-9707] Remove hardcoded unified harness image
---
sdks/python/apache_beam/runners/dataflow/internal/apiclient.py | 10 --
1 file changed, 10 deletions(-)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index 4fb7e6a..d474849 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -196,16 +196,6 @@ class Environment(object):
if job_type.startswith('FNAPI_'):
self.debug_options.experiments = self.debug_options.experiments or []
- # TODO(BEAM-9707) : Remove hardcoding runner_harness_container for
- # Unified worker.
- if _use_unified_worker(
- options) and not self.debug_options.lookup_experiment(
- 'runner_harness_container_image'
- ) and 'dev' in beam_version.__version__:
-self.debug_options.add_experiment(
-'runner_harness_container_image='
-'gcr.io/cloud-dataflow/v1beta3/unified-harness:20200409-rc00')
-
if self.debug_options.lookup_experiment(
'runner_harness_container_image') or _use_unified_worker(options):
# Default image is not used if user provides a runner harness image.
[beam] branch master updated (970b36f -> e7ca038)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 970b36f Merge pull request #11943 from [BEAM-10209] Add without_defaults to Mean combiner in Python new 771e018 added instruction for converting maven project to gradle new 6819ed8 reworded and reformattted instructions after viewing locally new 8a2180e added instructions for executing with gradle for runners: direct, dataflow, spark new e7ca038 Merge pull request #11964 from JustineKoa/jk-addGradleInstructions The 27475 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../site/content/en/get-started/quickstart-java.md | 82 ++ 1 file changed, 82 insertions(+)
[beam] branch master updated: [BEAM-10060] Upgrading Python containers to beam-master-20200521
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 393f1ba [BEAM-10060] Upgrading Python containers to beam-master-20200521 new 1560e02 Merge pull request #11783 from angoenka/beam-master-20200521 393f1ba is described below commit 393f1bacffa64360d359b646655ead3e090373f4 Author: Ankur Goenka AuthorDate: Thu May 21 15:03:38 2020 -0700 [BEAM-10060] Upgrading Python containers to beam-master-20200521 --- sdks/python/apache_beam/runners/dataflow/internal/names.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/names.py b/sdks/python/apache_beam/runners/dataflow/internal/names.py index 9be60ff..446f00c 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/names.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/names.py @@ -40,10 +40,10 @@ SERIALIZED_SOURCE_KEY = 'serialized_source' # Update this version to the next version whenever there is a change that will # require changes to legacy Dataflow worker execution environment. -BEAM_CONTAINER_VERSION = 'beam-master-20200518' +BEAM_CONTAINER_VERSION = 'beam-master-20200521' # Update this version to the next version whenever there is a change that # requires changes to SDK harness container or SDK harness launcher. -BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20200518' +BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20200521' # TODO(BEAM-5939): Remove these shared names once Dataflow worker is updated. PICKLED_MAIN_SESSION_FILE = 'pickled_main_session'
[beam] annotated tag dataflow_runner_v2 updated (d1e706e -> 63c5e5f)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to annotated tag dataflow_runner_v2 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag dataflow_runner_v2 was modified! *** from d1e706e (commit) to 63c5e5f (tag) tagging d1e706e2b0fdc6be268fb39a24fc149de312a976 (commit) replaces java-ulr-removal by Ankur Goenka on Fri Apr 10 17:29:25 2020 -0700 - Log - Tag for Green Dataflow Runner V2. --- No new revisions were added by this update. Summary of changes:
[beam] branch master updated: TOIL: Update Unified worker image
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new d1c04cb TOIL: Update Unified worker image new d1e706e Merge pull request #11385 from angoenka/update_um_image d1c04cb is described below commit d1c04cba461360431db35aa94eb4a382587df77e Author: Ankur Goenka AuthorDate: Fri Apr 10 12:49:36 2020 -0700 TOIL: Update Unified worker image --- sdks/python/apache_beam/runners/dataflow/internal/apiclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index 7651c46..73b20bf 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -205,7 +205,7 @@ class Environment(object): ) and 'dev' in beam_version.__version__: self.debug_options.add_experiment( 'runner_harness_container_image=' -'gcr.io/cloud-dataflow/v1beta3/unified-harness:20200402-rc00') +'gcr.io/cloud-dataflow/v1beta3/unified-harness:20200409-rc00') if self.debug_options.lookup_experiment( 'runner_harness_container_image') or _use_unified_worker(options):
[beam] annotated tag dataflow_runner_v2 updated (6f6809a -> d399b17)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to annotated tag dataflow_runner_v2 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag dataflow_runner_v2 was modified! *** from 6f6809a (commit) to d399b17 (tag) tagging 6f6809a3ddca1fc93ea0143d0f85eb004e268769 (commit) replaces java-ulr-removal by Ankur Goenka on Wed Apr 8 17:15:55 2020 -0700 - Log - Tag for Green Dataflow Runner V2. --- No new revisions were added by this update. Summary of changes:
[beam] branch master updated: [BEAM-9707] Hardcode Unified harness image for fixing dataflow VR 2 tests
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 1a217db [BEAM-9707] Hardcode Unified harness image for fixing
dataflow VR 2 tests
new 1f48ac6 Merge pull request #11326 from angoenka/uw_image
1a217db is described below
commit 1a217db81a7fa6ffabfd48b292e131fc5c7c5eec
Author: Ankur Goenka
AuthorDate: Mon Apr 6 14:25:49 2020 -0700
[BEAM-9707] Hardcode Unified harness image for fixing dataflow VR 2 tests
---
.../python/apache_beam/runners/dataflow/internal/apiclient.py | 11 +++
sdks/python/scripts/run_integration_test.sh | 2 --
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index 8b4e169..71cd0cc 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -196,6 +196,17 @@ class Environment(object):
# TODO: Use enumerated type instead of strings for job types.
if job_type.startswith('FNAPI_'):
self.debug_options.experiments = self.debug_options.experiments or []
+
+ # TODO(BEAM-9707) : Remove hardcoding runner_harness_container for
+ # Unified worker.
+ if _use_unified_worker(
+ options) and not self.debug_options.lookup_experiment(
+ 'runner_harness_container_image'
+ ) and 'dev' in beam_version.__version__:
+self.debug_options.add_experiment(
+'runner_harness_container_image='
+'gcr.io/cloud-dataflow/v1beta3/unified-harness:20200402-rc00')
+
if self.debug_options.lookup_experiment(
'runner_harness_container_image') or _use_unified_worker(options):
# Default image is not used if user provides a runner harness image.
diff --git a/sdks/python/scripts/run_integration_test.sh
b/sdks/python/scripts/run_integration_test.sh
index 386bb42..b41e9e8 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -232,8 +232,6 @@ if [[ -z $PIPELINE_OPTS ]]; then
# Add --runner_v2 if provided
if [[ "$RUNNER_V2" = true ]]; then
opts+=("--experiments=use_runner_v2")
-# Cleanup jira BEAM-9391
-
opts+=("--experiments=runner_harness_container_image=gcr.io/cloud-dataflow/v1beta3/unified-harness:20200227-rc01")
if [[ "$STREAMING" = true ]]; then
# Dataflow Runner V2 only supports streaming engine.
opts+=("--enable_streaming_engine")
[beam] branch master updated: [BEAM-9263] Bump up python sdk dataflow environment major versions
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9a17b33 [BEAM-9263] Bump up python sdk dataflow environment major
versions
new e9cd938 Merge pull request #10870 from y1chi/BEAM-9263
9a17b33 is described below
commit 9a17b334cf3c661b10a206acf7e5bd285c4b84d3
Author: Yichi Zhang
AuthorDate: Mon Feb 10 10:57:55 2020 -0800
[BEAM-9263] Bump up python sdk dataflow environment major versions
---
runners/google-cloud-dataflow-java/build.gradle| 4 ++--
sdks/python/apache_beam/runners/dataflow/internal/apiclient.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index 8fbc4da..9a8e003 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -37,8 +37,8 @@ evaluationDependsOn(":sdks:java:container")
processResources {
filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
-'dataflow.legacy_environment_major_version' : '7',
-'dataflow.fnapi_environment_major_version' : '7',
+'dataflow.legacy_environment_major_version' : '8',
+'dataflow.fnapi_environment_major_version' : '8',
'dataflow.container_version' : 'beam-master-20200227'
]
}
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index a46e140..8b4e169 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -70,8 +70,8 @@ from apache_beam.utils import proto_utils
# Environment version information. It is passed to the service during a
# a job submission and is used by the service to establish what features
# are expected by the workers.
-_LEGACY_ENVIRONMENT_MAJOR_VERSION = '7'
-_FNAPI_ENVIRONMENT_MAJOR_VERSION = '7'
+_LEGACY_ENVIRONMENT_MAJOR_VERSION = '8'
+_FNAPI_ENVIRONMENT_MAJOR_VERSION = '8'
_LOGGER = logging.getLogger(__name__)
[beam] branch master updated: [BEAM-9504] Sickbay streaming test for batch VR
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 61da25f [BEAM-9504] Sickbay streaming test for batch VR
new 08c975a Merge pull request #11129 from
angoenka/sickbay_test_streaming_pipeline_returns_expected_user_metrics_fnapi_it
61da25f is described below
commit 61da25f9fadba0d87283731c89c95db0922205d5
Author: Ankur Goenka
AuthorDate: Fri Mar 13 14:52:38 2020 -0700
[BEAM-9504] Sickbay streaming test for batch VR
---
.../dataflow/dataflow_exercise_streaming_metrics_pipeline_test.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/sdks/python/apache_beam/runners/dataflow/dataflow_exercise_streaming_metrics_pipeline_test.py
b/sdks/python/apache_beam/runners/dataflow/dataflow_exercise_streaming_metrics_pipeline_test.py
index 16fee92..353d32e 100644
---
a/sdks/python/apache_beam/runners/dataflow/dataflow_exercise_streaming_metrics_pipeline_test.py
+++
b/sdks/python/apache_beam/runners/dataflow/dataflow_exercise_streaming_metrics_pipeline_test.py
@@ -114,7 +114,8 @@ class
ExerciseStreamingMetricsPipelineTest(unittest.TestCase):
argv = self.test_pipeline.get_full_options_as_args(**extra_opts)
return dataflow_exercise_streaming_metrics_pipeline.run(argv)
- @attr('IT', 'ValidatesRunner')
+ # Need not run streaming test in batch mode.
+ @attr('IT', 'ValidatesRunner', 'sickbay-batch')
def test_streaming_pipeline_returns_expected_user_metrics_fnapi_it(self):
"""
Runs streaming Dataflow job and verifies that user metrics are reported
[beam] branch master updated: [BEAM-9499] Sickbay test_multi_triggered_gbk_side_input for streaming mode
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 4d13e34 [BEAM-9499] Sickbay test_multi_triggered_gbk_side_input for
streaming mode
new bbcb499 Merge pull request #11121 from angoenka/sickbay_for_streaming
4d13e34 is described below
commit 4d13e3428fbfd61d30c1605c273bbd6b32f3b664
Author: Ankur Goenka
AuthorDate: Fri Mar 13 02:16:13 2020 -0700
[BEAM-9499] Sickbay test_multi_triggered_gbk_side_input for streaming mode
---
sdks/python/apache_beam/transforms/sideinputs_test.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py
b/sdks/python/apache_beam/transforms/sideinputs_test.py
index 7c43e6b..6c943f0 100644
--- a/sdks/python/apache_beam/transforms/sideinputs_test.py
+++ b/sdks/python/apache_beam/transforms/sideinputs_test.py
@@ -340,7 +340,8 @@ class SideInputsTest(unittest.TestCase):
assert_that(results, equal_to(['a', 'b']))
pipeline.run()
- @attr('ValidatesRunner', 'sickbay-batch')
+ # TODO(BEAM-9499): Disable this test in streaming temporarily.
+ @attr('ValidatesRunner', 'sickbay-batch', 'sickbay-streaming')
def test_multi_triggered_gbk_side_input(self):
"""Test a GBK sideinput, with multiple triggering."""
# TODO(BEAM-9322): Remove use of this experiment.
[beam] branch master updated: [BEAM-9485] Raise error when transform urn is not implemented
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 6df42b8 [BEAM-9485] Raise error when transform urn is not implemented new 64fb477 Merge pull request #11097 from angoenka/raise_non_implemented 6df42b8 is described below commit 6df42b86e4aa2575ee51938133951a710788e09f Author: Ankur Goenka AuthorDate: Tue Mar 10 23:45:36 2020 -0700 [BEAM-9485] Raise error when transform urn is not implemented --- sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index 654ccfa..5965ba7 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -854,7 +854,8 @@ class DataflowRunner(PipelineRunner): if common_urns.primitives.PAR_DO.urn == urn: self.run_ParDo(transform_node, options) else: - NotImplementedError(urn) + raise NotImplementedError( + '%s uses unsupported URN: %s' % (transform_node.full_label, urn)) def run_ParDo(self, transform_node, options): transform = transform_node.transform
[beam] branch master updated: [BEAM-9287] disable validates runner test which uses teststreams for batch
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a3b046e [BEAM-9287] disable validates runner test which uses
teststreams for batch
new a84611c Merge pull request #8 from angoenka/sickbay_batch
a3b046e is described below
commit a3b046e87bdbe9015d744e44a4553f1c69f3aed9
Author: Ankur Goenka
AuthorDate: Thu Mar 12 17:55:03 2020 -0700
[BEAM-9287] disable validates runner test which uses teststreams for batch
---
sdks/python/apache_beam/transforms/sideinputs_test.py | 2 +-
sdks/python/test-suites/dataflow/py2/build.gradle | 2 +-
sdks/python/test-suites/dataflow/py35/build.gradle| 2 +-
sdks/python/test-suites/dataflow/py36/build.gradle| 2 +-
sdks/python/test-suites/dataflow/py37/build.gradle| 2 +-
5 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py
b/sdks/python/apache_beam/transforms/sideinputs_test.py
index ddec122..7c43e6b 100644
--- a/sdks/python/apache_beam/transforms/sideinputs_test.py
+++ b/sdks/python/apache_beam/transforms/sideinputs_test.py
@@ -340,7 +340,7 @@ class SideInputsTest(unittest.TestCase):
assert_that(results, equal_to(['a', 'b']))
pipeline.run()
- @attr('ValidatesRunner')
+ @attr('ValidatesRunner', 'sickbay-batch')
def test_multi_triggered_gbk_side_input(self):
"""Test a GBK sideinput, with multiple triggering."""
# TODO(BEAM-9322): Remove use of this experiment.
diff --git a/sdks/python/test-suites/dataflow/py2/build.gradle
b/sdks/python/test-suites/dataflow/py2/build.gradle
index eff60d7..6f5ee0a 100644
--- a/sdks/python/test-suites/dataflow/py2/build.gradle
+++ b/sdks/python/test-suites/dataflow/py2/build.gradle
@@ -95,7 +95,7 @@ task validatesRunnerBatchTests {
dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
- def testOpts = basicTestOpts + ["--attr=ValidatesRunner"]
+ def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-batch"]
def argMap = [
"test_opts" : testOpts,
"worker_jar" : dataflowWorkerJar,
diff --git a/sdks/python/test-suites/dataflow/py35/build.gradle
b/sdks/python/test-suites/dataflow/py35/build.gradle
index e8f78a4..62d3110 100644
--- a/sdks/python/test-suites/dataflow/py35/build.gradle
+++ b/sdks/python/test-suites/dataflow/py35/build.gradle
@@ -66,7 +66,7 @@ task validatesRunnerBatchTests {
dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
- def testOpts = basicTestOpts + ["--attr=ValidatesRunner"]
+ def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-batch"]
def argMap = [
"test_opts" : testOpts,
"worker_jar" : dataflowWorkerJar,
diff --git a/sdks/python/test-suites/dataflow/py36/build.gradle
b/sdks/python/test-suites/dataflow/py36/build.gradle
index d39a1bb..51e1f7e 100644
--- a/sdks/python/test-suites/dataflow/py36/build.gradle
+++ b/sdks/python/test-suites/dataflow/py36/build.gradle
@@ -66,7 +66,7 @@ task validatesRunnerBatchTests {
dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
- def testOpts = basicTestOpts + ["--attr=ValidatesRunner"]
+ def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-batch"]
def argMap = [
"test_opts" : testOpts,
"worker_jar" : dataflowWorkerJar,
diff --git a/sdks/python/test-suites/dataflow/py37/build.gradle
b/sdks/python/test-suites/dataflow/py37/build.gradle
index 789784d..7ace108 100644
--- a/sdks/python/test-suites/dataflow/py37/build.gradle
+++ b/sdks/python/test-suites/dataflow/py37/build.gradle
@@ -99,7 +99,7 @@ task validatesRunnerBatchTests {
dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
- def testOpts = basicTestOpts + ["--attr=ValidatesRunner"]
+ def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-batch"]
def argMap = [
"test_opts" : testOpts,
"worker_jar" : dataflowWorkerJar,
[beam] branch master updated: [BEAM-9402] Remove options overwrite
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new e6d1420 [BEAM-9402] Remove options overwrite new facb4ef Merge pull request #10997 from angoenka/fix_test_multi_triggered_gbk_side_input e6d1420 is described below commit e6d142087bd15a7b12661371fcb3572b218f7e42 Author: Ankur Goenka AuthorDate: Thu Feb 27 18:16:43 2020 -0800 [BEAM-9402] Remove options overwrite --- sdks/python/apache_beam/transforms/sideinputs_test.py | 11 --- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py b/sdks/python/apache_beam/transforms/sideinputs_test.py index 20d2f6e..ddec122 100644 --- a/sdks/python/apache_beam/transforms/sideinputs_test.py +++ b/sdks/python/apache_beam/transforms/sideinputs_test.py @@ -27,8 +27,6 @@ import unittest from nose.plugins.attrib import attr import apache_beam as beam -from apache_beam.options.pipeline_options import DebugOptions -from apache_beam.options.pipeline_options import StandardOptions from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.test_stream import TestStream from apache_beam.testing.util import assert_that @@ -349,11 +347,10 @@ class SideInputsTest(unittest.TestCase): # This flag is only necessary when using the multi-output TestStream b/c # it relies on using the PCollection output tags as the PCollection output # ids. -options = StandardOptions(streaming=True) -options.view_as(DebugOptions).add_experiment( -'passthrough_pcollection_output_ids') - -p = TestPipeline(options=options) +p = TestPipeline( +additional_pipeline_args=[ +'--experiments=' + 'passthrough_pcollection_output_ids' +]) test_stream = ( p
[beam] branch master updated: [BEAM-9465] Fire repeatedly in reshuffle
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 0a2cee9 [BEAM-9465] Fire repeatedly in reshuffle new 23165a6 Merge pull request #11066 from angoenka/fix_reshuffle 0a2cee9 is described below commit 0a2cee9415a7ecfb6bf747537d6a1ec6add70963 Author: Ankur Goenka AuthorDate: Fri Mar 6 14:14:43 2020 -0800 [BEAM-9465] Fire repeatedly in reshuffle --- sdks/python/apache_beam/transforms/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index 309aba5..369e884 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -64,6 +64,7 @@ from apache_beam.transforms.ptransform import ptransform_fn from apache_beam.transforms.timeutil import TimeDomain from apache_beam.transforms.trigger import AccumulationMode from apache_beam.transforms.trigger import AfterCount +from apache_beam.transforms.trigger import Repeatedly from apache_beam.transforms.userstate import BagStateSpec from apache_beam.transforms.userstate import CombiningValueStateSpec from apache_beam.transforms.userstate import TimerSpec @@ -678,7 +679,7 @@ class ReshufflePerKey(PTransform): # accept only standard coders. ungrouped._windowing = Windowing( window.GlobalWindows(), -triggerfn=AfterCount(1), +triggerfn=Repeatedly(AfterCount(1)), accumulation_mode=AccumulationMode.DISCARDING, timestamp_combiner=TimestampCombiner.OUTPUT_AT_EARLIEST) result = (
[beam] branch master updated (036560f -> 5ff5313)
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 036560f Merge pull request #10999: [BEAM-9345] Fix source of test
flakiness in FlinkSubmissionTest
add 5ff5313 [BEAM-9287] Add Postcommit tests for dataflow runner v2
(#10998)
No new revisions were added by this update.
Summary of changes:
.github/PULL_REQUEST_TEMPLATE.md | 2 +-
...mmit_Python_ValidatesRunner_Dataflow_V2.groovy} | 22 ++--
README.md | 2 +-
sdks/python/scripts/run_integration_test.sh| 10 +-
sdks/python/test-suites/dataflow/py2/build.gradle | 29 ++--
sdks/python/test-suites/dataflow/py35/build.gradle | 40 ++
sdks/python/test-suites/dataflow/py36/build.gradle | 40 ++
sdks/python/test-suites/dataflow/py37/build.gradle | 40 ++
8 files changed, 119 insertions(+), 66 deletions(-)
copy
.test-infra/jenkins/{job_PostCommit_Python_ValidatesRunner_Dataflow.groovy =>
job_PostCommit_Python_ValidatesRunner_Dataflow_V2.groovy} (66%)
[beam] branch master updated (aebc5a7 -> e9cbb8c)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from aebc5a7 [GoSDK] Fix race condition in statemgr & test (#10941) add e9cbb8c [BEAM-9347] Don't overwrite default runner harness for unified worker (#10919) No new revisions were added by this update. Summary of changes: .../runners/dataflow/internal/apiclient.py | 13 +-- .../runners/dataflow/internal/apiclient_test.py| 45 ++ 2 files changed, 55 insertions(+), 3 deletions(-)
[beam] branch master updated: [BEAM-9287] Add Python Streaming Validates runner tests for Unified Worker
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e2d1598 [BEAM-9287] Add Python Streaming Validates runner tests for
Unified Worker
new 96b2d90 Merge pull request #10863 from angoenka/uw_vr
e2d1598 is described below
commit e2d15981d6ea467a5a6ec88b62d9c2bc6d084cfe
Author: Ankur Goenka
AuthorDate: Mon Feb 10 13:51:39 2020 -0800
[BEAM-9287] Add Python Streaming Validates runner tests for Unified Worker
---
sdks/python/scripts/run_integration_test.sh | 11 +++
sdks/python/test-suites/dataflow/py2/build.gradle | 17 +++--
2 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/sdks/python/scripts/run_integration_test.sh
b/sdks/python/scripts/run_integration_test.sh
index 0804ace..a99930d 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -126,6 +126,11 @@ case $key in
shift # past argument
shift # past value
;;
+--runner_v2)
+RUNNER_V2="$2"
+shift # past argument
+shift # past value
+;;
--kms_key_name)
KMS_KEY_NAME="$2"
shift # past argument
@@ -223,6 +228,12 @@ if [[ -z $PIPELINE_OPTS ]]; then
opts+=("--dataflow_worker_jar=$WORKER_JAR")
fi
+ # Add --runner_v2 if provided
+ if [[ "$RUNNER_V2" = true ]]; then
+opts+=("--experiments=use_runner_v2")
+opts+=("--enable_streaming_engine")
+ fi
+
if [[ ! -z "$KMS_KEY_NAME" ]]; then
opts+=(
"--kms_key_name=$KMS_KEY_NAME"
diff --git a/sdks/python/test-suites/dataflow/py2/build.gradle
b/sdks/python/test-suites/dataflow/py2/build.gradle
index 248f71d..ec9345b 100644
--- a/sdks/python/test-suites/dataflow/py2/build.gradle
+++ b/sdks/python/test-suites/dataflow/py2/build.gradle
@@ -118,14 +118,19 @@ task validatesRunnerStreamingTests {
def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+ def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-streaming"]
+ def argMap = ["test_opts": testOpts,
+"streaming": "true",
+"worker_jar": dataflowWorkerJar,
+"sdk_location":
files(configurations.distTarBall.files).singleFile,
+"suite": "validatesRunnerStreamingTests-df"]
+ if (project.hasProperty('useRunnerV2'))
+argMap.put("runner_v2", "true")
+// KMS is not supported for streaming engine.
+argMap.put("kms_key_name", "")
+
doLast {
// TODO(BEAM-3544,BEAM-5025): Disable tests with 'sickbay-streaming' tag.
-def testOpts = basicTestOpts +
["--attr=ValidatesRunner,!sickbay-streaming"]
-def argMap = ["test_opts": testOpts,
- "streaming": "true",
- "worker_jar": dataflowWorkerJar,
- "sdk_location":
files(configurations.distTarBall.files).singleFile,
- "suite": "validatesRunnerStreamingTests-df"]
def cmdArgs = mapToArgString(argMap)
exec {
executable 'sh'
[beam] branch master updated (58cedf4 -> c4e5755)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 58cedf4 Merge pull request #10879: [BEAM-9326] Make JsonToRow transform input instead of add c4e5755 [BEAM-8575] Removed MAX_TIMESTAMP from testing data (#10835) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/transforms/util_test.py | 7 +++ 1 file changed, 3 insertions(+), 4 deletions(-)
[beam] branch master updated: [BEAM-9290] Support runner_harness_container_image in released python sdks.
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 87f8ff1 [BEAM-9290] Support runner_harness_container_image in
released python sdks.
new cec1094 Merge pull request #10827 from
angoenka/runner_harness_image_dev
87f8ff1 is described below
commit 87f8ff1996a5d7a787968516234f32837d25135d
Author: Ankur Goenka
AuthorDate: Mon Feb 10 17:28:20 2020 -0800
[BEAM-9290] Support runner_harness_container_image in released python sdks.
---
.../runners/dataflow/internal/apiclient.py | 11 +
.../runners/dataflow/internal/apiclient_test.py| 26 +-
2 files changed, 32 insertions(+), 5 deletions(-)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index 878dc55..b8e9caf 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -184,12 +184,15 @@ class Environment(object):
])
# TODO: Use enumerated type instead of strings for job types.
if job_type.startswith('FNAPI_'):
+ self.debug_options = self.debug_options or DebugOptions()
self.debug_options.experiments = self.debug_options.experiments or []
+ if not self.debug_options.lookup_experiment(
+ 'runner_harness_container_image'):
+runner_harness_override = (get_runner_harness_container_image())
+if runner_harness_override:
+ self.debug_options.add_experiment(
+ 'runner_harness_container_image=' + runner_harness_override)
debug_options_experiments = self.debug_options.experiments
- runner_harness_override = (get_runner_harness_container_image())
- if runner_harness_override:
-debug_options_experiments.append(
-'runner_harness_container_image=' + runner_harness_override)
# Add use_multiple_sdk_containers flag if it's not already present. Do
not
# add the flag if 'no_use_multiple_sdk_containers' is present.
# TODO: Cleanup use_multiple_sdk_containers once we deprecate Python SDK
diff --git
a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
index faaccc4..ee643a2 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
@@ -318,7 +318,7 @@ class UtilTest(unittest.TestCase):
'apache_beam.runners.dataflow.internal.apiclient.'
'beam_version.__version__',
'2.2.0')
- def test_harness_override_present_in_released_sdks(self):
+ def test_harness_override_default_in_released_sdks(self):
pipeline_options = PipelineOptions(
['--temp_location', 'gs://any-location/temp', '--streaming'])
override = ''.join([
@@ -335,6 +335,30 @@ class UtilTest(unittest.TestCase):
@mock.patch(
'apache_beam.runners.dataflow.internal.apiclient.'
'beam_version.__version__',
+ '2.2.0')
+ def test_harness_override_custom_in_released_sdks(self):
+pipeline_options = PipelineOptions([
+'--temp_location',
+'gs://any-location/temp',
+'--streaming',
+'--experiments=runner_harness_container_image=fake_image'
+])
+env = apiclient.Environment([], #packages
+pipeline_options,
+'2.0.0', #any environment version
+FAKE_PIPELINE_URL)
+self.assertEqual(
+1,
+len([
+x for x in env.proto.experiments
+if x.startswith('runner_harness_container_image=')
+]))
+self.assertIn(
+'runner_harness_container_image=fake_image', env.proto.experiments)
+
+ @mock.patch(
+ 'apache_beam.runners.dataflow.internal.apiclient.'
+ 'beam_version.__version__',
'2.2.0.rc1')
def test_harness_override_uses_base_version_in_rc_releases(self):
pipeline_options = PipelineOptions(
[beam] branch master updated: Fix race exception in python worker status thread dump
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 294b2e6 Fix race exception in python worker status thread dump new 78ab631 Merge pull request #10797 from y1chi/py_thread_dump 294b2e6 is described below commit 294b2e60c3bb0743accd33fa164539638f7efcc0 Author: Yichi Zhang AuthorDate: Thu Feb 6 17:07:35 2020 -0800 Fix race exception in python worker status thread dump --- sdks/python/apache_beam/runners/worker/worker_status.py | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/runners/worker/worker_status.py b/sdks/python/apache_beam/runners/worker/worker_status.py index 494f953..8c5ce76 100644 --- a/sdks/python/apache_beam/runners/worker/worker_status.py +++ b/sdks/python/apache_beam/runners/worker/worker_status.py @@ -41,7 +41,12 @@ def thread_dump(): frames = sys._current_frames() # pylint: disable=protected-access for t in threading.enumerate(): -stack_trace = ''.join(traceback.format_stack(frames[t.ident])) +try: + stack_trace = ''.join(traceback.format_stack(frames[t.ident])) +except KeyError: + # the thread may have been destroyed already while enumerating, in such + # case, skip to next thread. + continue thread_ident_name = (t.ident, t.name) stack_traces[stack_trace].append(thread_ident_name)
[beam] branch master updated: Remove managing late data not supported by python sdk note
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d852114 Remove managing late data not supported by python sdk note
new a8af0e1 Merge pull request #10761 from y1chi/fix_doc
d852114 is described below
commit d85211428f5e39ba59be72ec11510455e89e329e
Author: Yichi Zhang
AuthorDate: Mon Feb 3 18:09:45 2020 -0800
Remove managing late data not supported by python sdk note
---
website/src/documentation/programming-guide.md | 21 +
1 file changed, 17 insertions(+), 4 deletions(-)
diff --git a/website/src/documentation/programming-guide.md
b/website/src/documentation/programming-guide.md
index 510d7cd..ca12d67 100644
--- a/website/src/documentation/programming-guide.md
+++ b/website/src/documentation/programming-guide.md
@@ -2577,7 +2577,6 @@ elements.
7.4.1. Managing late data {#managing-late-data}
-> **Note:** Managing late data is not supported in the Beam SDK for Python.
You can allow late data by invoking the `.withAllowedLateness` operation when
you set your `PCollection`'s windowing strategy. The following code example
@@ -2591,6 +2590,15 @@ the end of a window.
.withAllowedLateness(Duration.standardDays(2)));
```
+```py
+ pc = [Initial PCollection]
+ pc | beam.WindowInto(
+ FixedWindows(60),
+ trigger=trigger_fn,
+ accumulation_mode=accumulation_mode,
+ timestamp_combiner=timestamp_combiner,
+ allowed_lateness=Duration(seconds=2*24*60*60)) # 2 days
+```
When you set `.withAllowedLateness` on a `PCollection`, that allowed lateness
propagates forward to any subsequent `PCollection` derived from the first
`PCollection` you applied allowed lateness to. If you want to change the
allowed
@@ -2858,7 +2866,6 @@ on each firing:
8.4.2. Handling late data {#handling-late-data}
-> The Beam SDK for Python does not currently support allowed lateness.
If you want your pipeline to process data that arrives after the watermark
passes the end of the window, you can apply an *allowed lateness* when you set
@@ -2877,7 +2884,13 @@ windowing function:
.withAllowedLateness(Duration.standardMinutes(30));
```
```py
- # The Beam SDK for Python does not currently support allowed lateness.
+ pc = [Initial PCollection]
+ pc | beam.WindowInto(
+FixedWindows(60),
+trigger=AfterProcessingTime(60),
+allowed_lateness=1800) # 30 minutes
+ | ...
+
```
This allowed lateness propagates to all `PCollection`s derived as a result of
@@ -3107,4 +3120,4 @@ public class MyMetricsDoFn extends DoFn
{
context.output(context.element());
}
}
-```
\ No newline at end of file
+```
[beam] branch master updated: Removing none instead of bare return
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 9293a81 Removing none instead of bare return new 6463d6e Merge pull request #10724 from angoenka/uw_option 9293a81 is described below commit 9293a819e554957c61d51018d7d1b910a09ff91d Author: Ankur AuthorDate: Wed Jan 29 21:58:52 2020 -0800 Removing none instead of bare return --- sdks/python/apache_beam/runners/dataflow/internal/apiclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index 6a5c46d..d3296df 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -891,7 +891,7 @@ def _use_fnapi(pipeline_options): def _use_unified_worker(pipeline_options): if not _use_fnapi(pipeline_options): -return +return False debug_options = pipeline_options.view_as(DebugOptions) use_unified_worker_flag = 'use_unified_worker'
[beam] branch master updated (0a37f19 -> d1b70d6)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 0a37f19 Merge pull request #10697 from angoenka/fix_lint add d1b70d6 [BEAM-8626] Implement status fn api handler in python sdk (#10598) No new revisions were added by this update. Summary of changes: .../apache_beam/runners/worker/data_plane_test.py | 26 +--- .../apache_beam/runners/worker/sdk_worker.py | 18 ++- .../apache_beam/runners/worker/sdk_worker_main.py | 29 ++-- .../runners/worker/sdk_worker_main_test.py | 4 +- .../apache_beam/runners/worker/worker_status.py| 148 + .../runners/worker/worker_status_test.py | 92 + sdks/python/apache_beam/testing/util.py| 42 ++ 7 files changed, 313 insertions(+), 46 deletions(-) create mode 100644 sdks/python/apache_beam/runners/worker/worker_status.py create mode 100644 sdks/python/apache_beam/runners/worker/worker_status_test.py
[beam] branch master updated: Fixing Lint
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 1f17361 Fixing Lint new 0a37f19 Merge pull request #10697 from angoenka/fix_lint 1f17361 is described below commit 1f17361c30b605c737e2cdd4c46368b8e758d6f8 Author: Ankur AuthorDate: Mon Jan 27 16:14:34 2020 -0800 Fixing Lint --- sdks/python/apache_beam/io/gcp/datastore/v1new/types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1new/types.py b/sdks/python/apache_beam/io/gcp/datastore/v1new/types.py index 497b76a..ca2cb63 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1new/types.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1new/types.py @@ -44,7 +44,8 @@ class Query(object): Args: kind: (str) The kind to query. project: (str) Required. Project associated with query. - namespace: (str, ValueProvider(str)) (Optional) Namespace to restrict results to. + namespace: (str, ValueProvider(str)) (Optional) Namespace to restrict +results to. ancestor: (:class:`~apache_beam.io.gcp.datastore.v1new.types.Key`) (Optional) key of the ancestor to which this query's results are restricted.
[beam] branch master updated (4590b1c -> b01113a)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 4590b1c Merge pull request #10657 from kjmrknsn/fix-writetobigquery-doc add b01113a [BEAM-9093] Log invalid overwrites in pipeline options (#10613) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/options/pipeline_options.py | 5 + sdks/python/apache_beam/options/pipeline_options_test.py | 12 2 files changed, 17 insertions(+)
[beam] 01/01: Revert "[BEAM-9122] Add uses_keyed_state step property in python dataflow run… (#10596)"
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch revert-10596-uses_keyed_state in repository https://gitbox.apache.org/repos/asf/beam.git commit 2269a5015dfb0bb3b1f8469d06f1ab622f85a3ab Author: Ankur AuthorDate: Tue Jan 21 13:43:04 2020 -0800 Revert "[BEAM-9122] Add uses_keyed_state step property in python dataflow run… (#10596)" This reverts commit 52b478eeccfd19bfc05fecd519f7dc54db8c67eb. --- sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 7 ++- sdks/python/apache_beam/runners/dataflow/internal/names.py | 1 - 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index ada700c..69b1fb8 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -52,7 +52,6 @@ from apache_beam.options.pipeline_options import TestOptions from apache_beam.options.pipeline_options import WorkerOptions from apache_beam.portability import common_urns from apache_beam.pvalue import AsSideInput -from apache_beam.runners.common import DoFnSignature from apache_beam.runners.dataflow.internal import names from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api from apache_beam.runners.dataflow.internal.names import PropertyNames @@ -953,10 +952,6 @@ class DataflowRunner(PipelineRunner): step.add_property(PropertyNames.RESTRICTION_ENCODING, self._get_cloud_encoding(restriction_coder)) -if options.view_as(StandardOptions).streaming and DoFnSignature( -transform.dofn).is_stateful_dofn(): - step.add_property(PropertyNames.USES_KEYED_STATE, "true") - @staticmethod def _pardo_fn_data(transform_node, get_label): transform = transform_node.transform @@ -1134,6 +1129,7 @@ class DataflowRunner(PipelineRunner): coders.registry.get_coder(transform_node.outputs[None].element_type), coders.coders.GlobalWindowCoder()) +from apache_beam.runners.dataflow.internal import apiclient step.encoding = self._get_cloud_encoding(coder) step.add_property( PropertyNames.OUTPUT_INFO, @@ -1219,6 +1215,7 @@ class DataflowRunner(PipelineRunner): # correct coder. coder = coders.WindowedValueCoder(transform.sink.coder, coders.coders.GlobalWindowCoder()) +from apache_beam.runners.dataflow.internal import apiclient step.encoding = self._get_cloud_encoding(coder) step.add_property(PropertyNames.ENCODING, step.encoding) step.add_property( diff --git a/sdks/python/apache_beam/runners/dataflow/internal/names.py b/sdks/python/apache_beam/runners/dataflow/internal/names.py index e9b34d4..7bc0295 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/names.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/names.py @@ -123,7 +123,6 @@ class PropertyNames(object): USE_INDEXED_FORMAT = 'use_indexed_format' USER_FN = 'user_fn' USER_NAME = 'user_name' - USES_KEYED_STATE = 'uses_keyed_state' VALIDATE_SINK = 'validate_sink' VALIDATE_SOURCE = 'validate_source' VALUE = 'value'
[beam] branch revert-10596-uses_keyed_state created (now 2269a50)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch revert-10596-uses_keyed_state in repository https://gitbox.apache.org/repos/asf/beam.git. at 2269a50 Revert "[BEAM-9122] Add uses_keyed_state step property in python dataflow run… (#10596)" This branch includes the following new commits: new 2269a50 Revert "[BEAM-9122] Add uses_keyed_state step property in python dataflow run… (#10596)" The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[beam] branch master updated (122e15d -> 52b478e)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 122e15d Merge pull request #10645 from apache/revert-10582-add-asfyaml add 52b478e [BEAM-9122] Add uses_keyed_state step property in python dataflow run… (#10596) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 7 +-- sdks/python/apache_beam/runners/dataflow/internal/names.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-)
[beam] branch master updated (ce797f0 -> d140b90)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from ce797f0 Merge pull request #10149 [BEAM-8739] Consistently use with Pipeline(...) syntax add d140b90 [BEAM-8625] Implement servlet for exposing sdk harness statuses in Da… (#10553) No new revisions were added by this update. Summary of changes: .../dataflow/worker/BatchDataflowWorker.java | 5 +- .../dataflow/worker/DataflowRunnerHarness.java | 47 -- .../dataflow/worker/StreamingDataflowWorker.java | 10 ++- .../dataflow/worker/status/DebugCapture.java | 5 +- .../worker/status/SdkWorkerStatusServlet.java | 99 ++ .../dataflow/worker/status/WorkerStatusPages.java | 16 +++- 6 files changed, 168 insertions(+), 14 deletions(-) create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/SdkWorkerStatusServlet.java
[beam] branch master updated (952c608 -> e2bb239)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 952c608 [BEAM-9013] TestStream fix for DataflowRunner (#10445) add e2bb239 [BEAM-8575] Refactor test_do_fn_with_windowing_in_finish_bundle to work with Dataflow runner (#10447) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/transforms/ptransform_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
[beam] branch master updated: [BEAM-8824] Add support to allow specify window allowed_lateness in python sdk
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 79b6e5b [BEAM-8824] Add support to allow specify window
allowed_lateness in python sdk
new fb0353e Merge pull request #10216 from y1chi/allowed_lateness
79b6e5b is described below
commit 79b6e5bb863c8cf96b690d1a96c8988d8bec72be
Author: Yichi Zhang
AuthorDate: Tue Nov 19 10:16:38 2019 -0800
[BEAM-8824] Add support to allow specify window allowed_lateness in python
sdk
---
.../apache_beam/examples/snippets/snippets_test.py | 2 ++
.../testing/data/trigger_transcripts.yaml | 22
.../python/apache_beam/testing/test_stream_test.py | 2 +-
sdks/python/apache_beam/transforms/core.py | 39 +-
sdks/python/apache_beam/transforms/trigger.py | 5 ++-
sdks/python/apache_beam/transforms/trigger_test.py | 29 ++--
6 files changed, 78 insertions(+), 21 deletions(-)
diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py
b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index f0f53e2..38bcb88 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -958,6 +958,7 @@ class SnippetsTest(unittest.TestCase):
| 'pair_with_one' >> beam.Map(lambda x: (x, 1))
| WindowInto(FixedWindows(15),
trigger=trigger,
+ allowed_lateness=20,
accumulation_mode=AccumulationMode.DISCARDING)
| 'group' >> beam.GroupByKey()
| 'count' >> beam.Map(
@@ -1014,6 +1015,7 @@ class SnippetsTest(unittest.TestCase):
FixedWindows(1 * 60),
trigger=AfterWatermark(
late=AfterProcessingTime(10 * 60)),
+ allowed_lateness=10,
accumulation_mode=AccumulationMode.DISCARDING)
# [END model_composite_triggers]
| 'group' >> beam.GroupByKey()
diff --git a/sdks/python/apache_beam/testing/data/trigger_transcripts.yaml
b/sdks/python/apache_beam/testing/data/trigger_transcripts.yaml
index cac0c74..b2d4e9a 100644
--- a/sdks/python/apache_beam/testing/data/trigger_transcripts.yaml
+++ b/sdks/python/apache_beam/testing/data/trigger_transcripts.yaml
@@ -30,6 +30,7 @@ name: fixed_default_late_data
window_fn: FixedWindows(10)
trigger_fn: Default
timestamp_combiner: OUTPUT_AT_EOW
+allowed_lateness: 100
transcript:
- input: [1, 2, 3, 10, 11, 25]
- watermark: 100
@@ -42,6 +43,26 @@ transcript:
- {window: [0, 9], values: [1, 2, 3, 7], timestamp: 9, late: true}
---
+name: fixed_drop_late_data_after_allowed_lateness
+window_fn: FixedWindows(10)
+trigger_fn: AfterWatermark(early=AfterCount(3), late=AfterCount(1))
+timestamp_combiner: OUTPUT_AT_EOW
+allowed_lateness: 20
+accumulation_mode: accumulating
+transcript:
+ - input: [1, 2, 10, 11, 80, 81]
+ - watermark: 100
+ - expect:
+ - {window: [0, 9], values: [1, 2], timestamp: 9, final: false}
+ - {window: [10, 19], values: [10, 11], timestamp: 19}
+ - {window: [80, 89], values: [80, 81], timestamp: 89, late: false}
+ - input: [7, 8] # no output
+ - input: [17, 18] # no output
+ - input: [82]
+ - expect:
+ - {window: [80, 89], values: [80, 81, 82], timestamp: 89, late: true}
+
+---
name: timestamp_combiner_earliest
window_fn: FixedWindows(10)
trigger_fn: Default
@@ -118,6 +139,7 @@ broken_on:
- SwitchingDirectRunner
window_fn: Sessions(10)
trigger_fn: AfterWatermark(early=AfterCount(2), late=AfterCount(3))
+allowed_lateness: 100
timestamp_combiner: OUTPUT_AT_EOW
transcript:
- input: [1, 2, 3]
diff --git a/sdks/python/apache_beam/testing/test_stream_test.py
b/sdks/python/apache_beam/testing/test_stream_test.py
index 26b54bd..bfadb5e 100644
--- a/sdks/python/apache_beam/testing/test_stream_test.py
+++ b/sdks/python/apache_beam/testing/test_stream_test.py
@@ -275,7 +275,7 @@ class TestStreamTest(unittest.TestCase):
p = TestPipeline(options=options)
records = (p
| test_stream
- | beam.WindowInto(FixedWindows(15))
+ | beam.WindowInto(FixedWindows(15), allowed_lateness=300)
| beam.Map(lambda x: ('k', x))
| beam.GroupByKey())
diff --git a/sdks/python/apache_beam/transforms/core.py
b/sdks/python/apache_beam/transforms/core.py
index 3cd5472..25cc91f 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -64,6 +64,7 @@ from apache_beam.typehints.trivial_inference import
element_type
from apache_beam.typehints.typehints import is_consistent_with
from apache_beam.utils import timestamp
from apache_beam.utils import urns
+from apa
[beam] branch master updated (99037e4 -> 794e58d)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 99037e4 Merge pull request #10388: [BEAM-7991] Fix cleanPython race with :clean add 794e58d [BEAM-8944] Change to use single thread in py sdk bundle progress report (#10387) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/runners/worker/sdk_worker.py | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-)
[beam] branch master updated: Reduce the verbosity of local_job_service_main
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 5a8b276 Reduce the verbosity of local_job_service_main new 0301cda Merge pull request #9813 from udim/portability-less-logging 5a8b276 is described below commit 5a8b2764f72f30d8c8301ad591a80df4a362d2fc Author: Udi Meiri AuthorDate: Wed Oct 16 17:28:10 2019 -0700 Reduce the verbosity of local_job_service_main Since the logs are coming from a subprocess, they are not caught by nose. This has been inflating the test logs on Jenkins, making them very sluggish to view on a browser. --- sdks/python/apache_beam/runners/portability/local_job_service_main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/runners/portability/local_job_service_main.py b/sdks/python/apache_beam/runners/portability/local_job_service_main.py index 4563769..70a33ff 100644 --- a/sdks/python/apache_beam/runners/portability/local_job_service_main.py +++ b/sdks/python/apache_beam/runners/portability/local_job_service_main.py @@ -45,5 +45,4 @@ def run(argv): if __name__ == '__main__': - logging.getLogger().setLevel(logging.INFO) run(sys.argv)
[beam] branch master updated: Reduce the verbosity of local_job_service_main
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 5a8b276 Reduce the verbosity of local_job_service_main new 0301cda Merge pull request #9813 from udim/portability-less-logging 5a8b276 is described below commit 5a8b2764f72f30d8c8301ad591a80df4a362d2fc Author: Udi Meiri AuthorDate: Wed Oct 16 17:28:10 2019 -0700 Reduce the verbosity of local_job_service_main Since the logs are coming from a subprocess, they are not caught by nose. This has been inflating the test logs on Jenkins, making them very sluggish to view on a browser. --- sdks/python/apache_beam/runners/portability/local_job_service_main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/runners/portability/local_job_service_main.py b/sdks/python/apache_beam/runners/portability/local_job_service_main.py index 4563769..70a33ff 100644 --- a/sdks/python/apache_beam/runners/portability/local_job_service_main.py +++ b/sdks/python/apache_beam/runners/portability/local_job_service_main.py @@ -45,5 +45,4 @@ def run(argv): if __name__ == '__main__': - logging.getLogger().setLevel(logging.INFO) run(sys.argv)
[beam] branch master updated: Reduce the verbosity of local_job_service_main
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 5a8b276 Reduce the verbosity of local_job_service_main new 0301cda Merge pull request #9813 from udim/portability-less-logging 5a8b276 is described below commit 5a8b2764f72f30d8c8301ad591a80df4a362d2fc Author: Udi Meiri AuthorDate: Wed Oct 16 17:28:10 2019 -0700 Reduce the verbosity of local_job_service_main Since the logs are coming from a subprocess, they are not caught by nose. This has been inflating the test logs on Jenkins, making them very sluggish to view on a browser. --- sdks/python/apache_beam/runners/portability/local_job_service_main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/runners/portability/local_job_service_main.py b/sdks/python/apache_beam/runners/portability/local_job_service_main.py index 4563769..70a33ff 100644 --- a/sdks/python/apache_beam/runners/portability/local_job_service_main.py +++ b/sdks/python/apache_beam/runners/portability/local_job_service_main.py @@ -45,5 +45,4 @@ def run(argv): if __name__ == '__main__': - logging.getLogger().setLevel(logging.INFO) run(sys.argv)
[beam] branch master updated (77f047b -> a927a3a)
This is an automated email from the ASF dual-hosted git repository. goenka pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 77f047b Merge pull request #9718: [BEAM-8343] Moved BeamTable classes to a more appropriate location add a927a3a [BEAM-7657] fix misleading sdk_worker_parallelism description (#9709) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/options/pipeline_options.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
[beam] branch master updated (19f8812 -> 0256f38)
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 19f8812 Merge pull request #9361: [BEAM-7990] Add ability to read
parquet files into PCollection
add 0256f38 [BEAM-8314] Add aggregation logic to beam_fn_api metric
counter updat… (#9679)
No new revisions were added by this update.
Summary of changes:
.../dataflow/worker/DataflowOperationContext.java | 3 +-
.../worker/MetricsToCounterUpdateConverter.java| 1 +
.../dataflow/worker/StreamingDataflowWorker.java | 43 +-
.../CounterUpdateAggregator.java} | 23 +++---
.../worker/counters/CounterUpdateAggregators.java | 75 +
.../DistributionCounterUpdateAggregator.java | 65 +++
.../counters/MeanCounterUpdateAggregator.java | 55 +
.../counters/SumCounterUpdateAggregator.java | 47 +++
...ntMonitoringInfoToCounterUpdateTransformer.java | 3 +-
...ecMonitoringInfoToCounterUpdateTransformer.java | 3 +-
...ntMonitoringInfoToCounterUpdateTransformer.java | 3 +-
...onMonitoringInfoToCounterUpdateTransformer.java | 3 +-
...erMonitoringInfoToCounterUpdateTransformer.java | 3 +-
.../worker/BatchModeExecutionContextTest.java | 11 +--
.../dataflow/worker/IsmSideInputReaderTest.java| 3 +-
.../worker/StreamingModeExecutionContextTest.java | 5 +-
.../worker/StreamingStepMetricsContainerTest.java | 2 +-
.../dataflow/worker/WorkItemStatusClientTest.java | 9 +-
.../counters/CounterUpdateAggregatorsTest.java | 96 ++
.../DistributionCounterUpdateAggregatorTest.java | 72
.../counters/MeanCounterUpdateAggregatorTest.java | 66 +++
.../counters/SumCounterUpdateAggregatorTest.java | 62 ++
22 files changed, 622 insertions(+), 31 deletions(-)
copy
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{fn/control/MonitoringInfoToCounterUpdateTransformer.java
=> counters/CounterUpdateAggregator.java} (56%)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterUpdateAggregators.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/DistributionCounterUpdateAggregator.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/MeanCounterUpdateAggregator.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/SumCounterUpdateAggregator.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/CounterUpdateAggregatorsTest.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/DistributionCounterUpdateAggregatorTest.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/MeanCounterUpdateAggregatorTest.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/SumCounterUpdateAggregatorTest.java
[beam] branch master updated: Add artifacts-dir param to flink runner
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 43bb1ba Add artifacts-dir param to flink runner
new ba9a7ff Merge pull request #9648 from angoenka/flink_options
43bb1ba is described below
commit 43bb1ba420c202f6756588a0d925b8950c0c10ee
Author: Ankur
AuthorDate: Mon Sep 23 21:56:29 2019 -0700
Add artifacts-dir param to flink runner
---
sdks/python/apache_beam/runners/portability/flink_runner.py | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/sdks/python/apache_beam/runners/portability/flink_runner.py
b/sdks/python/apache_beam/runners/portability/flink_runner.py
index 448e5ef..47546c2 100644
--- a/sdks/python/apache_beam/runners/portability/flink_runner.py
+++ b/sdks/python/apache_beam/runners/portability/flink_runner.py
@@ -42,6 +42,7 @@ class FlinkRunnerOptions(pipeline_options.PipelineOptions):
help='Flink version to use.')
parser.add_argument('--flink_job_server_jar',
help='Path or URL to a flink jobserver jar.')
+parser.add_argument('--artifacts_dir', default=None)
class FlinkJarJobServer(job_server.JavaJarJobServer):
@@ -51,6 +52,7 @@ class FlinkJarJobServer(job_server.JavaJarJobServer):
self._jar = options.flink_job_server_jar
self._master_url = options.flink_master_url
self._flink_version = options.flink_version
+self._artifacts_dir = options.artifacts_dir
def path_to_jar(self):
if self._jar:
@@ -62,7 +64,8 @@ class FlinkJarJobServer(job_server.JavaJarJobServer):
def java_arguments(self, job_port, artifacts_dir):
return [
'--flink-master-url', self._master_url,
-'--artifacts-dir', artifacts_dir,
+'--artifacts-dir', (self._artifacts_dir
+if self._artifacts_dir else artifacts_dir),
'--job-port', job_port,
'--artifact-port', 0,
'--expansion-port', 0
[beam] branch master updated: [BEAM-7859] set SDK worker parallelism to 1 in word count test
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 4dbfc6f [BEAM-7859] set SDK worker parallelism to 1 in word count test
new fba3524 Merge pull request #9421 from ibzib/sdk-worker-test
4dbfc6f is described below
commit 4dbfc6f0587f4ddea56e7ef5a1589f03b3acf8bd
Author: Kyle Weaver
AuthorDate: Fri Aug 23 14:34:57 2019 -0700
[BEAM-7859] set SDK worker parallelism to 1 in word count test
---
buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 1 +
1 file changed, 1 insertion(+)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 8a441662..1f353a2 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2002,6 +2002,7 @@ class BeamModulePlugin implements Plugin {
"--experiments=worker_threads=100",
"--parallelism=2",
"--shutdown_sources_on_final_watermark",
+ "--sdk_worker_parallelism=1",
]
if (isStreaming)
options += [
[beam] branch master updated: [BEAM-7972] Always use Global window in reshuffle and then apply window again.
This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new f22231c [BEAM-7972] Always use Global window in reshuffle and then apply window again. new d8c1146 Merge pull request #9334 from angoenka/fix_reshuffle f22231c is described below commit f22231c8a650db16524548e063f2c3e26fc48f4c Author: Ankur Goenka AuthorDate: Tue Aug 13 15:59:48 2019 -0700 [BEAM-7972] Always use Global window in reshuffle and then apply window again. --- sdks/python/apache_beam/transforms/util.py | 32 -- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index c2866d6..801f522 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -594,7 +594,6 @@ class ReshufflePerKey(PTransform): # In this (common) case we can use a trivial trigger driver # and avoid the (expensive) window param. globally_windowed = window.GlobalWindows.windowed_value(None) - window_fn = window.GlobalWindows() MIN_TIMESTAMP = window.MIN_TIMESTAMP def reify_timestamps(element, timestamp=DoFn.TimestampParam): @@ -612,29 +611,24 @@ class ReshufflePerKey(PTransform): for (value, timestamp) in values] else: - # The linter is confused. - # hash(1) is used to force "runtime" selection of _IdentityWindowFn - # pylint: disable=abstract-class-instantiated - cls = hash(1) and _IdentityWindowFn - window_fn = cls( - windowing_saved.windowfn.get_window_coder()) - - def reify_timestamps(element, timestamp=DoFn.TimestampParam): + def reify_timestamps(element, + timestamp=DoFn.TimestampParam, + window=DoFn.WindowParam): key, value = element -return key, TimestampedValue(value, timestamp) +# Transport the window as part of the value and restore it later. +return key, windowed_value.WindowedValue(value, timestamp, [window]) - def restore_timestamps(element, window=DoFn.WindowParam): -# Pass the current window since _IdentityWindowFn wouldn't know how -# to generate it. -key, values = element -return [ -windowed_value.WindowedValue( -(key, value.value), value.timestamp, [window]) -for value in values] + def restore_timestamps(element): +key, windowed_values = element +return [wv.with_value((key, wv.value)) for wv in windowed_values] ungrouped = pcoll | Map(reify_timestamps) + +# TODO(BEAM-8104) Using global window as one of the standard window. +# This is to mitigate the Dataflow Java Runner Harness limitation to +# accept only standard coders. ungrouped._windowing = Windowing( -window_fn, +window.GlobalWindows(), triggerfn=AfterCount(1), accumulation_mode=AccumulationMode.DISCARDING, timestamp_combiner=TimestampCombiner.OUTPUT_AT_EARLIEST)
[beam] branch master updated: [BEAM-7988] py: include runner name when runner is invalid
This is an automated email from the ASF dual-hosted git repository.
goenka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 1041db2 [BEAM-7988] py: include runner name when runner is invalid
new 862b04c Merge pull request #9355 from ibzib/bad-runner
1041db2 is described below
commit 1041db2c300aab7e94e7a008b71ea1a091045f9a
Author: Kyle Weaver
AuthorDate: Thu Aug 15 17:24:30 2019 -0700
[BEAM-7988] py: include runner name when runner is invalid
---
sdks/python/apache_beam/pipeline.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/sdks/python/apache_beam/pipeline.py
b/sdks/python/apache_beam/pipeline.py
index 25f9a1f..2a3c6a7 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -145,8 +145,8 @@ class Pipeline(object):
if isinstance(runner, str):
runner = create_runner(runner)
elif not isinstance(runner, PipelineRunner):
- raise TypeError('Runner must be a PipelineRunner object or the '
- 'name of a registered runner.')
+ raise TypeError('Runner %s is not a PipelineRunner object or the '
+ 'name of a registered runner.' % runner)
# Validate pipeline options
errors = PipelineOptionsValidator(self._options, runner).validate()
