(beam) branch master updated (f45b5d88e1d -> 54b882d285e)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from f45b5d88e1d Add SerializableSupplier to the core beam.sdk.util package (#31766) add 54b882d285e Replace LGPL dep in Go SDK with an MIT alternative (#31769) No new revisions were added by this update. Summary of changes: CHANGES.md | 1 + sdks/go.mod| 3 +- sdks/go.sum| 10 ++- sdks/go/pkg/beam/core/runtime/xlangx/expand.go | 33 +++--- .../integration/internal/containers/containers.go | 32 ++--- 5 files changed, 35 insertions(+), 44 deletions(-)
(beam) branch master updated: Add a test for getting state with MultimapSideInput StateKey (#31757)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new dbe72830b11 Add a test for getting state with MultimapSideInput
StateKey (#31757)
dbe72830b11 is described below
commit dbe72830b11c2603e062e32295dfb2e3efedbaa9
Author: mls3odp
AuthorDate: Wed Jul 3 11:55:16 2024 -0700
Add a test for getting state with MultimapSideInput StateKey (#31757)
---
.../runners/prism/internal/worker/worker_test.go | 81 +-
1 file changed, 80 insertions(+), 1 deletion(-)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/worker/worker_test.go
b/sdks/go/pkg/beam/runners/prism/internal/worker/worker_test.go
index e5b03214ae0..469e0e2f3d8 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/worker/worker_test.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/worker/worker_test.go
@@ -18,13 +18,14 @@ package worker
import (
"bytes"
"context"
- "github.com/google/go-cmp/cmp"
"net"
"sort"
"sync"
"testing"
"time"
+ "github.com/google/go-cmp/cmp"
+
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/coder"
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window"
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/exec"
@@ -386,3 +387,81 @@ func TestWorker_State_MultimapKeysSideInput(t *testing.T) {
})
}
}
+
+func TestWorker_State_MultimapSideInput(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ wtypex.Window
+ }{
+ {
+ name: "global window",
+ w:window.GlobalWindow{},
+ },
+ {
+ name: "interval window",
+ w: window.IntervalWindow{
+ Start: 1000,
+ End: 2000,
+ },
+ },
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ var encW []byte
+ if !tt.w.Equals(window.GlobalWindow{}) {
+ buf := bytes.Buffer{}
+ if err :=
exec.MakeWindowEncoder(coder.NewIntervalWindow()).EncodeSingle(tt.w, ); err
!= nil {
+ t.Fatalf("error encoding window: %v,
err: %v", tt.w, err)
+ }
+ encW = buf.Bytes()
+ }
+ wk, stateStream, done := serveTestWorkerStateStream(t)
+ defer done()
+ instID := wk.NextInst()
+ wk.activeInstructions[instID] = {
+ MultiMapSideInputData:
map[SideInputKey]map[typex.Window]map[string][][]byte{
+ SideInputKey{
+ TransformID: "transformID",
+ Local: "i1",
+ }: {
+ tt.w: map[string][][]byte{"a":
{{5}}, "b": {{12}}},
+ },
+ },
+ }
+ var testKey = []string{"a", "b", "x"}
+ expectedResult := map[string][]int{
+ "a": {5},
+ "b": {12},
+ }
+ for _, key := range testKey {
+ stateStream.Send({
+ Id:"first",
+ InstructionId: instID,
+ Request: _Get{
+ Get: {},
+ },
+ StateKey: {Type:
_MultimapSideInput_{
+ MultimapSideInput:
_MultimapSideInput{
+ TransformId:
"transformID",
+ SideInputId: "i1",
+ Window: encW,
+ Key:
[]byte(key),
+ },
+ }},
+ })
+
+
(beam) branch master updated: Disable SoftDeletePolicy when creating a default bucket (#31748)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 366db73bf19 Disable SoftDeletePolicy when creating a default bucket
(#31748)
366db73bf19 is described below
commit 366db73bf198090660a4c6a604ea4925b8ca694d
Author: ljj
AuthorDate: Tue Jul 2 12:17:59 2024 -0700
Disable SoftDeletePolicy when creating a default bucket (#31748)
---
sdks/go/pkg/beam/util/gcsx/gcs.go | 15 +--
sdks/go/pkg/beam/util/gcsx/gcs_test.go | 10 ++
2 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/sdks/go/pkg/beam/util/gcsx/gcs.go
b/sdks/go/pkg/beam/util/gcsx/gcs.go
index 83eeb823f4a..1dd85924447 100644
--- a/sdks/go/pkg/beam/util/gcsx/gcs.go
+++ b/sdks/go/pkg/beam/util/gcsx/gcs.go
@@ -63,9 +63,20 @@ func Upload(ctx context.Context, client *storage.Client,
project, bucket, object
}
-// CreateBucket creates a bucket in GCS.
+// Get BucketAttrs with RetentionDuration of SoftDeletePolicy set to zero for
disabling SoftDeletePolicy.
+func getDisableSoftDeletePolicyBucketAttrs() *storage.BucketAttrs {
+ attrs := {
+ SoftDeletePolicy: {
+ RetentionDuration: 0,
+ },
+ }
+ return attrs
+}
+
+// CreateBucket creates a bucket in GCS with RetentionDuration of zero to
disable SoftDeletePolicy.
func CreateBucket(ctx context.Context, client *storage.Client, project, bucket
string) error {
- return client.Bucket(bucket).Create(ctx, project, nil)
+ disableSoftDeletePolicyBucketAttrs :=
getDisableSoftDeletePolicyBucketAttrs()
+ return client.Bucket(bucket).Create(ctx, project,
disableSoftDeletePolicyBucketAttrs)
}
// BucketExists returns true iff the given bucket exists.
diff --git a/sdks/go/pkg/beam/util/gcsx/gcs_test.go
b/sdks/go/pkg/beam/util/gcsx/gcs_test.go
index 90fb4b59f2f..463ba3ea183 100644
--- a/sdks/go/pkg/beam/util/gcsx/gcs_test.go
+++ b/sdks/go/pkg/beam/util/gcsx/gcs_test.go
@@ -96,3 +96,13 @@ func TestJoin(t *testing.T) {
}
}
}
+
+func TestGetDisableSoftDeletePolicyBucketAttrs(t *testing.T) {
+ attrs := getDisableSoftDeletePolicyBucketAttrs()
+ if attrs == nil {
+ t.Errorf("Fail to getDisableSoftDeletePolicyBucketAttrs.")
+ }
+ if attrs != nil && attrs.SoftDeletePolicy.RetentionDuration != 0 {
+ t.Errorf("attrs has RetentionDuration %v which is not correct",
attrs.SoftDeletePolicy.RetentionDuration)
+ }
+}
(beam) branch master updated: [#31403] Python wrapper to download, use, or build and run prism. (#31583)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 2f81e16f3ea [#31403] Python wrapper to download, use, or build and run prism. (#31583) 2f81e16f3ea is described below commit 2f81e16f3eadbfb77bb8191dcc7e6ef9a4887c6f Author: Robert Burke AuthorDate: Fri Jun 28 15:10:50 2024 -0700 [#31403] Python wrapper to download, use, or build and run prism. (#31583) --- .../python/apache_beam/options/pipeline_options.py | 19 ++ .../runners/portability/prism_runner.py| 216 + .../runners/portability/prism_runner_test.py | 269 + 3 files changed, 504 insertions(+) diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py index b204adc7fc5..6b1dd8bb48c 100644 --- a/sdks/python/apache_beam/options/pipeline_options.py +++ b/sdks/python/apache_beam/options/pipeline_options.py @@ -515,6 +515,7 @@ class StandardOptions(PipelineOptions): 'apache_beam.runners.interactive.interactive_runner.InteractiveRunner', 'apache_beam.runners.portability.flink_runner.FlinkRunner', 'apache_beam.runners.portability.portable_runner.PortableRunner', + 'apache_beam.runners.portability.prism_runner.PrismRunner', 'apache_beam.runners.portability.spark_runner.SparkRunner', 'apache_beam.runners.test.TestDirectRunner', 'apache_beam.runners.test.TestDataflowRunner', @@ -1707,6 +1708,24 @@ class SparkRunnerOptions(PipelineOptions): help='Spark major version to use.') +class PrismRunnerOptions(PipelineOptions): + @classmethod + def _add_argparse_args(cls, parser): +parser.add_argument( +'--prism_location', +help='Path or URL to a prism binary, or zipped binary for the current ' +'platform (Operating System and Architecture). May also be an Apache ' +'Beam Github Release page URL, with a matching beam_version_override ' +'set. This option overrides all others for finding a prism binary.') +parser.add_argument( +'--prism_beam_version_override', +help= +'Override the SDK\'s version for deriving the Github Release URLs for ' +'downloading a zipped prism binary, for the current platform. If ' +'prism_location is set to a Github Release page URL, them it will use ' +'that release page as a base when constructing the download URL.') + + class TestOptions(PipelineOptions): @classmethod def _add_argparse_args(cls, parser): diff --git a/sdks/python/apache_beam/runners/portability/prism_runner.py b/sdks/python/apache_beam/runners/portability/prism_runner.py new file mode 100644 index 000..eeccaf5748c --- /dev/null +++ b/sdks/python/apache_beam/runners/portability/prism_runner.py @@ -0,0 +1,216 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A runner for executing portable pipelines on Apache Beam Prism.""" + +# this will make using the list parameterized generic happy +# on python 3.8 so we aren't revisiting this code after we +# sunset it +from __future__ import annotations + +import logging +import os +import platform +import shutil +import stat +import typing +import urllib +import zipfile +from urllib.error import URLError +from urllib.request import urlopen + +from apache_beam.io.filesystems import FileSystems +from apache_beam.options import pipeline_options +from apache_beam.runners.portability import job_server +from apache_beam.runners.portability import portable_runner +from apache_beam.transforms import environments +from apache_beam.utils import subprocess_server +from apache_beam.version import __version__ as beam_version + +# pytype: skip-file + +# Prefix for constructing a download URL +GITHUB_DOWNLOAD_PREFIX = 'https://github.com/apache/beam/releases/download/' +# Prefix for constructing a release URL, so we can derive a download URL +GITHUB_TAG_PREFIX = 'https://github.com/apache/beam/releases/tag/' + +_LOGGER = logging.getLogger(__name__) + + +class PrismRunn
(beam) branch master updated: Bump braces from 3.0.2 to 3.0.3 in /sdks/typescript (#31664)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 2cc2b8e4488 Bump braces from 3.0.2 to 3.0.3 in /sdks/typescript
(#31664)
2cc2b8e4488 is described below
commit 2cc2b8e448833d391d654503704a884f4b5f4e88
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
AuthorDate: Thu Jun 27 13:36:31 2024 -0700
Bump braces from 3.0.2 to 3.0.3 in /sdks/typescript (#31664)
Bumps [braces](https://github.com/micromatch/braces) from 3.0.2 to 3.0.3.
- [Changelog](https://github.com/micromatch/braces/blob/master/CHANGELOG.md)
- [Commits](https://github.com/micromatch/braces/compare/3.0.2...3.0.3)
---
updated-dependencies:
- dependency-name: braces
dependency-type: indirect
...
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot]
<49699333+dependabot[bot]@users.noreply.github.com>
---
sdks/typescript/package-lock.json | 32
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/sdks/typescript/package-lock.json
b/sdks/typescript/package-lock.json
index 1eb77b74800..4b58d334c97 100644
--- a/sdks/typescript/package-lock.json
+++ b/sdks/typescript/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "apache-beam",
- "version": "2.54.0-SNAPSHOT",
+ "version": "2.58.0-SNAPSHOT",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "apache-beam",
- "version": "2.54.0-SNAPSHOT",
+ "version": "2.58.0-SNAPSHOT",
"dependencies": {
"@google-cloud/pubsub": "^2.19.4",
"@grpc/grpc-js": "~1.4.6",
@@ -1033,12 +1033,12 @@
}
},
"node_modules/braces": {
- "version": "3.0.2",
- "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz;,
- "integrity":
"sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz;,
+ "integrity":
"sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
"dev": true,
"dependencies": {
-"fill-range": "^7.0.1"
+"fill-range": "^7.1.1"
},
"engines": {
"node": ">=8"
@@ -1845,9 +1845,9 @@
}
},
"node_modules/fill-range": {
- "version": "7.0.1",
- "resolved":
"https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz;,
- "integrity":
"sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+ "version": "7.1.1",
+ "resolved":
"https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz;,
+ "integrity":
"sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
"dev": true,
"dependencies": {
"to-regex-range": "^5.0.1"
@@ -4789,12 +4789,12 @@
}
},
"braces": {
- "version": "3.0.2",
- "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz;,
- "integrity":
"sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz;,
+ "integrity":
"sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
"dev": true,
"requires": {
-"fill-range": "^7.0.1"
+"fill-range": "^7.1.1"
}
},
"browser-stdout": {
@@ -5402,9 +5402,9 @@
}
},
"fill-range": {
- "version": "7.0.1",
- "resolved":
"https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz;,
- "integrity":
"sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+ "version": "7.1.1",
+ "resolved":
"https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz;,
+ "integrity":
"sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
"dev": true,
"requires": {
"to-regex-range": "^5.0.1"
(beam) branch master updated (bb296e46c06 -> ba04213af09)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from bb296e46c06 Create CsvIOParseError data class (#31700) add ba04213af09 [#28187][Prism] Relax or fix issues in Prism to allow Python pipelines to execute. (#31694) No new revisions were added by this update. Summary of changes: sdks/go/pkg/beam/runners/prism/internal/coders.go | 25 ++ .../beam/runners/prism/internal/handlerunner.go| 15 + .../prism/internal/jobservices/management.go | 15 ++--- .../pkg/beam/runners/prism/internal/preprocess.go | 5 - sdks/go/pkg/beam/runners/prism/internal/stage.go | 18 +++- .../runners/prism/internal/unimplemented_test.go | 10 + .../beam/runners/prism/internal/worker/worker.go | 11 +- 7 files changed, 85 insertions(+), 14 deletions(-)
(beam) branch master updated (8ec0de58f85 -> 832712ac053)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 8ec0de58f85 Bump github.com/spf13/cobra from 1.8.0 to 1.8.1 in /sdks (#31612) add 832712ac053 Bump github.com/tetratelabs/wazero from 1.7.0 to 1.7.3 in /sdks (#31672) No new revisions were added by this update. Summary of changes: sdks/go.mod | 2 +- sdks/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-)
(beam) branch master updated: Bump github.com/spf13/cobra from 1.8.0 to 1.8.1 in /sdks (#31612)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 8ec0de58f85 Bump github.com/spf13/cobra from 1.8.0 to 1.8.1 in /sdks (#31612) 8ec0de58f85 is described below commit 8ec0de58f85c76fb475ebcb5364ca70360bbff9b Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Thu Jun 27 09:54:04 2024 -0700 Bump github.com/spf13/cobra from 1.8.0 to 1.8.1 in /sdks (#31612) Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.8.0 to 1.8.1. - [Release notes](https://github.com/spf13/cobra/releases) - [Commits](https://github.com/spf13/cobra/compare/v1.8.0...v1.8.1) --- updated-dependencies: - dependency-name: github.com/spf13/cobra dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 2 +- sdks/go.sum | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index 727094e3931..60ce2baba38 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -48,7 +48,7 @@ require ( github.com/nats-io/nats-server/v2 v2.10.12 github.com/nats-io/nats.go v1.33.1 github.com/proullon/ramsql v0.1.3 - github.com/spf13/cobra v1.8.0 + github.com/spf13/cobra v1.8.1 github.com/testcontainers/testcontainers-go v0.26.0 github.com/tetratelabs/wazero v1.7.0 github.com/xitongsys/parquet-go v1.6.2 diff --git a/sdks/go.sum b/sdks/go.sum index 2701c133651..b04d350c63c 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -758,7 +758,7 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -1124,8 +1124,8 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
(beam) branch master updated: Bump cloud.google.com/go/datastore from 1.17.0 to 1.17.1 in /sdks (#31695)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 17a7b3ef398 Bump cloud.google.com/go/datastore from 1.17.0 to 1.17.1 in /sdks (#31695) 17a7b3ef398 is described below commit 17a7b3ef398e1c267362ade32967036f902c6ef6 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Thu Jun 27 09:50:26 2024 -0700 Bump cloud.google.com/go/datastore from 1.17.0 to 1.17.1 in /sdks (#31695) Bumps [cloud.google.com/go/datastore](https://github.com/googleapis/google-cloud-go) from 1.17.0 to 1.17.1. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/documentai/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/kms/v1.17.0...kms/v1.17.1) --- updated-dependencies: - dependency-name: cloud.google.com/go/datastore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 2 +- sdks/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index e7443172de0..727094e3931 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -25,7 +25,7 @@ go 1.20 require ( cloud.google.com/go/bigquery v1.61.0 cloud.google.com/go/bigtable v1.25.0 - cloud.google.com/go/datastore v1.17.0 + cloud.google.com/go/datastore v1.17.1 cloud.google.com/go/profiler v0.4.0 cloud.google.com/go/pubsub v1.38.0 cloud.google.com/go/spanner v1.63.0 diff --git a/sdks/go.sum b/sdks/go.sum index 19fd97db5e7..2701c133651 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -237,8 +237,8 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= -cloud.google.com/go/datastore v1.17.0 h1:UEmzuUdyDE58HV2jcb0BoqwCAwsJS2mtHapCsMmhVh0= -cloud.google.com/go/datastore v1.17.0/go.mod h1:RiRZU0G6VVlIVlv1HRo3vSAPFHULV0ddBNsXO+Sony4= +cloud.google.com/go/datastore v1.17.1 h1:6Me8ugrAOAxssGhSo8im0YSuy4YvYk4mbGvCadAH5aE= +cloud.google.com/go/datastore v1.17.1/go.mod h1:mtzZ2HcVtz90OVrEXXGDc2pO4NM1kiBQy8YV4qGe0ZM= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g=
(beam) branch master updated: [#31403] Relax prism constraints to allow python wordcount to execute. (#31644)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 29285d049f0 [#31403] Relax prism constraints to allow python wordcount
to execute. (#31644)
29285d049f0 is described below
commit 29285d049f06968a07968e69dd69668510f20546
Author: Robert Burke
AuthorDate: Thu Jun 20 14:43:35 2024 -0700
[#31403] Relax prism constraints to allow python wordcount to execute.
(#31644)
Co-authored-by: lostluck <13907733+lostl...@users.noreply.github.com>
---
.../prism/internal/jobservices/management.go | 67 +++---
1 file changed, 60 insertions(+), 7 deletions(-)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
index 3526ee00cc1..5760ce7871b 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
@@ -127,6 +127,9 @@ func (s *Server) Prepare(ctx context.Context, req
*jobpb.PrepareJobRequest) (*jo
urns.TransformCombinePerKey,
urns.TransformCombineGlobally, // Used by Java SDK
urns.TransformCombineGroupedValues, // Used by Java SDK
+ urns.TransformMerge,// Used directly by
Python SDK if "pre-optimized"
+ urns.TransformPreCombine, // Used directly by
Python SDK if "pre-optimized"
+ urns.TransformExtract, // Used directly by
Python SDK if "pre-optimized"
urns.TransformAssignWindows:
// Very few expected transforms types for submitted pipelines.
// Most URNs are for the runner to communicate back to the SDK
for execution.
@@ -165,12 +168,6 @@ func (s *Server) Prepare(ctx context.Context, req
*jobpb.PrepareJobRequest) (*jo
check("OnWindowExpirationTimerFamily",
pardo.GetOnWindowExpirationTimerFamilySpec(), "") // Unsupported for now.
- case "":
- // Composites can often have no spec
- if len(t.GetSubtransforms()) > 0 {
- continue
- }
- fallthrough
case urns.TransformTestStream:
var testStream pipepb.TestStreamPayload
if err := proto.Unmarshal(t.GetSpec().GetPayload(),
); err != nil {
@@ -179,7 +176,15 @@ func (s *Server) Prepare(ctx context.Context, req
*jobpb.PrepareJobRequest) (*jo
t.EnvironmentId = "" // Unset the environment, to
ensure it's handled prism side.
testStreamIds = append(testStreamIds, tid)
+
default:
+ // Composites can often have some unknown urn, permit
those.
+ // Eg. The Python SDK has urns
"beam:transform:generic_composite:v1", "beam:transform:pickled_python:v1", as
well as the deprecated "beam:transform:read:v1",
+ // but they are composites. Since we don't do anything
special with the high level, we simply use their internal subgraph.
+ if len(t.GetSubtransforms()) > 0 {
+ continue
+ }
+ // But if not, fail.
check("PTransform.Spec.Urn", urn+" "+t.GetUniqueName(),
"")
}
}
@@ -191,7 +196,8 @@ func (s *Server) Prepare(ctx context.Context, req
*jobpb.PrepareJobRequest) (*jo
// Inspect Windowing strategies for unsupported features.
for wsID, ws := range
job.Pipeline.GetComponents().GetWindowingStrategies() {
check("WindowingStrategy.AllowedLateness",
ws.GetAllowedLateness(), int64(0))
- check("WindowingStrategy.ClosingBehaviour",
ws.GetClosingBehavior(), pipepb.ClosingBehavior_EMIT_IF_NONEMPTY)
+ // Both Closing behaviors are identical without additional
trigger firings.
+ check("WindowingStrategy.ClosingBehaviour",
ws.GetClosingBehavior(), pipepb.ClosingBehavior_EMIT_IF_NONEMPTY,
pipepb.ClosingBehavior_EMIT_ALWAYS)
check("WindowingStrategy.AccumulationMode",
ws.GetAccumulationMode(), pipepb.AccumulationMode_DISCARDING)
if ws.GetWindowFn().GetUrn() != urns.WindowFnSession {
check("WindowingStrategy.MergeStatus",
ws.GetMergeStatus(), pipepb.MergeStatus_NON_MERGING)
@@ -398,3 +404,50 @@ func (s *Server) GetState(_
(beam) branch lostluck-patch-3 deleted (was de51848cf3e)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch lostluck-patch-3 in repository https://gitbox.apache.org/repos/asf/beam.git was de51848cf3e Update build_release_candidate.yml The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(beam) branch master updated: Update build_release_candidate.yml (#31638)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 3ed91c880f8 Update build_release_candidate.yml (#31638)
3ed91c880f8 is described below
commit 3ed91c880f85d09a45039e70d5136f1c2324916d
Author: Robert Burke
AuthorDate: Tue Jun 18 11:25:40 2024 -0700
Update build_release_candidate.yml (#31638)
Fix issue where the release version directory might not exist yet for the
SVN repo when the prism build step occurs.
```
...Adding prism artifacts to the Dev Apache SVN repo...
svn: E15: Can't find parent directory's node while trying to add
'/home/runner/work/beam/beam/beam/2.57.0/prism'
svn: E155010: The node '/home/runner/work/beam/beam/beam/2.57.0' was not
found.
```
Adding --parents makes SVN also add intermediate parents before making the
update.
---
.github/workflows/build_release_candidate.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/build_release_candidate.yml
b/.github/workflows/build_release_candidate.yml
index ab4b3859177..215f2566634 100644
--- a/.github/workflows/build_release_candidate.yml
+++ b/.github/workflows/build_release_candidate.yml
@@ -530,6 +530,6 @@ jobs:
echo "...Adding prism artifacts to the Dev Apache SVN
repo..."
- svn add --force prism
+ svn add --force --parents prism
svn status
svn commit -m "Staging Prism artifacts for Apache Beam ${RELEASE}
RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID
}}" --password "${{ github.event.inputs.APACHE_PASSWORD }}"
(beam) branch lostluck-patch-3 created (now de51848cf3e)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch lostluck-patch-3 in repository https://gitbox.apache.org/repos/asf/beam.git at de51848cf3e Update build_release_candidate.yml This branch includes the following new commits: new de51848cf3e Update build_release_candidate.yml The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(beam) 01/01: Update build_release_candidate.yml
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch lostluck-patch-3
in repository https://gitbox.apache.org/repos/asf/beam.git
commit de51848cf3e74b30dac87e70e8761a7721cc3206
Author: Robert Burke
AuthorDate: Tue Jun 18 10:36:19 2024 -0700
Update build_release_candidate.yml
Fix issue where the release version directory might not exist yet for the
SVN repo when the prism build step occurs.
```
...Adding prism artifacts to the Dev Apache SVN repo...
svn: E15: Can't find parent directory's node while trying to add
'/home/runner/work/beam/beam/beam/2.57.0/prism'
svn: E155010: The node '/home/runner/work/beam/beam/beam/2.57.0' was not
found.
```
Adding --parents makes SVN also add intermediate parents before making the
update.
---
.github/workflows/build_release_candidate.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/build_release_candidate.yml
b/.github/workflows/build_release_candidate.yml
index ab4b3859177..215f2566634 100644
--- a/.github/workflows/build_release_candidate.yml
+++ b/.github/workflows/build_release_candidate.yml
@@ -530,6 +530,6 @@ jobs:
echo "...Adding prism artifacts to the Dev Apache SVN
repo..."
- svn add --force prism
+ svn add --force --parents prism
svn status
svn commit -m "Staging Prism artifacts for Apache Beam ${RELEASE}
RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID
}}" --password "${{ github.event.inputs.APACHE_PASSWORD }}"
(beam) branch master updated (5feba0dc1ec -> 403ad563ed5)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 5feba0dc1ec Allow compression of Elasticsearch requests (#31601) add 403ad563ed5 Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks (#31613) No new revisions were added by this update. Summary of changes: sdks/go.mod | 20 ++-- sdks/go.sum | 40 2 files changed, 30 insertions(+), 30 deletions(-)
(beam) branch master updated: [#29697] Update the Release guide with currently manual GitHub release steps for prism. (#31582)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 791d55790da [#29697] Update the Release guide with currently manual
GitHub release steps for prism. (#31582)
791d55790da is described below
commit 791d55790da2aaa39e0fa443ecdfd32eb22bfc71
Author: Robert Burke
AuthorDate: Thu Jun 13 15:32:27 2024 -0700
[#29697] Update the Release guide with currently manual GitHub release
steps for prism. (#31582)
---
contributor-docs/release-guide.md | 42 +++
1 file changed, 38 insertions(+), 4 deletions(-)
diff --git a/contributor-docs/release-guide.md
b/contributor-docs/release-guide.md
index 52ce3fb0a29..c0e8e7c67ce 100644
--- a/contributor-docs/release-guide.md
+++ b/contributor-docs/release-guide.md
@@ -528,9 +528,36 @@ The following should be confirmed:
- [ ] There is a commit not on the release branch with the version adjusted.
- [ ] The RC tag points to that commit.
+### Create a draft, pre-release Github release for the RC Tag
+
+TODO: Automate these steps as a github action.
+
+If this is for the first release candidate, create a new, draft, pre-release
Github release.
+
+* Go to https://github.com/apache/beam/releases/new to start creating a Github
release.
+
+If this is for subsequent release candidates re-use the existing Github
release for this version.
+
+* Do not create a new release if one already exists, navigate to the existing
Github release for the previous RC.
+
+Once on the release page:
+
+* Update the Release tag to the current RC Tag.
+* Title the release "Beam ${RELEASE_VERSION} release".
+* The description may remain empty for now, but will eventually contain the
release blog post.
+* Set this release as a pre-release, by checking the `Set as pre-release` box
below the description box.
+
+Once configured properly, press the `Save draft` button.
+
+The following should be confirmed:
+
+- [ ] The Github release is configured as a draft, pre-release.
+- [ ] The Github release points to the current RC tag.
+
### Run build_release_candidate GitHub Action to create a release candidate
**Action**
[build_release_candidate](https://github.com/apache/beam/actions/workflows/build_release_candidate.yml)
(click `run workflow`)
+and update the JSON configuration fields with "yes".
**The action will:**
@@ -542,11 +569,15 @@ The following should be confirmed:
5. Build javadoc, pydoc, typedocs for a PR to update beam-site.
- **NOTE**: Do not merge this PR until after an RC has been approved (see
"Finalize the Release").
+6. Build Prism binaries for various platforms, and upload them into
[dist.apache.org](https://dist.apache.org/repos/dist/dev/beam)
+ and the Github Release with the matching RC tag.
-### Verify source distributions
+### Verify source and artifact distributions
- [ ] Verify that the source zip of the whole project is present in
[dist.apache.org](https://dist.apache.org/repos/dist/dev/beam).
- [ ] Verify that the Python binaries are present in
[dist.apache.org](https://dist.apache.org/repos/dist/dev/beam).
+ - [ ] Verify that the Prism binaries are present in
[dist.apache.org](https://dist.apache.org/repos/dist/dev/beam).
+ - [ ] Verify that the Prism binaries are attached to the Github Release
created in the previous step.
### Verify docker images
@@ -1189,9 +1220,12 @@ Merge all of the website pull requests
### Publish release to Github
-Once the tag is uploaded, publish the release notes to Github. From the [Beam
release page on Github](https://github.com/apache/beam/releases) select
-"Draft a new release." Title the release "Beam ${RELEASE_VERSION} release" and
set the release at the version tag created above. Use the content of the
-release blog post as the body of the release notes, set this version as the
latest release, and publish it.
+Once the tag is uploaded, publish the release notes to Github.
+From the [Beam release page on Github](https://github.com/apache/beam/releases)
+find and open the release for the final RC tag for for editing.
+Update the release with the final version tag created above.
+Use the content of the release blog post as the body of the release notes,
+set this version as the latest release, and publish it.
The release notes should now be visible on Github's
[Releases](https://github.com/apache/beam/releases) page.
(beam) branch master updated: Skip newly added TestStream VR tests for flink runner (#31538)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e5c2f6c5980 Skip newly added TestStream VR tests for flink runner
(#31538)
e5c2f6c5980 is described below
commit e5c2f6c59801569a45b28792b18eda142d19508d
Author: Yi Hu
AuthorDate: Fri Jun 7 00:54:23 2024 -0400
Skip newly added TestStream VR tests for flink runner (#31538)
---
...eam_PostCommit_XVR_Flink.json => beam_PostCommit_Go_VR_Flink.json} | 0
sdks/go/test/integration/integration.go | 4
2 files changed, 4 insertions(+)
diff --git a/.github/trigger_files/beam_PostCommit_XVR_Flink.json
b/.github/trigger_files/beam_PostCommit_Go_VR_Flink.json
similarity index 100%
rename from .github/trigger_files/beam_PostCommit_XVR_Flink.json
rename to .github/trigger_files/beam_PostCommit_Go_VR_Flink.json
diff --git a/sdks/go/test/integration/integration.go
b/sdks/go/test/integration/integration.go
index 12f7fdb6bd8..aec69036eeb 100644
--- a/sdks/go/test/integration/integration.go
+++ b/sdks/go/test/integration/integration.go
@@ -183,6 +183,10 @@ var flinkFilters = []string{
"TestTestStreamByteSliceSequence",
"TestTestStreamTwoUserTypeSequences",
"TestTestStreamInt16Sequence",
+ "TestTestStreamSimple",
+ "TestTestStreamSimple_InfinityDefault",
+ "TestTestStreamToGBK",
+ "TestTestStreamTimersEventTime",
"TestTimers_EventTime_Unbounded", // (failure when comparing on side
inputs (NPE on window lookup))
"TestTimers_ProcessingTime.*",// Flink doesn't support processing
time timers.
(beam) branch master updated: filter out test stream using test from spark suite (#31509)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new c37107a6ea6 filter out test stream using test from spark suite (#31509)
c37107a6ea6 is described below
commit c37107a6ea6bd55fb09cb836e1cff250483fe161
Author: Robert Burke
AuthorDate: Wed Jun 5 18:45:44 2024 -0700
filter out test stream using test from spark suite (#31509)
Co-authored-by: lostluck <13907733+lostl...@users.noreply.github.com>
---
sdks/go/test/integration/integration.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/sdks/go/test/integration/integration.go
b/sdks/go/test/integration/integration.go
index d8aee63718c..12f7fdb6bd8 100644
--- a/sdks/go/test/integration/integration.go
+++ b/sdks/go/test/integration/integration.go
@@ -259,7 +259,8 @@ var sparkFilters = []string{
"TestSetStateClear",
"TestSetState",
- "TestTimers_EventTime_Unbounded", // Side inputs in executable stage
not supported.
+ "TestTimers_EventTime_Unbounded", // Side inputs in executable
stage not supported.
+ "TestTimers_ProcessingTime_Infinity", // Spark doesn't support test
stream.
}
var dataflowFilters = []string{
(beam) branch release-2.57.0 updated: Update Go container build version to 1.21.11 (#31516)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch release-2.57.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.57.0 by this push:
new 9806447f2ee Update Go container build version to 1.21.11 (#31516)
9806447f2ee is described below
commit 9806447f2ee22dd33df52b83f0a65ca18fcdaec9
Author: Robert Burke
AuthorDate: Wed Jun 5 11:32:53 2024 -0700
Update Go container build version to 1.21.11 (#31516)
---
buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 7c692fcb3f4..493be3d7cc0 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2224,7 +2224,7 @@ class BeamModulePlugin implements Plugin {
// This sets the whole project Go version.
// The latest stable Go version can be checked at https://go.dev/dl/
- project.ext.goVersion = "go1.21.8"
+ project.ext.goVersion = "go1.21.11"
// Minor TODO: Figure out if we can pull out the GOCMD env variable
after goPrepare script
// completion, and avoid this GOBIN substitution.
(beam) branch lostluck-patch-3 deleted (was 73cdbaf92aa)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch lostluck-patch-3 in repository https://gitbox.apache.org/repos/asf/beam.git was 73cdbaf92aa Update Go container build version to 1.21.11 The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(beam) branch master updated: Update Go container build version to 1.21.11 (#31515)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9c9de4919b5 Update Go container build version to 1.21.11 (#31515)
9c9de4919b5 is described below
commit 9c9de4919b54e57f3b814a61203d1efee4a092c7
Author: Robert Burke
AuthorDate: Wed Jun 5 11:27:15 2024 -0700
Update Go container build version to 1.21.11 (#31515)
---
buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 7c692fcb3f4..493be3d7cc0 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2224,7 +2224,7 @@ class BeamModulePlugin implements Plugin {
// This sets the whole project Go version.
// The latest stable Go version can be checked at https://go.dev/dl/
- project.ext.goVersion = "go1.21.8"
+ project.ext.goVersion = "go1.21.11"
// Minor TODO: Figure out if we can pull out the GOCMD env variable
after goPrepare script
// completion, and avoid this GOBIN substitution.
(beam) branch lostluck-patch-3 created (now 73cdbaf92aa)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch lostluck-patch-3 in repository https://gitbox.apache.org/repos/asf/beam.git at 73cdbaf92aa Update Go container build version to 1.21.11 This branch includes the following new commits: new 73cdbaf92aa Update Go container build version to 1.21.11 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(beam) 01/01: Update Go container build version to 1.21.11
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch lostluck-patch-3
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 73cdbaf92aa097d1db1d6a4244fc84efb4b60662
Author: Robert Burke
AuthorDate: Wed Jun 5 11:05:08 2024 -0700
Update Go container build version to 1.21.11
---
buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 7c692fcb3f4..493be3d7cc0 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2224,7 +2224,7 @@ class BeamModulePlugin implements Plugin {
// This sets the whole project Go version.
// The latest stable Go version can be checked at https://go.dev/dl/
- project.ext.goVersion = "go1.21.8"
+ project.ext.goVersion = "go1.21.11"
// Minor TODO: Figure out if we can pull out the GOCMD env variable
after goPrepare script
// completion, and avoid this GOBIN substitution.
(beam) branch master updated: [#27839][Go SDK] Write pipeline options to a file, instead reading from a flag. (#31482)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e31e8855ad9 [#27839][Go SDK] Write pipeline options to a file, instead
reading from a flag. (#31482)
e31e8855ad9 is described below
commit e31e8855ad9a7767c79700e7cd5ea31a419a7997
Author: Robert Burke
AuthorDate: Tue Jun 4 05:17:22 2024 -0700
[#27839][Go SDK] Write pipeline options to a file, instead reading from a
flag. (#31482)
* [#27839] Move pipeline options file creation to tools package.
* Write options to a file in the container instead of burdening the command
line.
-
Co-authored-by: lostluck <13907733+lostl...@users.noreply.github.com>
---
sdks/go/container/boot.go | 4 ++-
sdks/go/container/tools/pipeline_options.go| 39 ++
sdks/go/pkg/beam/core/runtime/harness/init/init.go | 17 +-
sdks/java/container/boot.go| 24 +++--
4 files changed, 62 insertions(+), 22 deletions(-)
diff --git a/sdks/go/container/boot.go b/sdks/go/container/boot.go
index 1a5e154ace7..15f9ecc101c 100644
--- a/sdks/go/container/boot.go
+++ b/sdks/go/container/boot.go
@@ -137,7 +137,9 @@ func main() {
"--logging_endpoint=" + *loggingEndpoint,
"--control_endpoint=" + *controlEndpoint,
"--semi_persist_dir=" + *semiPersistDir,
- "--options=" + options,
+ }
+ if err := tools.MakePipelineOptionsFileAndEnvVar(options); err != nil {
+ logger.Fatalf(ctx, "Failed to load pipeline options to worker:
%v", err)
}
if info.GetStatusEndpoint() != nil {
os.Setenv("STATUS_ENDPOINT", info.GetStatusEndpoint().GetUrl())
diff --git a/sdks/go/container/tools/pipeline_options.go
b/sdks/go/container/tools/pipeline_options.go
new file mode 100644
index 000..7b46d8fa8c8
--- /dev/null
+++ b/sdks/go/container/tools/pipeline_options.go
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+//http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tools
+
+import (
+ "fmt"
+ "os"
+)
+
+// MakePipelineOptionsFileAndEnvVar writes the pipeline options to a file.
+// Assumes the options string is JSON formatted.
+//
+// Stores the file name in question in PIPELINE_OPTIONS_FILE for access by the
SDK.
+func MakePipelineOptionsFileAndEnvVar(options string) error {
+ fn := "pipeline_options.json"
+ f, err := os.Create(fn)
+ if err != nil {
+ return fmt.Errorf("unable to create %v: %w", fn, err)
+ }
+ defer f.Close()
+ if _, err := f.WriteString(options); err != nil {
+ return fmt.Errorf("error writing %v: %w", f.Name(), err)
+ }
+ os.Setenv("PIPELINE_OPTIONS_FILE", f.Name())
+ return nil
+}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/init/init.go
b/sdks/go/pkg/beam/core/runtime/harness/init/init.go
index 468708b2917..8a5b45fea5e 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/init/init.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/init/init.go
@@ -51,7 +51,7 @@ var (
controlEndpoint = flag.String("control_endpoint", "", "Local control
gRPC endpoint (required in worker mode).")
//lint:ignore U1000 semiPersistDir flag is passed in through the boot
container, will need to be removed later
semiPersistDir = flag.String("semi_persist_dir", "/tmp", "Local
semi-persistent directory (optional in worker mode).")
- options= flag.String("options", "", "JSON-encoded pipeline
options (required in worker mode).")
+ options= flag.String("options", "", "JSON-encoded pipeline
options (required in worker mode). (deprecated)")
)
type exitMode int
@@ -93,6 +93,21 @@ func hook() {
// will be captured by
(beam) branch master updated: [#30083] Add synthetic processing time to prism. (#30492)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 4daedbf5a8a [#30083] Add synthetic processing time to prism. (#30492)
4daedbf5a8a is described below
commit 4daedbf5a8ad762bc7e8c230a4b1c7c3fe13bab5
Author: Robert Burke
AuthorDate: Tue May 28 20:48:36 2024 -0700
[#30083] Add synthetic processing time to prism. (#30492)
* [prism] Add basic processing time queue.
* Initial residual handling refactor.
* Re-work teststream initilization. Remove pending element race.
* touch up
* rm merge duplicate
* Simplify watermark hold tracking.
* First successful run!
* Remove duplicated test run.
* Deduplicate processing time heap.
* rm debug text
* Remove some debug prints, cleanup.
* tiny todo cleanup
* ProcessingTime workming most of the time!
* Some cleanup
* try to get github suite to pass #1
* touch
* reduce counts a bit, filter tests some.
* Clean up unrelated state changes. Clean up comments somewhat.
* Filter out dataflow incompatible test.
* Refine processing time event comment.
* Remove test touch.
-
Co-authored-by: lostluck <13907733+lostl...@users.noreply.github.com>
---
.../prism/internal/engine/elementmanager.go| 284 +---
.../runners/prism/internal/engine/engine_test.go | 43 +++
.../beam/runners/prism/internal/engine/holds.go| 39 +--
.../prism/internal/engine/processingtime.go| 96 +++
.../prism/internal/engine/processingtime_test.go | 139 ++
.../runners/prism/internal/engine/teststream.go| 12 +-
.../beam/runners/prism/internal/engine/timers.go | 166
.../runners/prism/internal/engine/timers_test.go | 291 +
sdks/go/pkg/beam/runners/prism/internal/execute.go | 10 +-
.../prism/internal/jobservices/management.go | 7 +-
sdks/go/pkg/beam/runners/prism/internal/stage.go | 25 +-
.../beam/runners/prism/internal/worker/worker.go | 4 +-
sdks/go/test/integration/integration.go| 7 +-
sdks/go/test/integration/primitives/timers.go | 151 +++
sdks/go/test/integration/primitives/timers_test.go | 10 +-
15 files changed, 1206 insertions(+), 78 deletions(-)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
index 5d665edf286..76c60e810d4 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
@@ -169,6 +169,7 @@ type ElementManager struct {
livePending atomic.Int64 // An accessible live pending count.
DEBUG USE ONLY
pendingElements sync.WaitGroup // pendingElements counts all
unprocessed elements in a job. Jobs with no pending elements terminate
successfully.
+ processTimeEvents *stageRefreshQueue // Manages sequence of stage
updates when interfacing with processing time.
testStreamHandler *testStreamHandler // Optional test stream handler
when a test stream is in the pipeline.
}
@@ -192,6 +193,7 @@ func NewElementManager(config Config) *ElementManager {
watermarkRefreshes: set[string]{},
inprogressBundles: set[string]{},
refreshCond:sync.Cond{L: {}},
+ processTimeEvents: newStageRefreshQueue(),
}
}
@@ -227,6 +229,11 @@ func (em *ElementManager) StageStateful(ID string) {
em.stages[ID].stateful = true
}
+// StageProcessingTimeTimers indicates which timers are processingTime domain
timers.
+func (em *ElementManager) StageProcessingTimeTimers(ID string, ptTimers
map[string]bool) {
+ em.stages[ID].processingTimeTimersFamilies = ptTimers
+}
+
// AddTestStream provides a builder interface for the execution layer to build
the test stream from
// the protos.
func (em *ElementManager) AddTestStream(id string, tagToPCol
map[string]string) TestStreamBuilder {
@@ -305,8 +312,13 @@ func (em *ElementManager) Bundles(ctx context.Context,
nextBundID func() string)
for {
em.refreshCond.L.Lock()
- // If there are no watermark refreshes available, we
wait until there are.
- for len(em.watermarkRefreshes) == 0 {
+ // Check if processing time has advanced before the
wait loop.
+ emNow := em.ProcessingTimeNow()
+ ptRefreshed := em.processTimeEvents.AdvanceTo(emNow)
+ em.watermarkRefreshes.merge(ptRefreshed)
+
+ // If there are no wat
(beam) branch prismBuildRelease deleted (was 524762d25b8)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git was 524762d25b8 needs f The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(beam) branch master updated: [#29697] Add prism artifact building workflow. (#31369)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7d281558dd8 [#29697] Add prism artifact building workflow. (#31369)
7d281558dd8 is described below
commit 7d281558dd8e7900e4422612479a051776364aec
Author: Robert Burke
AuthorDate: Tue May 28 20:14:40 2024 -0700
[#29697] Add prism artifact building workflow. (#31369)
---
.github/workflows/build_release_candidate.yml | 188 ++
1 file changed, 160 insertions(+), 28 deletions(-)
diff --git a/.github/workflows/build_release_candidate.yml
b/.github/workflows/build_release_candidate.yml
index b9e62c35aad..37a70fd2f8d 100644
--- a/.github/workflows/build_release_candidate.yml
+++ b/.github/workflows/build_release_candidate.yml
@@ -19,29 +19,35 @@ on:
APACHE_PASSWORD:
description: Your Apache password. Required if you want to stage
artifacts into https://dist.apache.org/repos/dist/dev/beam/
required: false
- BEAM_SITE_TOKEN:
-description: Github Personal Access Token with apache/beam-site repo
permission if you want to create the beam-site docs PR. See
https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens.
+ REPO_TOKEN:
+description: |
+ Github Personal Access Token with repo permissions if you want to
create the beam-site docs PR,
+ create a draft release, or upload prism artifacts to that release.
+ See
https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens.
default: ''
- PUBLISH_JAVA_ARTIFACTS:
-description: Whether to publish java artifacts to
https://repository.apache.org/#stagingRepositories (yes/no)
+ STAGE:
+description: |
+ Configure which actions this workflow should perform, by setting the
desired fields to "yes".
+ --
+ java_artifacts: publish java artifacts to
https://repository.apache.org/#stagingRepositories
+ --
+ java_source: java source into
https://dist.apache.org/repos/dist/dev/beam/
+ --
+ docker_artifacts: stage SDK docker images to docker hub Apache
organization
+ --
+ python_artifacts: stage the python artifacts into
https://dist.apache.org/repos/dist/dev/beam/
+ --
+ beam_site_pr: create the documentation update PR against
apache/beam-site.
+ --
+ prism: build and upload the artifacts to the release for this tag
required: true
-default: 'no'
- STAGE_JAVA_SOURCE:
-description: Whether to stage the java source into
https://dist.apache.org/repos/dist/dev/beam/
-required: true
-default: 'no'
- STAGE_DOCKER_ARTIFACTS:
-description: Whether to stage SDK docker images to docker hub Apache
organization
-required: true
-default: 'no'
- STAGE_PYTHON_ARTIFACTS:
-description: Whether to stage the python artifacts into
https://dist.apache.org/repos/dist/dev/beam/
-required: true
-default: 'no'
- CREATE_BEAM_SITE_PR:
-description: Whether to create the documentation update PR against
apache/beam-site.
-required: true
-default: 'no'
+default: |
+ {java_artifacts: "no",
+java_source: "no",
+docker_artifacts: "no",
+python_artifacts: "no",
+beam_site_pr: "no",
+prism: "no"}
env:
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
@@ -50,7 +56,7 @@ env:
jobs:
publish_java_artifacts:
-if: ${{github.event.inputs.PUBLISH_JAVA_ARTIFACTS == 'yes'}}
+if: ${{ fromJson(github.event.inputs.STAGE).java_artifacts == 'yes'}}
runs-on: [self-hosted, ubuntu-20.04, main]
steps:
- name: Checkout
@@ -92,7 +98,7 @@ jobs:
stage_java_source:
-if: ${{github.event.inputs.STAGE_JAVA_SOURCE == 'yes'}}
+if: ${{ fromJson(github.event.inputs.STAGE).java_source == 'yes'}}
runs-on: ubuntu-latest
steps:
- name: Mask Apache Password
@@ -156,7 +162,7 @@ jobs:
svn commit -m "Staging Java artifacts for Apache Beam ${{
github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}"
--non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password
"${{ github.event.inputs.APACHE_PASSWORD }}"
stage_python_artifacts:
-if: ${{github.event.inputs.STAGE_PYTHON_ARTIFACTS == 'yes'}}
+if: ${{ fromJson(github.event.inputs.STAGE).python_artifacts == 'yes'}}
runs-on: ubuntu-latest
steps:
- name: Checkout
@@ -239,7 +245,7 @@ jobs:
stage_docker:
-if: ${{gi
(beam) branch prismBuildRelease updated (7e80b74295f -> 524762d25b8)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 7e80b74295f rm -r star add 524762d25b8 needs f No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
(beam) branch prismBuildRelease updated (2946d31228b -> 7e80b74295f)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 2946d31228b rm .* isn't right add 7e80b74295f rm -r star No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (29daf166121 -> 2946d31228b)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 29daf166121 match the version tag. add 2946d31228b rm .* isn't right No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (2102a4f6625 -> 29daf166121)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 2102a4f6625 use underscore for apache_beam add 29daf166121 match the version tag. No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (412a082b35e -> 2102a4f6625)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 412a082b35e Remove version control info from prism builds. add 2102a4f6625 use underscore for apache_beam No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (6a5e6d9f25c -> 412a082b35e)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 6a5e6d9f25c fix beam_site_pr field add 412a082b35e Remove version control info from prism builds. No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (94d3ea7c96c -> 6a5e6d9f25c)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 94d3ea7c96c Fix root_dir add adac6268131 Clean up add 6a5e6d9f25c fix beam_site_pr field No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 37 ++- 1 file changed, 19 insertions(+), 18 deletions(-)
(beam) branch prismBuildRelease updated (f8984bd2bfa -> 94d3ea7c96c)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from f8984bd2bfa more debug thoughts add 94d3ea7c96c Fix root_dir No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 22 +- 1 file changed, 5 insertions(+), 17 deletions(-)
(beam) branch prismBuildRelease updated (3f633ed86c7 -> f8984bd2bfa)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 3f633ed86c7 remove breaks, retry svn add f8984bd2bfa more debug thoughts No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 4 1 file changed, 4 insertions(+)
(beam) branch prismBuildRelease updated (c5a2776bf94 -> 3f633ed86c7)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from c5a2776bf94 try breaks, build to SVN directory add 3f633ed86c7 remove breaks, retry svn No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 40 --- 1 file changed, 24 insertions(+), 16 deletions(-)
(beam) branch prismBuildRelease updated (44d9b2998d5 -> c5a2776bf94)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 44d9b2998d5 Update the description a bit. add c5a2776bf94 try breaks, build to SVN directory No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 76 +++ 1 file changed, 43 insertions(+), 33 deletions(-)
(beam) branch prismBuildRelease updated (9ac0b0dcd09 -> 44d9b2998d5)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 9ac0b0dcd09 Uodate configurations. add 44d9b2998d5 Update the description a bit. No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 14 +++--- 1 file changed, 7 insertions(+), 7 deletions(-)
(beam) branch prismBuildRelease updated (72f4bb77268 -> 9ac0b0dcd09)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 72f4bb77268 The direct approach add 9ac0b0dcd09 Uodate configurations. No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 72 +-- 1 file changed, 34 insertions(+), 38 deletions(-)
(beam) branch prismBuildRelease updated (a57bb2db820 -> 72f4bb77268)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from a57bb2db820 slash coninuations add 72f4bb77268 The direct approach No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 9 + 1 file changed, 1 insertion(+), 8 deletions(-)
(beam) branch prismBuildRelease updated (32d6222f340 -> a57bb2db820)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 32d6222f340 Upload to release! add a57bb2db820 slash coninuations No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (ccdea3a441c -> 32d6222f340)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from ccdea3a441c Retry with github token. add 32d6222f340 Upload to release! No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 33 +++ 1 file changed, 18 insertions(+), 15 deletions(-)
(beam) branch prismBuildRelease updated (2a755e5cec8 -> ccdea3a441c)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 2a755e5cec8 better jq add ccdea3a441c Retry with github token. No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (77802c77ec3 -> 2a755e5cec8)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 77802c77ec3 gh id add 2a755e5cec8 better jq No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 10 ++ 1 file changed, 2 insertions(+), 8 deletions(-)
(beam) branch prismBuildRelease updated (1305ac86ded -> 77802c77ec3)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 1305ac86ded bonus curl to check add 77802c77ec3 gh id No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
(beam) branch prismBuildRelease updated (0d26c3d1034 -> 1305ac86ded)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 0d26c3d1034 switch to curl add 1305ac86ded bonus curl to check No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 6 ++ 1 file changed, 6 insertions(+)
(beam) branch prismBuildRelease updated (2095d9ab2e5 -> 0d26c3d1034)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 2095d9ab2e5 explicit repo add 0d26c3d1034 switch to curl No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 13 + 1 file changed, 9 insertions(+), 4 deletions(-)
(beam) branch prismBuildRelease updated (d98896274df -> 2095d9ab2e5)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from d98896274df try release list add 2095d9ab2e5 explicit repo No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (6e5ec547fd1 -> d98896274df)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 6e5ec547fd1 Use the passed in token for gh access. add d98896274df try release list No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (5cb57d4948e -> 6e5ec547fd1)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 5cb57d4948e test json config add 6e5ec547fd1 Use the passed in token for gh access. No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 11 +++ 1 file changed, 7 insertions(+), 4 deletions(-)
(beam) branch prismBuildRelease updated (4491500bf10 -> 5cb57d4948e)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 4491500bf10 Different token env var add 5cb57d4948e test json config No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 7 +-- 1 file changed, 5 insertions(+), 2 deletions(-)
(beam) branch prismBuildRelease updated (77f5adfad28 -> 4491500bf10)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 77f5adfad28 But what if I use the "USER_TOKEN"? add 4491500bf10 Different token env var No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (aa3dcb45806 -> 77f5adfad28)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from aa3dcb45806 print tag for debugging add 77f5adfad28 But what if I use the "USER_TOKEN"? No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (3a115673e1e -> aa3dcb45806)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 3a115673e1e Use GH to find the release id for this tag. add aa3dcb45806 print tag for debugging No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (278285253be -> 3a115673e1e)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 278285253be fix leftover local flags add 3a115673e1e Use GH to find the release id for this tag. No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 7 +++ 1 file changed, 7 insertions(+)
(beam) branch prismBuildRelease updated (11461d22111 -> 278285253be)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 11461d22111 remove parsing completely. add 278285253be fix leftover local flags No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (dfccef5ca1a -> 11461d22111)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from dfccef5ca1a Remove parsing. add 11461d22111 remove parsing completely. No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 19 --- 1 file changed, 19 deletions(-)
(beam) branch prismBuildRelease updated (e5c4ee9ff20 -> dfccef5ca1a)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from e5c4ee9ff20 fix read command flags add dfccef5ca1a Remove parsing. No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 74 --- 1 file changed, 33 insertions(+), 41 deletions(-)
(beam) branch prismBuildRelease updated (a761aaacc0a -> e5c4ee9ff20)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from a761aaacc0a fix debug echos to not use backtics add e5c4ee9ff20 fix read command flags No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (4afe3195c20 -> a761aaacc0a)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 4afe3195c20 add more debuging add a761aaacc0a fix debug echos to not use backtics No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 8 1 file changed, 4 insertions(+), 4 deletions(-)
(beam) branch prismBuildRelease updated (6450fa23391 -> 4afe3195c20)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 6450fa23391 fix sdks directory add 4afe3195c20 add more debuging No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 9 + 1 file changed, 9 insertions(+)
(beam) branch prismBuildRelease updated (15af30a8030 -> 6450fa23391)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 15af30a8030 Actually checkout the repo. add 6450fa23391 fix sdks directory No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
(beam) branch prismBuildRelease updated (4f91db52fa3 -> 15af30a8030)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 4f91db52fa3 reduce dispate input set add 15af30a8030 Actually checkout the repo. No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 5 + 1 file changed, 5 insertions(+)
(beam) branch prismBuildRelease updated (980db343808 -> 4f91db52fa3)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git from 980db343808 Check for jq. add 4f91db52fa3 reduce dispate input set No new revisions were added by this update. Summary of changes: .github/workflows/build_release_candidate.yml | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-)
(beam) branch prismBuildRelease created (now 980db343808)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch prismBuildRelease in repository https://gitbox.apache.org/repos/asf/beam.git at 980db343808 Check for jq. No new revisions were added by this update.
(beam) branch master updated: Implement CoderNullable pullDecoderNoAlloc case (#31340)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new fed64891240 Implement CoderNullable pullDecoderNoAlloc case (#31340)
fed64891240 is described below
commit fed6489124000b3f222dc444136009ab22e4846e
Author: Damon
AuthorDate: Mon May 20 08:59:14 2024 -0700
Implement CoderNullable pullDecoderNoAlloc case (#31340)
---
sdks/go/pkg/beam/runners/prism/internal/coders.go | 14 ++
1 file changed, 14 insertions(+)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/coders.go
b/sdks/go/pkg/beam/runners/prism/internal/coders.go
index 6deaab65d36..b7157b8598d 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/coders.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/coders.go
@@ -212,6 +212,20 @@ func pullDecoderNoAlloc(c *pipepb.Coder, coders
map[string]*pipepb.Coder) func(i
l, _ := coder.DecodeVarInt(r)
ioutilx.ReadN(r, int(l))
}
+ case urns.CoderNullable:
+ return func(r io.Reader) {
+ b, _ := ioutilx.ReadN(r, 1)
+ if len(b) == 0 {
+ return
+ }
+ // Nullable coder is prefixed with 0 or 1 to indicate
whether there exists remaining data.
+ prefix := b[0]
+ if prefix == 0 {
+ return
+ }
+ l, _ := coder.DecodeVarInt(r)
+ ioutilx.ReadN(r, int(l))
+ }
case urns.CoderVarInt:
return func(r io.Reader) {
coder.DecodeVarInt(r)
(beam) branch master updated: Bump google.golang.org/grpc from 1.62.1 to 1.63.2 in /sdks (#30900)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 070fc903571 Bump google.golang.org/grpc from 1.62.1 to 1.63.2 in /sdks (#30900) 070fc903571 is described below commit 070fc9035710aa537c6e5b5b8885314099a8b0f7 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Mon May 6 12:36:30 2024 -0700 Bump google.golang.org/grpc from 1.62.1 to 1.63.2 in /sdks (#30900) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.62.1 to 1.63.2. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.62.1...v1.63.2) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 3 +-- sdks/go.sum | 7 ++- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index a57295eca6d..11a17c3faf1 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -61,7 +61,7 @@ require ( golang.org/x/text v0.14.0 google.golang.org/api v0.171.0 google.golang.org/genproto v0.0.0-20240308144416-29370a3891b7 - google.golang.org/grpc v1.62.1 + google.golang.org/grpc v1.63.2 google.golang.org/protobuf v1.33.0 gopkg.in/retry.v1 v1.0.3 gopkg.in/yaml.v2 v2.4.0 @@ -136,7 +136,6 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa // indirect github.com/containerd/containerd v1.7.11 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect diff --git a/sdks/go.sum b/sdks/go.sum index e8b35c72189..402763321ff 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -142,9 +142,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk= -github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= @@ -723,8 +720,8 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
(beam) branch master updated: [#30083][prism] Factor out hold tracking to dedicated structures (#31105)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 673da546c14 [#30083][prism] Factor out hold tracking to dedicated
structures (#31105)
673da546c14 is described below
commit 673da546c1465c931fdbbc5769e7d566ff55b4d8
Author: Robert Burke
AuthorDate: Fri Apr 26 15:11:39 2024 -0700
[#30083][prism] Factor out hold tracking to dedicated structures (#31105)
* [prism] Factor out hold tracking to dedicated structures
* review comment-reorder move code out of ladder.
-
Co-authored-by: lostluck <13907733+lostl...@users.noreply.github.com>
---
.../prism/internal/engine/elementmanager.go| 77 +++---
.../prism/internal/engine/elementmanager_test.go | 2 +-
.../beam/runners/prism/internal/engine/holds.go| 105 +++
.../runners/prism/internal/engine/holds_test.go| 115 +
.../runners/prism/internal/engine/teststream.go| 12 +--
5 files changed, 236 insertions(+), 75 deletions(-)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
index e40f5513dae..5d665edf286 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
@@ -414,7 +414,7 @@ func (em *ElementManager) checkForQuiescence(advanced
set[string]) {
outW := ss.OutputWatermark()
upPCol, upW := ss.UpstreamWatermark()
upS := em.pcolParents[upPCol]
- stageState = append(stageState, fmt.Sprintln(id, "watermark
in", inW, "out", outW, "upstream", upW, "from", upS, "pending", ss.pending,
"byKey", ss.pendingByKeys, "inprogressKeys", ss.inprogressKeys, "byBundle",
ss.inprogressKeysByBundle, "holds", ss.watermarkHoldHeap, "holdCounts",
ss.watermarkHoldsCounts))
+ stageState = append(stageState, fmt.Sprintln(id, "watermark
in", inW, "out", outW, "upstream", upW, "from", upS, "pending", ss.pending,
"byKey", ss.pendingByKeys, "inprogressKeys", ss.inprogressKeys, "byBundle",
ss.inprogressKeysByBundle, "holds", ss.watermarkHolds.heap, "holdCounts",
ss.watermarkHolds.counts))
}
panic(fmt.Sprintf("nothing in progress and no refreshes with non zero
pending elements: %v\n%v", v, strings.Join(stageState, "")))
}
@@ -706,18 +706,7 @@ func (em *ElementManager) PersistBundle(rb RunBundle,
col2Coders map[string]PCol
delete(stage.inprogressKeysByBundle, rb.BundleID)
for hold, v := range stage.inprogressHoldsByBundle[rb.BundleID] {
- n := stage.watermarkHoldsCounts[hold] - v
- if n == 0 {
- delete(stage.watermarkHoldsCounts, hold)
- for i, h := range stage.watermarkHoldHeap {
- if hold == h {
- heap.Remove(, i)
- break
- }
- }
- } else {
- stage.watermarkHoldsCounts[hold] = n
- }
+ stage.watermarkHolds.Drop(hold, v)
}
delete(stage.inprogressHoldsByBundle, rb.BundleID)
@@ -918,8 +907,7 @@ type stageState struct {
// We track the count of timers with the same hold, and clear it from
// the map and heap when the count goes to zero.
// This avoids scanning the heap to remove or access a hold for each
element.
- watermarkHoldsCountsmap[mtime.Time]int
- watermarkHoldHeap holdHeap
+ watermarkHolds *holdTracker
inprogressHoldsByBundle map[string]map[mtime.Time]int // bundle to
associated holds.
}
@@ -940,37 +928,15 @@ type dataAndTimers struct {
timers map[timerKey]timerTimes
}
-// holdHeap orders holds based on their timestamps
-// so we can always find the minimum timestamp of pending holds.
-type holdHeap []mtime.Time
-
-func (h holdHeap) Len() int { return len(h) }
-func (h holdHeap) Less(i, j int) bool { return h[i] < h[j] }
-func (h holdHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
-
-func (h *holdHeap) Push(x any) {
- // Push and Pop use pointer receivers because they modify the slice's
length,
- // not just its contents.
- *h = append(*h, x.(mtime.Time))
-}
-
-func (h *holdHeap) Pop() any {
- old := *h
- n := len(old)
- x := old[n-1]
- *h = old[0 : n-1]
- return x
-}
-
// makeStageState produ
(beam) branch master updated (58bea3f19e5 -> ab27f6f0358)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 58bea3f19e5 Fix building release candidate (#31084) add ab27f6f0358 Refactor Residuals to prepare for processing time scheduling. (#31057) No new revisions were added by this update. Summary of changes: .../prism/internal/engine/elementmanager.go| 110 + .../prism/internal/engine/elementmanager_test.go | 18 ++-- sdks/go/pkg/beam/runners/prism/internal/stage.go | 48 ++--- 3 files changed, 113 insertions(+), 63 deletions(-)
(beam) branch master updated: Bump golang.org/x/net from 0.22.0 to 0.24.0 in /sdks (#31065)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new a0dad088980 Bump golang.org/x/net from 0.22.0 to 0.24.0 in /sdks (#31065) a0dad088980 is described below commit a0dad0889803ffd95ac32e1c3e27cb26f4edac2f Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Mon Apr 22 14:24:21 2024 -0700 Bump golang.org/x/net from 0.22.0 to 0.24.0 in /sdks (#31065) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.22.0 to 0.24.0. - [Commits](https://github.com/golang/net/compare/v0.22.0...v0.24.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 6 +++--- sdks/go.sum | 12 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index 5d3f6cfbd58..a57295eca6d 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -54,10 +54,10 @@ require ( github.com/xitongsys/parquet-go v1.6.2 github.com/xitongsys/parquet-go-source v0.0.0-20220315005136-aec0fe3e777c go.mongodb.org/mongo-driver v1.13.1 - golang.org/x/net v0.23.0 + golang.org/x/net v0.24.0 golang.org/x/oauth2 v0.18.0 golang.org/x/sync v0.6.0 - golang.org/x/sys v0.18.0 + golang.org/x/sys v0.19.0 golang.org/x/text v0.14.0 google.golang.org/api v0.171.0 google.golang.org/genproto v0.0.0-20240308144416-29370a3891b7 @@ -182,7 +182,7 @@ require ( github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/mod v0.13.0 // indirect golang.org/x/tools v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect diff --git a/sdks/go.sum b/sdks/go.sum index d824a1a1113..e8b35c72189 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -497,8 +497,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -556,8 +556,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-2022202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -610,8 +610,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAY
(beam) branch master updated (718af0d7652 -> 5a86b3fc922)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 718af0d7652 Don't use isolated builds when building an sdist for a pipeline package supplied in --setup_file option. (#31070) add 5a86b3fc922 Bump golang.org/x/net from 0.17.0 to 0.23.0 in /learning/katas/go (#31050) No new revisions were added by this update. Summary of changes: learning/katas/go/go.mod | 2 +- learning/katas/go/go.sum | 20 2 files changed, 13 insertions(+), 9 deletions(-)
(beam) branch master updated (5a86b3fc922 -> 3dbbad98c80)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 5a86b3fc922 Bump golang.org/x/net from 0.17.0 to 0.23.0 in /learning/katas/go (#31050) add 3dbbad98c80 Bump golang.org/x/net from 0.17.0 to 0.23.0 in /.test-infra/mock-apis (#31051) No new revisions were added by this update. Summary of changes: .test-infra/mock-apis/go.mod | 6 +++--- .test-infra/mock-apis/go.sum | 12 ++-- 2 files changed, 9 insertions(+), 9 deletions(-)
(beam) branch master updated: [#30083][prism] Stabilize additional teststream cases. (#31046)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 8dcbf967a1a [#30083][prism] Stabilize additional teststream cases.
(#31046)
8dcbf967a1a is described below
commit 8dcbf967a1a692da4aef7dec79922795e83013dc
Author: Robert Burke
AuthorDate: Mon Apr 22 10:54:33 2024 -0700
[#30083][prism] Stabilize additional teststream cases. (#31046)
* Stabilize additional teststream cases.
* Update sdks/go/test/integration/primitives/teststream_test.go
Co-authored-by: Ritesh Ghorse
* Update sdks/go/test/integration/primitives/teststream.go
Co-authored-by: Ritesh Ghorse
* Update sdks/go/test/integration/primitives/teststream_test.go
Co-authored-by: Ritesh Ghorse
-
Co-authored-by: lostluck <13907733+lostl...@users.noreply.github.com>
Co-authored-by: Ritesh Ghorse
---
.../prism/internal/engine/elementmanager.go| 26 +++-
.../runners/prism/internal/engine/engine_test.go | 5 ++
.../runners/prism/internal/engine/teststream.go| 49 --
.../beam/runners/prism/internal/worker/bundle.go | 5 +-
sdks/go/test/integration/primitives/teststream.go | 74 ++
.../test/integration/primitives/teststream_test.go | 20 ++
sdks/go/test/integration/primitives/timers.go | 2 +-
7 files changed, 172 insertions(+), 9 deletions(-)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
index 28ea75ac9e5..ba0ab6f13d2 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
@@ -153,7 +153,8 @@ type Config struct {
type ElementManager struct {
config Config
- stages map[string]*stageState // The state for each stage.
+ impulses set[string]// List of impulse stages.
+ stages map[string]*stageState // The state for each stage.
consumers map[string][]string // Map from pcollectionID to stageIDs
that consumes them as primary input.
sideConsumers map[string][]LinkID // Map from pcollectionID to the
stage+transform+input that consumes them as side input.
@@ -254,6 +255,14 @@ func (em *ElementManager) Impulse(stageID string) {
em.addPending(count)
}
refreshes := stage.updateWatermarks(em)
+
+ // Since impulses are synthetic, we need to simulate them properly
+ // if a pipeline is only test stream driven.
+ if em.impulses == nil {
+ em.impulses = refreshes
+ } else {
+ em.impulses.merge(refreshes)
+ }
em.addRefreshes(refreshes)
}
@@ -286,6 +295,13 @@ func (em *ElementManager) Bundles(ctx context.Context,
nextBundID func() string)
// Watermark evaluation goroutine.
go func() {
defer close(runStageCh)
+
+ // If we have a test stream, clear out existing refreshes, so
the test stream can
+ // insert any elements it needs.
+ if em.testStreamHandler != nil {
+ em.watermarkRefreshes =
singleSet(em.testStreamHandler.ID)
+ }
+
for {
em.refreshCond.L.Lock()
// If there are no watermark refreshes available, we
wait until there are.
@@ -370,7 +386,13 @@ func (em *ElementManager) checkForQuiescence(advanced
set[string]) {
nextEvent.Execute(em)
// Decrement pending for the event being processed.
em.addPending(-1)
- return
+ // If there are refreshes scheduled, then test stream permitted
execution to continue.
+ // Note: it's a prism bug if test stream never causes a refresh
to occur for a given event.
+ // It's not correct to move to the next event if no refreshes
would occur.
+ if len(em.watermarkRefreshes) > 0 {
+ return
+ }
+ // If there are no refreshes, then there's no mechanism to
make progress, so it's time to fast fail.
}
v := em.livePending.Load()
diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/engine_test.go
b/sdks/go/pkg/beam/runners/prism/internal/engine/engine_test.go
index 04269e3dd6a..0c042d731d6 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/engine/engine_test.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/engine/engine_test.go
@@ -186,6 +186,11 @@ func TestTestStream(t *testing.T) {
{pipeline: primitives.TestStreamTwoFloat64Sequences},
{pipeline: primitives.TestStreamTwoInt64Sequences},
(beam) branch master updated (38a8b5f7047 -> a207e4e8bbe)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 38a8b5f7047 Create YAML Join Transform (#30734) add a207e4e8bbe Revert "Docker - Remove Deprecated Types (#30826)" (#31010) No new revisions were added by this update. Summary of changes: sdks/go/pkg/beam/runners/prism/internal/environments.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
(beam) branch master updated: [Dataflow Streaming] Add workToken to thread name for easier debugging (#30786)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 08253ea8a6e [Dataflow Streaming] Add workToken to thread name for
easier debugging (#30786)
08253ea8a6e is described below
commit 08253ea8a6ec5c9a6c8f530fb390b0d4e0702356
Author: Arun Pandian
AuthorDate: Fri Mar 29 10:10:17 2024 -0700
[Dataflow Streaming] Add workToken to thread name for easier debugging
(#30786)
Co-authored-by: Arun Pandian
---
.../beam/runners/dataflow/worker/util/BoundedQueueExecutor.java | 8
1 file changed, 8 insertions(+)
diff --git
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BoundedQueueExecutor.java
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BoundedQueueExecutor.java
index b1a0e087ef6..f7f6fd91a8c 100644
---
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BoundedQueueExecutor.java
+++
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BoundedQueueExecutor.java
@@ -22,6 +22,7 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.beam.runners.dataflow.worker.streaming.Work;
import
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Monitor;
import
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Monitor.Guard;
@@ -185,10 +186,17 @@ public class BoundedQueueExecutor {
try {
executor.execute(
() -> {
+String threadName = Thread.currentThread().getName();
try {
+ if (work instanceof Work) {
+String workToken =
+String.format("%016x", ((Work)
work).getWorkItem().getWorkToken());
+Thread.currentThread().setName(threadName + ":" + workToken);
+ }
work.run();
} finally {
decrementCounters(workBytes);
+ Thread.currentThread().setName(threadName);
}
});
} catch (RuntimeException e) {
(beam) branch master updated (7097443e27c -> 62200da9df8)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 7097443e27c Add link to official reference on beam site. (#30768)
add 62200da9df8 factor out failure handling logic from
StreamingDataflowWorker (#30695)
No new revisions were added by this update.
Summary of changes:
.../dataflow/worker/StreamingDataflowWorker.java | 634 +
.../runners/dataflow/worker/WorkUnitClient.java| 2 +-
.../streaming/harness/StreamingCounters.java | 101
.../harness/StreamingWorkerStatusReporter.java | 396 +
.../dataflow/worker/util/BoundedQueueExecutor.java | 6 +
.../work/processing/failures/FailureTracker.java | 104
.../work/processing/failures/HeapDumper.java | 11 +-
.../failures/StreamingApplianceFailureTracker.java | 60 ++
.../failures/StreamingApplianceStatsReporter.java} | 10 +-
.../failures/StreamingEngineFailureTracker.java} | 30 +-
.../processing/failures/WorkFailureProcessor.java | 185 ++
.../worker/StreamingDataflowWorkerTest.java| 41 +-
.../StreamingApplianceFailureTrackerTest.java | 97
.../StreamingEngineFailureTrackerTest.java | 91 +++
.../failures/WorkFailureProcessorTest.java | 204 +++
15 files changed, 1456 insertions(+), 516 deletions(-)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/harness/StreamingCounters.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/harness/StreamingWorkerStatusReporter.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/failures/FailureTracker.java
copy learning/tour-of-beam/frontend/test_driver/integration_test.dart =>
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/failures/HeapDumper.java
(78%)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/failures/StreamingApplianceFailureTracker.java
copy
runners/{direct-java/src/main/java/org/apache/beam/runners/direct/Clock.java =>
google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/failures/StreamingApplianceStatsReporter.java}
(73%)
copy
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/{client/grpc/stubs/WindmillStubFactory.java
=> work/processing/failures/StreamingEngineFailureTracker.java} (51%)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/failures/WorkFailureProcessor.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/failures/StreamingApplianceFailureTrackerTest.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/failures/StreamingEngineFailureTrackerTest.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/failures/WorkFailureProcessorTest.java
(beam) branch master updated (ea7a27daf85 -> c8ad120bde8)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from ea7a27daf85 Bump cloud.google.com/go/bigtable from 1.21.0 to 1.22.0 in /sdks (#30713) add c8ad120bde8 Fix all tour examples from part 1 (#30478) No new revisions were added by this update. Summary of changes: .../aggregation/count/go-example/main.go | 4 +- .../aggregation/max/go-example/main.go | 41 ++- .../aggregation/mean/go-example/main.go| 40 ++- .../aggregation/min/go-example/main.go | 40 ++- .../aggregation/sum/go-example/main.go | 4 +- .../common-transforms/filter/go-example/main.go| 44 ++-- .../motivating-challenge/go-challenge/main.go | 8 ++- .../motivating-challenge/go-solution/main.go | 8 ++- .../from-memory/go-example/from_memory.go | 36 +- .../reading-from-csv/go-example/csvExample.go | 73 ++- .../reading-from-text/go-example/textIo.go | 83 -- .../creating-pipeline/go-example/main.go | 34 - .../setting-pipeline/go-example/main.go| 60 13 files changed, 254 insertions(+), 221 deletions(-)
(beam) branch master updated: Bump cloud.google.com/go/bigtable from 1.21.0 to 1.22.0 in /sdks (#30713)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new ea7a27daf85 Bump cloud.google.com/go/bigtable from 1.21.0 to 1.22.0 in /sdks (#30713) ea7a27daf85 is described below commit ea7a27daf85b406d80788d76ce86503585b58b4c Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Mon Mar 25 15:15:48 2024 -0700 Bump cloud.google.com/go/bigtable from 1.21.0 to 1.22.0 in /sdks (#30713) Bumps [cloud.google.com/go/bigtable](https://github.com/googleapis/google-cloud-go) from 1.21.0 to 1.22.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/documentai/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.21.0...pubsub/v1.22.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/bigtable dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 16 ++-- sdks/go.sum | 24 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index 27ab3ded1b1..60e998d82d2 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -24,12 +24,12 @@ go 1.20 require ( cloud.google.com/go/bigquery v1.59.1 - cloud.google.com/go/bigtable v1.21.0 + cloud.google.com/go/bigtable v1.22.0 cloud.google.com/go/datastore v1.15.0 cloud.google.com/go/profiler v0.4.0 cloud.google.com/go/pubsub v1.36.2 - cloud.google.com/go/spanner v1.56.0 - cloud.google.com/go/storage v1.37.0 + cloud.google.com/go/spanner v1.57.0 + cloud.google.com/go/storage v1.38.0 github.com/aws/aws-sdk-go-v2 v1.26.0 github.com/aws/aws-sdk-go-v2/config v1.27.4 github.com/aws/aws-sdk-go-v2/credentials v1.17.8 @@ -60,7 +60,7 @@ require ( golang.org/x/sys v0.18.0 golang.org/x/text v0.14.0 google.golang.org/api v0.171.0 - google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 + google.golang.org/genproto v0.0.0-20240308144416-29370a3891b7 google.golang.org/grpc v1.62.1 google.golang.org/protobuf v1.33.0 gopkg.in/retry.v1 v1.0.3 @@ -78,7 +78,6 @@ require ( github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/apache/arrow/go/v14 v14.0.2 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.5.0 // indirect github.com/frankban/quicktest v1.14.0 // indirect github.com/go-logr/logr v1.4.1 // indirect @@ -93,12 +92,9 @@ require ( github.com/nats-io/jwt/v2 v2.5.5 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/shirou/gopsutil/v3 v3.23.9 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/stretchr/objx v0.5.0 // indirect - github.com/stretchr/testify v1.8.4 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect @@ -107,13 +103,13 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/sdk v1.21.0 // indirect + go.opentelemetry.io/otel/sdk v1.22.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/time v0.5.0 // indirect ) require ( - cloud.google.com/go v0.112.0 // indirect + cloud.google.com/go v0.112.1 // indirect cloud.google.com/go/compute v1.24.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.6 // indirect diff --git a/sdks/go.sum b/sdks/go.sum index ffb33a636b5..f951e22762e 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -8,15 +8,15 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go
(beam) branch dependabot/go_modules/sdks/github.com/docker/docker-25.0.5incompatible deleted (was 38a0341ccf7)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch dependabot/go_modules/sdks/github.com/docker/docker-25.0.5incompatible in repository https://gitbox.apache.org/repos/asf/beam.git was 38a0341ccf7 Bump github.com/docker/docker in /sdks The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(beam) branch master updated: Bump github.com/docker/docker in /sdks (#30684)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 8206d8976d5 Bump github.com/docker/docker in /sdks (#30684) 8206d8976d5 is described below commit 8206d8976d524853613a9ab350a7061088eda50b Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Wed Mar 20 09:57:22 2024 -0700 Bump github.com/docker/docker in /sdks (#30684) Bumps [github.com/docker/docker](https://github.com/docker/docker) from 25.0.3+incompatible to 25.0.5+incompatible. - [Release notes](https://github.com/docker/docker/releases) - [Commits](https://github.com/docker/docker/compare/v25.0.3...v25.0.5) --- updated-dependencies: - dependency-name: github.com/docker/docker dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 2 +- sdks/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index c5887395a14..26e9bba48cd 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -143,7 +143,7 @@ require ( github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa // indirect github.com/containerd/containerd v1.7.11 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect - github.com/docker/docker v25.0.3+incompatible // but required to resolve issue docker has with go1.20 + github.com/docker/docker v25.0.5+incompatible // but required to resolve issue docker has with go1.20 github.com/docker/go-units v0.5.0 // indirect github.com/envoyproxy/go-control-plane v0.12.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect diff --git a/sdks/go.sum b/sdks/go.sum index eaa713f7d06..d59d0ba2173 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -160,8 +160,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v25.0.3+incompatible h1:D5fy/lYmY7bvZa0XTZ5/UJPljor41F+vdyJG5luQLfQ= -github.com/docker/docker v25.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v25.0.5+incompatible h1:UmQydMduGkrD5nQde1mecF/YnSbTOaPeFIeP5C4W+DE= +github.com/docker/docker v25.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
(beam) branch master updated: Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks (#30669)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new a3e5ac86eea Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks (#30669) a3e5ac86eea is described below commit a3e5ac86eeade9fbef391a2c19d67825335938e6 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Tue Mar 19 09:52:59 2024 -0700 Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks (#30669) Bumps [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) from 1.17.4 to 1.17.8. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/v1.17.8/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.17.4...v1.17.8) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 18 +- sdks/go.sum | 36 ++-- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index 4042a9a848d..a44a90adba7 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -30,9 +30,9 @@ require ( cloud.google.com/go/pubsub v1.36.2 cloud.google.com/go/spanner v1.56.0 cloud.google.com/go/storage v1.37.0 - github.com/aws/aws-sdk-go-v2 v1.25.2 + github.com/aws/aws-sdk-go-v2 v1.26.0 github.com/aws/aws-sdk-go-v2/config v1.27.4 - github.com/aws/aws-sdk-go-v2/credentials v1.17.4 + github.com/aws/aws-sdk-go-v2/credentials v1.17.8 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.8 github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2 github.com/aws/smithy-go v1.20.1 @@ -124,18 +124,18 @@ require ( github.com/apache/thrift v0.17.0 // indirect github.com/aws/aws-sdk-go v1.34.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/sdks/go.sum b/sdks/go.sum index dc0ac9189bd..72b1037f254 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -78,26 +78,26 @@ github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZve github.com/aws/aws-sdk-go v1.34.0 h1:brux2dRrlwCF5JhTL7MUT3WUwo9zfDHZZp3+g3Mvlmo= github.com/aws/aws-sdk-go v1.34.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250= -github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w= -github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo= +github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA= +github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= github.com/aws/aws-sdk-go-v2/aws/protocol
(beam) branch lostluck-goversion deleted (was bbc6d490d65)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch lostluck-goversion in repository https://gitbox.apache.org/repos/asf/beam.git was bbc6d490d65 Update run_with_go_version.sh The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(beam) branch master updated: Update Go Version for artifacts to Go v1.21.8 (#30501)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0440587898c Update Go Version for artifacts to Go v1.21.8 (#30501)
0440587898c is described below
commit 0440587898cb7b3dbf7e6fe95d0a6ff958e44643
Author: Robert Burke
AuthorDate: Tue Mar 5 12:21:14 2024 -0800
Update Go Version for artifacts to Go v1.21.8 (#30501)
* Update BeamModulePlugin.groovy go version to 1.21.8
* Update run_with_go_version.sh
---
buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
sdks/go/run_with_go_version.sh | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 3157eaaf7ea..fae25d67b1b 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2202,7 +2202,7 @@ class BeamModulePlugin implements Plugin {
// This sets the whole project Go version.
// The latest stable Go version can be checked at https://go.dev/dl/
- project.ext.goVersion = "go1.21.6"
+ project.ext.goVersion = "go1.21.8"
// Minor TODO: Figure out if we can pull out the GOCMD env variable
after goPrepare script
// completion, and avoid this GOBIN substitution.
diff --git a/sdks/go/run_with_go_version.sh b/sdks/go/run_with_go_version.sh
index aa31399193a..03d8d74e9f5 100755
--- a/sdks/go/run_with_go_version.sh
+++ b/sdks/go/run_with_go_version.sh
@@ -37,7 +37,7 @@ set -e
#
# This variable is also used as the execution command downscript.
# The list of downloadable versions are at https://go.dev/dl/
-GOVERS=go1.21.0
+GOVERS=go1.21.8
if ! command -v go &> /dev/null
then
(beam) branch lostluck-goversion updated: Update run_with_go_version.sh
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch lostluck-goversion in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/lostluck-goversion by this push: new bbc6d490d65 Update run_with_go_version.sh bbc6d490d65 is described below commit bbc6d490d653161aec88b41af1a1a7a12514d7e5 Author: Robert Burke AuthorDate: Tue Mar 5 10:14:18 2024 -0800 Update run_with_go_version.sh --- sdks/go/run_with_go_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/go/run_with_go_version.sh b/sdks/go/run_with_go_version.sh index aa31399193a..03d8d74e9f5 100755 --- a/sdks/go/run_with_go_version.sh +++ b/sdks/go/run_with_go_version.sh @@ -37,7 +37,7 @@ set -e # # This variable is also used as the execution command downscript. # The list of downloadable versions are at https://go.dev/dl/ -GOVERS=go1.21.0 +GOVERS=go1.21.8 if ! command -v go &> /dev/null then
(beam) 01/01: Update BeamModulePlugin.groovy go version to 1.21.8
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch lostluck-goversion
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 8f3b062fb3e731cd4695ab7b3fdff322b8f75620
Author: Robert Burke
AuthorDate: Tue Mar 5 10:12:41 2024 -0800
Update BeamModulePlugin.groovy go version to 1.21.8
---
buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 3157eaaf7ea..fae25d67b1b 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2202,7 +2202,7 @@ class BeamModulePlugin implements Plugin {
// This sets the whole project Go version.
// The latest stable Go version can be checked at https://go.dev/dl/
- project.ext.goVersion = "go1.21.6"
+ project.ext.goVersion = "go1.21.8"
// Minor TODO: Figure out if we can pull out the GOCMD env variable
after goPrepare script
// completion, and avoid this GOBIN substitution.
(beam) branch lostluck-goversion created (now 8f3b062fb3e)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch lostluck-goversion in repository https://gitbox.apache.org/repos/asf/beam.git at 8f3b062fb3e Update BeamModulePlugin.groovy go version to 1.21.8 This branch includes the following new commits: new 8f3b062fb3e Update BeamModulePlugin.groovy go version to 1.21.8 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(beam) branch lostluck-patch-3 deleted (was 87a3ae086c7)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch lostluck-patch-3 in repository https://gitbox.apache.org/repos/asf/beam.git was 87a3ae086c7 [Go SDK] filter out TestTestStreamInt16Sequence from flink VR tests. The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(beam) branch master updated (6cae6b874a5 -> 168d06bade5)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 6cae6b874a5 make the flaky_test_detection script read only be default (#30461) add 168d06bade5 [Go SDK] filter out TestTestStreamInt16Sequence from flink VR tests. (#30462) No new revisions were added by this update. Summary of changes: sdks/go/test/integration/integration.go | 1 + 1 file changed, 1 insertion(+)
(beam) branch lostluck-patch-3 created (now 87a3ae086c7)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch lostluck-patch-3 in repository https://gitbox.apache.org/repos/asf/beam.git at 87a3ae086c7 [Go SDK] filter out TestTestStreamInt16Sequence from flink VR tests. No new revisions were added by this update.
(beam) branch robertwb-patch-1 deleted (was 564279fc2ac)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch robertwb-patch-1 in repository https://gitbox.apache.org/repos/asf/beam.git was 564279fc2ac Update beam-2.54.0.md about v2 default. The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(beam) branch master updated (6c3eccd49b7 -> 3728f5c23c3)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 6c3eccd49b7 Duet AI Prompts: Apache Beam Runners (#30346) add 3728f5c23c3 Update flink description.md to not refer to Dataflow (#30401) No new revisions were added by this update. Summary of changes: .../introduction/introduction-concepts/runner-concepts/description.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
svn commit: r67502 - /release/beam/vendor/grpc-1_60_1/0.2/
Author: lostluck Date: Thu Feb 22 23:09:54 2024 New Revision: 67502 Log: Adding vendored artifacts for grpc-1_60_1:0.2 Added: release/beam/vendor/grpc-1_60_1/0.2/ release/beam/vendor/grpc-1_60_1/0.2/apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip (with props) release/beam/vendor/grpc-1_60_1/0.2/apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip.asc release/beam/vendor/grpc-1_60_1/0.2/apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip.sha512 Added: release/beam/vendor/grpc-1_60_1/0.2/apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip == Binary file - no diff available. Propchange: release/beam/vendor/grpc-1_60_1/0.2/apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip -- svn:mime-type = application/octet-stream Added: release/beam/vendor/grpc-1_60_1/0.2/apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip.asc == --- release/beam/vendor/grpc-1_60_1/0.2/apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip.asc (added) +++ release/beam/vendor/grpc-1_60_1/0.2/apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip.asc Thu Feb 22 23:09:54 2024 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEknCCeZHC9kOgdMKQ/P0VKBG/FXgFAmXGHlQACgkQ/P0VKBG/ +FXiFphAAjYSx8wl8K48nrkNAIaTegTxkF65mT/WKnIIfSG2plQuaONSjfjZs0/am +EKIGqSuAQ/frbAQZzKpxk9yltmZ63VDC00lF8V28vvTS8tipMHfxdKgCkoSy6D69 +5bUtwLCUmc/+LRe/ZcRrxklsUA+2vJ43Ukr2dlmGDpxQnL1peSu892xZ+RgVYeGn +zg2zJhYWdYOP9v4iKtbWMdRyhdmxCcxeoJkFzEFKo3gAjI8lCYzZPS/N7qMu5pZh +LbI1lDii4Br2lNfNvrPmz/FjZPQ4KWQADDtjbMOjOB+ZGuVBu50ocxb3aWAT1dQo +Rh98eLq+noe5DqEKaIXOSNez63VDQDGqsVVq0Z9fcEAlH4VnXc2Qp15peXVaKzQw +uogC/oGd0qwMcNwnIFBVT5m2GNhJBFYdBDC64x/bWYhIWa1zq/fP0IpZTPxdpkaV +d8X4RUUu5hpWgpeS1+f00ge37B3IAp26T+3uHo+gMEpVzdgS4rzNWnfcNn/UR++G +FrabU9lnSvV8olVxOoOg8foU/NoI5e8ouiZP59GFTdRrQqsSerRoBtwCnzXtvOzp +dNILUk341d1+CTWWH3izaZJDA26A0LVoj0CM4TFFtcez67/sW90Y/Iy+2x25kpvZ +tZtML1JOzv5U6MbmteHYTYODI8dvXpHoNmE7OIRgzlBFoWPRr78= +=JqHN +-END PGP SIGNATURE- Added: release/beam/vendor/grpc-1_60_1/0.2/apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip.sha512 == --- release/beam/vendor/grpc-1_60_1/0.2/apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip.sha512 (added) +++ release/beam/vendor/grpc-1_60_1/0.2/apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip.sha512 Thu Feb 22 23:09:54 2024 @@ -0,0 +1 @@ +db84d3c6179b667016675e0b26a6129ae2f83c72948ae95a651a90023e005bea03f729ce81f8f84f866024fd5013bbef02b14e49dcc6d9f7ce9f31cb090e51f9 apache-beam-2d08b32e674a1046ba7be0ae5f1e4b7b05b73488-source-release.zip
(beam) branch dependabot/go_modules/sdks/github.com/docker/docker-25.0.3incompatible deleted (was 8842262fca6)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch dependabot/go_modules/sdks/github.com/docker/docker-25.0.3incompatible in repository https://gitbox.apache.org/repos/asf/beam.git was 8842262fca6 Bump github.com/docker/docker in /sdks The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(beam) branch master updated: Bump github.com/docker/docker in /sdks (#30308)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 67f930048d3 Bump github.com/docker/docker in /sdks (#30308) 67f930048d3 is described below commit 67f930048d353629aad8bc8201595946731e95ec Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Tue Feb 20 13:40:08 2024 -0800 Bump github.com/docker/docker in /sdks (#30308) Bumps [github.com/docker/docker](https://github.com/docker/docker) from 24.0.7+incompatible to 25.0.3+incompatible. - [Release notes](https://github.com/docker/docker/releases) - [Commits](https://github.com/docker/docker/compare/v24.0.7...v25.0.3) --- updated-dependencies: - dependency-name: github.com/docker/docker dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 6 +++--- sdks/go.sum | 16 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index 7eb08fd1705..df02e9a9061 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -78,6 +78,7 @@ require ( github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.5.0 // indirect github.com/frankban/quicktest v1.14.0 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -85,6 +86,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/minio/highwayhash v1.0.2 // indirect + github.com/moby/sys/user v0.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/nats-io/jwt/v2 v2.5.3 // indirect @@ -141,8 +143,7 @@ require ( github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 // indirect github.com/containerd/containerd v1.7.11 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.7+incompatible // but required to resolve issue docker has with go1.20 + github.com/docker/docker v25.0.3+incompatible // but required to resolve issue docker has with go1.20 github.com/docker/go-units v0.5.0 // indirect github.com/envoyproxy/go-control-plane v0.11.1 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect @@ -174,7 +175,6 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect - github.com/opencontainers/runc v1.1.12 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/xattr v0.4.9 // indirect diff --git a/sdks/go.sum b/sdks/go.sum index 275bec50e40..50b6d2039fc 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -161,10 +161,10 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= -github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v25.0.3+incompatible h1:D5fy/lYmY7bvZa0XTZ5/UJPljor41F+vdyJG5luQLfQ= +github.com/docker/docker v25.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -288,6 +288,7 @@ github.com/gorilla/handler
(beam) branch master updated: [#29917][prism] Initial TestStream support (#30072)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 48adde999b9 [#29917][prism] Initial TestStream support (#30072)
48adde999b9 is described below
commit 48adde999b9212c5bae8a330111fe8739fc1fbde
Author: Robert Burke
AuthorDate: Fri Feb 16 11:49:29 2024 -0800
[#29917][prism] Initial TestStream support (#30072)
---
.../prism/internal/engine/elementmanager.go| 99 ++--
.../runners/prism/internal/engine/engine_test.go | 47
.../runners/prism/internal/engine/teststream.go| 269 +
sdks/go/pkg/beam/runners/prism/internal/execute.go | 54 +
.../prism/internal/jobservices/management.go | 18 ++
.../runners/prism/internal/unimplemented_test.go | 43 +++-
sdks/go/pkg/beam/testing/teststream/teststream.go | 8 +-
sdks/go/test/integration/integration.go| 7 +-
sdks/go/test/integration/primitives/teststream.go | 43 +++-
.../test/integration/primitives/teststream_test.go | 10 +
10 files changed, 548 insertions(+), 50 deletions(-)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
index 077d6386315..28ea75ac9e5 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
@@ -166,6 +166,8 @@ type ElementManager struct {
livePending atomic.Int64 // An accessible live pending count.
DEBUG USE ONLY
pendingElements sync.WaitGroup // pendingElements counts all
unprocessed elements in a job. Jobs with no pending elements terminate
successfully.
+
+ testStreamHandler *testStreamHandler // Optional test stream handler
when a test stream is in the pipeline.
}
func (em *ElementManager) addPending(v int) {
@@ -223,6 +225,15 @@ func (em *ElementManager) StageStateful(ID string) {
em.stages[ID].stateful = true
}
+// AddTestStream provides a builder interface for the execution layer to build
the test stream from
+// the protos.
+func (em *ElementManager) AddTestStream(id string, tagToPCol
map[string]string) TestStreamBuilder {
+ impl := {em: em}
+ impl.initHandler(id)
+ impl.TagsToPCollections(tagToPCol)
+ return impl
+}
+
// Impulse marks and initializes the given stage as an impulse which
// is a root transform that starts processing.
func (em *ElementManager) Impulse(stageID string) {
@@ -319,37 +330,72 @@ func (em *ElementManager) Bundles(ctx context.Context,
nextBundID func() string)
em.refreshCond.L.Lock()
}
}
- if len(em.inprogressBundles) == 0 &&
len(em.watermarkRefreshes) == 0 {
- v := em.livePending.Load()
- slog.Debug("Bundles: nothing in progress and no
refreshes", slog.Int64("pendingElementCount", v))
- if v > 0 {
- var stageState []string
- ids := maps.Keys(em.stages)
- sort.Strings(ids)
- for _, id := range ids {
- ss := em.stages[id]
- inW := ss.InputWatermark()
- outW := ss.OutputWatermark()
- upPCol, upW :=
ss.UpstreamWatermark()
- upS := em.pcolParents[upPCol]
- stageState = append(stageState,
fmt.Sprintln(id, "watermark in", inW, "out", outW, "upstream", upW, "from",
upS, "pending", ss.pending, "byKey", ss.pendingByKeys, "inprogressKeys",
ss.inprogressKeys, "byBundle", ss.inprogressKeysByBundle, "holds",
ss.watermarkHoldHeap, "holdCounts", ss.watermarkHoldsCounts))
- }
- panic(fmt.Sprintf("nothing in progress
and no refreshes with non zero pending elements: %v\n%v", v,
strings.Join(stageState, "")))
- }
- } else if len(em.inprogressBundles) == 0 {
- v := em.livePending.Load()
- slog.Debug("Bundles: nothing in progress after
advance",
- slog.Any("advanced", advanced),
(beam-starter-java) branch main updated (fa8c358 -> 3ab23c3)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/beam-starter-java.git from fa8c358 Merge pull request #95 from jrmccluskey/versionUpdate add f8f29e5 Update to beam 2.54.0 add b343097 Update build.sbt add 9a8ba6b Update pom.xml new 3ab23c3 Merge pull request #96 from apache/lostluck-254 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: build.gradle | 4 ++-- build.sbt| 2 +- pom.xml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-)
(beam-starter-java) 01/01: Merge pull request #96 from apache/lostluck-254
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/beam-starter-java.git commit 3ab23c3b205c5a2a687e116b37abe6bd33504a1b Merge: fa8c358 9a8ba6b Author: Robert Burke AuthorDate: Thu Feb 15 17:20:13 2024 -0800 Merge pull request #96 from apache/lostluck-254 Update to beam 2.54.0 build.gradle | 4 ++-- build.sbt| 2 +- pom.xml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-)
(beam-starter-go) branch main updated: Update README.md to point to beam site quickstart guide.
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/beam-starter-go.git The following commit(s) were added to refs/heads/main by this push: new 45f8ea3 Update README.md to point to beam site quickstart guide. 45f8ea3 is described below commit 45f8ea321ddf1e29f00914345963a96c426f08b0 Author: Robert Burke AuthorDate: Thu Feb 15 15:58:10 2024 -0800 Update README.md to point to beam site quickstart guide. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e558c67..2a257f9 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ If you want to clone this repository to start your own project, you can choose the license you prefer and feel free to delete anything related to the license you are dropping. +See also https://beam.apache.org/get-started/quickstart/go/ for additional instructions for this quickstart. ## Before you begin
(beam-starter-go) branch main updated (dddbbca -> 1744ef5)
This is an automated email from the ASF dual-hosted git repository. lostluck pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/beam-starter-go.git from dddbbca [#29] Update go.mod to require go 1.22, to fix dependabot. add 73ebc1b Bump github.com/apache/beam/sdks/v2 from 2.52.0 to 2.54.0 new 1744ef5 Merge pull request #30 from apache/dependabot/go_modules/github.com/apache/beam/sdks/v2-2.54.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: go.mod | 44 - go.sum | 135 - 2 files changed, 117 insertions(+), 62 deletions(-)
