[beam] branch master updated: [BEAM-13015] Allow decoding a set of elements until we hit the block boundary. (#16220)
This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new ffb18c7 [BEAM-13015] Allow decoding a set of elements until we hit
the block boundary. (#16220)
ffb18c7 is described below
commit ffb18c79127e85faa9dea7104c5d3e145fdfaf9a
Author: Lukasz Cwik
AuthorDate: Mon Dec 13 22:42:03 2021 -0800
[BEAM-13015] Allow decoding a set of elements until we hit the block
boundary. (#16220)
This is in preparation for loading blocks at a time into a cache.
---
.../org/apache/beam/sdk/fn/stream/DataStreams.java | 29 +++-
.../apache/beam/sdk/fn/stream/DataStreamsTest.java | 84 ++
2 files changed, 97 insertions(+), 16 deletions(-)
diff --git
a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DataStreams.java
b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DataStreams.java
index a5e485e..52af762 100644
---
a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DataStreams.java
+++
b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DataStreams.java
@@ -22,7 +22,9 @@ import static
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Prec
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import java.util.NoSuchElementException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.vendor.grpc.v1p36p0.com.google.protobuf.ByteString;
@@ -179,6 +181,24 @@ public class DataStreams {
this.inbound = new Inbound();
}
+public void seekToNextByteString() {
+ inbound.currentStream = inputByteStrings.next().newInput();
+ inbound.position = 0;
+}
+
+public List decodeTillAtChunkBoundary() {
+ try {
+InputStream previousStream = inbound.currentStream;
+List rvals = new ArrayList<>();
+while (previousStream == inbound.currentStream &&
inbound.currentStream.available() != 0) {
+ rvals.add(next());
+}
+return rvals;
+ } catch (IOException e) {
+throw new IllegalStateException(e);
+ }
+}
+
@Override
public boolean isReady() {
try {
@@ -212,9 +232,10 @@ public class DataStreams {
try {
long previousPosition = inbound.position;
+InputStream previousStream = inbound.currentStream;
T next = coder.decode(inbound);
// Skip one byte if decoding the value consumed 0 bytes.
-if (inbound.position - previousPosition == 0) {
+if (previousPosition == inbound.position && previousStream ==
inbound.currentStream) {
checkState(inbound.read() != -1, "Unexpected EOF reached");
}
return next;
@@ -237,7 +258,7 @@ public class DataStreams {
* Closing this input stream has no effect.
*/
private class Inbound extends InputStream {
- private long position;
+ private int position; // Position within the current input stream.
private InputStream currentStream;
public Inbound() {
@@ -256,6 +277,7 @@ public class DataStreams {
return true;
}
currentStream = inputByteStrings.next().newInput();
+ position = 0;
}
return true;
}
@@ -269,6 +291,7 @@ public class DataStreams {
return true;
}
currentStream = inputByteStrings.next().newInput();
+ position = 0;
}
return false;
}
@@ -282,6 +305,7 @@ public class DataStreams {
return -1;
}
currentStream = inputByteStrings.next().newInput();
+ position = 0;
}
position += 1;
return read;
@@ -302,6 +326,7 @@ public class DataStreams {
return bytesRead > 0 ? bytesRead : -1;
}
currentStream = inputByteStrings.next().newInput();
+position = 0;
}
remainingLen -= read;
}
diff --git
a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DataStreamsTest.java
b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DataStreamsTest.java
index 8d032bc..f3a6379 100644
---
a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DataStreamsTest.java
+++
b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DataStreamsTest.java
@@ -19,6 +19,8 @@ package org.apache.beam.sdk.fn.stream;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.is;
import static org.hamcrest.collection.IsCollectionWithSize.hasSize;
import static org.junit.Assert.assertArrayEquals;
import static
[beam] branch master updated (d41ae47 -> 6a552dc)
This is an automated email from the ASF dual-hosted git repository. danoliveira pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from d41ae47 [BEAM-12976] Pipeline visitor to discover pushdown opportunities. (#16176) add a8bcfc2 [BEAM-13321] Pass TempLocation as pipeline option to Dataflow Go for xlang. add 6a552dc Merge pull request #16069: [BEAM-13321] Pass TempLocation as pipeline option to Dataflow Go for XLang. No new revisions were added by this update. Summary of changes: sdks/go/pkg/beam/runners/dataflow/dataflowlib/job.go | 17 +++-- 1 file changed, 11 insertions(+), 6 deletions(-)
svn commit: r51396 - /dev/beam/2.35.0/python/
Author: tvalentyn Date: Tue Dec 14 04:20:10 2021 New Revision: 51396 Log: Staging Python artifacts for Apache Beam 2.35.0 RC4 Modified: dev/beam/2.35.0/python/apache-beam-2.35.0.zip dev/beam/2.35.0/python/apache-beam-2.35.0.zip.asc dev/beam/2.35.0/python/apache-beam-2.35.0.zip.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-macosx_10_9_x86_64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-macosx_10_9_x86_64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux1_i686.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux1_i686.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux1_i686.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux1_x86_64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux1_x86_64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux1_x86_64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux2010_i686.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux2010_i686.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux2010_i686.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux2010_x86_64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux2010_x86_64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux2010_x86_64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux2014_aarch64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux2014_aarch64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-manylinux2014_aarch64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-win32.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-win32.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-win32.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-win_amd64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-win_amd64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp36-cp36m-win_amd64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-macosx_10_9_x86_64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-macosx_10_9_x86_64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-macosx_10_9_x86_64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux1_i686.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux1_i686.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux1_i686.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux1_x86_64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux1_x86_64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux1_x86_64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux2010_i686.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux2010_i686.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux2010_i686.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux2010_x86_64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux2010_x86_64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux2010_x86_64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux2014_aarch64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux2014_aarch64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-manylinux2014_aarch64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-win32.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-win32.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-win32.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-win_amd64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-win_amd64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp37-cp37m-win_amd64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp38-cp38-macosx_10_9_x86_64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp38-cp38-macosx_10_9_x86_64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp38-cp38-macosx_10_9_x86_64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp38-cp38-manylinux1_i686.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp38-cp38-manylinux1_i686.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp38-cp38-manylinux1_i686.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-cp38-cp38-manylinux1_x86_64.whl dev/beam/2.35.0/python/apache_beam-2.35.0-cp38-cp38-manylinux1_x86_64.whl.asc dev/beam/2.35.0/python/apache_beam-2.35.0-cp38-cp38-manylinux1_x86_64.whl.sha512 dev/beam/2.35.0/python/apache_beam-2.35.0-c
[beam] branch nightly-refs/heads/master updated (6e2d39f -> d41ae47)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch nightly-refs/heads/master in repository https://gitbox.apache.org/repos/asf/beam.git. from 6e2d39f [BEAM-13015] Plumb the cache through contexts and transform executors. (#16188) add 2547dbe [BEAM-13441] Use quiet delete for S3 batch deletes. In quiet mode only errors will be returned in the service response. add 3985d8d Merge pull request #16197: [BEAM-13441] Use S3 quiet mode for deletes add cc11936 Merge pull request #16140 from [BEAM-13377][Playground] Update CI/CD python scripts to work with specific SDK add 0257792 Merge pull request #16120 from [BEAM-1][Playground] Save Python logs to the cache add e183c14 Merge pull request #16185 from [BEAM-13425][Playground][Bugfix] Support logs while storing to bucket add b225f60 change context type add 79d6418 Merge pull request #16211 from [BEAM-13207][Playground] Change context type add 97a2acc Merge pull request #16121 from [BEAM-13334][Playground] Save Go logs to the cache add 290fc87 [BEAM-13423]: fix frontend failure if no examples add ad28b00 Merge pull request #16193 from [BEAM-13423] [Bugiix] [Playground]: fix frontend failure if no examples add 9a68b80 Docs for validators tests add bf77781 Merge pull request #16206 from [BEAM-13449][Playground] Docs for validators tests add 22e761c change return type of 2 methods add ba35c86 Merge pull request #16196 from [BEAM-13408][Playground] Change return type of 2 methods add d7e1a50 Merge pull request #16179 from [BEAM-13344][Playground] support python unit tests add 3e5630f Merge pull request #16204 from [BEAM-13444][Playground] Add processing of an error add 626e7a9 Merge pull request #16208 from [BEAM-13442][Playground] Filepath to log error add f969a7b [BEAM-13445] Correctly set data limit when flushing S3 upload buffer and avoid temporary copy to array when writing source buffer to upload buffer add e2ac013 Merge pull request #16205: [BEAM-13445] Correctly set data limit when flushing S3 upload buffer add cf35f72 [BEAM-13276] bump jackson-core to 2.13.0 for .test-infra (#16062) add 512ce86 Change Pub/Sub Lite PollResult to set explicit watermark (#16216) add 3cfec5b Updating Grafana from v8.1.2 to v8.1.6 add 37d1d99 Merge pull request #16201 from apache/aaltay-patch-1 add e056d73 Include name of missing tag in error message. add 25e6008 Merge pull request #16175 from ibzib/tag-name add 98c5266 Better type hints for Count combiners. add 0111905 Merge pull request #16145 Better type hints for Count combiners. add da11b84 [BEAM-13454] Fix and test dataframe read_fwf. (#16064) add d41ae47 [BEAM-12976] Pipeline visitor to discover pushdown opportunities. (#16176) No new revisions were added by this update. Summary of changes: .test-infra/metrics/grafana/Dockerfile | 2 +- .test-infra/validate-runner/build.gradle | 4 +- playground/backend/cmd/server/controller.go| 7 +- playground/backend/configs/SDK_PYTHON.json | 4 +- .../internal/cloud_bucket/precompiled_objects.go | 14 +- .../internal/code_processing/code_processing.go| 113 ++--- .../internal/preparators/python_preparators.go | 77 +- .../preparators/python_preparators_test.go | 101 .../internal/setup_tools/builder/setup_builder.go | 2 +- .../setup_tools/builder/setup_builder_test.go | 3 + .../internal/streaming/run_output_writer.go| 3 + playground/backend/internal/utils/cache_utils.go | 2 +- .../backend/internal/utils/validators_utils.go | 2 +- .../internal/validators/python_validators.go | 34 ++- .../backend/internal/validators/validator_test.go | 3 + .../components/playground_page_providers.dart | 3 + playground/frontend/pubspec.lock | 10 +- playground/infrastructure/api/v1/api_pb2_grpc.py | 2 +- playground/infrastructure/cd_helper.py | 15 +- playground/infrastructure/ci_cd.py | 67 +++-- playground/infrastructure/ci_helper.py | 18 +- playground/infrastructure/config.py| 12 +- playground/infrastructure/grpc_client.py | 15 ++ playground/infrastructure/helper.py| 68 +++-- playground/infrastructure/requirements.txt | 1 + playground/infrastructure/test_cd_helper.py| 35 +-- playground/infrastructure/test_ci_cd.py| 42 +--- playground/infrastructure/test_helper.py | 129 +- .../graph/ProjectionProducerVisitor.java | 83 +++ .../graph/ProjectionProducerVisitorTest.java | 274 + .../apache/beam/sdk/values/PCollectionTuple.java | 3 +- .../apache/beam/sdk/io/aws/s3/S3FileSystem.java| 3 +- .../beam/
[beam] branch master updated (da11b84 -> d41ae47)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from da11b84 [BEAM-13454] Fix and test dataframe read_fwf. (#16064) add d41ae47 [BEAM-12976] Pipeline visitor to discover pushdown opportunities. (#16176) No new revisions were added by this update. Summary of changes: .../graph/ProjectionProducerVisitor.java | 83 +++ .../graph/ProjectionProducerVisitorTest.java | 274 + 2 files changed, 357 insertions(+) create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitor.java create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitorTest.java
[beam] annotated tag sdks/v2.35.0-RC4 updated (2af80c2 -> 2de42b1)
This is an automated email from the ASF dual-hosted git repository. tvalentyn pushed a change to annotated tag sdks/v2.35.0-RC4 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag sdks/v2.35.0-RC4 was modified! *** from 2af80c2 (commit) to 2de42b1 (tag) tagging 2af80c27686419a18bdb563136169f38c33ae31d (commit) replaces jupyterlab-sidepanel-v2.0.0 by Valentyn Tymofieiev on Mon Dec 13 16:05:39 2021 -0800 - Log - Go SDK v2.35.0-RC4 --- No new revisions were added by this update. Summary of changes:
[beam] annotated tag v2.35.0-RC4 updated (2af80c2 -> f1752a6)
This is an automated email from the ASF dual-hosted git repository. tvalentyn pushed a change to annotated tag v2.35.0-RC4 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.35.0-RC4 was modified! *** from 2af80c2 (commit) to f1752a6 (tag) tagging 2af80c27686419a18bdb563136169f38c33ae31d (commit) replaces jupyterlab-sidepanel-v2.0.0 by Valentyn Tymofieiev on Mon Dec 13 16:05:39 2021 -0800 - Log - v2.35.0-RC4 --- No new revisions were added by this update. Summary of changes:
[beam] branch master updated: [BEAM-13454] Fix and test dataframe read_fwf. (#16064)
This is an automated email from the ASF dual-hosted git repository.
robertwb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new da11b84 [BEAM-13454] Fix and test dataframe read_fwf. (#16064)
da11b84 is described below
commit da11b84ba29f75a5764826f465b5bca9fad72a0e
Author: Robert Bradshaw
AuthorDate: Mon Dec 13 15:58:37 2021 -0800
[BEAM-13454] Fix and test dataframe read_fwf. (#16064)
---
sdks/python/apache_beam/dataframe/io.py | 47 +---
sdks/python/apache_beam/dataframe/io_test.py | 29 -
2 files changed, 71 insertions(+), 5 deletions(-)
diff --git a/sdks/python/apache_beam/dataframe/io.py
b/sdks/python/apache_beam/dataframe/io.py
index f3d436e..ab315f1 100644
--- a/sdks/python/apache_beam/dataframe/io.py
+++ b/sdks/python/apache_beam/dataframe/io.py
@@ -72,7 +72,7 @@ def read_csv(path, *args, splittable=False, **kwargs):
args,
kwargs,
incremental=True,
- splitter=_CsvSplitter(args, kwargs) if splittable else None)
+ splitter=_TextFileSplitter(args, kwargs) if splittable else None)
def _as_pc(df, label=None):
@@ -93,7 +93,14 @@ def to_csv(df, path, transform_label=None, *args, **kwargs):
@frame_base.with_docs_from(pd)
def read_fwf(path, *args, **kwargs):
- return _ReadFromPandas(pd.read_fwf, path, args, kwargs, incremental=True)
+ return _ReadFromPandas(
+ pd.read_fwf,
+ path,
+ args,
+ kwargs,
+ incremental=True,
+ binary=False,
+ splitter=_TextFileSplitter(args, kwargs))
@frame_base.with_docs_from(pd)
@@ -350,7 +357,7 @@ def _maybe_encode(str_or_bytes):
return str_or_bytes
-class _CsvSplitter(_DelimSplitter):
+class _TextFileSplitter(_DelimSplitter):
"""Splitter for dynamically sharding CSV files and newline record boundaries.
Currently does not handle quoted newlines, so is off by default, but such
@@ -442,6 +449,7 @@ class _TruncatingFileHandle(object):
self._done = False
self._header, self._buffer = self._splitter.read_header(self._underlying)
self._buffer_start_pos = len(self._header)
+self._iterator = None
start = self._tracker.current_restriction().start
# Seek to first delimiter after the start position.
if start > len(self._header):
@@ -471,9 +479,40 @@ class _TruncatingFileHandle(object):
def __iter__(self):
# For pandas is_file_like.
-raise NotImplementedError()
+return self
+
+ def __next__(self):
+if self._iterator is None:
+ self._iterator = self._line_iterator()
+return next(self._iterator)
+
+ def readline(self):
+# This attribute is checked, but unused, by pandas.
+return next(self)
+
+ def _line_iterator(self):
+line_start = 0
+chunk = self._read()
+while True:
+ line_end = chunk.find(self._splitter._delim, line_start)
+ while line_end == -1:
+more = self._read()
+if not more:
+ if line_start < len(chunk):
+yield chunk[line_start:]
+ return
+chunk = chunk[line_start:] + more
+line_start = 0
+line_end = chunk.find(self._splitter._delim, line_start)
+ yield chunk[line_start:line_end + 1]
+ line_start = line_end + 1
def read(self, size=-1):
+if self._iterator:
+ raise NotImplementedError('Cannot call read after iterating.')
+return self._read(size)
+
+ def _read(self, size=-1):
if self._header:
res = self._header
self._header = None
diff --git a/sdks/python/apache_beam/dataframe/io_test.py
b/sdks/python/apache_beam/dataframe/io_test.py
index d525b40..681642c 100644
--- a/sdks/python/apache_beam/dataframe/io_test.py
+++ b/sdks/python/apache_beam/dataframe/io_test.py
@@ -38,6 +38,7 @@ from apache_beam.dataframe import convert
from apache_beam.dataframe import io
from apache_beam.io import restriction_trackers
from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
class SimpleRow(typing.NamedTuple):
@@ -76,6 +77,19 @@ class IOTest(unittest.TestCase):
if delete:
os.remove(path)
+ def test_read_fwf(self):
+input = self.temp_dir(
+{'all.fwf': '''
+A B
+11a 0
+37a 1
+389a 2
+'''.strip()})
+with beam.Pipeline() as p:
+ df = p | io.read_fwf(input + 'all.fwf')
+ rows = convert.to_pcollection(df) | beam.Map(tuple)
+ assert_that(rows, equal_to([('11a', 0), ('37a', 1), ('389a', 2)]))
+
def test_read_write_csv(self):
input = self.temp_dir({'1.csv': 'a,b\n1,2\n', '2.csv': 'a,b\n3,4\n'})
output = self.temp_dir()
@@ -229,6 +243,19 @@ class IOTest(unittest.TestCase):
self.assertGreater(
min(len(s) for s in splits), len(numbers) * 0.9**20 * 0.1)
+ def _run_truncating_file_handle_iter_test(self, s, delim=' ', chunk_size=10):
+tracker = restriction_trackers.OffsetRestrictionTracke
[beam] branch master updated (25e6008 -> 0111905)
This is an automated email from the ASF dual-hosted git repository. robertwb pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 25e6008 Merge pull request #16175 from ibzib/tag-name add 98c5266 Better type hints for Count combiners. add 0111905 Merge pull request #16145 Better type hints for Count combiners. No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/transforms/combiners.py | 6 ++ sdks/python/apache_beam/transforms/ptransform_test.py | 10 +++--- 2 files changed, 9 insertions(+), 7 deletions(-)
[beam] 01/01: [BEAM-13434] Bump up Apache log4j2 version in tests to 2.15.0 #16217
This is an automated email from the ASF dual-hosted git repository. tvalentyn pushed a commit to branch release-2.35.0 in repository https://gitbox.apache.org/repos/asf/beam.git commit 033fc383f3a14e1177ecdd43ef5c46b10fed0d0c Merge: 343e140 59b11f3 Author: tvalentyn AuthorDate: Mon Dec 13 15:46:51 2021 -0800 [BEAM-13434] Bump up Apache log4j2 version in tests to 2.15.0 #16217 sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/build.gradle | 2 +- sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/build.gradle | 2 +- sdks/java/io/elasticsearch-tests/elasticsearch-tests-7/build.gradle | 2 +- .../java/io/elasticsearch-tests/elasticsearch-tests-common/build.gradle | 2 +- sdks/java/io/hcatalog/build.gradle | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-)
[beam] branch release-2.35.0 updated (343e140 -> 033fc38)
This is an automated email from the ASF dual-hosted git repository. tvalentyn pushed a change to branch release-2.35.0 in repository https://gitbox.apache.org/repos/asf/beam.git. from 343e140 [BEAM-13352] Cherrypick #16146 onto 2.35.0 release branch. (#16166) add 59b11f3 [BEAM-13434] Bump up Apache log4j2 vulnerability to 2.15.0 new 033fc38 [BEAM-13434] Bump up Apache log4j2 version in tests to 2.15.0 #16217 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/build.gradle | 2 +- sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/build.gradle | 2 +- sdks/java/io/elasticsearch-tests/elasticsearch-tests-7/build.gradle | 2 +- .../java/io/elasticsearch-tests/elasticsearch-tests-common/build.gradle | 2 +- sdks/java/io/hcatalog/build.gradle | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-)
[beam] branch master updated: Include name of missing tag in error message.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e056d73 Include name of missing tag in error message.
new 25e6008 Merge pull request #16175 from ibzib/tag-name
e056d73 is described below
commit e056d73ca8d8f0bab6d0439bfc5f498081319976
Author: Kyle Weaver
AuthorDate: Wed Dec 8 14:16:33 2021 -0800
Include name of missing tag in error message.
---
.../src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
index fef7a01..178644c 100644
---
a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
+++
b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
@@ -221,7 +221,8 @@ public class PCollectionTuple implements PInput, POutput {
@SuppressWarnings("unchecked")
PCollection pcollection = (PCollection) pcollectionMap.get(tag);
if (pcollection == null) {
- throw new IllegalArgumentException("TupleTag not found in this
PCollectionTuple tuple");
+ throw new IllegalArgumentException(
+ String.format("TupleTag %s not found in this PCollectionTuple
tuple", tag));
}
return pcollection;
}
[beam] branch master updated (512ce86 -> 37d1d99)
This is an automated email from the ASF dual-hosted git repository. altay pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 512ce86 Change Pub/Sub Lite PollResult to set explicit watermark (#16216) add 3cfec5b Updating Grafana from v8.1.2 to v8.1.6 add 37d1d99 Merge pull request #16201 from apache/aaltay-patch-1 No new revisions were added by this update. Summary of changes: .test-infra/metrics/grafana/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[beam] branch tvalentyn-patch-1 created (now 267106d)
This is an automated email from the ASF dual-hosted git repository. tvalentyn pushed a change to branch tvalentyn-patch-1 in repository https://gitbox.apache.org/repos/asf/beam.git. at 267106d Removes the comment that seems no longer relevant. This branch includes the following new commits: new 267106d Removes the comment that seems no longer relevant. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[beam] 01/01: Removes the comment that seems no longer relevant.
This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a commit to branch tvalentyn-patch-1
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 267106d1d09555dee82d265c3eedb1aee0c64fcb
Author: tvalentyn
AuthorDate: Mon Dec 13 11:08:59 2021 -0800
Removes the comment that seems no longer relevant.
---
sdks/java/io/hcatalog/build.gradle | 3 ---
1 file changed, 3 deletions(-)
diff --git a/sdks/java/io/hcatalog/build.gradle
b/sdks/java/io/hcatalog/build.gradle
index e76bc3b..87c687a2 100644
--- a/sdks/java/io/hcatalog/build.gradle
+++ b/sdks/java/io/hcatalog/build.gradle
@@ -44,9 +44,6 @@ test {
configurations.testRuntimeClasspath {
resolutionStrategy {
def log4j_version = "2.15.0"
-// Beam's build system forces a uniform log4j version resolution for all
modules, however for
-// the HCatalog case the current version of log4j produces
NoClassDefFoundError so we need to
-// force an old version on the tests runtime classpath
force "org.apache.logging.log4j:log4j-api:${log4j_version}"
force "org.apache.logging.log4j:log4j-core:${log4j_version}"
}
[beam] branch master updated (cf35f72 -> 512ce86)
This is an automated email from the ASF dual-hosted git repository. lcwik pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from cf35f72 [BEAM-13276] bump jackson-core to 2.13.0 for .test-infra (#16062) add 512ce86 Change Pub/Sub Lite PollResult to set explicit watermark (#16216) No new revisions were added by this update. Summary of changes: .../sdk/io/gcp/pubsublite/internal/SubscriptionPartitionLoader.java| 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
[beam] branch master updated (e2ac013 -> cf35f72)
This is an automated email from the ASF dual-hosted git repository. emilyye pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from e2ac013 Merge pull request #16205: [BEAM-13445] Correctly set data limit when flushing S3 upload buffer add cf35f72 [BEAM-13276] bump jackson-core to 2.13.0 for .test-infra (#16062) No new revisions were added by this update. Summary of changes: .test-infra/validate-runner/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[beam] branch master updated (626e7a9 -> e2ac013)
This is an automated email from the ASF dual-hosted git repository. aromanenko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 626e7a9 Merge pull request #16208 from [BEAM-13442][Playground] Filepath to log error add f969a7b [BEAM-13445] Correctly set data limit when flushing S3 upload buffer and avoid temporary copy to array when writing source buffer to upload buffer add e2ac013 Merge pull request #16205: [BEAM-13445] Correctly set data limit when flushing S3 upload buffer No new revisions were added by this update. Summary of changes: .../beam/sdk/io/aws/s3/S3WritableByteChannel.java | 25 ++ .../sdk/io/aws/s3/S3WritableByteChannelTest.java | 39 +- .../beam/sdk/io/aws2/s3/S3WritableByteChannel.java | 25 ++ .../apache/beam/sdk/io/aws2/s3/S3FileSystemIT.java | 18 +- .../sdk/io/aws2/s3/S3WritableByteChannelTest.java | 17 ++ 5 files changed, 72 insertions(+), 52 deletions(-)
[beam] branch master updated (3e5630f -> 626e7a9)
This is an automated email from the ASF dual-hosted git repository. pabloem pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 3e5630f Merge pull request #16204 from [BEAM-13444][Playground] Add processing of an error add 626e7a9 Merge pull request #16208 from [BEAM-13442][Playground] Filepath to log error No new revisions were added by this update. Summary of changes: playground/infrastructure/ci_helper.py | 18 -- 1 file changed, 8 insertions(+), 10 deletions(-)
[beam] branch master updated (d7e1a50 -> 3e5630f)
This is an automated email from the ASF dual-hosted git repository. pabloem pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from d7e1a50 Merge pull request #16179 from [BEAM-13344][Playground] support python unit tests add 3e5630f Merge pull request #16204 from [BEAM-13444][Playground] Add processing of an error No new revisions were added by this update. Summary of changes:
[beam] branch master updated (ba35c86 -> d7e1a50)
This is an automated email from the ASF dual-hosted git repository. pabloem pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from ba35c86 Merge pull request #16196 from [BEAM-13408][Playground] Change return type of 2 methods add d7e1a50 Merge pull request #16179 from [BEAM-13344][Playground] support python unit tests No new revisions were added by this update. Summary of changes: playground/backend/configs/SDK_PYTHON.json | 4 +- .../internal/code_processing/code_processing.go| 52 ++ .../internal/setup_tools/builder/setup_builder.go | 2 +- .../internal/streaming/run_output_writer.go| 3 ++ .../backend/internal/utils/validators_utils.go | 2 +- .../internal/validators/python_validators.go | 34 -- 6 files changed, 61 insertions(+), 36 deletions(-)
[beam] branch master updated: change return type of 2 methods
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 22e761c change return type of 2 methods
new ba35c86 Merge pull request #16196 from [BEAM-13408][Playground]
Change return type of 2 methods
22e761c is described below
commit 22e761c7af32c48027e82004e6fe49a50a83b365
Author: daria-malkova
AuthorDate: Fri Dec 10 16:45:53 2021 +0300
change return type of 2 methods
---
playground/backend/cmd/server/controller.go| 4 ++--
.../backend/internal/cloud_bucket/precompiled_objects.go | 14 +++---
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/playground/backend/cmd/server/controller.go
b/playground/backend/cmd/server/controller.go
index 64f2d3d..706613b 100644
--- a/playground/backend/cmd/server/controller.go
+++ b/playground/backend/cmd/server/controller.go
@@ -226,7 +226,7 @@ func (controller *playgroundController)
GetPrecompiledObjectCode(ctx context.Con
logger.Errorf("GetPrecompiledObject(): cloud storage error:
%s", err.Error())
return nil, errors.InternalError("GetPrecompiledObjects(): ",
err.Error())
}
- response := pb.GetPrecompiledObjectCodeResponse{Code: *codeString}
+ response := pb.GetPrecompiledObjectCodeResponse{Code: codeString}
return &response, nil
}
@@ -238,6 +238,6 @@ func (controller *playgroundController)
GetPrecompiledObjectOutput(ctx context.C
logger.Errorf("GetPrecompiledObjectOutput(): cloud storage
error: %s", err.Error())
return nil, errors.InternalError("GetPrecompiledObjectOutput():
", err.Error())
}
- response := pb.GetRunOutputResponse{Output: *output}
+ response := pb.GetRunOutputResponse{Output: output}
return &response, nil
}
diff --git a/playground/backend/internal/cloud_bucket/precompiled_objects.go
b/playground/backend/internal/cloud_bucket/precompiled_objects.go
index 8b612c6..c9265f9 100644
--- a/playground/backend/internal/cloud_bucket/precompiled_objects.go
+++ b/playground/backend/internal/cloud_bucket/precompiled_objects.go
@@ -93,27 +93,27 @@ func New() *CloudStorage {
}
// GetPrecompiledObject returns the source code of the example
-func (cd *CloudStorage) GetPrecompiledObject(ctx context.Context,
precompiledObjectPath string) (*string, error) {
+func (cd *CloudStorage) GetPrecompiledObject(ctx context.Context,
precompiledObjectPath string) (string, error) {
extension, err := getFileExtensionBySdk(precompiledObjectPath)
if err != nil {
- return nil, err
+ return "", err
}
data, err := cd.getFileFromBucket(ctx, precompiledObjectPath, extension)
if err != nil {
- return nil, err
+ return "", err
}
result := string(data)
- return &result, nil
+ return result, nil
}
// GetPrecompiledObjectOutput returns the run output of the example
-func (cd *CloudStorage) GetPrecompiledObjectOutput(ctx context.Context,
precompiledObjectPath string) (*string, error) {
+func (cd *CloudStorage) GetPrecompiledObjectOutput(ctx context.Context,
precompiledObjectPath string) (string, error) {
data, err := cd.getFileFromBucket(ctx, precompiledObjectPath,
OutputExtension)
if err != nil {
- return nil, err
+ return "", err
}
result := string(data)
- return &result, nil
+ return result, nil
}
// GetPrecompiledObjects returns stored at the cloud storage bucket
precompiled objects for the target category
[beam] branch master updated: Docs for validators tests
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9a68b80 Docs for validators tests
new bf77781 Merge pull request #16206 from [BEAM-13449][Playground] Docs
for validators tests
9a68b80 is described below
commit 9a68b80c73d11acb29b168c451a6d961016adb8f
Author: daria-malkova
AuthorDate: Mon Dec 13 14:58:25 2021 +0300
Docs for validators tests
---
playground/backend/internal/validators/validator_test.go | 3 +++
1 file changed, 3 insertions(+)
diff --git a/playground/backend/internal/validators/validator_test.go
b/playground/backend/internal/validators/validator_test.go
index 04f5c5c..cf1ccff 100644
--- a/playground/backend/internal/validators/validator_test.go
+++ b/playground/backend/internal/validators/validator_test.go
@@ -21,6 +21,9 @@ import (
"testing"
)
+// TestMain setups and teardown all necessary functionality for tests
+// in 'validators' package (i.e. for java_validators_test, go_validators_test,
+// python_validators_test)
func TestMain(m *testing.M) {
setup()
defer teardown()
[beam] branch master updated: [BEAM-13423]: fix frontend failure if no examples
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 290fc87 [BEAM-13423]: fix frontend failure if no examples
new ad28b00 Merge pull request #16193 from [BEAM-13423] [Bugiix]
[Playground]: fix frontend failure if no examples
290fc87 is described below
commit 290fc87b59be43d140107bc1383e66a9abb54e0e
Author: Aydar Farrakhov
AuthorDate: Fri Dec 10 11:53:47 2021 +0300
[BEAM-13423]: fix frontend failure if no examples
---
.../pages/playground/components/playground_page_providers.dart | 3 +++
playground/frontend/pubspec.lock | 10 +-
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git
a/playground/frontend/lib/pages/playground/components/playground_page_providers.dart
b/playground/frontend/lib/pages/playground/components/playground_page_providers.dart
index 66ae46d..20dfbc5 100644
---
a/playground/frontend/lib/pages/playground/components/playground_page_providers.dart
+++
b/playground/frontend/lib/pages/playground/components/playground_page_providers.dart
@@ -89,6 +89,9 @@ class PlaygroundPageProviders extends StatelessWidget {
.expand((sdkCategory) => sdkCategory.map((e) => e.examples))
.expand((element) => element)
.toList();
+if (allExamples?.isEmpty ?? true) {
+ return null;
+}
final defaultExample = exampleState.defaultExamplesMap![playground.sdk]!;
return allExamples?.firstWhere(
(example) => example.path == examplePath,
diff --git a/playground/frontend/pubspec.lock b/playground/frontend/pubspec.lock
index c883ed9..5d9ab7a 100644
--- a/playground/frontend/pubspec.lock
+++ b/playground/frontend/pubspec.lock
@@ -35,7 +35,7 @@ packages:
name: async
url: "https://pub.dartlang.org";
source: hosted
-version: "2.8.1"
+version: "2.8.2"
boolean_selector:
dependency: transitive
description:
@@ -105,7 +105,7 @@ packages:
name: characters
url: "https://pub.dartlang.org";
source: hosted
-version: "1.1.0"
+version: "1.2.0"
charcode:
dependency: transitive
description:
@@ -372,7 +372,7 @@ packages:
name: matcher
url: "https://pub.dartlang.org";
source: hosted
-version: "0.12.10"
+version: "0.12.11"
meta:
dependency: transitive
description:
@@ -601,7 +601,7 @@ packages:
name: test_api
url: "https://pub.dartlang.org";
source: hosted
-version: "0.4.2"
+version: "0.4.3"
timing:
dependency: transitive
description:
@@ -664,7 +664,7 @@ packages:
name: vector_math
url: "https://pub.dartlang.org";
source: hosted
-version: "2.1.0"
+version: "2.1.1"
watcher:
dependency: transitive
description:
[beam] branch master updated: Merge pull request #16121 from [BEAM-13334][Playground] Save Go logs to the cache
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 97a2acc Merge pull request #16121 from [BEAM-13334][Playground] Save
Go logs to the cache
97a2acc is described below
commit 97a2acc599982f3bbd5ae498b5a8ff5968e84ca6
Author: Aydar Zainutdinov
AuthorDate: Mon Dec 13 17:27:18 2021 +0300
Merge pull request #16121 from [BEAM-13334][Playground] Save Go logs to the
cache
* [BEAM-13334][Playground]
Implement adding go code processing logs to the cache
* [BEAM-13334][Playground]
Changed string concatenation to string formatting
* [BEAM-13334][Playground]
Add of using t.Error() instead of panic(err)
* [BEAM-13334][Playground]
Added comments;
Renaming
---
.../internal/code_processing/code_processing.go| 31 +++---
.../setup_tools/builder/setup_builder_test.go | 3 +++
2 files changed, 30 insertions(+), 4 deletions(-)
diff --git a/playground/backend/internal/code_processing/code_processing.go
b/playground/backend/internal/code_processing/code_processing.go
index ad4336d..ea7e611 100644
--- a/playground/backend/internal/code_processing/code_processing.go
+++ b/playground/backend/internal/code_processing/code_processing.go
@@ -170,7 +170,22 @@ func Process(ctx context.Context, cacheService
cache.Cache, lc *fs_tool.LifeCycl
var runError bytes.Buffer
runOutput := streaming.RunOutputWriter{Ctx: ctxWithTimeout,
CacheService: cacheService, PipelineId: pipelineId}
go readLogFile(ctxWithTimeout, cacheService,
lc.GetAbsoluteLogFilePath(), pipelineId, stopReadLogsChannel,
finishReadLogsChannel)
- runCmdWithOutput(runCmd, &runOutput, &runError, successChannel,
errorChannel)
+
+ if sdkEnv.ApacheBeamSdk == pb.Sdk_SDK_GO {
+ // For go SDK all logs are placed to stdErr.
+ file, err := os.Create(lc.GetAbsoluteLogFilePath())
+ if err != nil {
+ // If some error with creating a log file do the same
as with other SDK.
+ logger.Errorf("%s: error during create log file (go
sdk): %s", pipelineId, err.Error())
+ runCmdWithOutput(runCmd, &runOutput, &runError,
successChannel, errorChannel)
+ } else {
+ // Use the log file to write all stdErr into it.
+ runCmdWithOutput(runCmd, &runOutput, file,
successChannel, errorChannel)
+ }
+ } else {
+ // Other SDKs write logs to the log file on their own.
+ runCmdWithOutput(runCmd, &runOutput, &runError, successChannel,
errorChannel)
+ }
// Start of the monitoring of background tasks (run
step/cancellation/timeout)
ok, err = reconcileBackgroundTask(ctxWithTimeout, pipelineId,
cacheService, cancelChannel, successChannel)
@@ -179,6 +194,14 @@ func Process(ctx context.Context, cacheService
cache.Cache, lc *fs_tool.LifeCycl
}
if !ok {
// Run step is finished, but code contains some error (divide
by 0 for example)
+ if sdkEnv.ApacheBeamSdk == pb.Sdk_SDK_GO {
+ // For Go SDK stdErr was redirected to the log file.
+ errData, err := os.ReadFile(lc.GetAbsoluteLogFilePath())
+ if err != nil {
+ logger.Errorf("%s: error during read errors
from log file (go sdk): %s", pipelineId, err.Error())
+ }
+ runError.Write(errData)
+ }
_ = processRunError(ctxWithTimeout, errorChannel,
runError.Bytes(), pipelineId, cacheService, stopReadLogsChannel,
finishReadLogsChannel)
return
}
@@ -272,7 +295,7 @@ func GetLastIndex(ctx context.Context, cacheService
cache.Cache, key uuid.UUID,
}
// runCmdWithOutput runs command with keeping stdOut and stdErr
-func runCmdWithOutput(cmd *exec.Cmd, stdOutput io.Writer, stdError
*bytes.Buffer, successChannel chan bool, errorChannel chan error) {
+func runCmdWithOutput(cmd *exec.Cmd, stdOutput io.Writer, stdError io.Writer,
successChannel chan bool, errorChannel chan error) {
cmd.Stdout = stdOutput
cmd.Stderr = stdError
go func(cmd *exec.Cmd, successChannel chan bool, errChannel chan error)
{
@@ -411,7 +434,7 @@ func processCompileError(ctx context.Context, errorChannel
chan error, errorOutp
err := <-errorChannel
logger.Errorf("%s: Compile(): err: %s, output: %s\n", pipelineId,
err.Error(), errorOutput)
- if err := utils.SetToCache(ctx, cacheService, pipelineId,
cache.CompileOutput, "error: "+err.Error()+", output: "+string(errorOutput));
err != nil {
+ if err := utils.SetToCache(ctx, cacheService, pipelineId,
cache.Comp
[beam] branch master updated: change context type
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new b225f60 change context type
new 79d6418 Merge pull request #16211 from [BEAM-13207][Playground]
Change context type
b225f60 is described below
commit b225f60d815f9ff2cfe76765b37396c26b4cf169
Author: daria-malkova
AuthorDate: Mon Dec 13 16:24:37 2021 +0300
change context type
---
playground/backend/cmd/server/controller.go | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/playground/backend/cmd/server/controller.go
b/playground/backend/cmd/server/controller.go
index 64f2d3d..dcf80b4 100644
--- a/playground/backend/cmd/server/controller.go
+++ b/playground/backend/cmd/server/controller.go
@@ -80,8 +80,7 @@ func (controller *playgroundController) RunCode(ctx
context.Context, info *pb.Ru
return nil, errors.InternalError("Run code()", "Error during
set expiration to cache: %s", err.Error())
}
- // TODO change using of context.TODO() to context.Background()
- go code_processing.Process(context.TODO(), controller.cacheService, lc,
pipelineId, &controller.env.ApplicationEnvs, &controller.env.BeamSdkEnvs,
info.PipelineOptions)
+ go code_processing.Process(context.Background(),
controller.cacheService, lc, pipelineId, &controller.env.ApplicationEnvs,
&controller.env.BeamSdkEnvs, info.PipelineOptions)
pipelineInfo := pb.RunCodeResponse{PipelineUuid: pipelineId.String()}
return &pipelineInfo, nil
[beam] branch master updated (0257792 -> e183c14)
This is an automated email from the ASF dual-hosted git repository. pabloem pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 0257792 Merge pull request #16120 from [BEAM-1][Playground] Save Python logs to the cache add e183c14 Merge pull request #16185 from [BEAM-13425][Playground][Bugfix] Support logs while storing to bucket No new revisions were added by this update. Summary of changes: playground/infrastructure/cd_helper.py | 13 + playground/infrastructure/config.py | 1 + playground/infrastructure/grpc_client.py | 15 +++ playground/infrastructure/helper.py | 1 + 4 files changed, 30 insertions(+)
[beam] branch master updated (cc11936 -> 0257792)
This is an automated email from the ASF dual-hosted git repository. pabloem pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from cc11936 Merge pull request #16140 from [BEAM-13377][Playground] Update CI/CD python scripts to work with specific SDK add 0257792 Merge pull request #16120 from [BEAM-1][Playground] Save Python logs to the cache No new revisions were added by this update. Summary of changes: .../internal/code_processing/code_processing.go| 36 ++-- .../internal/preparators/python_preparators.go | 77 +++- .../preparators/python_preparators_test.go | 101 + playground/backend/internal/utils/cache_utils.go | 2 +- 4 files changed, 204 insertions(+), 12 deletions(-) create mode 100644 playground/backend/internal/preparators/python_preparators_test.go
[beam] branch master updated (3985d8d -> cc11936)
This is an automated email from the ASF dual-hosted git repository. pabloem pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 3985d8d Merge pull request #16197: [BEAM-13441] Use S3 quiet mode for deletes add cc11936 Merge pull request #16140 from [BEAM-13377][Playground] Update CI/CD python scripts to work with specific SDK No new revisions were added by this update. Summary of changes: playground/infrastructure/api/v1/api_pb2_grpc.py | 2 +- playground/infrastructure/cd_helper.py | 2 +- playground/infrastructure/ci_cd.py | 67 +--- playground/infrastructure/config.py | 11 +- playground/infrastructure/helper.py | 67 +--- playground/infrastructure/requirements.txt | 1 + playground/infrastructure/test_cd_helper.py | 35 +++--- playground/infrastructure/test_ci_cd.py | 42 +--- playground/infrastructure/test_helper.py | 129 +++ 9 files changed, 176 insertions(+), 180 deletions(-)
[beam] branch master updated (6e2d39f -> 3985d8d)
This is an automated email from the ASF dual-hosted git repository. aromanenko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 6e2d39f [BEAM-13015] Plumb the cache through contexts and transform executors. (#16188) add 2547dbe [BEAM-13441] Use quiet delete for S3 batch deletes. In quiet mode only errors will be returned in the service response. add 3985d8d Merge pull request #16197: [BEAM-13441] Use S3 quiet mode for deletes No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java | 3 ++- .../src/main/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystem.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-)
