[GitHub] [beam] youngoli merged pull request #12845: [BEAM-6928] Update changelog with WriteToBigQuery changed requirements.
youngoli merged pull request #12845: URL: https://github.com/apache/beam/pull/12845 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] youngoli commented on a change in pull request #12588: [BEAM-7009] Add Go SDK Standard Coders yaml tests.
youngoli commented on a change in pull request #12588:
URL: https://github.com/apache/beam/pull/12588#discussion_r489139814
##
File path: sdks/go/test/regression/coders/fromyaml/fromyaml.go
##
@@ -0,0 +1,415 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+//http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// fromyaml generates a resource file from the standard_coders.yaml
+// file for use in these coder regression tests.
+//
+// It expects to be run in it's test directory, or via it's go test.
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "math"
+ "reflect"
+ "runtime/debug"
+ "strconv"
+ "strings"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/core/graph/mtime"
+ "github.com/apache/beam/sdks/go/pkg/beam/core/graph/window"
+ "github.com/apache/beam/sdks/go/pkg/beam/core/runtime/exec"
+ "github.com/apache/beam/sdks/go/pkg/beam/core/runtime/graphx"
+ "github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
+ pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+ "github.com/google/go-cmp/cmp"
+ "golang.org/x/text/encoding/charmap"
+ yaml "gopkg.in/yaml.v2"
+)
+
+var unimplementedCoders = map[string]bool{
+ "beam:coder:param_windowed_value:v1": true,
+ "beam:coder:timer:v1":true,
+}
+
+// Coder is a representation a serialized beam coder.
+type Coder struct {
+ Urn string `yaml:"urn,omitempty"`
+ Payload string `yaml:"payload,omitempty"`
+ Components []Coder `yaml:"components,omitempty"`
+ NonDeterministic bool`yaml:"non_deterministic,omitempty"`
+}
+
+type logger interface {
+ Errorf(string, ...interface{})
+ Logf(string, ...interface{})
+}
+
+// Spec is a set of conditions that a coder must pass.
+type Spec struct {
+ CoderCoder `yaml:"coder,omitempty"`
+ Nested *bool `yaml:"nested,omitempty"`
+ Examples yaml.MapSlice `yaml:"examples,omitempty"`
+ Log logger
+
+ id int // for generating coder ids.
+ coderPBs map[string]*pipepb.Coder
+}
+
+func (s *Spec) nextID() string {
+ ret := fmt.Sprintf("%d", s.id)
+ s.id++
+ return ret
+}
+
+func (s *Spec) testStandardCoder() (err error) {
+ if unimplementedCoders[s.Coder.Urn] {
+ log.Printf("skipping unimplemented coder urn: %v", s.Coder.Urn)
+ return nil
+ }
+ // Construct the coder proto equivalents.
+
+ // Only nested tests need to be run, since nestedness is a
pre-portability
+ // concept.
+ // For legacy Java reasons, the row coder examples are all marked
nested: false
+ // so we need to check that before skipping unnested tests.
+ if s.Coder.Urn != "beam:coder:row:v1" && s.Nested != nil && !*s.Nested {
+ log.Printf("skipping unnested coder spec: %v\n", s.Coder)
+ return nil
+ }
+
+ s.coderPBs = make(map[string]*pipepb.Coder)
+ id := s.parseCoder(s.Coder)
+ b := graphx.NewCoderUnmarshaller(s.coderPBs)
+ underTest, err := b.Coder(id)
+ if err != nil {
+ return fmt.Errorf("unable to create coder: %v", err)
+ }
+
+ defer func() {
+ if e := recover(); e != nil {
+ err = fmt.Errorf("panicked on coder %v || %v:\n\t%v
:\n%s", underTest, s.Coder, e, debug.Stack())
+ }
+ }()
+
+ var decFails, encFails int
+ for _, eg := range s.Examples {
+
+ // Test Decoding
+ // Ideally we'd use the beam package coders, but KVs make that
complicated.
+ // This can be cleaned up once a type parametered beam.KV type
exists.
+ dec := exec.MakeElementDecoder(underTest)
+ encoded := eg.Key.(string)
+ var elem exec.FullValue
+
+ // What I would have expected.
+ // r :=
charmap.ISO8859_1.NewDecoder().Reader(strings.NewReader(encoded))
+ recoded, err := charmap.ISO8859_1.NewEncoder().String(encoded)
+ if err != nil {
+ return err
+
[GitHub] [beam] youngoli commented on a change in pull request #12588: [BEAM-7009] Add Go SDK Standard Coders yaml tests.
youngoli commented on a change in pull request #12588:
URL: https://github.com/apache/beam/pull/12588#discussion_r489138265
##
File path: sdks/go/pkg/beam/core/runtime/exec/coder.go
##
@@ -434,6 +600,267 @@ func convertIfNeeded(v interface{}, allocated *FullValue)
*FullValue {
return allocated
}
+type iterableEncoder struct {
+ t reflect.Type
+ enc ElementEncoder
+}
+
+func (c *iterableEncoder) Encode(val *FullValue, w io.Writer) error {
+ // Do a reflect, get the length.
+ rv := reflect.ValueOf(val.Elm)
+ size := rv.Len()
+ if err := coder.EncodeInt32((int32)(size), w); err != nil {
+ return err
+ }
+ var e FullValue
+ for i := 0; i < size; i++ {
+ e.Elm = rv.Index(i).Interface()
+ err := c.enc.Encode(, w)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+type iterableDecoder struct {
+ t reflect.Type
+ dec ElementDecoder
+}
+
+func (c *iterableDecoder) DecodeTo(r io.Reader, fv *FullValue) error {
+ // (1) Read count prefixed encoded data
+
+ size, err := coder.DecodeInt32(r)
+ if err != nil {
+ return err
+ }
+ n := int(size)
+ switch {
+ case n >= 0:
+ rv, err := c.decodeToSlice(int(n), r)
+ if err != nil {
+ return err
+ }
+ *fv = FullValue{Elm: rv.Interface()}
+ return nil
+ case n == -1:
+ rv := reflect.MakeSlice(c.t, 0, 0)
+ chunk, err := coder.DecodeVarInt(r)
+ if err != nil {
+ return err
+ }
+ for chunk != 0 {
+ rvi, err := c.decodeToSlice(int(chunk), r)
+ if err != nil {
+ return err
+ }
+ rv = reflect.AppendSlice(rv, rvi)
+ chunk, err = coder.DecodeVarInt(r)
+ if err != nil {
+ return err
+ }
+ }
+ *fv = FullValue{Elm: rv.Interface()}
+ }
+
+ return nil
+}
+
+func (c *iterableDecoder) decodeToSlice(n int, r io.Reader) (reflect.Value,
error) {
+ var e FullValue
+ rv := reflect.MakeSlice(c.t, n, n)
+ for i := 0; i < int(n); i++ {
+ err := c.dec.DecodeTo(r, )
+ if err != nil {
+ return reflect.Value{}, err
+ }
+ if e.Elm != nil {
+ rv.Index(i).Set(reflect.ValueOf(e.Elm))
+ } else {
+ rv.Index(i).Set(reflect.ValueOf(e.Windows[0]))
Review comment:
Ah that makes sense now. Yeah definitely could use a comment.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] youngoli commented on a change in pull request #12588: [BEAM-7009] Add Go SDK Standard Coders yaml tests.
youngoli commented on a change in pull request #12588:
URL: https://github.com/apache/beam/pull/12588#discussion_r489137571
##
File path: sdks/go/pkg/beam/core/runtime/exec/coder.go
##
@@ -81,24 +82,82 @@ func MakeElementEncoder(c *coder.Coder) ElementEncoder {
return {}
case coder.Custom:
- return {
+ enc := {
t: c.Custom.Type,
enc: makeEncoder(c.Custom.Enc.Fn),
}
+ if c.Custom.Name != "schema" {
+ return enc
+ }
+ // Custom schema coding is shorthand for using beam
infrastructure
+ // wrapped in a custom coder.
Review comment:
Yeah that makes more sense to me.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] udim commented on a change in pull request #12832: Add basic ToRows transform.
udim commented on a change in pull request #12832:
URL: https://github.com/apache/beam/pull/12832#discussion_r489114329
##
File path: sdks/python/apache_beam/transforms/core.py
##
@@ -2519,6 +2520,45 @@ def expand(self, pcoll):
(*(key + value
+class ToRows(PTransform):
+ """Converts the elements of a PCollection into a schema'd PCollection of
Rows.
+
+ `ToRow(...)` is roughly equivalent to `Map(lambda x: Row(...))` where each
+ argument (which may be a string or callable) of `ToRow` is applied to `x`.
+ For example,
+
+ pcoll | beam.ToRow('a', b=lambda x: foo(x))
Review comment:
Nit: For simplicity I'd only support the kwargs format.
(My personal preference would be to always use kwargs so it's clear what's
being mapped to what.)
##
File path: sdks/python/apache_beam/typehints/trivial_inference.py
##
@@ -320,6 +325,10 @@ def infer_return_type(c, input_types, debug=False,
depth=5):
dict: typehints.Dict[Any, Any]
}[c]
return c
+elif (c == getattr and len(input_types) == 2 and
Review comment:
I recommend testing internally since this may break some tests.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] udim commented on a change in pull request #12702: [BEAM-10603] Fix BCJ termination conditions.
udim commented on a change in pull request #12702: URL: https://github.com/apache/beam/pull/12702#discussion_r489117331 ## File path: sdks/python/apache_beam/runners/interactive/background_caching_job.py ## @@ -87,11 +87,10 @@ def _should_end_condition_checker(self): return any([l.is_triggered() for l in self._limiters]) def is_done(self): -is_terminated = self._pipeline_result.state is PipelineState.DONE +is_terminated = self._pipeline_result.state in ( +PipelineState.DONE, PipelineState.CANCELLED) Review comment: Codecov reporting was buggy, should be better now. :) This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] mouyang commented on pull request #12730: [BEAM-10532] convert NUMERIC field in TableSchema
mouyang commented on pull request #12730: URL: https://github.com/apache/beam/pull/12730#issuecomment-693114519 > Seems like this will be covered by #12711 > > cc: @robinyqiu It does. This has additional unit tests not in that PR. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12822: [BEAM-10880] Log error counts to debug BigQuery streaming insert requ…
codecov[bot] edited a comment on pull request #12822: URL: https://github.com/apache/beam/pull/12822#issuecomment-690950662 # [Codecov](https://codecov.io/gh/apache/beam/pull/12822?src=pr=h1) Report > Merging [#12822](https://codecov.io/gh/apache/beam/pull/12822?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/57055262e7a6bff447eef2df1e6efcda754939ca?el=desc) will **increase** coverage by `41.90%`. > The diff coverage is `76.74%`. [](https://codecov.io/gh/apache/beam/pull/12822?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12822 +/- ## === + Coverage 40.38% 82.29% +41.90% === Files 451 451 Lines 5372153849 +128 === + Hits2169644315+22619 + Misses 32025 9534-22491 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12822?src=pr=tree) | Coverage Δ | | |---|---|---| | [...dks/python/apache\_beam/options/pipeline\_options.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vb3B0aW9ucy9waXBlbGluZV9vcHRpb25zLnB5) | `94.98% <ø> (+38.99%)` | :arrow_up: | | [sdks/python/apache\_beam/io/gcp/bigquery.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL2JpZ3F1ZXJ5LnB5) | `79.08% <73.68%> (+51.98%)` | :arrow_up: | | [sdks/python/apache\_beam/io/gcp/bigquery\_tools.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL2JpZ3F1ZXJ5X3Rvb2xzLnB5) | `88.05% <100.00%> (+59.48%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | [sdks/python/apache\_beam/utils/profiler.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvcHJvZmlsZXIucHk=) | `32.11% <0.00%> (+1.83%)` | :arrow_up: | | [...on/apache\_beam/runners/direct/sdf\_direct\_runner.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kaXJlY3Qvc2RmX2RpcmVjdF9ydW5uZXIucHk=) | `36.06% <0.00%> (+2.32%)` | :arrow_up: | | ... and [274 more](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12822?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12822?src=pr=footer). Last update [5705526...3bbeb31](https://codecov.io/gh/apache/beam/pull/12822?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12505: [BEAM-8106] Add version to java container image name
codecov[bot] edited a comment on pull request #12505: URL: https://github.com/apache/beam/pull/12505#issuecomment-684502100 # [Codecov](https://codecov.io/gh/apache/beam/pull/12505?src=pr=h1) Report > Merging [#12505](https://codecov.io/gh/apache/beam/pull/12505?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/8f829ddffed87dcee477e5c3114bec734840d954?el=desc) will **increase** coverage by `0.00%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12505?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12505 +/- ## === Coverage 82.30% 82.30% === Files 451 451 Lines 5381853818 === + Hits4429544296+1 + Misses 9523 9522-1 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12505?src=pr=tree) | Coverage Δ | | |---|---|---| | [.../python/apache\_beam/transforms/periodicsequence.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9wZXJpb2RpY3NlcXVlbmNlLnB5) | `96.49% <0.00%> (-1.76%)` | :arrow_down: | | [...hon/apache\_beam/runners/worker/bundle\_processor.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy93b3JrZXIvYnVuZGxlX3Byb2Nlc3Nvci5weQ==) | `94.45% <0.00%> (-0.14%)` | :arrow_down: | | [sdks/python/apache\_beam/runners/common.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9jb21tb24ucHk=) | `89.20% <0.00%> (+0.44%)` | :arrow_up: | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12505?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12505?src=pr=footer). Last update [2031203...00fb543](https://codecov.io/gh/apache/beam/pull/12505?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12822: [BEAM-10880] Log error counts to debug BigQuery streaming insert requ…
codecov[bot] edited a comment on pull request #12822: URL: https://github.com/apache/beam/pull/12822#issuecomment-690950662 # [Codecov](https://codecov.io/gh/apache/beam/pull/12822?src=pr=h1) Report > Merging [#12822](https://codecov.io/gh/apache/beam/pull/12822?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/57055262e7a6bff447eef2df1e6efcda754939ca?el=desc) will **increase** coverage by `41.90%`. > The diff coverage is `76.74%`. [](https://codecov.io/gh/apache/beam/pull/12822?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12822 +/- ## === + Coverage 40.38% 82.29% +41.90% === Files 451 451 Lines 5372153849 +128 === + Hits2169644315+22619 + Misses 32025 9534-22491 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12822?src=pr=tree) | Coverage Δ | | |---|---|---| | [...dks/python/apache\_beam/options/pipeline\_options.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vb3B0aW9ucy9waXBlbGluZV9vcHRpb25zLnB5) | `94.98% <ø> (+38.99%)` | :arrow_up: | | [sdks/python/apache\_beam/io/gcp/bigquery.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL2JpZ3F1ZXJ5LnB5) | `79.08% <73.68%> (+51.98%)` | :arrow_up: | | [sdks/python/apache\_beam/io/gcp/bigquery\_tools.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL2JpZ3F1ZXJ5X3Rvb2xzLnB5) | `88.05% <100.00%> (+59.48%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | [sdks/python/apache\_beam/utils/profiler.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvcHJvZmlsZXIucHk=) | `32.11% <0.00%> (+1.83%)` | :arrow_up: | | [...on/apache\_beam/runners/direct/sdf\_direct\_runner.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kaXJlY3Qvc2RmX2RpcmVjdF9ydW5uZXIucHk=) | `36.06% <0.00%> (+2.32%)` | :arrow_up: | | ... and [274 more](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12822?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12822?src=pr=footer). Last update [5705526...3bbeb31](https://codecov.io/gh/apache/beam/pull/12822?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] udim edited a comment on pull request #12832: Add basic ToRows transform.
udim edited a comment on pull request #12832: URL: https://github.com/apache/beam/pull/12832#issuecomment-693106765 Please add a JIRA to the title This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] udim commented on pull request #12832: Add basic ToRows transform.
udim commented on pull request #12832: URL: https://github.com/apache/beam/pull/12832#issuecomment-693106765 Please add a JIRA to the description This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12505: [BEAM-8106] Add version to java container image name
codecov[bot] edited a comment on pull request #12505: URL: https://github.com/apache/beam/pull/12505#issuecomment-684502100 # [Codecov](https://codecov.io/gh/apache/beam/pull/12505?src=pr=h1) Report > Merging [#12505](https://codecov.io/gh/apache/beam/pull/12505?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/8f829ddffed87dcee477e5c3114bec734840d954?el=desc) will **increase** coverage by `0.00%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12505?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12505 +/- ## === Coverage 82.30% 82.30% === Files 451 451 Lines 5381853818 === + Hits4429544296+1 + Misses 9523 9522-1 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12505?src=pr=tree) | Coverage Δ | | |---|---|---| | [.../python/apache\_beam/transforms/periodicsequence.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9wZXJpb2RpY3NlcXVlbmNlLnB5) | `96.49% <0.00%> (-1.76%)` | :arrow_down: | | [...hon/apache\_beam/runners/worker/bundle\_processor.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy93b3JrZXIvYnVuZGxlX3Byb2Nlc3Nvci5weQ==) | `94.45% <0.00%> (-0.14%)` | :arrow_down: | | [sdks/python/apache\_beam/runners/common.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9jb21tb24ucHk=) | `89.20% <0.00%> (+0.44%)` | :arrow_up: | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12505?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12505?src=pr=footer). Last update [2031203...00fb543](https://codecov.io/gh/apache/beam/pull/12505?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] ihji commented on a change in pull request #12822: [BEAM-10880] Log error counts to debug BigQuery streaming insert requ…
ihji commented on a change in pull request #12822: URL: https://github.com/apache/beam/pull/12822#discussion_r489094641 ## File path: sdks/python/apache_beam/io/gcp/bigquery.py ## @@ -1210,20 +1236,32 @@ def process(self, element, *schema_side_inputs): return self._flush_all_batches() def finish_bundle(self): -if BigQueryWriteFn.LATENCY_LOGGING_LOCK.acquire(False): +if BigQueryWriteFn.STREAMING_API_LOGGING_LOCK.acquire(False): Review comment: splitted into a separate helper method. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12822: [BEAM-10880] Log error counts to debug BigQuery streaming insert requ…
codecov[bot] edited a comment on pull request #12822: URL: https://github.com/apache/beam/pull/12822#issuecomment-690950662 # [Codecov](https://codecov.io/gh/apache/beam/pull/12822?src=pr=h1) Report > Merging [#12822](https://codecov.io/gh/apache/beam/pull/12822?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/57055262e7a6bff447eef2df1e6efcda754939ca?el=desc) will **increase** coverage by `41.98%`. > The diff coverage is `75.00%`. [](https://codecov.io/gh/apache/beam/pull/12822?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12822 +/- ## === + Coverage 40.38% 82.37% +41.98% === Files 451 451 Lines 5372153897 +176 === + Hits2169644398+22702 + Misses 32025 9499-22526 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12822?src=pr=tree) | Coverage Δ | | |---|---|---| | [...dks/python/apache\_beam/options/pipeline\_options.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vb3B0aW9ucy9waXBlbGluZV9vcHRpb25zLnB5) | `94.98% <ø> (+38.99%)` | :arrow_up: | | [sdks/python/apache\_beam/io/gcp/bigquery.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL2JpZ3F1ZXJ5LnB5) | `79.02% <71.42%> (+51.91%)` | :arrow_up: | | [sdks/python/apache\_beam/io/gcp/bigquery\_tools.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL2JpZ3F1ZXJ5X3Rvb2xzLnB5) | `88.05% <100.00%> (+59.48%)` | :arrow_up: | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | [sdks/python/apache\_beam/utils/profiler.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvcHJvZmlsZXIucHk=) | `32.11% <0.00%> (+1.83%)` | :arrow_up: | | [...on/apache\_beam/runners/direct/sdf\_direct\_runner.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kaXJlY3Qvc2RmX2RpcmVjdF9ydW5uZXIucHk=) | `36.21% <0.00%> (+2.46%)` | :arrow_up: | | [sdks/python/apache\_beam/coders/coder\_impl.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL2NvZGVyX2ltcGwucHk=) | `95.25% <0.00%> (+2.64%)` | :arrow_up: | | [sdks/python/apache\_beam/io/azure/blobstorageio.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vYXp1cmUvYmxvYnN0b3JhZ2Vpby5weQ==) | `26.95% <0.00%> (+3.19%)` | :arrow_up: | | [sdks/python/apache\_beam/utils/proto\_utils.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvcHJvdG9fdXRpbHMucHk=) | `63.07% <0.00%> (+4.61%)` | :arrow_up: | | [...pache\_beam/examples/cookbook/bigquery\_tornadoes.py](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZXhhbXBsZXMvY29va2Jvb2svYmlncXVlcnlfdG9ybmFkb2VzLnB5) | `45.00% <0.00%> (+5.00%)` | :arrow_up: | | ... and [271 more](https://codecov.io/gh/apache/beam/pull/12822/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12822?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12822?src=pr=footer). Last update [5705526...3bbeb31](https://codecov.io/gh/apache/beam/pull/12822?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] emilymye commented on pull request #12505: [BEAM-8106] Add version to java container image name
emilymye commented on pull request #12505: URL: https://github.com/apache/beam/pull/12505#issuecomment-693100689 Will update once I've run the x-lang example This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12505: [BEAM-8106] Add version to java container image name
codecov[bot] edited a comment on pull request #12505: URL: https://github.com/apache/beam/pull/12505#issuecomment-684502100 # [Codecov](https://codecov.io/gh/apache/beam/pull/12505?src=pr=h1) Report > Merging [#12505](https://codecov.io/gh/apache/beam/pull/12505?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/8f829ddffed87dcee477e5c3114bec734840d954?el=desc) will **increase** coverage by `0.05%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12505?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12505 +/- ## == + Coverage 82.30% 82.35% +0.05% == Files 451 450 -1 Lines 5381853702 -116 == - Hits4429544228 -67 + Misses 9523 9474 -49 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12505?src=pr=tree) | Coverage Δ | | |---|---|---| | [sdks/python/apache\_beam/dataframe/partitionings.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL3BhcnRpdGlvbmluZ3MucHk=) | `83.60% <0.00%> (-5.44%)` | :arrow_down: | | [...ache\_beam/runners/interactive/recording\_manager.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9yZWNvcmRpbmdfbWFuYWdlci5weQ==) | `92.85% <0.00%> (-5.20%)` | :arrow_down: | | [.../python/apache\_beam/testing/test\_stream\_service.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy90ZXN0X3N0cmVhbV9zZXJ2aWNlLnB5) | `88.63% <0.00%> (-4.55%)` | :arrow_down: | | [...hon/apache\_beam/runners/direct/test\_stream\_impl.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kaXJlY3QvdGVzdF9zdHJlYW1faW1wbC5weQ==) | `91.17% <0.00%> (-2.95%)` | :arrow_down: | | [...che\_beam/runners/interactive/interactive\_runner.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9pbnRlcmFjdGl2ZV9ydW5uZXIucHk=) | `90.90% <0.00%> (-1.82%)` | :arrow_down: | | [.../python/apache\_beam/transforms/periodicsequence.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9wZXJpb2RpY3NlcXVlbmNlLnB5) | `96.49% <0.00%> (-1.76%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/expressions.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2V4cHJlc3Npb25zLnB5) | `88.07% <0.00%> (-1.64%)` | :arrow_down: | | [...am/runners/interactive/options/capture\_limiters.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9vcHRpb25zL2NhcHR1cmVfbGltaXRlcnMucHk=) | `91.93% <0.00%> (-1.62%)` | :arrow_down: | | [...ive/messaging/interactive\_environment\_inspector.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9tZXNzYWdpbmcvaW50ZXJhY3RpdmVfZW52aXJvbm1lbnRfaW5zcGVjdG9yLnB5) | `96.15% <0.00%> (-1.29%)` | :arrow_down: | | [...eam/runners/interactive/caching/streaming\_cache.py](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9jYWNoaW5nL3N0cmVhbWluZ19jYWNoZS5weQ==) | `94.06% <0.00%> (-1.28%)` | :arrow_down: | | ... and [19 more](https://codecov.io/gh/apache/beam/pull/12505/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12505?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12505?src=pr=footer). Last update [2031203...00fb543](https://codecov.io/gh/apache/beam/pull/12505?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] rosetn edited a comment on pull request #12835: [BEAM-10889] Add a note about BatchElements on GroupIntoBatches pages.
rosetn edited a comment on pull request #12835: URL: https://github.com/apache/beam/pull/12835#issuecomment-693100165 Ideally, we should have an entry in the Transforms catalog for BatchElements, but I can understand needing this reference now. FYI @davidcavazos This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] emilymye commented on a change in pull request #12505: [BEAM-8106] Add version to java container image name
emilymye commented on a change in pull request #12505:
URL: https://github.com/apache/beam/pull/12505#discussion_r489092740
##
File path:
runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java
##
@@ -357,4 +354,14 @@ private static File zipDirectory(File directory) throws
IOException {
return env;
}
}
+
+ private static String getDefaultJavaSdkHarnessContainerUrl() {
+String javaVersionId =
+Float.parseFloat(System.getProperty("java.specification.version")) >=
9 ? "java11" : "java8";
Review comment:
I went ahead and added an Enum to Environments, and removed similar
checks in DataflowRunner
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] rosetn commented on pull request #12835: [BEAM-10889] Add a note about BatchElements on GroupIntoBatches pages.
rosetn commented on pull request #12835: URL: https://github.com/apache/beam/pull/12835#issuecomment-693100165 Ideally, we should have an entry in the Transforms catalog for BatchElements, but I can understand needing this reference now. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] pabloem commented on pull request #12721: [BEAM-10871] Add deidentify for FhirIO connector
pabloem commented on pull request #12721: URL: https://github.com/apache/beam/pull/12721#issuecomment-693099477 Run Java PreCommit This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] apilloud commented on a change in pull request #12643: [BEAM-10438] Update SupportedZetaSqlBuiltinFunctions and support math functions
apilloud commented on a change in pull request #12643:
URL: https://github.com/apache/beam/pull/12643#discussion_r489084327
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamBigQuerySqlDialect.java
##
@@ -87,16 +87,17 @@
.put("$extract_time", "TIME")
.put("$extract_datetime", "DATETIME")
.build();
- public static final String DOUBLE_POSITIVE_INF_FUNCTION =
"double_positive_inf";
- public static final String DOUBLE_NEGATIVE_INF_FUNCTION =
"double_negative_inf";
- public static final String DOUBLE_NAN_FUNCTION = "double_nan";
- private static final Map DOUBLE_FUNCTIONS =
+ public static final String DOUBLE_POSITIVE_INF_WRAPPER =
"double_positive_inf";
Review comment:
nit: Renames like this are probably a net negative when you take into
account the total cost of a change (time authoring and reviewing, increased
diffs when debugging future issues, potential upgrade friction for customers).
It is hard to know exactly where the line is so feel free to push forward with
these as much of the work is already done. In the future I would suggest
avoiding non-functional changes, particularly if they are something that would
be at most a nit on a code review. (These went in recently and didn't even get
a nit comment.)
##
File path:
sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SupportedZetaSqlBuiltinFunctions.java
##
@@ -475,17 +512,22 @@
// FunctionSignatureId.FN_RANGE_BUCKET, // range_bucket(T,
array) -> int64
- // FunctionSignatureId.FN_RAND, // rand() -> double
+ FunctionSignatureId.FN_RAND // rand() -> double
Review comment:
This one scares me just a little. In particular, I'm worried that we
might accidentally optimize such that we end up with a constant. Is that
something we need to test? https://xkcd.com/221/
##
File path:
sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlTypesUtils.java
##
@@ -24,6 +24,11 @@
@Internal
public class ZetaSqlTypesUtils {
+ public static final BigDecimal NUMERIC_MAX_VALUE =
Review comment:
nit: these constants are only used in `ZetaSqlMathFunctionsTest.java`,
do they belong there?
##
File path:
sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SupportedZetaSqlBuiltinFunctions.java
##
@@ -277,29 +293,31 @@
FunctionSignatureId.FN_PARSE_DATETIME, // parse_datetime
FunctionSignatureId.FN_PARSE_TIME, // parse_time
FunctionSignatureId.FN_PARSE_TIMESTAMP, // parse_timestamp
+ // FunctionSignatureId.FN_LAST_DAY_DATE, // last_day date
+ // FunctionSignatureId.FN_LAST_DAY_DATETIME, // last_day datetime
// Math functions
- // FunctionSignatureId.FN_ABS_INT64, // abs
- // FunctionSignatureId.FN_ABS_DOUBLE, // abs
- // FunctionSignatureId.FN_ABS_NUMERIC, // abs
+ FunctionSignatureId.FN_ABS_INT64, // abs
Review comment:
nit: This PR has a interesting mix of minor refactors and addition of
new functionality. These should probably be separate changes.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] ihji removed a comment on pull request #12852: [BEAM-10890] Log error counts to debug BigQuery streaming insert requ…
ihji removed a comment on pull request #12852: URL: https://github.com/apache/beam/pull/12852#issuecomment-693091570 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] ihji commented on pull request #12852: [BEAM-10890] Log error counts to debug BigQuery streaming insert requ…
ihji commented on pull request #12852: URL: https://github.com/apache/beam/pull/12852#issuecomment-693091533 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] ihji commented on a change in pull request #12822: [BEAM-10880] Log error counts to debug BigQuery streaming insert requ…
ihji commented on a change in pull request #12822:
URL: https://github.com/apache/beam/pull/12822#discussion_r48908
##
File path: sdks/python/apache_beam/io/gcp/bigquery_tools.py
##
@@ -538,6 +539,13 @@ def _insert_all_rows(
try:
response = self.client.tabledata.InsertAll(request)
# response.insertErrors is not [] if errors encountered.
+except HttpError as exn:
+ if error_counter:
+content = json.loads(exn.content)
+error_counter.record(
+'%s(%s)' %
+(content['error']['errors'][0]['reason'], exn.status_code))
Review comment:
It records string error codes and numeric http codes from BigQuery API
JSON response https://cloud.google.com/bigquery/docs/error-messages. For
example:
```
{quotaExceeded(403): 461, rateLimitExceeded(403): 2724}
```
I think string error codes here are human readable and clearly shows the
cause of errors. Do you want to append any other string messages to them?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] apilloud commented on a change in pull request #12596: [BEAM-10498] [WIP] Eliminate nullability errors from :sdks:java:extensions:sql:zetasql
apilloud commented on a change in pull request #12596:
URL: https://github.com/apache/beam/pull/12596#discussion_r489077798
##
File path:
sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java
##
@@ -132,6 +132,8 @@
public static final SqlOperator DATE_OP =
createUdfOperator("DATE", BeamBuiltinMethods.DATE_METHOD);
+ // TODO: Fix Later
+ @SuppressWarnings("nullness")
public static final SqlUserDefinedFunction CAST_OP =
new SqlUserDefinedFunction(
new SqlIdentifier("CAST", SqlParserPos.ZERO),
Review comment:
null is correct.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] monicadsong commented on a change in pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
monicadsong commented on a change in pull request #12851: URL: https://github.com/apache/beam/pull/12851#discussion_r489074298 ## File path: sdks/python/apache_beam/transforms/stats_test.py ## @@ -89,6 +90,12 @@ def setUp(self): None, 0.1, 'assert:global_by_error_with_large_population'), + ( + 'numpy_input_data', + np.array(range(10)), Review comment: Ah sorry, just saw this. I didn't like the assumption I made about the type of elements in the PCollection either, and the use case you mention is definitely unsupported with this draft edit but would not alert the user if they made that mistake. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] boyuanzz commented on a change in pull request #12836: [BEAM-10887] Expose clear() to FnApiTimer.
boyuanzz commented on a change in pull request #12836: URL: https://github.com/apache/beam/pull/12836#discussion_r489074544 ## File path: sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java ## @@ -81,6 +81,9 @@ */ void setRelative(); + /** Clears the timer. */ Review comment: There are 2 conditions that we know a cleared timer should never been fired: - Timer(key = A) is set first then cleared within the same bundle. - A never-set timer is cleared. One condition that we know a cleared timer will be fired: - Timer(key = A) is cleared first then set again. And the contract is vague under certain conditions(e.g, the runner may or may not clear the timer depends on the implementation details of the runner): - Timer(key = A) is being scheduled by service meanwhile user code just clears Timer(key = A). This happens when set and clear calls happen on different bundles. I'll add ValidatesRunner tests to cover these. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12806: [BEAM-10869] Make WriteToPubsub output serialized PubsubMessage proto bytes when using runner v2
codecov[bot] edited a comment on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-692876818 # [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=h1) Report > Merging [#12806](https://codecov.io/gh/apache/beam/pull/12806?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/2bb60c323095340240ec4982c1e9dabc397107e5?el=desc) will **increase** coverage by `41.87%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12806 +/- ## === + Coverage 40.43% 82.30% +41.87% === Files 449 451+2 Lines 5353053817 +287 === + Hits2164344294+22651 + Misses 31887 9523-22364 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) | Coverage Δ | | |---|---|---| | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.54% <ø> (+51.53%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_util.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya191dGlsLnB5) | `0.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...eam/testing/benchmarks/nexmark/nexmark\_launcher.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19sYXVuY2hlci5weQ==) | `0.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_perf.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19wZXJmLnB5) | `0.00% <0.00%> (ø)` | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/utils/histogram.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvaGlzdG9ncmFtLnB5) | `94.28% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | ... and [275 more](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=footer). Last update [b6d0abb...d01ad96](https://codecov.io/gh/apache/beam/pull/12806?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] robertwb commented on pull request #12819: [BEAM-9561] Initial framework for testing pandas website docs.
robertwb commented on pull request #12819: URL: https://github.com/apache/beam/pull/12819#issuecomment-693021255 The apache_beam.transforms.ptransform_test.PTransformTest.test_flatten_no_pcollections failure looks unrelated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] apilloud merged pull request #12818: [BEAM-10875] Support NUMERIC in spanner schema parser
apilloud merged pull request #12818: URL: https://github.com/apache/beam/pull/12818 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] chamikaramj commented on pull request #12730: [BEAM-10532] convert NUMERIC field in TableSchema
chamikaramj commented on pull request #12730: URL: https://github.com/apache/beam/pull/12730#issuecomment-693011358 Seems like this will be covered by https://github.com/apache/beam/pull/12711 cc: @robinyqiu This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] chamikaramj commented on pull request #12767: Remove experimental annotations for BQ storage API source
chamikaramj commented on pull request #12767: URL: https://github.com/apache/beam/pull/12767#issuecomment-693010464 Retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] chamikaramj commented on pull request #12767: Remove experimental annotations for BQ storage API source
chamikaramj commented on pull request #12767: URL: https://github.com/apache/beam/pull/12767#issuecomment-693010547 LGTM This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] boyuanzz commented on pull request #12806: [BEAM-10869] Make WriteToPubsub output serialized PubsubMessage proto bytes when using runner v2
boyuanzz commented on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-693007091 Run Python PreCommit This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] ibzib merged pull request #12820: [BEAM-9575] Only copy the Spark runner jar, not whatever other jars h…
ibzib merged pull request #12820: URL: https://github.com/apache/beam/pull/12820 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12806: [BEAM-10869] Make WriteToPubsub output serialized PubsubMessage proto bytes when using runner v2
codecov[bot] edited a comment on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-692876818 # [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=h1) Report > Merging [#12806](https://codecov.io/gh/apache/beam/pull/12806?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/2bb60c323095340240ec4982c1e9dabc397107e5?el=desc) will **increase** coverage by `41.86%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12806 +/- ## === + Coverage 40.43% 82.29% +41.86% === Files 449 451+2 Lines 5353053817 +287 === + Hits2164344289+22646 + Misses 31887 9528-22359 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) | Coverage Δ | | |---|---|---| | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.54% <ø> (+51.53%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_util.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya191dGlsLnB5) | `0.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...eam/testing/benchmarks/nexmark/nexmark\_launcher.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19sYXVuY2hlci5weQ==) | `0.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/utils/histogram.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvaGlzdG9ncmFtLnB5) | `94.28% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_perf.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19wZXJmLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | ... and [275 more](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=footer). Last update [b6d0abb...d01ad96](https://codecov.io/gh/apache/beam/pull/12806?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] ihji opened a new pull request #12852: [BEAM-10890] Log error counts to debug BigQuery streaming insert requ…
ihji opened a new pull request #12852: URL: https://github.com/apache/beam/pull/12852 …ests for Java SDK Thank you for your contribution! Follow this checklist to help us incorporate your contribution quickly and easily: - [ ] [**Choose reviewer(s)**](https://beam.apache.org/contribute/#make-your-change) and mention them in a comment (`R: @username`). - [ ] Format the pull request title like `[BEAM-XXX] Fixes bug in ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA issue, if applicable. This will automatically link the pull request to the issue. - [ ] Update `CHANGES.md` with noteworthy changes. - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf). See the [Contributor Guide](https://beam.apache.org/contribute) for more tips on [how to make review process smoother](https://beam.apache.org/contribute/#make-reviewers-job-easier). Post-Commit Tests Status (on master branch) Lang | SDK | Dataflow | Flink | Samza | Spark | Twister2 --- | --- | --- | --- | --- | --- | --- Go | [](https://ci-beam.apache.org/job/beam_PostCommit_Go/lastCompletedBuild/) | --- | [](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/lastCompletedBuild/) | --- | [](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark/lastCompletedBuild/) | --- Java | [](https://ci-beam.apache.org/job/beam_PostCommit_Java/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow_Java11/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink_Java11/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Flink_Batch/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Flink_Streaming/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Samza/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Spark/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Spark_Batch/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Twister2/lastCompletedBuild/) Python | [](https://ci-beam.apache.org/job/beam_PostCommit_Python2/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Python35/lastCompletedBuild/)[![Build
[GitHub] [beam] codecov[bot] edited a comment on pull request #12806: [BEAM-10869] Make WriteToPubsub output serialized PubsubMessage proto bytes when using runner v2
codecov[bot] edited a comment on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-692876818 # [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=h1) Report > Merging [#12806](https://codecov.io/gh/apache/beam/pull/12806?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/2bb60c323095340240ec4982c1e9dabc397107e5?el=desc) will **increase** coverage by `41.86%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12806 +/- ## === + Coverage 40.43% 82.29% +41.86% === Files 449 451+2 Lines 5353053817 +287 === + Hits2164344289+22646 + Misses 31887 9528-22359 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) | Coverage Δ | | |---|---|---| | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.54% <ø> (+51.53%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_util.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya191dGlsLnB5) | `0.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...eam/testing/benchmarks/nexmark/nexmark\_launcher.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19sYXVuY2hlci5weQ==) | `0.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_perf.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19wZXJmLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/utils/histogram.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvaGlzdG9ncmFtLnB5) | `94.28% <0.00%> (ø)` | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | ... and [275 more](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=footer). Last update [b6d0abb...d01ad96](https://codecov.io/gh/apache/beam/pull/12806?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] amaliujia commented on a change in pull request #12843: [BEAM-10895] Support UNNEST an (possibly nested) array field of an struct column
amaliujia commented on a change in pull request #12843:
URL: https://github.com/apache/beam/pull/12843#discussion_r488992286
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnnestRel.java
##
@@ -111,31 +112,41 @@ public RelWriter explainTerms(RelWriter pw) {
Schema joinedSchema = CalciteUtils.toSchema(getRowType());
return outer
- .apply(ParDo.of(new UnnestFn(joinedSchema, unnestIndex)))
+ .apply(ParDo.of(new UnnestFn(joinedSchema, unnestIndices)))
.setRowSchema(joinedSchema);
}
}
private static class UnnestFn extends DoFn {
private final Schema outputSchema;
-private final int unnestIndex;
+private final List unnestIndices;
-private UnnestFn(Schema outputSchema, int unnestIndex) {
+private UnnestFn(Schema outputSchema, List unnestIndices) {
this.outputSchema = outputSchema;
- this.unnestIndex = unnestIndex;
+ this.unnestIndices = unnestIndices;
}
@ProcessElement
public void process(@Element Row row, OutputReceiver out) {
-
- @Nullable Collection rawValues = row.getArray(unnestIndex);
+ Row rowWithArrayField = row;
+ Schema schemaWithArrayField = outputSchema;
+ for (int i = unnestIndices.size() - 1; i > 0; i--) {
+rowWithArrayField = rowWithArrayField.getRow(unnestIndices.get(i));
Review comment:
So it looks like guaranteed to have nested row except for the last
index? If not better to add validation here.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] amaliujia commented on a change in pull request #12843: [BEAM-10895] Support UNNEST an (possibly nested) array field of an struct column
amaliujia commented on a change in pull request #12843:
URL: https://github.com/apache/beam/pull/12843#discussion_r488992286
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnnestRel.java
##
@@ -111,31 +112,41 @@ public RelWriter explainTerms(RelWriter pw) {
Schema joinedSchema = CalciteUtils.toSchema(getRowType());
return outer
- .apply(ParDo.of(new UnnestFn(joinedSchema, unnestIndex)))
+ .apply(ParDo.of(new UnnestFn(joinedSchema, unnestIndices)))
.setRowSchema(joinedSchema);
}
}
private static class UnnestFn extends DoFn {
private final Schema outputSchema;
-private final int unnestIndex;
+private final List unnestIndices;
-private UnnestFn(Schema outputSchema, int unnestIndex) {
+private UnnestFn(Schema outputSchema, List unnestIndices) {
this.outputSchema = outputSchema;
- this.unnestIndex = unnestIndex;
+ this.unnestIndices = unnestIndices;
}
@ProcessElement
public void process(@Element Row row, OutputReceiver out) {
-
- @Nullable Collection rawValues = row.getArray(unnestIndex);
+ Row rowWithArrayField = row;
+ Schema schemaWithArrayField = outputSchema;
+ for (int i = unnestIndices.size() - 1; i > 0; i--) {
+rowWithArrayField = rowWithArrayField.getRow(unnestIndices.get(i));
Review comment:
So it looks like guaranteed to have nested row except for the last index?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12806: [BEAM-10869] Make WriteToPubsub output serialized PubsubMessage proto bytes when using runner v2
codecov[bot] edited a comment on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-692876818 # [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=h1) Report > Merging [#12806](https://codecov.io/gh/apache/beam/pull/12806?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/2bb60c323095340240ec4982c1e9dabc397107e5?el=desc) will **increase** coverage by `41.86%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12806 +/- ## === + Coverage 40.43% 82.29% +41.86% === Files 449 451+2 Lines 5353053817 +287 === + Hits2164344289+22646 + Misses 31887 9528-22359 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) | Coverage Δ | | |---|---|---| | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.54% <ø> (+51.53%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_util.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya191dGlsLnB5) | `0.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...eam/testing/benchmarks/nexmark/nexmark\_launcher.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19sYXVuY2hlci5weQ==) | `0.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [sdks/python/apache\_beam/utils/histogram.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvaGlzdG9ncmFtLnB5) | `94.28% <0.00%> (ø)` | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_perf.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19wZXJmLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | ... and [275 more](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=footer). Last update [b6d0abb...d01ad96](https://codecov.io/gh/apache/beam/pull/12806?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12806: [BEAM-10869] Make WriteToPubsub output serialized PubsubMessage proto bytes when using runner v2
codecov[bot] edited a comment on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-692876818 # [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=h1) Report > Merging [#12806](https://codecov.io/gh/apache/beam/pull/12806?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/2bb60c323095340240ec4982c1e9dabc397107e5?el=desc) will **increase** coverage by `41.83%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12806 +/- ## === + Coverage 40.43% 82.27% +41.83% === Files 449 451+2 Lines 5353053785 +255 === + Hits2164344249+22606 + Misses 31887 9536-22351 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) | Coverage Δ | | |---|---|---| | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.54% <ø> (+51.53%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_util.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya191dGlsLnB5) | `0.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...eam/testing/benchmarks/nexmark/nexmark\_launcher.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19sYXVuY2hlci5weQ==) | `0.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/utils/histogram.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvaGlzdG9ncmFtLnB5) | `94.28% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_perf.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19wZXJmLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | ... and [275 more](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=footer). Last update [b6d0abb...d01ad96](https://codecov.io/gh/apache/beam/pull/12806?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] ibzib commented on pull request #12820: [BEAM-9575] Only copy the Spark runner jar, not whatever other jars h…
ibzib commented on pull request #12820: URL: https://github.com/apache/beam/pull/12820#issuecomment-692979803 R: @boyuanzz This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] rohdesamuel commented on pull request #12799: [BEAM-10603] Add record_pipeline, clear to RM and fix duration limiter
rohdesamuel commented on pull request #12799: URL: https://github.com/apache/beam/pull/12799#issuecomment-692976118 R: @pabloem This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12799: [BEAM-10603] Add record_pipeline, clear to RM and fix duration limiter
codecov[bot] edited a comment on pull request #12799: URL: https://github.com/apache/beam/pull/12799#issuecomment-692960218 # [Codecov](https://codecov.io/gh/apache/beam/pull/12799?src=pr=h1) Report > Merging [#12799](https://codecov.io/gh/apache/beam/pull/12799?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/a2126e571cfddc90e3819e66ef1f2da3d52833ed?el=desc) will **decrease** coverage by `0.06%`. > The diff coverage is `58.56%`. [](https://codecov.io/gh/apache/beam/pull/12799?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12799 +/- ## == - Coverage 82.36% 82.30% -0.07% == Files 450 451 +1 Lines 5370853849 +141 == + Hits4423844319 +81 - Misses 9470 9530 +60 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12799?src=pr=tree) | Coverage Δ | | |---|---|---| | [...beam/runners/interactive/background\_caching\_job.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9iYWNrZ3JvdW5kX2NhY2hpbmdfam9iLnB5) | `96.52% <ø> (+1.73%)` | :arrow_up: | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [...pache\_beam/runners/interactive/interactive\_beam.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9pbnRlcmFjdGl2ZV9iZWFtLnB5) | `76.02% <64.40%> (-10.53%)` | :arrow_down: | | [...runners/interactive/display/pcoll\_visualization.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9kaXNwbGF5L3Bjb2xsX3Zpc3VhbGl6YXRpb24ucHk=) | `85.26% <87.50%> (-0.88%)` | :arrow_down: | | [...ive/messaging/interactive\_environment\_inspector.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9tZXNzYWdpbmcvaW50ZXJhY3RpdmVfZW52aXJvbm1lbnRfaW5zcGVjdG9yLnB5) | `97.43% <100.00%> (ø)` | | | [...am/runners/interactive/options/capture\_limiters.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9vcHRpb25zL2NhcHR1cmVfbGltaXRlcnMucHk=) | `90.47% <100.00%> (-3.08%)` | :arrow_down: | | [...ache\_beam/runners/interactive/recording\_manager.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9yZWNvcmRpbmdfbWFuYWdlci5weQ==) | `98.33% <100.00%> (+5.47%)` | :arrow_up: | | [...ks/python/apache\_beam/runners/interactive/utils.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS91dGlscy5weQ==) | `92.38% <100.00%> (+0.30%)` | :arrow_up: | | [...ks/python/apache\_beam/runners/worker/data\_plane.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy93b3JrZXIvZGF0YV9wbGFuZS5weQ==) | `88.68% <0.00%> (-1.23%)` | :arrow_down: | | ... and [19 more](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12799?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12799?src=pr=footer). Last update [5206131...9ba76c5](https://codecov.io/gh/apache/beam/pull/12799?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12799: [BEAM-10603] Add record_pipeline, clear to RM and fix duration limiter
codecov[bot] edited a comment on pull request #12799: URL: https://github.com/apache/beam/pull/12799#issuecomment-692960218 # [Codecov](https://codecov.io/gh/apache/beam/pull/12799?src=pr=h1) Report > Merging [#12799](https://codecov.io/gh/apache/beam/pull/12799?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/a2126e571cfddc90e3819e66ef1f2da3d52833ed?el=desc) will **decrease** coverage by `0.06%`. > The diff coverage is `58.56%`. [](https://codecov.io/gh/apache/beam/pull/12799?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12799 +/- ## == - Coverage 82.36% 82.30% -0.07% == Files 450 451 +1 Lines 5370853849 +141 == + Hits4423844319 +81 - Misses 9470 9530 +60 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12799?src=pr=tree) | Coverage Δ | | |---|---|---| | [...beam/runners/interactive/background\_caching\_job.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9iYWNrZ3JvdW5kX2NhY2hpbmdfam9iLnB5) | `96.52% <ø> (+1.73%)` | :arrow_up: | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [...pache\_beam/runners/interactive/interactive\_beam.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9pbnRlcmFjdGl2ZV9iZWFtLnB5) | `76.02% <64.40%> (-10.53%)` | :arrow_down: | | [...runners/interactive/display/pcoll\_visualization.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9kaXNwbGF5L3Bjb2xsX3Zpc3VhbGl6YXRpb24ucHk=) | `85.26% <87.50%> (-0.88%)` | :arrow_down: | | [...ive/messaging/interactive\_environment\_inspector.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9tZXNzYWdpbmcvaW50ZXJhY3RpdmVfZW52aXJvbm1lbnRfaW5zcGVjdG9yLnB5) | `97.43% <100.00%> (ø)` | | | [...am/runners/interactive/options/capture\_limiters.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9vcHRpb25zL2NhcHR1cmVfbGltaXRlcnMucHk=) | `90.47% <100.00%> (-3.08%)` | :arrow_down: | | [...ache\_beam/runners/interactive/recording\_manager.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9yZWNvcmRpbmdfbWFuYWdlci5weQ==) | `98.33% <100.00%> (+5.47%)` | :arrow_up: | | [...ks/python/apache\_beam/runners/interactive/utils.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS91dGlscy5weQ==) | `92.38% <100.00%> (+0.30%)` | :arrow_up: | | [...ks/python/apache\_beam/runners/worker/data\_plane.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy93b3JrZXIvZGF0YV9wbGFuZS5weQ==) | `88.68% <0.00%> (-1.23%)` | :arrow_down: | | ... and [19 more](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12799?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12799?src=pr=footer). Last update [5206131...9ba76c5](https://codecov.io/gh/apache/beam/pull/12799?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] commented on pull request #12799: [BEAM-10603] Add record_pipeline, clear to RM and fix duration limiter
codecov[bot] commented on pull request #12799: URL: https://github.com/apache/beam/pull/12799#issuecomment-692960218 # [Codecov](https://codecov.io/gh/apache/beam/pull/12799?src=pr=h1) Report > Merging [#12799](https://codecov.io/gh/apache/beam/pull/12799?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/a2126e571cfddc90e3819e66ef1f2da3d52833ed?el=desc) will **decrease** coverage by `0.06%`. > The diff coverage is `58.56%`. [](https://codecov.io/gh/apache/beam/pull/12799?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12799 +/- ## == - Coverage 82.36% 82.30% -0.07% == Files 450 451 +1 Lines 5370853849 +141 == + Hits4423844319 +81 - Misses 9470 9530 +60 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12799?src=pr=tree) | Coverage Δ | | |---|---|---| | [...beam/runners/interactive/background\_caching\_job.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9iYWNrZ3JvdW5kX2NhY2hpbmdfam9iLnB5) | `96.52% <ø> (+1.73%)` | :arrow_up: | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [...pache\_beam/runners/interactive/interactive\_beam.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9pbnRlcmFjdGl2ZV9iZWFtLnB5) | `76.02% <64.40%> (-10.53%)` | :arrow_down: | | [...runners/interactive/display/pcoll\_visualization.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9kaXNwbGF5L3Bjb2xsX3Zpc3VhbGl6YXRpb24ucHk=) | `85.26% <87.50%> (-0.88%)` | :arrow_down: | | [...ive/messaging/interactive\_environment\_inspector.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9tZXNzYWdpbmcvaW50ZXJhY3RpdmVfZW52aXJvbm1lbnRfaW5zcGVjdG9yLnB5) | `97.43% <100.00%> (ø)` | | | [...am/runners/interactive/options/capture\_limiters.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9vcHRpb25zL2NhcHR1cmVfbGltaXRlcnMucHk=) | `90.47% <100.00%> (-3.08%)` | :arrow_down: | | [...ache\_beam/runners/interactive/recording\_manager.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9yZWNvcmRpbmdfbWFuYWdlci5weQ==) | `98.33% <100.00%> (+5.47%)` | :arrow_up: | | [...ks/python/apache\_beam/runners/interactive/utils.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS91dGlscy5weQ==) | `92.38% <100.00%> (+0.30%)` | :arrow_up: | | [...ks/python/apache\_beam/runners/worker/data\_plane.py](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy93b3JrZXIvZGF0YV9wbGFuZS5weQ==) | `88.68% <0.00%> (-1.23%)` | :arrow_down: | | ... and [19 more](https://codecov.io/gh/apache/beam/pull/12799/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12799?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12799?src=pr=footer). Last update [5206131...9ba76c5](https://codecov.io/gh/apache/beam/pull/12799?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] robertwb commented on pull request #12819: [BEAM-9561] Initial framework for testing pandas website docs.
robertwb commented on pull request #12819: URL: https://github.com/apache/beam/pull/12819#issuecomment-692945434 Run Python PreCommit This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] robertwb commented on a change in pull request #12782: Overriding Dataflow Native BQSource.
robertwb commented on a change in pull request #12782:
URL: https://github.com/apache/beam/pull/12782#discussion_r488902538
##
File path: sdks/python/apache_beam/io/gcp/bigquery_test.py
##
@@ -168,7 +168,8 @@ def test_invalid_json_neg_inf(self):
@unittest.skipIf(HttpError is None, 'GCP dependencies are not installed')
class TestBigQuerySource(unittest.TestCase):
def test_display_data_item_on_validate_true(self):
-source = beam.io.BigQuerySource('dataset.table', validate=True)
+source = beam.io.BigQuerySource(
Review comment:
How many of these tests fail if we don't force the native source?
##
File path: sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
##
@@ -451,7 +451,9 @@ def test_read_from_table(self):
client.jobs.GetQueryResults.return_value =
bigquery.GetQueryResultsResponse(
jobComplete=True, rows=table_rows, schema=schema)
actual_rows = []
-with beam.io.BigQuerySource('dataset.table').reader(client) as reader:
+with beam.io.BigQuerySource(
Review comment:
Could you file a JIRA to update these to not depend on the deprecated
source?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] robertwb commented on pull request #12782: Overriding Dataflow Native BQSource.
robertwb commented on pull request #12782: URL: https://github.com/apache/beam/pull/12782#issuecomment-692921764 Run Portable_Python PreCommit This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] TheNeuralBit commented on pull request #12705: [BEAM-10720][WIP] Implement StringMethods
TheNeuralBit commented on pull request #12705: URL: https://github.com/apache/beam/pull/12705#issuecomment-692919644 > Can you rebase this? Done This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] monicadsong closed pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
monicadsong closed pull request #12851: URL: https://github.com/apache/beam/pull/12851 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] tvalentyn commented on a change in pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
tvalentyn commented on a change in pull request #12851: URL: https://github.com/apache/beam/pull/12851#discussion_r488878863 ## File path: sdks/python/apache_beam/transforms/stats_test.py ## @@ -89,6 +90,12 @@ def setUp(self): None, 0.1, 'assert:global_by_error_with_large_population'), + ( + 'numpy_input_data', + np.array(range(10)), Review comment: I see. At this point I am curious about how users use ApproximateUnique. The use-case you are addressing is when users pass a PCollection of elements, where each element is a single value stored in a numpy datatype. Since it's a single value, we convert it to a scalar. Is that right? I am wondering if there is also a use-case when users pass a PCollection of numpy arrays (perhaps erroneously). In which case the current combiner will pick the first element of the array, so approximation may not be very precise. I wonder if a more precise implementation makes sense or this use-case is not common. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
codecov[bot] edited a comment on pull request #12851: URL: https://github.com/apache/beam/pull/12851#issuecomment-692881256 # [Codecov](https://codecov.io/gh/apache/beam/pull/12851?src=pr=h1) Report > Merging [#12851](https://codecov.io/gh/apache/beam/pull/12851?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/803efdd931699c24e60350e5ccf6e54482f5916f?el=desc) will **decrease** coverage by `0.10%`. > The diff coverage is `87.12%`. [](https://codecov.io/gh/apache/beam/pull/12851?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12851 +/- ## == - Coverage 82.38% 82.27% -0.11% == Files 451 451 Lines 5377553730 -45 == - Hits4430344207 -96 - Misses 9472 9523 +51 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12851?src=pr=tree) | Coverage Δ | | |---|---|---| | [...n/apache\_beam/runners/dataflow/dataflow\_metrics.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kYXRhZmxvdy9kYXRhZmxvd19tZXRyaWNzLnB5) | `74.32% <33.33%> (-0.85%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/combiners.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9jb21iaW5lcnMucHk=) | `92.14% <50.00%> (-0.02%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frame\_base.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lX2Jhc2UucHk=) | `84.70% <57.14%> (ø)` | | | [sdks/python/apache\_beam/dataframe/doctests.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2RvY3Rlc3RzLnB5) | `96.83% <83.33%> (-0.93%)` | :arrow_down: | | [sdks/python/apache\_beam/typehints/opcodes.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL29wY29kZXMucHk=) | `87.65% <83.33%> (-0.35%)` | :arrow_down: | | [sdks/python/apache\_beam/typehints/schemas.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL3NjaGVtYXMucHk=) | `93.20% <86.66%> (-3.41%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/stats.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9zdGF0cy5weQ==) | `90.47% <88.00%> (+3.08%)` | :arrow_up: | | [sdks/python/apache\_beam/coders/row\_coder.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Jvd19jb2Rlci5weQ==) | `94.36% <90.90%> (-0.64%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frames.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lcy5weQ==) | `90.48% <100.00%> (+0.07%)` | :arrow_up: | | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.58% <100.00%> (+1.27%)` | :arrow_up: | | ... and [31 more](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12851?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12851?src=pr=footer). Last update [63f54fd...4f71b9b](https://codecov.io/gh/apache/beam/pull/12851?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12806: [BEAM-10869] Make WriteToPubsub output serialized PubsubMessage proto bytes when using runner v2
codecov[bot] edited a comment on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-692876818 # [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=h1) Report > Merging [#12806](https://codecov.io/gh/apache/beam/pull/12806?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/2bb60c323095340240ec4982c1e9dabc397107e5?el=desc) will **increase** coverage by `41.83%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12806 +/- ## === + Coverage 40.43% 82.27% +41.83% === Files 449 451+2 Lines 5353053785 +255 === + Hits2164344249+22606 + Misses 31887 9536-22351 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) | Coverage Δ | | |---|---|---| | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.54% <ø> (+51.53%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_util.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya191dGlsLnB5) | `0.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...eam/testing/benchmarks/nexmark/nexmark\_launcher.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19sYXVuY2hlci5weQ==) | `0.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [sdks/python/apache\_beam/utils/histogram.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvaGlzdG9ncmFtLnB5) | `94.28% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_perf.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19wZXJmLnB5) | `0.00% <0.00%> (ø)` | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | ... and [275 more](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=footer). Last update [b6d0abb...60b0229](https://codecov.io/gh/apache/beam/pull/12806?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12576: [BEAM-10671] Add environment configuration fields as first-class pipeline options.
codecov[bot] edited a comment on pull request #12576: URL: https://github.com/apache/beam/pull/12576#issuecomment-692353567 # [Codecov](https://codecov.io/gh/apache/beam/pull/12576?src=pr=h1) Report > Merging [#12576](https://codecov.io/gh/apache/beam/pull/12576?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/803efdd931699c24e60350e5ccf6e54482f5916f?el=desc) will **decrease** coverage by `0.08%`. > The diff coverage is `89.89%`. [](https://codecov.io/gh/apache/beam/pull/12576?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12576 +/- ## == - Coverage 82.38% 82.30% -0.09% == Files 451 451 Lines 5377553807 +32 == - Hits4430344284 -19 - Misses 9472 9523 +51 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12576?src=pr=tree) | Coverage Δ | | |---|---|---| | [...n/apache\_beam/runners/dataflow/dataflow\_metrics.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kYXRhZmxvdy9kYXRhZmxvd19tZXRyaWNzLnB5) | `74.32% <33.33%> (-0.85%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/combiners.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9jb21iaW5lcnMucHk=) | `92.14% <50.00%> (-0.02%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frame\_base.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lX2Jhc2UucHk=) | `84.70% <57.14%> (ø)` | | | [sdks/python/apache\_beam/dataframe/doctests.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2RvY3Rlc3RzLnB5) | `96.83% <83.33%> (-0.93%)` | :arrow_down: | | [sdks/python/apache\_beam/typehints/opcodes.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL29wY29kZXMucHk=) | `87.65% <83.33%> (-0.35%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/stats.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9zdGF0cy5weQ==) | `90.41% <86.36%> (+3.02%)` | :arrow_up: | | [sdks/python/apache\_beam/typehints/schemas.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL3NjaGVtYXMucHk=) | `93.20% <86.66%> (-3.41%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/environments.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9lbnZpcm9ubWVudHMucHk=) | `84.19% <90.47%> (+0.37%)` | :arrow_up: | | [sdks/python/apache\_beam/coders/row\_coder.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Jvd19jb2Rlci5weQ==) | `94.36% <90.90%> (-0.64%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frames.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lcy5weQ==) | `90.48% <100.00%> (+0.07%)` | :arrow_up: | | ... and [33 more](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12576?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12576?src=pr=footer). Last update [91a18f1...2710a0e](https://codecov.io/gh/apache/beam/pull/12576?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12576: [BEAM-10671] Add environment configuration fields as first-class pipeline options.
codecov[bot] edited a comment on pull request #12576: URL: https://github.com/apache/beam/pull/12576#issuecomment-692353567 # [Codecov](https://codecov.io/gh/apache/beam/pull/12576?src=pr=h1) Report > Merging [#12576](https://codecov.io/gh/apache/beam/pull/12576?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/803efdd931699c24e60350e5ccf6e54482f5916f?el=desc) will **decrease** coverage by `0.08%`. > The diff coverage is `89.89%`. [](https://codecov.io/gh/apache/beam/pull/12576?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12576 +/- ## == - Coverage 82.38% 82.30% -0.09% == Files 451 451 Lines 5377553807 +32 == - Hits4430344284 -19 - Misses 9472 9523 +51 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12576?src=pr=tree) | Coverage Δ | | |---|---|---| | [...n/apache\_beam/runners/dataflow/dataflow\_metrics.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kYXRhZmxvdy9kYXRhZmxvd19tZXRyaWNzLnB5) | `74.32% <33.33%> (-0.85%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/combiners.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9jb21iaW5lcnMucHk=) | `92.14% <50.00%> (-0.02%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frame\_base.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lX2Jhc2UucHk=) | `84.70% <57.14%> (ø)` | | | [sdks/python/apache\_beam/dataframe/doctests.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2RvY3Rlc3RzLnB5) | `96.83% <83.33%> (-0.93%)` | :arrow_down: | | [sdks/python/apache\_beam/typehints/opcodes.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL29wY29kZXMucHk=) | `87.65% <83.33%> (-0.35%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/stats.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9zdGF0cy5weQ==) | `90.41% <86.36%> (+3.02%)` | :arrow_up: | | [sdks/python/apache\_beam/typehints/schemas.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL3NjaGVtYXMucHk=) | `93.20% <86.66%> (-3.41%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/environments.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9lbnZpcm9ubWVudHMucHk=) | `84.19% <90.47%> (+0.37%)` | :arrow_up: | | [sdks/python/apache\_beam/coders/row\_coder.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Jvd19jb2Rlci5weQ==) | `94.36% <90.90%> (-0.64%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frames.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lcy5weQ==) | `90.48% <100.00%> (+0.07%)` | :arrow_up: | | ... and [33 more](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12576?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12576?src=pr=footer). Last update [91a18f1...2710a0e](https://codecov.io/gh/apache/beam/pull/12576?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] ibzib commented on pull request #12576: [BEAM-10671] Add environment configuration fields as first-class pipeline options.
ibzib commented on pull request #12576: URL: https://github.com/apache/beam/pull/12576#issuecomment-692885360 (Filed BEAM-10901 for test flakes.) This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] pabloem commented on pull request #12721: [BEAM-10871] Add deidentify for FhirIO connector
pabloem commented on pull request #12721: URL: https://github.com/apache/beam/pull/12721#issuecomment-692884656 Run Java PreCommit This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] monicadsong commented on a change in pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
monicadsong commented on a change in pull request #12851: URL: https://github.com/apache/beam/pull/12851#discussion_r488863704 ## File path: sdks/python/apache_beam/transforms/stats_test.py ## @@ -89,6 +90,12 @@ def setUp(self): None, 0.1, 'assert:global_by_error_with_large_population'), + ( + 'numpy_input_data', + np.array(range(10)), Review comment: No. For numpy input I am assuming that for ApproxUniqueCombineFn.add_input(self, accumulator, element), element is a numpy scalar or numpy array with 1 element. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] monicadsong commented on a change in pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
monicadsong commented on a change in pull request #12851: URL: https://github.com/apache/beam/pull/12851#discussion_r488865232 ## File path: sdks/python/apache_beam/transforms/stats_test.py ## @@ -89,6 +90,12 @@ def setUp(self): None, 0.1, 'assert:global_by_error_with_large_population'), + ( + 'numpy_input_data', + np.array(range(10)), Review comment: I can allow multidimensional numpy arrays as input if you want. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] monicadsong commented on a change in pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
monicadsong commented on a change in pull request #12851: URL: https://github.com/apache/beam/pull/12851#discussion_r488863704 ## File path: sdks/python/apache_beam/transforms/stats_test.py ## @@ -89,6 +90,12 @@ def setUp(self): None, 0.1, 'assert:global_by_error_with_large_population'), + ( + 'numpy_input_data', + np.array(range(10)), Review comment: No. For numpy input I am assuming that for ApproxUniqueCombineFn.add_input(self, accumulator, element), element is a numpy scalar (not a numpy array). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] robertwb commented on pull request #12705: [BEAM-10720][WIP] Implement StringMethods
robertwb commented on pull request #12705: URL: https://github.com/apache/beam/pull/12705#issuecomment-692882655 Can you rebase this? This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] monicadsong commented on a change in pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
monicadsong commented on a change in pull request #12851: URL: https://github.com/apache/beam/pull/12851#discussion_r488863704 ## File path: sdks/python/apache_beam/transforms/stats_test.py ## @@ -89,6 +90,12 @@ def setUp(self): None, 0.1, 'assert:global_by_error_with_large_population'), + ( + 'numpy_input_data', + np.array(range(10)), Review comment: No. For numpy input I am assuming that for ApproxUniqueCombineFn.add_input(self, accumulator, element), element is a numpy scalar (not a numpy array). However, if the element is a multidimensional numpy array, then calling element.item(0) returns the value at the first index of the array as a python type. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] robertwb merged pull request #12787: [BEAM-10641] Add eliminate_common_key_with_none graph optimizer
robertwb merged pull request #12787: URL: https://github.com/apache/beam/pull/12787 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] ibzib commented on a change in pull request #12827: [BEAM-10885] Add Avro support to Kafka table provider
ibzib commented on a change in pull request #12827:
URL: https://github.com/apache/beam/pull/12827#discussion_r488862816
##
File path:
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableTest.java
##
@@ -41,27 +40,49 @@
import
org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.rel.type.RelDataTypeSystem;
import
org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.type.SqlTypeName;
import
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
-import org.apache.commons.csv.CSVFormat;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
/** Test for BeamKafkaCSVTable. */
-public class BeamKafkaCSVTableTest {
+public abstract class BeamKafkaTableTest {
@Rule public TestPipeline pipeline = TestPipeline.create();
- private static final Row ROW1 = Row.withSchema(genSchema()).addValues(1L, 1,
1.0).build();
+ protected static final Schema BEAM_SQL_SCHEMA =
+ TestTableUtils.buildBeamSqlSchema(
+ Schema.FieldType.INT32,
+ "order_id",
+ Schema.FieldType.INT32,
+ "site_id",
+ Schema.FieldType.INT32,
+ "price");
- private static final Row ROW2 = Row.withSchema(genSchema()).addValues(2L, 2,
2.0).build();
+ protected static final List TOPICS = ImmutableList.of("topic1",
"topic2");
+
+ protected static final Schema SCHEMA = genSchema();
+
+ protected static final Row ROW1 = Row.withSchema(SCHEMA).addValues(1L, 1,
1.0).build();
+
+ protected static final Row ROW2 = Row.withSchema(SCHEMA).addValues(2L, 2,
2.0).build();
+
+ private static final Map tables = new HashMap<>();
- private static Map tables = new HashMap<>();
protected static BeamSqlEnv env = BeamSqlEnv.readOnly("test", tables);
+ protected abstract KafkaTestRecord createKafkaTestRecord(
Review comment:
Nit: Can we change this to `createKafkaTestRecord(String key, List
values, int timestamp)`?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] codecov[bot] commented on pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
codecov[bot] commented on pull request #12851: URL: https://github.com/apache/beam/pull/12851#issuecomment-692881256 # [Codecov](https://codecov.io/gh/apache/beam/pull/12851?src=pr=h1) Report > Merging [#12851](https://codecov.io/gh/apache/beam/pull/12851?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/803efdd931699c24e60350e5ccf6e54482f5916f?el=desc) will **decrease** coverage by `0.10%`. > The diff coverage is `87.12%`. [](https://codecov.io/gh/apache/beam/pull/12851?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12851 +/- ## == - Coverage 82.38% 82.27% -0.11% == Files 451 451 Lines 5377553730 -45 == - Hits4430344207 -96 - Misses 9472 9523 +51 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12851?src=pr=tree) | Coverage Δ | | |---|---|---| | [...n/apache\_beam/runners/dataflow/dataflow\_metrics.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kYXRhZmxvdy9kYXRhZmxvd19tZXRyaWNzLnB5) | `74.32% <33.33%> (-0.85%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/combiners.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9jb21iaW5lcnMucHk=) | `92.14% <50.00%> (-0.02%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frame\_base.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lX2Jhc2UucHk=) | `84.70% <57.14%> (ø)` | | | [sdks/python/apache\_beam/dataframe/doctests.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2RvY3Rlc3RzLnB5) | `96.83% <83.33%> (-0.93%)` | :arrow_down: | | [sdks/python/apache\_beam/typehints/opcodes.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL29wY29kZXMucHk=) | `87.65% <83.33%> (-0.35%)` | :arrow_down: | | [sdks/python/apache\_beam/typehints/schemas.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL3NjaGVtYXMucHk=) | `93.20% <86.66%> (-3.41%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/stats.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9zdGF0cy5weQ==) | `90.47% <88.00%> (+3.08%)` | :arrow_up: | | [sdks/python/apache\_beam/coders/row\_coder.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Jvd19jb2Rlci5weQ==) | `94.36% <90.90%> (-0.64%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frames.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lcy5weQ==) | `90.48% <100.00%> (+0.07%)` | :arrow_up: | | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.58% <100.00%> (+1.27%)` | :arrow_up: | | ... and [31 more](https://codecov.io/gh/apache/beam/pull/12851/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12851?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12851?src=pr=footer). Last update [63f54fd...4f71b9b](https://codecov.io/gh/apache/beam/pull/12851?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12576: [BEAM-10671] Add environment configuration fields as first-class pipeline options.
codecov[bot] edited a comment on pull request #12576: URL: https://github.com/apache/beam/pull/12576#issuecomment-692353567 # [Codecov](https://codecov.io/gh/apache/beam/pull/12576?src=pr=h1) Report > Merging [#12576](https://codecov.io/gh/apache/beam/pull/12576?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/803efdd931699c24e60350e5ccf6e54482f5916f?el=desc) will **decrease** coverage by `0.08%`. > The diff coverage is `89.89%`. [](https://codecov.io/gh/apache/beam/pull/12576?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12576 +/- ## == - Coverage 82.38% 82.30% -0.09% == Files 451 451 Lines 5377553807 +32 == - Hits4430344284 -19 - Misses 9472 9523 +51 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12576?src=pr=tree) | Coverage Δ | | |---|---|---| | [...n/apache\_beam/runners/dataflow/dataflow\_metrics.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kYXRhZmxvdy9kYXRhZmxvd19tZXRyaWNzLnB5) | `74.32% <33.33%> (-0.85%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/combiners.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9jb21iaW5lcnMucHk=) | `92.14% <50.00%> (-0.02%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frame\_base.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lX2Jhc2UucHk=) | `84.70% <57.14%> (ø)` | | | [sdks/python/apache\_beam/dataframe/doctests.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2RvY3Rlc3RzLnB5) | `96.83% <83.33%> (-0.93%)` | :arrow_down: | | [sdks/python/apache\_beam/typehints/opcodes.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL29wY29kZXMucHk=) | `87.65% <83.33%> (-0.35%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/stats.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9zdGF0cy5weQ==) | `90.41% <86.36%> (+3.02%)` | :arrow_up: | | [sdks/python/apache\_beam/typehints/schemas.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL3NjaGVtYXMucHk=) | `93.20% <86.66%> (-3.41%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/environments.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9lbnZpcm9ubWVudHMucHk=) | `84.19% <90.47%> (+0.37%)` | :arrow_up: | | [sdks/python/apache\_beam/coders/row\_coder.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Jvd19jb2Rlci5weQ==) | `94.36% <90.90%> (-0.64%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frames.py](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lcy5weQ==) | `90.48% <100.00%> (+0.07%)` | :arrow_up: | | ... and [33 more](https://codecov.io/gh/apache/beam/pull/12576/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12576?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12576?src=pr=footer). Last update [91a18f1...2710a0e](https://codecov.io/gh/apache/beam/pull/12576?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12806: [BEAM-10869] Make WriteToPubsub output serialized PubsubMessage proto bytes when using runner v2
codecov[bot] edited a comment on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-692876818 # [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=h1) Report > Merging [#12806](https://codecov.io/gh/apache/beam/pull/12806?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/2bb60c323095340240ec4982c1e9dabc397107e5?el=desc) will **increase** coverage by `41.83%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12806 +/- ## === + Coverage 40.43% 82.27% +41.83% === Files 449 451+2 Lines 5353053785 +255 === + Hits2164344249+22606 + Misses 31887 9536-22351 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) | Coverage Δ | | |---|---|---| | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.54% <ø> (+51.53%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_util.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya191dGlsLnB5) | `0.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...eam/testing/benchmarks/nexmark/nexmark\_launcher.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19sYXVuY2hlci5weQ==) | `0.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/utils/histogram.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvaGlzdG9ncmFtLnB5) | `94.28% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_perf.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19wZXJmLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | ... and [275 more](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=footer). Last update [b6d0abb...60b0229](https://codecov.io/gh/apache/beam/pull/12806?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12806: [BEAM-10869] Make WriteToPubsub output serialized PubsubMessage proto bytes when using runner v2
codecov[bot] edited a comment on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-692876818 # [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=h1) Report > Merging [#12806](https://codecov.io/gh/apache/beam/pull/12806?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/2bb60c323095340240ec4982c1e9dabc397107e5?el=desc) will **increase** coverage by `41.83%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12806 +/- ## === + Coverage 40.43% 82.27% +41.83% === Files 449 451+2 Lines 5353053785 +255 === + Hits2164344249+22606 + Misses 31887 9536-22351 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) | Coverage Δ | | |---|---|---| | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.54% <ø> (+51.53%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_util.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya191dGlsLnB5) | `0.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...eam/testing/benchmarks/nexmark/nexmark\_launcher.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19sYXVuY2hlci5weQ==) | `0.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [sdks/python/apache\_beam/utils/histogram.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvaGlzdG9ncmFtLnB5) | `94.28% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_perf.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19wZXJmLnB5) | `0.00% <0.00%> (ø)` | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | ... and [275 more](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=footer). Last update [b6d0abb...60b0229](https://codecov.io/gh/apache/beam/pull/12806?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12806: [BEAM-10869] Make WriteToPubsub output serialized PubsubMessage proto bytes when using runner v2
codecov[bot] edited a comment on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-692876818 # [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=h1) Report > Merging [#12806](https://codecov.io/gh/apache/beam/pull/12806?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/2bb60c323095340240ec4982c1e9dabc397107e5?el=desc) will **increase** coverage by `41.83%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12806 +/- ## === + Coverage 40.43% 82.27% +41.83% === Files 449 451+2 Lines 5353053785 +255 === + Hits2164344249+22606 + Misses 31887 9536-22351 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) | Coverage Δ | | |---|---|---| | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.54% <ø> (+51.53%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_util.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya191dGlsLnB5) | `0.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...eam/testing/benchmarks/nexmark/nexmark\_launcher.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19sYXVuY2hlci5weQ==) | `0.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [sdks/python/apache\_beam/utils/histogram.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvaGlzdG9ncmFtLnB5) | `94.28% <0.00%> (ø)` | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_perf.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19wZXJmLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | ... and [275 more](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=footer). Last update [b6d0abb...60b0229](https://codecov.io/gh/apache/beam/pull/12806?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] mxm commented on pull request #12576: [BEAM-10671] Add environment configuration fields as first-class pipeline options.
mxm commented on pull request #12576: URL: https://github.com/apache/beam/pull/12576#issuecomment-692877960 Thanks! I'll have another look tomorrow but generally looks good to me. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12819: [BEAM-9561] Initial framework for testing pandas website docs.
codecov[bot] edited a comment on pull request #12819: URL: https://github.com/apache/beam/pull/12819#issuecomment-691336298 # [Codecov](https://codecov.io/gh/apache/beam/pull/12819?src=pr=h1) Report > Merging [#12819](https://codecov.io/gh/apache/beam/pull/12819?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/803efdd931699c24e60350e5ccf6e54482f5916f?el=desc) will **increase** coverage by `0.03%`. > The diff coverage is `88.93%`. [](https://codecov.io/gh/apache/beam/pull/12819?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12819 +/- ## == + Coverage 82.38% 82.42% +0.03% == Files 451 451 Lines 5377553983 +208 == + Hits4430344493 +190 - Misses 9472 9490 +18 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12819?src=pr=tree) | Coverage Δ | | |---|---|---| | [...n/apache\_beam/runners/dataflow/dataflow\_metrics.py](https://codecov.io/gh/apache/beam/pull/12819/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kYXRhZmxvdy9kYXRhZmxvd19tZXRyaWNzLnB5) | `74.32% <33.33%> (-0.85%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frame\_base.py](https://codecov.io/gh/apache/beam/pull/12819/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lX2Jhc2UucHk=) | `84.70% <57.14%> (ø)` | | | [sdks/python/apache\_beam/typehints/opcodes.py](https://codecov.io/gh/apache/beam/pull/12819/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL29wY29kZXMucHk=) | `87.65% <83.33%> (-0.35%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/stats.py](https://codecov.io/gh/apache/beam/pull/12819/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9zdGF0cy5weQ==) | `87.39% <86.36%> (ø)` | | | [sdks/python/apache\_beam/typehints/schemas.py](https://codecov.io/gh/apache/beam/pull/12819/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL3NjaGVtYXMucHk=) | `93.20% <86.66%> (-3.41%)` | :arrow_down: | | [sdks/python/apache\_beam/coders/row\_coder.py](https://codecov.io/gh/apache/beam/pull/12819/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Jvd19jb2Rlci5weQ==) | `94.36% <90.90%> (-0.64%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/doctests.py](https://codecov.io/gh/apache/beam/pull/12819/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2RvY3Rlc3RzLnB5) | `96.39% <95.09%> (-1.37%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frames.py](https://codecov.io/gh/apache/beam/pull/12819/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lcy5weQ==) | `90.48% <100.00%> (+0.07%)` | :arrow_up: | | [...n/apache\_beam/runners/direct/evaluation\_context.py](https://codecov.io/gh/apache/beam/pull/12819/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kaXJlY3QvZXZhbHVhdGlvbl9jb250ZXh0LnB5) | `91.91% <100.00%> (ø)` | | | [.../python/apache\_beam/transforms/periodicsequence.py](https://codecov.io/gh/apache/beam/pull/12819/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9wZXJpb2RpY3NlcXVlbmNlLnB5) | `96.49% <0.00%> (-1.76%)` | :arrow_down: | | ... and [9 more](https://codecov.io/gh/apache/beam/pull/12819/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12819?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12819?src=pr=footer). Last update [8e7014d...ce69f94](https://codecov.io/gh/apache/beam/pull/12819?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] commented on pull request #12806: [BEAM-10869] Make WriteToPubsub output serialized PubsubMessage proto bytes when using runner v2
codecov[bot] commented on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-692876818 # [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=h1) Report > Merging [#12806](https://codecov.io/gh/apache/beam/pull/12806?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/2bb60c323095340240ec4982c1e9dabc397107e5?el=desc) will **increase** coverage by `41.83%`. > The diff coverage is `n/a`. [](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) ```diff @@ Coverage Diff @@ ## master #12806 +/- ## === + Coverage 40.43% 82.27% +41.83% === Files 449 451+2 Lines 5353053785 +255 === + Hits2164344249+22606 + Misses 31887 9536-22351 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12806?src=pr=tree) | Coverage Δ | | |---|---|---| | [sdks/python/apache\_beam/io/gcp/pubsub.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vaW8vZ2NwL3B1YnN1Yi5weQ==) | `93.54% <ø> (+51.53%)` | :arrow_up: | | [...apache\_beam/portability/api/beam\_runner\_api\_pb2.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjIucHk=) | `100.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_util.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya191dGlsLnB5) | `0.00% <0.00%> (ø)` | | | [...e\_beam/portability/api/beam\_runner\_api\_pb2\_urns.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcG9ydGFiaWxpdHkvYXBpL2JlYW1fcnVubmVyX2FwaV9wYjJfdXJucy5weQ==) | `100.00% <0.00%> (ø)` | | | [...eam/testing/benchmarks/nexmark/nexmark\_launcher.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19sYXVuY2hlci5weQ==) | `0.00% <0.00%> (ø)` | | | [...ache\_beam/runners/interactive/pipeline\_analyzer.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9waXBlbGluZV9hbmFseXplci5weQ==) | | | | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [...he\_beam/testing/benchmarks/nexmark/nexmark\_perf.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvbmV4bWFya19wZXJmLnB5) | `0.00% <0.00%> (ø)` | | | [sdks/python/apache\_beam/utils/histogram.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdXRpbHMvaGlzdG9ncmFtLnB5) | `94.28% <0.00%> (ø)` | | | [sdks/python/apache\_beam/coders/slow\_stream.py](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Nsb3dfc3RyZWFtLnB5) | `92.43% <0.00%> (+1.68%)` | :arrow_up: | | ... and [275 more](https://codecov.io/gh/apache/beam/pull/12806/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12806?src=pr=footer). Last update [b6d0abb...60b0229](https://codecov.io/gh/apache/beam/pull/12806?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] tvalentyn commented on a change in pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
tvalentyn commented on a change in pull request #12851: URL: https://github.com/apache/beam/pull/12851#discussion_r488853550 ## File path: sdks/python/apache_beam/transforms/stats_test.py ## @@ -89,6 +90,12 @@ def setUp(self): None, 0.1, 'assert:global_by_error_with_large_population'), + ( + 'numpy_input_data', + np.array(range(10)), Review comment: Do multidimensional arrays also work? This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] boyuanzz commented on pull request #12806: [BEAM-10869] Use PubsubMessagePayloadOnlyCoder when writing to Pubsub.
boyuanzz commented on pull request #12806: URL: https://github.com/apache/beam/pull/12806#issuecomment-692865584 r: @chamikaramj This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12779: [BEAM-10856] Support for NestedValueProvider for Python SDK
codecov[bot] edited a comment on pull request #12779: URL: https://github.com/apache/beam/pull/12779#issuecomment-692856347 # [Codecov](https://codecov.io/gh/apache/beam/pull/12779?src=pr=h1) Report > Merging [#12779](https://codecov.io/gh/apache/beam/pull/12779?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/a2126e571cfddc90e3819e66ef1f2da3d52833ed?el=desc) will **decrease** coverage by `0.08%`. > The diff coverage is `44.11%`. [](https://codecov.io/gh/apache/beam/pull/12779?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12779 +/- ## == - Coverage 82.36% 82.28% -0.09% == Files 450 451 +1 Lines 5370853738 +30 == - Hits4423844217 -21 - Misses 9470 9521 +51 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12779?src=pr=tree) | Coverage Δ | | |---|---|---| | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [...pache\_beam/runners/interactive/interactive\_beam.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9pbnRlcmFjdGl2ZV9iZWFtLnB5) | `76.02% <64.40%> (-10.53%)` | :arrow_down: | | [...runners/interactive/display/pcoll\_visualization.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9kaXNwbGF5L3Bjb2xsX3Zpc3VhbGl6YXRpb24ucHk=) | `85.26% <87.50%> (-0.88%)` | :arrow_down: | | [sdks/python/apache\_beam/options/value\_provider.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vb3B0aW9ucy92YWx1ZV9wcm92aWRlci5weQ==) | `91.76% <93.33%> (+0.21%)` | :arrow_up: | | [...ive/messaging/interactive\_environment\_inspector.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9tZXNzYWdpbmcvaW50ZXJhY3RpdmVfZW52aXJvbm1lbnRfaW5zcGVjdG9yLnB5) | `97.43% <100.00%> (ø)` | | | [...eam/runners/interactive/options/capture\_control.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9vcHRpb25zL2NhcHR1cmVfY29udHJvbC5weQ==) | `92.00% <0.00%> (-8.00%)` | :arrow_down: | | [...ks/python/apache\_beam/runners/worker/sdk\_worker.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy93b3JrZXIvc2RrX3dvcmtlci5weQ==) | `88.98% <0.00%> (-0.36%)` | :arrow_down: | | [...hon/apache\_beam/runners/worker/bundle\_processor.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy93b3JrZXIvYnVuZGxlX3Byb2Nlc3Nvci5weQ==) | `94.45% <0.00%> (-0.14%)` | :arrow_down: | | [sdks/python/apache\_beam/pvalue.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcHZhbHVlLnB5) | `92.01% <0.00%> (+0.38%)` | :arrow_up: | | ... and [5 more](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12779?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12779?src=pr=footer). Last update [5206131...af2c14c](https://codecov.io/gh/apache/beam/pull/12779?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] robertwb merged pull request #12812: [BEAM-10873] Stronger testing of dataframe partitioning declarations.
robertwb merged pull request #12812: URL: https://github.com/apache/beam/pull/12812 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] robertwb commented on pull request #12812: [BEAM-10873] Stronger testing of dataframe partitioning declarations.
robertwb commented on pull request #12812: URL: https://github.com/apache/beam/pull/12812#issuecomment-692861888 The github actions failures look completely unrelated and the tests all pass on jenkins (and locally). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] ibzib commented on a change in pull request #12576: [BEAM-10671] Add environment configuration fields as first-class pipeline options.
ibzib commented on a change in pull request #12576:
URL: https://github.com/apache/beam/pull/12576#discussion_r488839465
##
File path: sdks/python/apache_beam/options/pipeline_options.py
##
@@ -1044,7 +1044,12 @@ def _add_argparse_args(cls, parser):
'form {"os": "", "arch": "", "command": '
'"", "env":{"": '
'""} }. All fields in the json are optional except '
-'command.'))
+'command.\n\nPlease consider using the following '
+'options instead, depending on the environment type:\n'
+' --docker_env_container_image\n'
+' --process_env_command\n'
+' --process_env_variables\n'
+' --external_env_service_address'))
Review comment:
Done.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] monicadsong commented on pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
monicadsong commented on pull request #12851: URL: https://github.com/apache/beam/pull/12851#issuecomment-692856430 R: @tvalentyn This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] codecov[bot] commented on pull request #12779: [BEAM-10856] Support for NestedValueProvider for Python SDK
codecov[bot] commented on pull request #12779: URL: https://github.com/apache/beam/pull/12779#issuecomment-692856347 # [Codecov](https://codecov.io/gh/apache/beam/pull/12779?src=pr=h1) Report > Merging [#12779](https://codecov.io/gh/apache/beam/pull/12779?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/a2126e571cfddc90e3819e66ef1f2da3d52833ed?el=desc) will **decrease** coverage by `0.08%`. > The diff coverage is `44.11%`. [](https://codecov.io/gh/apache/beam/pull/12779?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12779 +/- ## == - Coverage 82.36% 82.28% -0.09% == Files 450 451 +1 Lines 5370853738 +30 == - Hits4423844217 -21 - Misses 9470 9521 +51 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12779?src=pr=tree) | Coverage Δ | | |---|---|---| | [...beam/testing/benchmarks/nexmark/queries/query10.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdGVzdGluZy9iZW5jaG1hcmtzL25leG1hcmsvcXVlcmllcy9xdWVyeTEwLnB5) | `0.00% <0.00%> (ø)` | | | [...pache\_beam/runners/interactive/interactive\_beam.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9pbnRlcmFjdGl2ZV9iZWFtLnB5) | `76.02% <64.40%> (-10.53%)` | :arrow_down: | | [...runners/interactive/display/pcoll\_visualization.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9kaXNwbGF5L3Bjb2xsX3Zpc3VhbGl6YXRpb24ucHk=) | `85.26% <87.50%> (-0.88%)` | :arrow_down: | | [sdks/python/apache\_beam/options/value\_provider.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vb3B0aW9ucy92YWx1ZV9wcm92aWRlci5weQ==) | `91.76% <93.33%> (+0.21%)` | :arrow_up: | | [...ive/messaging/interactive\_environment\_inspector.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9tZXNzYWdpbmcvaW50ZXJhY3RpdmVfZW52aXJvbm1lbnRfaW5zcGVjdG9yLnB5) | `97.43% <100.00%> (ø)` | | | [...eam/runners/interactive/options/capture\_control.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9vcHRpb25zL2NhcHR1cmVfY29udHJvbC5weQ==) | `92.00% <0.00%> (-8.00%)` | :arrow_down: | | [...ks/python/apache\_beam/runners/worker/sdk\_worker.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy93b3JrZXIvc2RrX3dvcmtlci5weQ==) | `88.98% <0.00%> (-0.36%)` | :arrow_down: | | [...hon/apache\_beam/runners/worker/bundle\_processor.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy93b3JrZXIvYnVuZGxlX3Byb2Nlc3Nvci5weQ==) | `94.45% <0.00%> (-0.14%)` | :arrow_down: | | [sdks/python/apache\_beam/pvalue.py](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcHZhbHVlLnB5) | `92.01% <0.00%> (+0.38%)` | :arrow_up: | | ... and [5 more](https://codecov.io/gh/apache/beam/pull/12779/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12779?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12779?src=pr=footer). Last update [5206131...af2c14c](https://codecov.io/gh/apache/beam/pull/12779?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] monicadsong opened a new pull request #12851: [BEAM-10900] add ability for ApproximateUniqueCombineFn to handle numpy input data
monicadsong opened a new pull request #12851: URL: https://github.com/apache/beam/pull/12851 If element for ApproximateUniqueCombineFn.add_input() is numpy type, convert to python type. Thank you for your contribution! Follow this checklist to help us incorporate your contribution quickly and easily: - [ ] [**Choose reviewer(s)**](https://beam.apache.org/contribute/#make-your-change) and mention them in a comment (`R: @username`). - [ ] Format the pull request title like `[BEAM-XXX] Fixes bug in ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA issue, if applicable. This will automatically link the pull request to the issue. - [ ] Update `CHANGES.md` with noteworthy changes. - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf). See the [Contributor Guide](https://beam.apache.org/contribute) for more tips on [how to make review process smoother](https://beam.apache.org/contribute/#make-reviewers-job-easier). Post-Commit Tests Status (on master branch) Lang | SDK | Dataflow | Flink | Samza | Spark | Twister2 --- | --- | --- | --- | --- | --- | --- Go | [](https://ci-beam.apache.org/job/beam_PostCommit_Go/lastCompletedBuild/) | --- | [](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/lastCompletedBuild/) | --- | [](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark/lastCompletedBuild/) | --- Java | [](https://ci-beam.apache.org/job/beam_PostCommit_Java/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow_Java11/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink_Java11/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Flink_Batch/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Flink_Streaming/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Samza/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Spark/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Spark_Batch/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Twister2/lastCompletedBuild/) Python | [](https://ci-beam.apache.org/job/beam_PostCommit_Python2/lastCompletedBuild/)[![Build
[GitHub] [beam] tvalentyn commented on pull request #12811: [BEAM-10705] Fixes a bug when passing whl files in --sdk_location from https locations
tvalentyn commented on pull request #12811: URL: https://github.com/apache/beam/pull/12811#issuecomment-692850902 Thanks, @samatix ! This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] tvalentyn merged pull request #12811: [BEAM-10705] Fixes a bug when passing whl files in --sdk_location from https locations
tvalentyn merged pull request #12811: URL: https://github.com/apache/beam/pull/12811 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] tvalentyn commented on a change in pull request #12811: [BEAM-10705] Fixes a bug when passing whl files in --sdk_location from https locations
tvalentyn commented on a change in pull request #12811: URL: https://github.com/apache/beam/pull/12811#discussion_r488824939 ## File path: sdks/python/apache_beam/runners/portability/stager_test.py ## @@ -448,7 +448,8 @@ def test_sdk_location_remote_source_file(self, *unused_mocks): def test_sdk_location_remote_wheel_file(self, *unused_mocks): staging_dir = self.make_temp_dir() sdk_filename = 'apache_beam-1.0.0-cp27-cp27mu-manylinux1_x86_64.whl' -sdk_location = 'https://storage.googleapis.com/my-gcs-bucket/' + sdk_filename +sdk_location = 'https://storage.googleapis.com/my-gcs-bucket/' + \ Review comment: Nit: implicit line joining (with parenthesis instead of backslash) is preferable by PIP8. Since Beam does not officially use PIP8, I will merge as is, just FYI. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] lukecwik commented on a change in pull request #12617: [BEAM-10670] Update Samza to be opt-out for SplittableDoFn powering the Read transform.
lukecwik commented on a change in pull request #12617:
URL: https://github.com/apache/beam/pull/12617#discussion_r488825043
##
File path:
runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaRunner.java
##
@@ -106,6 +107,7 @@ public PortablePipelineResult
runPortablePipeline(RunnerApi.Pipeline pipeline) {
@Override
public SamzaPipelineResult run(Pipeline pipeline) {
+SplittableParDo.validateNoPrimitiveReads(pipeline);
Review comment:
Yes since portable pipelines only support SDF. This is about migrating
non-portable pipeline runner implementations to use SDF.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] amaliujia commented on pull request #12843: [BEAM-10895] Support UNNEST an (possibly nested) array field of an struct column
amaliujia commented on pull request #12843: URL: https://github.com/apache/beam/pull/12843#issuecomment-692848083 Thanks @robinyqiu will take a look soon. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] kw2542 commented on a change in pull request #12617: [BEAM-10670] Update Samza to be opt-out for SplittableDoFn powering the Read transform.
kw2542 commented on a change in pull request #12617:
URL: https://github.com/apache/beam/pull/12617#discussion_r488820872
##
File path:
runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaRunner.java
##
@@ -106,6 +107,7 @@ public PortablePipelineResult
runPortablePipeline(RunnerApi.Pipeline pipeline) {
@Override
public SamzaPipelineResult run(Pipeline pipeline) {
+SplittableParDo.validateNoPrimitiveReads(pipeline);
Review comment:
is it expected that we only validate in non-portable mode? i.e. this
validation does not exist in runPortablePipeline()
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] tvalentyn commented on pull request #12811: [BEAM-10705] Fixes a bug when passing whl files in --sdk_location from https locations
tvalentyn commented on pull request #12811: URL: https://github.com/apache/beam/pull/12811#issuecomment-692845652 Actually found the previous green PostCommit result: https://ci-beam.apache.org/job/beam_PostCommit_Python36_PR/93/ This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] tvalentyn commented on pull request #12811: [BEAM-10705] Fixes a bug when passing whl files in --sdk_location from https locations
tvalentyn commented on pull request #12811: URL: https://github.com/apache/beam/pull/12811#issuecomment-692844750 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] tvalentyn commented on pull request #12811: [BEAM-10705] Fixes a bug when passing whl files in --sdk_location from https locations
tvalentyn commented on pull request #12811: URL: https://github.com/apache/beam/pull/12811#issuecomment-692844451 > able to reproduce the last two issues (they seem to be random). Any ideas on what went wrong? Those seem to be flaky tests :(. Found this in the logs: ``` self = def test_time_based_flush_grpc_data_channel(self): > self._grpc_data_channel_test(True) apache_beam/runners/worker/data_plane_test.py:44: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ apache_beam/runners/worker/data_plane_test.py:74: in _grpc_data_channel_test data_channel_service, data_channel_client, time_based_flush) apache_beam/runners/worker/data_plane_test.py:86: in _data_channel_test self._data_channel_test_one_direction(server, client, time_based_flush) apache_beam/runners/worker/data_plane_test.py:129: in _data_channel_test_one_direction instruction_id='2', transform_id=transform_2, data=b'ghi') E AssertionError: Lists differ: [inst[26 chars]id: "2" E data: "ghi" E , instruction_id: "2" E tran[22 chars]ef" E ] != [inst[26 chars]id: "1" E data: "def" E , instruction_id: "2" E tran[22 chars]hi" E ] E E First differing element 0: E instruction_id: "2" E transform_id: "2" E data: "ghi" E E instruction_id: "2" E transform_id: "1" E data: "def" E E E [instruction_id: "2" E + transform_id: "1" E + data: "def" E + , E + instruction_id: "2" E transform_id: "2" E data: "ghi" E - , E - instruction_id: "2" E - transform_id: "1" E - data: "def" E ] ``` There is an open issue about it: https://issues.apache.org/jira/browse/BEAM-10768. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] chamikaramj commented on a change in pull request #12505: [BEAM-8106] Add version to java container image name
chamikaramj commented on a change in pull request #12505: URL: https://github.com/apache/beam/pull/12505#discussion_r488818825 ## File path: website/www/site/content/en/documentation/runtime/environments.md ## @@ -116,8 +116,8 @@ By default, no licenses/notices are added to the docker images. To examine the containers that you built, run `docker images` from anywhere in the command line. If you successfully built all of the container images, the command prints a table like the following: ``` -REPOSITORY TAG IMAGE ID CREATED SIZE -apache/beam_java_sdk latest 16ca619d489e2 weeks ago550MB +REPOSITORY TAG IMAGE ID CREATED SIZE +apache/beam_java_8sdk latest 16ca619d489e2 weeks ago550MB Review comment: Let's make sure that the correct container name is set in the environment send to the remote SDKs that use Java transforms as cross-language transforms. (We don't have integration tests setup for this by we can just try existing Kafka/SQL x-lang examples to confirm) https://github.com/apache/beam/tree/master/sdks/python/apache_beam/examples/kafkataxi https://github.com/apache/beam/blob/master/sdks/python/apache_beam/examples/sql_taxi.py This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] tvalentyn commented on pull request #12637: [BEAM-10768] Don't assert the order in which elements are received.
tvalentyn commented on pull request #12637: URL: https://github.com/apache/beam/pull/12637#issuecomment-692843176 seeing this flake in precommits, thanks for fixing it. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] tvalentyn edited a comment on pull request #12779: [BEAM-10856] Support for NestedValueProvider for Python SDK
tvalentyn edited a comment on pull request #12779: URL: https://github.com/apache/beam/pull/12779#issuecomment-692833522 > A user has written a feature that they would find useful, and that will not change the experience for other users (if anything, it should improve it). The feature looks correct, and similar to what we do in Java. If we reject the PR, we may push the user to run on a fork. Can we let this in? @tvalentyn I agree with this assessment, feel free to merge once tests & linter pass. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] TheNeuralBit commented on a change in pull request #12505: [BEAM-8106] Add version to java container image name
TheNeuralBit commented on a change in pull request #12505:
URL: https://github.com/apache/beam/pull/12505#discussion_r488809349
##
File path: sdks/java/container/Dockerfile-java11
##
@@ -16,6 +16,10 @@
# limitations under the License.
###
+##
+# NOTE: This image is now deprecated.
+# Use Dockerfile with the appropriate java_version build argument.
+##
Review comment:
I'm +1 for just removing this. Looks like it was added in
https://github.com/apache/beam/pull/8053 in order to start running a job for
testing Java 11. As long as the jobs that use Java 11 still pass after removing
this I think it's fine.
I don't think users should be referencing this Dockerfile directly, if
they're building a Java 11 container at all it should be through the gradle
command (and even then we don't support Java 11 so they shouldn't be doing it).
##
File path: website/www/site/content/en/documentation/runtime/environments.md
##
@@ -116,8 +116,8 @@ By default, no licenses/notices are added to the docker
images.
To examine the containers that you built, run `docker images` from anywhere in
the command line. If you successfully built all of the container images, the
command prints a table like the following:
```
-REPOSITORY TAG IMAGE ID
CREATED SIZE
-apache/beam_java_sdk latest 16ca619d489e2
weeks ago550MB
+REPOSITORY TAG IMAGE ID
CREATED SIZE
+apache/beam_java_8sdk latest 16ca619d489e2
weeks ago550MB
Review comment:
typo here:
```suggestion
apache/beam_java8_sdk latest 16ca619d489e2
weeks ago550MB
```
I'm assuming this change and the others like it are the result of a search
to find and update all the references to `beam_java_sdk`, so we don't need to
worry about there being other references?
##
File path:
runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java
##
@@ -357,4 +354,14 @@ private static File zipDirectory(File directory) throws
IOException {
return env;
}
}
+
+ private static String getDefaultJavaSdkHarnessContainerUrl() {
+String javaVersionId =
+Float.parseFloat(System.getProperty("java.specification.version")) >=
9 ? "java11" : "java8";
Review comment:
nit: could you make this an exact check for 8 and 11 and throw an
exception for other (unsupported) versions
##
File path: sdks/java/container/Dockerfile-java11
##
@@ -16,6 +16,10 @@
# limitations under the License.
###
+##
+# NOTE: This image is now deprecated.
+# Use Dockerfile with the appropriate java_version build argument.
+##
Review comment:
Also the fact that this Dockerfile doesn't copy LICENSE and NOTICE is
problematic.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] codecov[bot] edited a comment on pull request #12841: [BEAM-10894] Basic CSV reading and writing.
codecov[bot] edited a comment on pull request #12841: URL: https://github.com/apache/beam/pull/12841#issuecomment-692395796 # [Codecov](https://codecov.io/gh/apache/beam/pull/12841?src=pr=h1) Report > Merging [#12841](https://codecov.io/gh/apache/beam/pull/12841?src=pr=desc) into [master](https://codecov.io/gh/apache/beam/commit/803efdd931699c24e60350e5ccf6e54482f5916f?el=desc) will **decrease** coverage by `0.10%`. > The diff coverage is `87.09%`. [](https://codecov.io/gh/apache/beam/pull/12841?src=pr=tree) ```diff @@Coverage Diff @@ ## master #12841 +/- ## == - Coverage 82.38% 82.27% -0.11% == Files 451 452 +1 Lines 5377553838 +63 == - Hits4430344296 -7 - Misses 9472 9542 +70 ``` | [Impacted Files](https://codecov.io/gh/apache/beam/pull/12841?src=pr=tree) | Coverage Δ | | |---|---|---| | [...n/apache\_beam/runners/dataflow/dataflow\_metrics.py](https://codecov.io/gh/apache/beam/pull/12841/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9kYXRhZmxvdy9kYXRhZmxvd19tZXRyaWNzLnB5) | `74.32% <33.33%> (-0.85%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frame\_base.py](https://codecov.io/gh/apache/beam/pull/12841/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lX2Jhc2UucHk=) | `84.70% <57.14%> (ø)` | | | [sdks/python/apache\_beam/dataframe/doctests.py](https://codecov.io/gh/apache/beam/pull/12841/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2RvY3Rlc3RzLnB5) | `96.83% <83.33%> (-0.93%)` | :arrow_down: | | [sdks/python/apache\_beam/typehints/opcodes.py](https://codecov.io/gh/apache/beam/pull/12841/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL29wY29kZXMucHk=) | `87.65% <83.33%> (-0.35%)` | :arrow_down: | | [sdks/python/apache\_beam/transforms/stats.py](https://codecov.io/gh/apache/beam/pull/12841/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHJhbnNmb3Jtcy9zdGF0cy5weQ==) | `90.41% <86.36%> (+3.02%)` | :arrow_up: | | [sdks/python/apache\_beam/typehints/schemas.py](https://codecov.io/gh/apache/beam/pull/12841/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vdHlwZWhpbnRzL3NjaGVtYXMucHk=) | `93.20% <86.66%> (-3.41%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/io.py](https://codecov.io/gh/apache/beam/pull/12841/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2lvLnB5) | `89.90% <89.90%> (ø)` | | | [sdks/python/apache\_beam/coders/row\_coder.py](https://codecov.io/gh/apache/beam/pull/12841/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vY29kZXJzL3Jvd19jb2Rlci5weQ==) | `94.36% <90.90%> (-0.64%)` | :arrow_down: | | [sdks/python/apache\_beam/dataframe/frames.py](https://codecov.io/gh/apache/beam/pull/12841/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vZGF0YWZyYW1lL2ZyYW1lcy5weQ==) | `90.60% <100.00%> (+0.18%)` | :arrow_up: | | [...pache\_beam/runners/interactive/interactive\_beam.py](https://codecov.io/gh/apache/beam/pull/12841/diff?src=pr=tree#diff-c2Rrcy9weXRob24vYXBhY2hlX2JlYW0vcnVubmVycy9pbnRlcmFjdGl2ZS9pbnRlcmFjdGl2ZV9iZWFtLnB5) | `76.02% <0.00%> (-10.53%)` | :arrow_down: | | ... and [29 more](https://codecov.io/gh/apache/beam/pull/12841/diff?src=pr=tree-more) | | -- [Continue to review full report at Codecov](https://codecov.io/gh/apache/beam/pull/12841?src=pr=continue). > **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta) > `Δ = absolute (impact)`, `ø = not affected`, `? = missing data` > Powered by [Codecov](https://codecov.io/gh/apache/beam/pull/12841?src=pr=footer). Last update [8e7014d...ab66e1c](https://codecov.io/gh/apache/beam/pull/12841?src=pr=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] ajamato commented on a change in pull request #12822: [BEAM-10880] Log error counts to debug BigQuery streaming insert requ…
ajamato commented on a change in pull request #12822: URL: https://github.com/apache/beam/pull/12822#discussion_r488813424 ## File path: sdks/python/apache_beam/io/gcp/bigquery_tools.py ## @@ -538,6 +539,13 @@ def _insert_all_rows( try: response = self.client.tabledata.InsertAll(request) # response.insertErrors is not [] if errors encountered. +except HttpError as exn: + if error_counter: +content = json.loads(exn.content) +error_counter.record( +'%s(%s)' % +(content['error']['errors'][0]['reason'], exn.status_code)) Review comment: Please convert this to the human readable format instead of the numeric http code. See the doc I shared with you. At least before logging https://cloud.google.com/apis/design/errors This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] timclemons opened a new pull request #12850: [BEAM-10481] Ensure registration of the accumulator occurs.
timclemons opened a new pull request #12850: URL: https://github.com/apache/beam/pull/12850 Move registration of accumulator so as to include those recovered from checkpoints. Prior to this, if the accumulator was recovered from a checkpoint it would not be registered, resulting in a runtime exception. R: @jbonofre Thank you for your contribution! Follow this checklist to help us incorporate your contribution quickly and easily: - [X] [**Choose reviewer(s)**](https://beam.apache.org/contribute/#make-your-change) and mention them in a comment (`R: @username`). - [X] Format the pull request title like `[BEAM-XXX] Fixes bug in ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA issue, if applicable. This will automatically link the pull request to the issue. - [ ] Update `CHANGES.md` with noteworthy changes. - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf). See the [Contributor Guide](https://beam.apache.org/contribute) for more tips on [how to make review process smoother](https://beam.apache.org/contribute/#make-reviewers-job-easier). Post-Commit Tests Status (on master branch) Lang | SDK | Dataflow | Flink | Samza | Spark | Twister2 --- | --- | --- | --- | --- | --- | --- Go | [](https://ci-beam.apache.org/job/beam_PostCommit_Go/lastCompletedBuild/) | --- | [](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/lastCompletedBuild/) | --- | [](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark/lastCompletedBuild/) | --- Java | [](https://ci-beam.apache.org/job/beam_PostCommit_Java/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow_Java11/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink_Java11/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Flink_Batch/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Flink_Streaming/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Samza/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Spark/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Spark_Batch/lastCompletedBuild/)[](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming/lastCompletedBuild/) | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Twister2/lastCompletedBuild/) Python | [](https://ci-beam.apache.org/job/beam_PostCommit_Python2/lastCompletedBuild/)[