lostluck commented on a change in pull request #13272:
URL: https://github.com/apache/beam/pull/13272#discussion_r522347395
##########
File path: sdks/go/pkg/beam/core/metrics/metrics.go
##########
@@ -448,3 +453,116 @@ func (m *gauge) get() (int64, time.Time) {
defer m.mu.Unlock()
return m.v, m.t
}
+
+// GaugeValue is the value of a Gauge metric.
+type GaugeValue struct {
+ Value int64
+ Timestamp time.Time
+}
+
+// Results represents all metrics gathered during the job's execution.
+// It allows for querying metrics using a provided filter.
+type Results struct {
+ counters []CounterResult
+ distributions []DistributionResult
+ gauges []GaugeResult
+}
+
+// NewResults creates a new Results.
+func NewResults(
+ counters []CounterResult,
+ distributions []DistributionResult,
+ gauges []GaugeResult) *Results {
+ return &Results{counters, distributions, gauges}
+}
+
+// AllMetrics returns all metrics from a Results instance.
+func (mr Results) AllMetrics() QueryResults {
+ return QueryResults{mr.counters, mr.distributions, mr.gauges}
+}
+
+// TODO(BEAM-11217): Implement Query(Filter) and metrics filtering
+
+// QueryResults is the result of a query. Allows accessing all of the
+// metrics that matched the filter.
+type QueryResults struct {
+ counters []CounterResult
+ distributions []DistributionResult
+ gauges []GaugeResult
+}
+
+// Counters returns an array of counter metrics.
Review comment:
In go comments, it's reasonable to call this "a slice". In particular,
arrays in Go are fixed size and have different implications compared to
variable sized slices. Here and below.
```
var slice []int // a slice of ints
var array [4]int // an array of ints of length 4
slice = []int{1,2,3,4}
array = [4]int{1,2,3,4}
array = [...]int{1,2,3,4} // Have the compiler determine the array size
based on the construction time parameters
// array = [2]int{1, 2} // a compiler error
slice = array[1:len(array)] // a slice of array, excluding the first
element.
// This is backed by array, which will lead to aliasing issues if care isn't
taken...
```
##########
File path: sdks/go/pkg/beam/core/runtime/metricsx/metricsx.go
##########
@@ -0,0 +1,181 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metricsx
+
+import (
+ "bytes"
+ "fmt"
+ "log"
+ "time"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder"
+ "github.com/apache/beam/sdks/go/pkg/beam/core/metrics"
+ pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+)
+
+// FromMonitoringInfos extracts metrics from monitored states and
+// groups them into counters, distributions and gauges.
+func FromMonitoringInfos(attempted []*pipepb.MonitoringInfo, committed
[]*pipepb.MonitoringInfo) *metrics.Results {
+ ac, ad, ag := groupByType(attempted)
+ cc, cd, cg := groupByType(committed)
+
+ return metrics.NewResults(mergeCounters(ac, cc), mergeDistributions(ad,
cd), mergeGauges(ag, cg))
+}
+
+func groupByType(minfos []*pipepb.MonitoringInfo) (
+ map[metrics.StepKey]int64,
+ map[metrics.StepKey]metrics.DistributionValue,
+ map[metrics.StepKey]metrics.GaugeValue) {
+ counters := make(map[metrics.StepKey]int64)
+ distributions := make(map[metrics.StepKey]metrics.DistributionValue)
+ gauges := make(map[metrics.StepKey]metrics.GaugeValue)
+
+ for _, minfo := range minfos {
+ key, err := extractKey(minfo)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+
+ r := bytes.NewReader(minfo.GetPayload())
+
+ switch minfo.GetType() {
+ case "beam:metrics:sum_int64:v1":
+ value, err := extractCounterValue(r)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ counters[key] = value
+ case "beam:metrics:distribution_int64:v1":
+ value, err := extractDistributionValue(r)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ distributions[key] = value
+ case
+ "beam:metrics:latest_int64:v1",
+ "beam:metrics:top_n_int64:v1",
+ "beam:metrics:bottom_n_int64:v1":
+ value, err := extractGaugeValue(r)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ gauges[key] = value
+ default:
+ log.Println("unknown metric type")
+ }
+ }
+ return counters, distributions, gauges
+}
+
+func mergeCounters(
+ attempted map[metrics.StepKey]int64,
+ committed map[metrics.StepKey]int64) []metrics.CounterResult {
+ res := make([]metrics.CounterResult, 0)
+
+ for k := range attempted {
+ v, ok := committed[k]
+ if !ok {
+ v = -1
+ }
+ res = append(res, metrics.CounterResult{Attempted:
attempted[k], Committed: v, Key: k})
+ }
+ return res
+}
+
+func mergeDistributions(
+ attempted map[metrics.StepKey]metrics.DistributionValue,
+ committed map[metrics.StepKey]metrics.DistributionValue)
[]metrics.DistributionResult {
+ res := make([]metrics.DistributionResult, 0)
+
+ for k := range attempted {
+ v, ok := committed[k]
+ if !ok {
+ v = metrics.DistributionValue{}
+ }
Review comment:
Since this is a map to values (rather than pointers), v will already be
the zero value for the type. In short, this code is identical to simply `v :=
commited[k]`
https://golang.org/doc/effective_go.html#maps
##########
File path: sdks/go/pkg/beam/core/runtime/metricsx/metricsx.go
##########
@@ -0,0 +1,181 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metricsx
+
+import (
+ "bytes"
+ "fmt"
+ "log"
+ "time"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder"
+ "github.com/apache/beam/sdks/go/pkg/beam/core/metrics"
+ pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+)
+
+// FromMonitoringInfos extracts metrics from monitored states and
+// groups them into counters, distributions and gauges.
+func FromMonitoringInfos(attempted []*pipepb.MonitoringInfo, committed
[]*pipepb.MonitoringInfo) *metrics.Results {
+ ac, ad, ag := groupByType(attempted)
+ cc, cd, cg := groupByType(committed)
+
+ return metrics.NewResults(mergeCounters(ac, cc), mergeDistributions(ad,
cd), mergeGauges(ag, cg))
+}
+
+func groupByType(minfos []*pipepb.MonitoringInfo) (
+ map[metrics.StepKey]int64,
+ map[metrics.StepKey]metrics.DistributionValue,
+ map[metrics.StepKey]metrics.GaugeValue) {
+ counters := make(map[metrics.StepKey]int64)
+ distributions := make(map[metrics.StepKey]metrics.DistributionValue)
+ gauges := make(map[metrics.StepKey]metrics.GaugeValue)
+
+ for _, minfo := range minfos {
+ key, err := extractKey(minfo)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+
+ r := bytes.NewReader(minfo.GetPayload())
+
+ switch minfo.GetType() {
+ case "beam:metrics:sum_int64:v1":
+ value, err := extractCounterValue(r)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ counters[key] = value
+ case "beam:metrics:distribution_int64:v1":
+ value, err := extractDistributionValue(r)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ distributions[key] = value
+ case
+ "beam:metrics:latest_int64:v1",
+ "beam:metrics:top_n_int64:v1",
+ "beam:metrics:bottom_n_int64:v1":
+ value, err := extractGaugeValue(r)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ gauges[key] = value
+ default:
+ log.Println("unknown metric type")
+ }
+ }
+ return counters, distributions, gauges
+}
+
+func mergeCounters(
+ attempted map[metrics.StepKey]int64,
+ committed map[metrics.StepKey]int64) []metrics.CounterResult {
+ res := make([]metrics.CounterResult, 0)
+
+ for k := range attempted {
+ v, ok := committed[k]
+ if !ok {
+ v = -1
+ }
Review comment:
Is the -1 part of the spec here? Wouldn't a 0 notionally be correct if
there are no commited values for a key?
##########
File path: sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metricsx
+
+import (
+ "testing"
+ "time"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/core/metrics"
+ pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+)
+
+func TestCounterExtraction(t *testing.T) {
+ var value int64 = 15
+ want := metrics.CounterResult{
+ Attempted: 15,
+ Committed: -1,
+ Key: metrics.StepKey{
+ Step: "main.customDoFn",
+ Name: "customCounter",
+ Namespace: "customDoFn",
+ }}
+
+ payload, err := Int64Counter(value)
+ if err != nil {
+ panic(err)
Review comment:
Same comment here, prefer a t.Fatalf here instead of a panic.
##########
File path: sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metricsx
+
+import (
+ "testing"
+ "time"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/core/metrics"
+ pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+)
+
+func TestCounterExtraction(t *testing.T) {
+ var value int64 = 15
+ want := metrics.CounterResult{
+ Attempted: 15,
+ Committed: -1,
+ Key: metrics.StepKey{
+ Step: "main.customDoFn",
+ Name: "customCounter",
+ Namespace: "customDoFn",
+ }}
+
+ payload, err := Int64Counter(value)
+ if err != nil {
+ panic(err)
+ }
+
+ labels := map[string]string{
+ "PTRANSFORM": "main.customDoFn",
+ "NAMESPACE": "customDoFn",
+ "NAME": "customCounter",
+ }
+
+ mInfo := &pipepb.MonitoringInfo{
+ Urn: UrnToString(UrnUserSumInt64),
+ Type: UrnToType(UrnUserSumInt64),
+ Labels: labels,
+ Payload: payload,
+ }
+
+ attempted := []*pipepb.MonitoringInfo{mInfo}
+ committed := []*pipepb.MonitoringInfo{}
+
+ got := FromMonitoringInfos(attempted, committed).AllMetrics().Counters()
+ size := len(got)
+ if size < 1 {
+ t.Fatalf("Invalid array's size: got: %v, expected: %v", size, 1)
+ }
+ if got[0] != want {
+ t.Fatalf("Invalid counter: got: %v, want: %v",
+ got[0], want)
+ }
+}
+
+func TestDistributionExtraction(t *testing.T) {
+ var count, sum, min, max int64 = 100, 5, -12, 30
+
+ want := metrics.DistributionResult{
+ Attempted: metrics.DistributionValue{
+ Count: 100,
+ Sum: 5,
+ Min: -12,
+ Max: 30,
+ },
+ Committed: metrics.DistributionValue{},
+ Key: metrics.StepKey{
+ Step: "main.customDoFn",
+ Name: "customDist",
+ Namespace: "customDoFn",
+ }}
+
+ payload, err := Int64Distribution(count, sum, min, max)
+ if err != nil {
+ panic(err)
Review comment:
Prefer a t.Fatal here instead, and be clear that the encoding the
Int64Distribution returned an error.
As a rule, never panic if you don't have to. In tests, it's better to
clearly explain why you're failing the test run anyway.
https://golang.org/doc/effective_go.html#panic
##########
File path: sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metricsx
+
+import (
+ "testing"
+ "time"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/core/metrics"
+ pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+)
+
+func TestCounterExtraction(t *testing.T) {
+ var value int64 = 15
+ want := metrics.CounterResult{
+ Attempted: 15,
+ Committed: -1,
+ Key: metrics.StepKey{
+ Step: "main.customDoFn",
+ Name: "customCounter",
+ Namespace: "customDoFn",
+ }}
+
+ payload, err := Int64Counter(value)
+ if err != nil {
+ panic(err)
+ }
+
+ labels := map[string]string{
+ "PTRANSFORM": "main.customDoFn",
+ "NAMESPACE": "customDoFn",
+ "NAME": "customCounter",
+ }
+
+ mInfo := &pipepb.MonitoringInfo{
+ Urn: UrnToString(UrnUserSumInt64),
+ Type: UrnToType(UrnUserSumInt64),
+ Labels: labels,
+ Payload: payload,
+ }
+
+ attempted := []*pipepb.MonitoringInfo{mInfo}
+ committed := []*pipepb.MonitoringInfo{}
+
+ got := FromMonitoringInfos(attempted, committed).AllMetrics().Counters()
+ size := len(got)
+ if size < 1 {
+ t.Fatalf("Invalid array's size: got: %v, expected: %v", size, 1)
+ }
+ if got[0] != want {
+ t.Fatalf("Invalid counter: got: %v, want: %v",
+ got[0], want)
+ }
+}
+
+func TestDistributionExtraction(t *testing.T) {
+ var count, sum, min, max int64 = 100, 5, -12, 30
+
+ want := metrics.DistributionResult{
+ Attempted: metrics.DistributionValue{
+ Count: 100,
+ Sum: 5,
+ Min: -12,
+ Max: 30,
+ },
+ Committed: metrics.DistributionValue{},
+ Key: metrics.StepKey{
+ Step: "main.customDoFn",
+ Name: "customDist",
+ Namespace: "customDoFn",
+ }}
+
+ payload, err := Int64Distribution(count, sum, min, max)
+ if err != nil {
+ panic(err)
+ }
+
+ labels := map[string]string{
+ "PTRANSFORM": "main.customDoFn",
+ "NAMESPACE": "customDoFn",
+ "NAME": "customDist",
+ }
+
+ mInfo := &pipepb.MonitoringInfo{
+ Urn: UrnToString(UrnUserDistInt64),
+ Type: UrnToType(UrnUserDistInt64),
+ Labels: labels,
+ Payload: payload,
+ }
+
+ attempted := []*pipepb.MonitoringInfo{mInfo}
+ committed := []*pipepb.MonitoringInfo{}
+
+ got := FromMonitoringInfos(attempted,
committed).AllMetrics().Distributions()
+ size := len(got)
+ if size < 1 {
+ t.Fatalf("Invalid array's size: got: %v, expected: %v", size, 1)
+ }
+ if got[0] != want {
+ t.Fatalf("Invalid distribution: got: %v, want: %v",
+ got[0], want)
+ }
+}
+
+func TestGaugeExtraction(t *testing.T) {
+ var value int64 = 100
+ loc, _ := time.LoadLocation("Local")
+ tm := time.Date(2020, 11, 9, 17, 52, 28, 462*int(time.Millisecond), loc)
+
+ want := metrics.GaugeResult{
+ Attempted: metrics.GaugeValue{
+ Value: 100,
+ Timestamp: tm,
+ },
+ Committed: metrics.GaugeValue{},
+ Key: metrics.StepKey{
+ Step: "main.customDoFn",
+ Name: "customGauge",
+ Namespace: "customDoFn",
+ }}
+
+ payload, err := Int64Latest(tm, value)
+ if err != nil {
+ panic(err)
Review comment:
Same comment here, prefer a t.Fatalf here instead of a panic.
##########
File path: sdks/go/pkg/beam/core/metrics/metrics.go
##########
@@ -448,3 +453,116 @@ func (m *gauge) get() (int64, time.Time) {
defer m.mu.Unlock()
return m.v, m.t
}
+
+// GaugeValue is the value of a Gauge metric.
+type GaugeValue struct {
+ Value int64
+ Timestamp time.Time
+}
+
+// Results represents all metrics gathered during the job's execution.
+// It allows for querying metrics using a provided filter.
+type Results struct {
+ counters []CounterResult
+ distributions []DistributionResult
+ gauges []GaugeResult
+}
+
+// NewResults creates a new Results.
+func NewResults(
+ counters []CounterResult,
+ distributions []DistributionResult,
+ gauges []GaugeResult) *Results {
+ return &Results{counters, distributions, gauges}
+}
+
+// AllMetrics returns all metrics from a Results instance.
+func (mr Results) AllMetrics() QueryResults {
+ return QueryResults{mr.counters, mr.distributions, mr.gauges}
+}
+
+// TODO(BEAM-11217): Implement Query(Filter) and metrics filtering
+
+// QueryResults is the result of a query. Allows accessing all of the
+// metrics that matched the filter.
+type QueryResults struct {
+ counters []CounterResult
+ distributions []DistributionResult
+ gauges []GaugeResult
+}
+
+// Counters returns an array of counter metrics.
+func (qr QueryResults) Counters() []CounterResult {
+ out := make([]CounterResult, len(qr.counters))
+ copy(out, qr.counters)
+ return out
+}
+
+// Distributions returns an array of distribution metrics.
+func (qr QueryResults) Distributions() []DistributionResult {
+ out := make([]DistributionResult, len(qr.distributions))
+ copy(out, qr.distributions)
+ return out
+}
+
+// Gauges returns an array of gauge metrics.
+func (qr QueryResults) Gauges() []GaugeResult {
+ out := make([]GaugeResult, len(qr.gauges))
+ copy(out, qr.gauges)
+ return out
+}
+
+// CounterResult is an attempted and a commited value of a counter metric plus
+// key.
+type CounterResult struct {
+ Attempted, Committed int64
+ Key StepKey
+}
+
+// Result returns committed metrics. Falls back to attempted metrics if
committed
+// are not populated (e.g. due to not being supported on a given runner).
+func (r CounterResult) Result() int64 {
+ if r.Committed != -1 {
+ return r.Committed
+ }
+ return r.Attempted
+}
+
+// DistributionResult is an attempted and a commited value of a distribution
+// metric plus key.
+type DistributionResult struct {
+ Attempted, Committed DistributionValue
+ Key StepKey
+}
+
+// Result returns committed metrics. Falls back to attempted metrics if
committed
+// are not populated (e.g. due to not being supported on a given runner).
+func (r DistributionResult) Result() DistributionValue {
+ empty := DistributionValue{}
+ if r.Committed != empty {
+ return r.Committed
+ }
+ return r.Attempted
+}
+
+// GaugeResult is an attempted and a commited value of a gauge metric plus
+// key.
+type GaugeResult struct {
+ Attempted, Committed GaugeValue
+ Key StepKey
+}
+
+// Result returns committed metrics. Falls back to attempted metrics if
committed
+// are not populated (e.g. due to not being supported on a given runner).
+func (r GaugeResult) Result() GaugeValue {
+ empty := GaugeValue{}
+ if r.Committed != empty {
+ return r.Committed
+ }
+ return r.Attempted
+}
+
+// StepKey uniquely identifies a metric.
Review comment:
```suggestion
// StepKey uniquely identifies a metric within a pipeline graph.
```
##########
File path: sdks/go/pkg/beam/core/runtime/metricsx/metricsx.go
##########
@@ -0,0 +1,181 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metricsx
+
+import (
+ "bytes"
+ "fmt"
+ "log"
+ "time"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder"
+ "github.com/apache/beam/sdks/go/pkg/beam/core/metrics"
+ pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+)
+
+// FromMonitoringInfos extracts metrics from monitored states and
+// groups them into counters, distributions and gauges.
+func FromMonitoringInfos(attempted []*pipepb.MonitoringInfo, committed
[]*pipepb.MonitoringInfo) *metrics.Results {
+ ac, ad, ag := groupByType(attempted)
+ cc, cd, cg := groupByType(committed)
+
+ return metrics.NewResults(mergeCounters(ac, cc), mergeDistributions(ad,
cd), mergeGauges(ag, cg))
+}
+
+func groupByType(minfos []*pipepb.MonitoringInfo) (
+ map[metrics.StepKey]int64,
+ map[metrics.StepKey]metrics.DistributionValue,
+ map[metrics.StepKey]metrics.GaugeValue) {
+ counters := make(map[metrics.StepKey]int64)
+ distributions := make(map[metrics.StepKey]metrics.DistributionValue)
+ gauges := make(map[metrics.StepKey]metrics.GaugeValue)
+
+ for _, minfo := range minfos {
+ key, err := extractKey(minfo)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+
+ r := bytes.NewReader(minfo.GetPayload())
+
+ switch minfo.GetType() {
+ case "beam:metrics:sum_int64:v1":
+ value, err := extractCounterValue(r)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ counters[key] = value
+ case "beam:metrics:distribution_int64:v1":
+ value, err := extractDistributionValue(r)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ distributions[key] = value
+ case
+ "beam:metrics:latest_int64:v1",
+ "beam:metrics:top_n_int64:v1",
+ "beam:metrics:bottom_n_int64:v1":
+ value, err := extractGaugeValue(r)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ gauges[key] = value
+ default:
+ log.Println("unknown metric type")
+ }
+ }
+ return counters, distributions, gauges
+}
+
+func mergeCounters(
+ attempted map[metrics.StepKey]int64,
+ committed map[metrics.StepKey]int64) []metrics.CounterResult {
+ res := make([]metrics.CounterResult, 0)
+
+ for k := range attempted {
+ v, ok := committed[k]
+ if !ok {
+ v = -1
+ }
+ res = append(res, metrics.CounterResult{Attempted:
attempted[k], Committed: v, Key: k})
+ }
+ return res
+}
+
+func mergeDistributions(
+ attempted map[metrics.StepKey]metrics.DistributionValue,
+ committed map[metrics.StepKey]metrics.DistributionValue)
[]metrics.DistributionResult {
+ res := make([]metrics.DistributionResult, 0)
+
+ for k := range attempted {
+ v, ok := committed[k]
+ if !ok {
+ v = metrics.DistributionValue{}
+ }
+ res = append(res, metrics.DistributionResult{Attempted:
attempted[k], Committed: v, Key: k})
+ }
+ return res
+}
+
+func mergeGauges(
+ attempted map[metrics.StepKey]metrics.GaugeValue,
+ committed map[metrics.StepKey]metrics.GaugeValue) []metrics.GaugeResult
{
+ res := make([]metrics.GaugeResult, 0)
+
+ for k := range attempted {
+ v, ok := committed[k]
+ if !ok {
+ v = metrics.GaugeValue{}
+ }
+ res = append(res, metrics.GaugeResult{Attempted: attempted[k],
Committed: v, Key: k})
+ }
+ return res
+}
+
+func extractKey(mi *pipepb.MonitoringInfo) (metrics.StepKey, error) {
+ labels := newLabels(mi.GetLabels())
+ stepName := labels.Transform()
+ if stepName == "" {
+ return metrics.StepKey{}, fmt.Errorf("Failed to deduce Step
from MonitoringInfo: %v", mi)
+ }
+ return metrics.StepKey{Step: stepName, Name: labels.Name(), Namespace:
labels.Namespace()}, nil
+}
+
+func extractCounterValue(reader *bytes.Reader) (int64, error) {
+ value, err := coder.DecodeVarInt(reader)
+ if err != nil {
+ return -1, err
Review comment:
In Go, if a non-nil error is returned, conventionally the non-error
returns should be ignored. There's no need to have a special marker value for
an error, since that's what the error is for. Specifically if the value
returned is still valid in some way when an error is returned, that is the
exception, and should be documented.
(As it literally doesn't matter, no change required here, so just pointing
it out for teaching purposes.)
##########
File path: sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metricsx
+
+import (
+ "testing"
+ "time"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/core/metrics"
+ pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+)
+
+func TestCounterExtraction(t *testing.T) {
+ var value int64 = 15
+ want := metrics.CounterResult{
+ Attempted: 15,
+ Committed: -1,
+ Key: metrics.StepKey{
+ Step: "main.customDoFn",
+ Name: "customCounter",
+ Namespace: "customDoFn",
+ }}
+
+ payload, err := Int64Counter(value)
+ if err != nil {
+ panic(err)
+ }
+
+ labels := map[string]string{
+ "PTRANSFORM": "main.customDoFn",
+ "NAMESPACE": "customDoFn",
+ "NAME": "customCounter",
+ }
+
+ mInfo := &pipepb.MonitoringInfo{
+ Urn: UrnToString(UrnUserSumInt64),
+ Type: UrnToType(UrnUserSumInt64),
+ Labels: labels,
+ Payload: payload,
+ }
+
+ attempted := []*pipepb.MonitoringInfo{mInfo}
+ committed := []*pipepb.MonitoringInfo{}
+
+ got := FromMonitoringInfos(attempted, committed).AllMetrics().Counters()
+ size := len(got)
+ if size < 1 {
+ t.Fatalf("Invalid array's size: got: %v, expected: %v", size, 1)
+ }
+ if got[0] != want {
Review comment:
Do this comparison using the cmp package instead.
"github.com/google/go-cmp/cmp"
```
if d := cmp.Diff(want, got[0]); d != "" {
t.Fatalf("Invalid counter: %v", d)
}
```
Here and below.
##########
File path: sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metricsx
+
+import (
+ "testing"
+ "time"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/core/metrics"
+ pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+)
+
+func TestCounterExtraction(t *testing.T) {
+ var value int64 = 15
+ want := metrics.CounterResult{
+ Attempted: 15,
+ Committed: -1,
+ Key: metrics.StepKey{
+ Step: "main.customDoFn",
+ Name: "customCounter",
+ Namespace: "customDoFn",
+ }}
+
+ payload, err := Int64Counter(value)
+ if err != nil {
+ panic(err)
+ }
+
+ labels := map[string]string{
+ "PTRANSFORM": "main.customDoFn",
+ "NAMESPACE": "customDoFn",
+ "NAME": "customCounter",
+ }
+
+ mInfo := &pipepb.MonitoringInfo{
+ Urn: UrnToString(UrnUserSumInt64),
+ Type: UrnToType(UrnUserSumInt64),
+ Labels: labels,
+ Payload: payload,
+ }
+
+ attempted := []*pipepb.MonitoringInfo{mInfo}
+ committed := []*pipepb.MonitoringInfo{}
+
+ got := FromMonitoringInfos(attempted, committed).AllMetrics().Counters()
+ size := len(got)
+ if size < 1 {
+ t.Fatalf("Invalid array's size: got: %v, expected: %v", size, 1)
Review comment:
In go tests, never say "expected" when you can simply say "want".
Otherwise this test output is excellent. Here and below.
##########
File path: sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metricsx
+
+import (
+ "testing"
+ "time"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/core/metrics"
+ pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+)
+
+func TestCounterExtraction(t *testing.T) {
Review comment:
Test names are related to the code as much as possible. In this case,
the thing we're properly testing is FromMonitoringInfos, and specifically
Counters so `func TestFromMonitoringInfos_Counters` is an idiomatic test name.
There's also an opportunity for subtests... but that's difficult here
without generics right now., and that's not until 2022. You can see more here
https://gobyexample.com/testing and https://blog.golang.org/subtests
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
