[GitHub] [beam] ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update protos related to MonitoringInfo.
ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update
protos related to MonitoringInfo.
URL: https://github.com/apache/beam/pull/11184#discussion_r395893496
##
File path: model/pipeline/src/main/proto/metrics.proto
##
@@ -229,101 +215,127 @@ message MonitoringInfo {
NAMESPACE = 5 [(label_props) = { name: "NAMESPACE" }];
NAME = 6 [(label_props) = { name: "NAME" }];
}
+
// A set of key+value labels which define the scope of the metric.
// Either a well defined entity id for matching the enum names in
// the MonitoringInfoLabels enum or any arbitrary label
// set by a custom metric or user metric.
+ //
// A monitoring system is expected to be able to aggregate the metrics
// together for all updates having the same URN and labels. Some systems such
// as Stackdriver will be able to aggregate the metrics using a subset of the
// provided labels
- map labels = 5;
-
- // The walltime of the most recent update.
- // Useful for aggregation for latest types such as LatestInt64.
- google.protobuf.Timestamp timestamp = 6;
+ map labels = 4;
}
message MonitoringInfoTypeUrns {
enum Enum {
+// Represents an integer counter where values are summed across bundles.
+//
+// Encoding:
+// - value: beam:coder:varint:v1
SUM_INT64_TYPE = 0 [(org.apache.beam.model.pipeline.v1.beam_urn) =
-"beam:metrics:sum_int_64"];
-
-DISTRIBUTION_INT64_TYPE = 1 [(org.apache.beam.model.pipeline.v1.beam_urn) =
- "beam:metrics:distribution_int_64"];
-
-LATEST_INT64_TYPE = 2 [(org.apache.beam.model.pipeline.v1.beam_urn) =
- "beam:metrics:latest_int_64"];
+"beam:metrics:sum_int64:v1"];
+
+// Represents a double counter where values are summed across bundles.
+//
+// Encoding:
+// value: beam:coder:double:v1
+SUM_DOUBLE_TYPE = 1 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+"beam:metrics:sum_int64:v1"];
+
+// Represents a distribution of an integer value where:
+// - count: represents the number of values seen across all bundles
+// - sum: represents the total of the value across all bundles
+// - min: represents the smallest value seen across all bundles
+// - max: represents the largest value seen across all bundles
+//
+// Encoding:
+// - count: beam:coder:varint:v1
+// - sum: beam:coder:varint:v1
+// - min: beam:coder:varint:v1
+// - max: beam:coder:varint:v1
+DISTRIBUTION_INT64_TYPE = 2 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+ "beam:metrics:distribution_int64:v1"];
+
+// Represents a distribution of a double value where:
+// - count: represents the number of values seen across all bundles
+// - sum: represents the total of the value across all bundles
+// - min: represents the smallest value seen across all bundles
+// - max: represents the largest value seen across all bundles
+//
+// Encoding:
+// - count: beam:coder:varint:v1
+// - sum: beam:coder:double:v1
+// - min: beam:coder:double:v1
+// - max: beam:coder:double:v1
+DISTRIBUTION_DOUBLE_TYPE = 3 [(org.apache.beam.model.pipeline.v1.beam_urn)
=
+ "beam:metrics:distribution_int64:v1"];
+
+// Represents the latest seen integer value. The timestamp is used to
+// provide an "ordering" over multiple values to determine which is the
+// latest.
+//
+// Encoding:
+// - timestamp: beam:coder:varint:v1 (milliseconds since epoch)
+// - value: beam:coder:varint:v1
+LATEST_INT64_TYPE = 4 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+ "beam:metrics:latest_int64:v1"];
+
+// Represents the latest seen integer value. The timestamp is used to
+// provide an "ordering" over multiple values to determine which is the
+// latest.
+//
+// Encoding:
+// - timestamp: beam:coder:varint:v1 (milliseconds since epoch)
+// - value: beam:coder:double:v1
+LATEST_DOUBLE_TYPE = 5 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+ "beam:metrics:latest_int64:v1"];
+
+// Represents the largest set of integer values seen across bundles.
+//
+// Encoding: ...
+// - valueX: beam:coder:varint:v1
+TOP_N_INT64_TYPE = 6 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+ "beam:metrics:top_n_int64:v1"];
+
+// Represents the largest set of double values seen across bundles.
+//
+// Encoding: ...
+// - valueX: beam:coder:double:v1
+TOP_N_DOUBLE_TYPE = 7 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+"beam:metrics:top_n_int64:v1"];
+
+// Represents the smallest set of integer
[GitHub] [beam] ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update protos related to MonitoringInfo.
ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update
protos related to MonitoringInfo.
URL: https://github.com/apache/beam/pull/11184#discussion_r395892463
##
File path: model/pipeline/src/main/proto/metrics.proto
##
@@ -229,101 +215,127 @@ message MonitoringInfo {
NAMESPACE = 5 [(label_props) = { name: "NAMESPACE" }];
NAME = 6 [(label_props) = { name: "NAME" }];
}
+
// A set of key+value labels which define the scope of the metric.
// Either a well defined entity id for matching the enum names in
// the MonitoringInfoLabels enum or any arbitrary label
// set by a custom metric or user metric.
+ //
// A monitoring system is expected to be able to aggregate the metrics
// together for all updates having the same URN and labels. Some systems such
// as Stackdriver will be able to aggregate the metrics using a subset of the
// provided labels
- map labels = 5;
-
- // The walltime of the most recent update.
- // Useful for aggregation for latest types such as LatestInt64.
- google.protobuf.Timestamp timestamp = 6;
+ map labels = 4;
}
message MonitoringInfoTypeUrns {
Review comment:
Let's add some comments to make it clear the type is referring to what is
collected in each MonitoringInfo update, and how they should be aggregated
together
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [beam] ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update protos related to MonitoringInfo.
ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update
protos related to MonitoringInfo.
URL: https://github.com/apache/beam/pull/11184#discussion_r395888595
##
File path: model/pipeline/src/main/proto/metrics.proto
##
@@ -229,101 +215,127 @@ message MonitoringInfo {
NAMESPACE = 5 [(label_props) = { name: "NAMESPACE" }];
NAME = 6 [(label_props) = { name: "NAME" }];
}
+
// A set of key+value labels which define the scope of the metric.
// Either a well defined entity id for matching the enum names in
// the MonitoringInfoLabels enum or any arbitrary label
// set by a custom metric or user metric.
+ //
// A monitoring system is expected to be able to aggregate the metrics
// together for all updates having the same URN and labels. Some systems such
// as Stackdriver will be able to aggregate the metrics using a subset of the
// provided labels
- map labels = 5;
-
- // The walltime of the most recent update.
- // Useful for aggregation for latest types such as LatestInt64.
- google.protobuf.Timestamp timestamp = 6;
+ map labels = 4;
}
message MonitoringInfoTypeUrns {
enum Enum {
+// Represents an integer counter where values are summed across bundles.
+//
+// Encoding:
+// - value: beam:coder:varint:v1
SUM_INT64_TYPE = 0 [(org.apache.beam.model.pipeline.v1.beam_urn) =
Review comment:
I think there are only a few of these types being used now. SUM_INT64_TYPE
and DISTRIBUTION_INT64_TYPE. I hope we can make it very simple to add new ones
of these with minimal changes (Adding MonitoringInfoSpec and reusing existing
framework/libraries in the SDK, runners can mostly pass them through to a
service to aggregate across multiple workers)
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [beam] ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update protos related to MonitoringInfo.
ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update
protos related to MonitoringInfo.
URL: https://github.com/apache/beam/pull/11184#discussion_r395889104
##
File path: model/pipeline/src/main/proto/metrics.proto
##
@@ -194,33 +188,25 @@ extend google.protobuf.EnumValueOptions {
}
message MonitoringInfo {
- // The name defining the metric or monitored state.
+ // The name defining the semantic meaning of the metric or monitored state.
+ //
+ // See MonitoringInfoSpecs.Enum for the set of well known metrics/monitored
+ // state.
string urn = 1;
- // This is specified as a URN that implies:
- // A message class: (Distribution, Counter, Extrema, MonitoringDataTable).
- // Sub types like field formats - int64, double, string.
- // Aggregation methods - SUM, LATEST, TOP-N, BOTTOM-N, DISTRIBUTION
- // valid values are:
- // beam:metrics:[sum_int_64|latest_int_64|top_n_int_64|bottom_n_int_64|
- // sum_double|latest_double|top_n_double|bottom_n_double|
- // distribution_int_64|distribution_double|monitoring_data_table|
- // latest_doubles
+ // This is specified as a URN that implies the encoding and aggregation
+ // method. See MonitoringInfoTypeUrns.Enum for the set of well known types.
string type = 2;
- // The Metric or monitored state.
- oneof data {
-MonitoringTableData monitoring_table_data = 3;
-Metric metric = 4;
-bytes payload = 7;
- }
+ // The monitored state encoded as per the specification defined by the type.
+ bytes payload = 3;
Review comment:
My biggest concern is losing the ability to print debug strings, which are
helpful when people are trying to learn how these are populated. But maybe we
can just add a few obvious places to dump debug logs, debug files, etc with the
MonitoringInfors parses properly.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [beam] ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update protos related to MonitoringInfo.
ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update
protos related to MonitoringInfo.
URL: https://github.com/apache/beam/pull/11184#discussion_r395888786
##
File path: model/pipeline/src/main/proto/metrics.proto
##
@@ -194,33 +188,25 @@ extend google.protobuf.EnumValueOptions {
}
message MonitoringInfo {
- // The name defining the metric or monitored state.
+ // The name defining the semantic meaning of the metric or monitored state.
+ //
+ // See MonitoringInfoSpecs.Enum for the set of well known metrics/monitored
+ // state.
string urn = 1;
- // This is specified as a URN that implies:
- // A message class: (Distribution, Counter, Extrema, MonitoringDataTable).
- // Sub types like field formats - int64, double, string.
- // Aggregation methods - SUM, LATEST, TOP-N, BOTTOM-N, DISTRIBUTION
- // valid values are:
- // beam:metrics:[sum_int_64|latest_int_64|top_n_int_64|bottom_n_int_64|
- // sum_double|latest_double|top_n_double|bottom_n_double|
- // distribution_int_64|distribution_double|monitoring_data_table|
- // latest_doubles
+ // This is specified as a URN that implies the encoding and aggregation
+ // method. See MonitoringInfoTypeUrns.Enum for the set of well known types.
string type = 2;
- // The Metric or monitored state.
- oneof data {
-MonitoringTableData monitoring_table_data = 3;
-Metric metric = 4;
-bytes payload = 7;
- }
+ // The monitored state encoded as per the specification defined by the type.
+ bytes payload = 3;
Review comment:
Probably you are already planning on doing this. But having helper functions
to easily encode/decode these would be great.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [beam] ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update protos related to MonitoringInfo.
ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update
protos related to MonitoringInfo.
URL: https://github.com/apache/beam/pull/11184#discussion_r395888153
##
File path: model/pipeline/src/main/proto/metrics.proto
##
@@ -229,101 +215,127 @@ message MonitoringInfo {
NAMESPACE = 5 [(label_props) = { name: "NAMESPACE" }];
NAME = 6 [(label_props) = { name: "NAME" }];
}
+
// A set of key+value labels which define the scope of the metric.
// Either a well defined entity id for matching the enum names in
// the MonitoringInfoLabels enum or any arbitrary label
// set by a custom metric or user metric.
+ //
// A monitoring system is expected to be able to aggregate the metrics
// together for all updates having the same URN and labels. Some systems such
// as Stackdriver will be able to aggregate the metrics using a subset of the
// provided labels
- map labels = 5;
-
- // The walltime of the most recent update.
- // Useful for aggregation for latest types such as LatestInt64.
- google.protobuf.Timestamp timestamp = 6;
+ map labels = 4;
}
message MonitoringInfoTypeUrns {
enum Enum {
+// Represents an integer counter where values are summed across bundles.
+//
+// Encoding:
+// - value: beam:coder:varint:v1
SUM_INT64_TYPE = 0 [(org.apache.beam.model.pipeline.v1.beam_urn) =
-"beam:metrics:sum_int_64"];
-
-DISTRIBUTION_INT64_TYPE = 1 [(org.apache.beam.model.pipeline.v1.beam_urn) =
- "beam:metrics:distribution_int_64"];
-
-LATEST_INT64_TYPE = 2 [(org.apache.beam.model.pipeline.v1.beam_urn) =
- "beam:metrics:latest_int_64"];
+"beam:metrics:sum_int64:v1"];
+
+// Represents a double counter where values are summed across bundles.
+//
+// Encoding:
+// value: beam:coder:double:v1
+SUM_DOUBLE_TYPE = 1 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+"beam:metrics:sum_int64:v1"];
+
+// Represents a distribution of an integer value where:
+// - count: represents the number of values seen across all bundles
+// - sum: represents the total of the value across all bundles
+// - min: represents the smallest value seen across all bundles
+// - max: represents the largest value seen across all bundles
+//
+// Encoding:
+// - count: beam:coder:varint:v1
+// - sum: beam:coder:varint:v1
+// - min: beam:coder:varint:v1
+// - max: beam:coder:varint:v1
+DISTRIBUTION_INT64_TYPE = 2 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+ "beam:metrics:distribution_int64:v1"];
+
+// Represents a distribution of a double value where:
+// - count: represents the number of values seen across all bundles
+// - sum: represents the total of the value across all bundles
+// - min: represents the smallest value seen across all bundles
+// - max: represents the largest value seen across all bundles
+//
+// Encoding:
+// - count: beam:coder:varint:v1
+// - sum: beam:coder:double:v1
+// - min: beam:coder:double:v1
+// - max: beam:coder:double:v1
+DISTRIBUTION_DOUBLE_TYPE = 3 [(org.apache.beam.model.pipeline.v1.beam_urn)
=
+ "beam:metrics:distribution_int64:v1"];
+
+// Represents the latest seen integer value. The timestamp is used to
+// provide an "ordering" over multiple values to determine which is the
+// latest.
+//
+// Encoding:
+// - timestamp: beam:coder:varint:v1 (milliseconds since epoch)
+// - value: beam:coder:varint:v1
+LATEST_INT64_TYPE = 4 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+ "beam:metrics:latest_int64:v1"];
+
+// Represents the latest seen integer value. The timestamp is used to
+// provide an "ordering" over multiple values to determine which is the
+// latest.
+//
+// Encoding:
+// - timestamp: beam:coder:varint:v1 (milliseconds since epoch)
+// - value: beam:coder:double:v1
+LATEST_DOUBLE_TYPE = 5 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+ "beam:metrics:latest_int64:v1"];
+
+// Represents the largest set of integer values seen across bundles.
+//
+// Encoding: ...
+// - valueX: beam:coder:varint:v1
+TOP_N_INT64_TYPE = 6 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+ "beam:metrics:top_n_int64:v1"];
+
+// Represents the largest set of double values seen across bundles.
+//
+// Encoding: ...
+// - valueX: beam:coder:double:v1
+TOP_N_DOUBLE_TYPE = 7 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+"beam:metrics:top_n_int64:v1"];
+
+// Represents the smallest set of integer
[GitHub] [beam] ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update protos related to MonitoringInfo.
ajamato commented on a change in pull request #11184: [WIP][BEAM-4374] Update
protos related to MonitoringInfo.
URL: https://github.com/apache/beam/pull/11184#discussion_r395887788
##
File path: model/pipeline/src/main/proto/metrics.proto
##
@@ -139,7 +137,7 @@ message MonitoringInfoSpecs {
USER_DISTRIBUTION_COUNTER = 6 [(monitoring_info_spec) = {
urn: "beam:metric:user_distribution",
- type_urn: "beam:metrics:distribution_int_64",
+ type_urn: "beam:metrics:distribution_int64",
Review comment:
Add a :v1 here
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
