[
https://issues.apache.org/jira/browse/BEAM-6138?focusedWorklogId=170991&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-170991
]
ASF GitHub Bot logged work on BEAM-6138:
----------------------------------------
Author: ASF GitHub Bot
Created on: 30/Nov/18 06:22
Start Date: 30/Nov/18 06:22
Worklog Time Spent: 10m
Work Description: Ardagan commented on a change in pull request #6799:
[BEAM-6138] Add User Counter Metric Support to Java SDK
URL: https://github.com/apache/beam/pull/6799#discussion_r237750978
##########
File path:
runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SimpleMonitoringInfoBuilder.java
##########
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.metrics;
+
+import com.google.common.base.Splitter;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.MonitoringInfo;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.MonitoringInfoSpec;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.MonitoringInfoSpecs;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.MonitoringInfoTypeUrns;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.MonitoringInfoUrns;
+import org.apache.beam.runners.core.construction.BeamUrns;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Simplified building of MonitoringInfo fields, allows setting one field at a
time with simpler
+ * method calls, without needing to dive into the details of the nested protos.
+ *
+ * <p>There is no need to set the type field, by setting the appropriate value
field: (i.e.
+ * setInt64Value), the typeUrn field is automatically set.
+ *
+ * <p>Additionally, if validateAndDropInvalid is set to true in the ctor, then
MonitoringInfos will
+ * be returned as null when build() is called if any fields are not properly
set. This is based on
+ * comparing the fields which are set to the MonitoringInfoSpec in
beam_fn_api.proto.
+ *
+ * <p>Example Usage (ElementCount counter):
+ *
+ * <p>SimpleMonitoringInfoBuilder builder = new SimpleMonitoringInfoBuilder();
+ * builder.setUrn(SimpleMonitoringInfoBuilder.ELEMENT_COUNT_URN);
builder.setInt64Value(1);
+ * builder.setPTransformLabel("myTransform");
builder.setPCollectionLabel("myPcollection");
+ * MonitoringInfo mi = builder.build();
+ *
+ * <p>Example Usage (ElementCount counter):
+ *
+ * <p>SimpleMonitoringInfoBuilder builder = new SimpleMonitoringInfoBuilder();
+ *
builder.setUrn(SimpleMonitoringInfoBuilder.setUrnForUserMetric("myNamespace",
"myName"));
+ * builder.setInt64Value(1); MonitoringInfo mi = builder.build();
+ */
+public class SimpleMonitoringInfoBuilder {
+ public static final String ELEMENT_COUNT_URN =
+ BeamUrns.getUrn(MonitoringInfoUrns.Enum.ELEMENT_COUNT);
+ public static final String USER_COUNTER_URN_PREFIX =
+ BeamUrns.getUrn(MonitoringInfoUrns.Enum.USER_COUNTER_URN_PREFIX);
+ public static final String SUM_INT64_TYPE_URN =
+ BeamUrns.getUrn(MonitoringInfoTypeUrns.Enum.SUM_INT64_TYPE);
+
+ private static final HashMap<String, MonitoringInfoSpec> specs =
+ new HashMap<String, MonitoringInfoSpec>();
+
+ private final boolean validateAndDropInvalid;
+
+ private static final Logger LOG =
LoggerFactory.getLogger(SimpleMonitoringInfoBuilder.class);
+
+ private MonitoringInfo.Builder builder;
+
+ static {
+ for (MonitoringInfoSpecs.Enum val : MonitoringInfoSpecs.Enum.values()) {
+ // Ignore the UNRECOGNIZED = -1 value;
+ if (!((Enum) val).name().equals("UNRECOGNIZED")) {
+ MonitoringInfoSpec spec =
+
val.getValueDescriptor().getOptions().getExtension(BeamFnApi.monitoringInfoSpec);
+ SimpleMonitoringInfoBuilder.specs.put(spec.getUrn(), spec);
+ }
+ }
+ }
+
+ public SimpleMonitoringInfoBuilder() {
+ this(true);
+ }
+
+ public SimpleMonitoringInfoBuilder(boolean validateAndDropInvalid) {
+ this.builder = MonitoringInfo.newBuilder();
+ this.validateAndDropInvalid = validateAndDropInvalid;
+ }
+
+ /** @return True if the MonitoringInfo has valid fields set, matching the
spec */
+ private boolean validate() {
+ String urn = this.builder.getUrn();
+ if (urn == null || urn.isEmpty()) {
+ LOG.warn("Dropping MonitoringInfo since no URN was specified.");
+ return false;
+ }
+
+ MonitoringInfoSpec spec;
+ // If it's a user counter, and it has this prefix.
+ if (urn.startsWith(USER_COUNTER_URN_PREFIX)) {
+ spec = SimpleMonitoringInfoBuilder.specs.get(USER_COUNTER_URN_PREFIX);
+ List<String> split = Splitter.on(':').splitToList(urn);
+ if (split.size() != 4) {
+ LOG.warn(
+ "Dropping MonitoringInfo for URN %s, UserMetric namespaces and "
+ + "name cannot contain ':' characters.",
+ urn);
+ return false;
+ }
+ } else if (!SimpleMonitoringInfoBuilder.specs.containsKey(urn)) {
+ // Succeed for unknown URNs, this is an extensible metric.
+ // TODO: Allow adding your own spec file to validate your metrics.
+ return true;
+ } else {
+ spec = SimpleMonitoringInfoBuilder.specs.get(urn);
+ }
+
+ if (!this.builder.getType().equals(spec.getTypeUrn())) {
+ LOG.warn(
+ "Dropping MonitoringInfo since for URN %s with invalid type field.
Expected: %s"
+ + " Actual: %s",
+ this.builder.getUrn(), spec.getTypeUrn(), this.builder.getType());
+ return false;
+ }
+
+ Set<String> requiredLabels = new
HashSet<String>(spec.getRequiredLabelsList());
+ if (!this.builder.getLabels().keySet().equals(requiredLabels)) {
+ LOG.warn(
+ "Dropping MonitoringInfo since for URN %s with invalid labels.
Expected: %s"
+ + " Actual: %s",
+ this.builder.getUrn(), requiredLabels,
this.builder.getLabels().keySet());
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * @param namespace The namespace of the metric.
Review comment:
[Ignore if adding @param tags is required by linter]
Remove these param specs, since you do not provide any extra information
here.
You can rename name and namespace to metricName and metricNamespace
respectively, if you think this clarification is required.
Same for other cases.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 170991)
Time Spent: 0.5h (was: 20m)
> Add User Metric Support to Java SDK
> -----------------------------------
>
> Key: BEAM-6138
> URL: https://issues.apache.org/jira/browse/BEAM-6138
> Project: Beam
> Issue Type: New Feature
> Components: java-fn-execution
> Reporter: Alex Amato
> Priority: Major
> Time Spent: 0.5h
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
