Repository: ambari Updated Branches: refs/heads/branch-2.2 0a76a076d -> ebee7108e
AMBARI-15214. Add ability to Create pre-built Grafana dashboards. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ebee7108 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ebee7108 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ebee7108 Branch: refs/heads/branch-2.2 Commit: ebee7108e43fd39c9257d1f39e68f542f1b4dc29 Parents: 0a76a07 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Fri Feb 26 22:21:17 2016 -0800 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Sat Feb 27 10:20:44 2016 -0800 ---------------------------------------------------------------------- .../grafana-hbase-master.json | 241 +++++++++ .../grafana-dashboards/grafana-hbase-rs.json | 241 +++++++++ .../grafana-dashboards/grafana-hdfs-dn.json | 199 ++++++++ .../grafana-dashboards/grafana-hdfs-nn.json | 199 ++++++++ .../grafana-dashboards/grafana-yarn-apps.json | 135 +++++ .../grafana-dashboards/grafana-yarn-nm.json | 315 ++++++++++++ .../grafana-yarn-overview.json | 488 +++++++++++++++++++ .../grafana-dashboards/grafana-yarn-rm.json | 125 +++++ .../0.1.0/package/scripts/metrics_grafana.py | 5 +- .../package/scripts/metrics_grafana_util.py | 310 ++++++++---- .../0.1.0/package/scripts/params.py | 44 +- .../python/stacks/2.0.6/configs/default.json | 4 + .../2.0.6/configs/default_ams_embedded.json | 1 + 13 files changed, 2217 insertions(+), 90 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hbase-master.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hbase-master.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hbase-master.json new file mode 100644 index 0000000..0918628 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hbase-master.json @@ -0,0 +1,241 @@ +{ + "id": null, + "title": "HBase Master", + "originalTitle": "HBase Master", + "tags": [ "builtin" ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.MemHeapCommittedM", + "refId": "A" + }, + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.MemHeapUsedM", + "refId": "B" + }, + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.MemHeapMaxM", + "refId": "C" + }, + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.MemNonHeapMaxM", + "refId": "D" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "JVM Heap", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + } + ], + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.GcTimeMillis", + "refId": "A" + }, + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep", + "refId": "B" + }, + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.GcTimeMillisParNew", + "refId": "C" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "JVM GC Times", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + } + ], + "title": "New row" + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 8, + "version": 5, + "links": [] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hbase-rs.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hbase-rs.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hbase-rs.json new file mode 100644 index 0000000..afe594e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hbase-rs.json @@ -0,0 +1,241 @@ +{ + "id": 9, + "title": "HBase RegionServer", + "originalTitle": "HBase RegionServer", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.MemHeapCommittedM", + "refId": "A" + }, + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.MemHeapUsedM", + "refId": "B" + }, + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.MemHeapMaxM", + "refId": "C" + }, + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.MemNonHeapMaxM", + "refId": "D" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "JVM Heap", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + } + ], + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.GcTimeMillis", + "refId": "A" + }, + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep", + "refId": "B" + }, + { + "aggregator": "avg", + "app": "hbase", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.GcTimeMillisParNew", + "refId": "C" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "JVM GC Times", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + } + ], + "title": "New row" + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 8, + "version": 5, + "links": [] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hdfs-dn.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hdfs-dn.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hdfs-dn.json new file mode 100644 index 0000000..55ca0a3 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hdfs-dn.json @@ -0,0 +1,199 @@ +{ + "id": 12, + "title": "HDFS Datanode", + "originalTitle": "HDFS Datanode", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "rpc.rpc.RpcQueueTimeNumOps on sid-ams-1.novalocal", + "yaxis": 2 + } + ], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "namenode", + "downsampleAggregator": "avg", + "errors": {}, + "hosts": "", + "metric": "rpc.rpc.RpcQueueTimeAvgTime", + "refId": "A", + "templatedHost": "sid-ams-1.novalocal" + }, + { + "aggregator": "avg", + "app": "namenode", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "rpc.rpc.RpcQueueTimeNumOps", + "refId": "B", + "shouldComputeRate": true, + "templatedHost": "sid-ams-1.novalocal" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Average Queue Time", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "id": 2, + "interval": null, + "isNew": true, + "links": [], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "span": 6, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "aggregator": "avg", + "app": "namenode", + "downsampleAggregator": "avg", + "errors": {}, + "hosts": "sid-ams-1.novalocal", + "metric": "rpc.rpc.CallQueueLength", + "refId": "A" + } + ], + "thresholds": "", + "title": "RPC Call Queue Length", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "title": "Row" + } + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 8, + "version": 8, + "links": [] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hdfs-nn.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hdfs-nn.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hdfs-nn.json new file mode 100644 index 0000000..14f6240 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hdfs-nn.json @@ -0,0 +1,199 @@ +{ + "id": null, + "title": "HDFS Namenode", + "originalTitle": "HDFS Namenode", + "tags": [ "builtin" ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "rpc.rpc.RpcQueueTimeNumOps on sid-ams-1.novalocal", + "yaxis": 2 + } + ], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "namenode", + "downsampleAggregator": "avg", + "errors": {}, + "hosts": "", + "metric": "rpc.rpc.RpcQueueTimeAvgTime", + "refId": "A", + "templatedHost": "sid-ams-1.novalocal" + }, + { + "aggregator": "avg", + "app": "namenode", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "rpc.rpc.RpcQueueTimeNumOps", + "refId": "B", + "shouldComputeRate": true, + "templatedHost": "sid-ams-1.novalocal" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Average Queue Time", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "id": 2, + "interval": null, + "isNew": true, + "links": [], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "span": 6, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "aggregator": "avg", + "app": "namenode", + "downsampleAggregator": "avg", + "errors": {}, + "hosts": "sid-ams-1.novalocal", + "metric": "rpc.rpc.CallQueueLength", + "refId": "A" + } + ], + "thresholds": "", + "title": "RPC Call Queue Length", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "title": "Row" + } + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 8, + "version": 8, + "links": [] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-apps.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-apps.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-apps.json new file mode 100644 index 0000000..f4cde0e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-apps.json @@ -0,0 +1,135 @@ +{ + "id": 5, + "title": "YARN Applications", + "originalTitle": "YARN Applications", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "AMS", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.ClusterMetrics.AMLaunchDelayAvgTime", + "refId": "A" + }, + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.ClusterMetrics.AMRegisterDelayAvgTime", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "AM launch / register delay", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "µs", + "short" + ] + } + ], + "title": "Row" + } + ], + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 8, + "version": 2, + "links": [] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-nm.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-nm.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-nm.json new file mode 100644 index 0000000..b9c1518 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-nm.json @@ -0,0 +1,315 @@ +{ + "id": 4, + "title": "Node Managers", + "originalTitle": "Node Managers", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "content": "This dashboard is managed by Ambari.\nIf you make any changes to this, Ambari may refresh this dashboard anytime and you could lose your change.\nIf you want to customize this dashboard, please make a copy first.", + "editable": true, + "error": false, + "id": 4, + "isNew": true, + "links": [], + "mode": "markdown", + "span": 6, + "style": {}, + "title": "Node Managers Dashboard", + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "AMS", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 1, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "nodemanager", + "downsampleAggregator": "avg", + "errors": {}, + "hosts": "", + "metric": "yarn.NodeManagerMetrics.AllocatedGB", + "refId": "A", + "templatedHost": "perf-a-2.c.pramod-thangali.internal" + }, + { + "aggregator": "avg", + "app": "nodemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.NodeManagerMetrics.AllocatedVCores", + "refId": "B", + "templatedHost": "perf-a-2.c.pramod-thangali.internal" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory / Vcore", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "AMS", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 3, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "nodemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.NodeManagerMetrics.GoodLocalDirsDiskUtilizationPerc", + "refId": "A", + "templatedHost": "perf-a-2.c.pramod-thangali.internal" + }, + { + "aggregator": "avg", + "app": "nodemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.NodeManagerMetrics.GoodLogDirsDiskUtilizationPerc", + "refId": "B", + "templatedHost": "perf-a-2.c.pramod-thangali.internal" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Utilization Percentage", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "AMS", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "nodemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.NodeManagerMetrics.BadLocalDirs", + "refId": "A", + "templatedHost": "perf-a-2.c.pramod-thangali.internal" + }, + { + "aggregator": "avg", + "app": "nodemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.NodeManagerMetrics.BadLogDirs", + "refId": "B", + "templatedHost": "perf-a-2.c.pramod-thangali.internal" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Bad local / log dir", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + } + ], + "title": "Row" + } + ], + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 8, + "version": 11, + "links": [] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-overview.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-overview.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-overview.json new file mode 100644 index 0000000..f802548 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-overview.json @@ -0,0 +1,488 @@ +{ + "id": 3, + "title": "YARN Home", + "originalTitle": "YARN Home", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "200px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "id": 3, + "interval": null, + "isNew": true, + "links": [], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "targets": [ + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.ClusterMetrics.NumActiveNMs", + "refId": "A" + } + ], + "thresholds": "", + "title": "Active nodes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "id": 2, + "interval": null, + "isNew": true, + "links": [], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "targets": [ + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.QueueMetrics.Queue=root.AppsRunning", + "precision": "seconds", + "refId": "A", + "shouldAddPrecision": false + } + ], + "thresholds": "", + "title": "Running apps", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "mbytes", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "targets": [ + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "jvm.JvmMetrics.MemHeapUsedM", + "refId": "C" + } + ], + "thresholds": "", + "title": "ResourceManager Heap", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "dashUri": "db/node-managers", + "dashboard": "Node Managers", + "title": "Node Managers", + "type": "dashboard" + } + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.ClusterMetrics.NumActiveNMs", + "refId": "A" + }, + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.ClusterMetrics.NumUnhealthyNMs", + "refId": "B" + }, + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.ClusterMetrics.NumLostNMs", + "refId": "C" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Nodes", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 4, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "dashUri": "db/yarn-applications", + "dashboard": "YARN Applications", + "title": "YARN Applications", + "type": "dashboard" + } + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.QueueMetrics.Queue=root.AppsRunning", + "refId": "A" + }, + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.QueueMetrics.Queue=root.AppsPending", + "refId": "B" + }, + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.QueueMetrics.Queue=root.AppsFailed", + "refId": "C" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Apps", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 9, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "resourcemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "yarn.QueueMetrics.Queue=root.AllocatedMB", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Cluster utilization", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "mbytes", + "short" + ] + } + ], + "title": "New row" + } + ], + "time": { + "from": "now-3d", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 8, + "version": 22, + "links": [] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-rm.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-rm.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-rm.json new file mode 100644 index 0000000..bd11be9 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-rm.json @@ -0,0 +1,125 @@ +{ + "id": 6, + "title": "YARN ResourceManager", + "originalTitle": "YARN ResourceManager", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "hcube", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "app": "nodemanager", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + } + ], + "title": "Row" + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 8, + "version": 1, + "links": [] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py index d96309c..76e8e20 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py @@ -22,7 +22,7 @@ from resource_management import Script, Execute from resource_management.libraries.functions import format from status import check_service_status from ams import ams -from metrics_grafana_util import create_ams_datasource +from metrics_grafana_util import create_ams_datasource, create_ams_dashboards class AmsGrafana(Script): def install(self, env): @@ -47,7 +47,10 @@ class AmsGrafana(Script): Execute(start_cmd, user=params.ams_user ) + # Create datasource create_ams_datasource() + # Create pre-built dashboards + create_ams_dashboards() def stop(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py index 37d403d..3309878 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py @@ -21,135 +21,271 @@ import httplib from resource_management.core.logger import Logger from resource_management.core.base import Fail from resource_management import Template - +from collections import namedtuple +from urlparse import urlparse import time import socket -import json +import ambari_simplejson as json import network -def create_ams_datasource(): +GRAFANA_CONNECT_TRIES = 5 +GRAFANA_CONNECT_TIMEOUT = 15 +GRAFANA_SEARCH_BULTIN_DASHBOARDS = "/api/search?tag=builtin" +GRAFANA_DATASOURCE_URL = "/api/datasources" +GRAFANA_DASHBOARDS_URL = "/api/dashboards/db" +METRICS_GRAFANA_DATASOURCE_NAME = "AMBARI_METRICS" + +Server = namedtuple('Server', [ 'protocol', 'host', 'port' ]) + +def perform_grafana_get_call(url, server): + grafana_https_enabled = server.protocol.lower() == 'https' + + conn = network.get_http_connection(server.host, + int(server.port), + grafana_https_enabled) + + Logger.info("Connecting (GET) to %s:%s%s" % (server.host, + server.port, + url)) + conn.request("GET", url) + response = conn.getresponse() + Logger.info("Http response: %s %s" % (response.status, response.reason)) + + return response + +def perform_grafana_put_call(url, id, payload, server): + response = None + data = None + headers = {"Content-Type": "application/json"} + grafana_https_enabled = server.protocol.lower() == 'https' + + for i in xrange(0, GRAFANA_CONNECT_TRIES): + try: + conn = network.get_http_connection(server.host, int(server.port), grafana_https_enabled) + conn.request("PUT", url + "/" + str(id), payload, headers) + response = conn.getresponse() + data = response.read() + Logger.info("Http data: %s" % data) + conn.close() + break + except (httplib.HTTPException, socket.error) as ex: + if i < GRAFANA_CONNECT_TRIES - 1: + time.sleep(GRAFANA_CONNECT_TIMEOUT) + Logger.info("Connection to Grafana failed. Next retry in %s seconds." + % (GRAFANA_CONNECT_TIMEOUT)) + continue + else: + raise Fail("Ambari Metrics Grafana update failed") + + return (response, data) + +def perform_grafana_post_call(url, payload, server): + response = None + data = None + Logger.debug('POST payload: %s' % payload) + headers = {"Content-Type": "application/json", "Content-Length" : len(payload)} + grafana_https_enabled = server.protocol.lower() == 'https' + for i in xrange(0, GRAFANA_CONNECT_TRIES): + try: + Logger.info("Connecting (POST) to %s:%s%s" % (server.host, server.port, url)) + conn = network.get_http_connection(server.host, + int(server.port), + grafana_https_enabled) + + conn.request("POST", url, payload, headers) + + response = conn.getresponse() + Logger.info("Http response: %s %s" % (response.status, response.reason)) + data = response.read() + Logger.info("Http data: %s" % data) + conn.close() + break + except (httplib.HTTPException, socket.error) as ex: + if i < GRAFANA_CONNECT_TRIES - 1: + time.sleep(GRAFANA_CONNECT_TIMEOUT) + Logger.info("Connection to Grafana failed. Next retry in %s seconds." + % (GRAFANA_CONNECT_TIMEOUT)) + continue + else: + raise Fail("Ambari Metrics Grafana create failed.") + + return (response, data) + +def is_unchanged_datasource_url(datasource_url): + import params + parsed_url = urlparse(datasource_url) + Logger.debug("parsed url, scheme = %s, host = %s, port = %s" % ( + parsed_url.scheme, parsed_url.hostname, parsed_url.port)) + Logger.debug("collector, scheme = %s, host = %s, port = %s" % + (params.metric_collector_protocol, params.metric_collector_host, + params.metric_collector_port)) + + return (parsed_url.scheme == params.metric_collector_protocol.strip()) and \ + (parsed_url.hostname == params.metric_collector_host.strip()) and \ + (parsed_url.port == params.metric_collector_port) + +def create_ams_datasource(): import params - GRAFANA_CONNECT_TRIES = 5 - GRAFANA_CONNECT_TIMEOUT = 15 - GRAFANA_URL = "/api/datasources" - METRICS_GRAFANA_DATASOURCE_NAME = "AMBARI_METRICS" - grafana_https_enabled = params.ams_grafana_protocol.lower() == 'https' - headers = {"Content-type": "application/json"} + server = Server(protocol = params.ams_grafana_protocol.strip(), + host = params.ams_grafana_host.strip(), + port = params.ams_grafana_port) + """ + Create AMS datasource in Grafana, if exsists make sure the collector url is accurate + """ ams_datasource_json = Template('metrics_grafana_datasource.json.j2', - ams_datasource_name=METRICS_GRAFANA_DATASOURCE_NAME)\ - .get_content() + ams_datasource_name=METRICS_GRAFANA_DATASOURCE_NAME).get_content() Logger.info("Checking if AMS Grafana datasource already exists") - Logger.info("Connecting (GET) to %s:%s%s" % (params.hostname, - params.ams_grafana_port, - GRAFANA_URL)) - conn = network.get_http_connection(params.hostname, - int(params.ams_grafana_port), - grafana_https_enabled) - conn.request("GET", GRAFANA_URL) - response = conn.getresponse() - Logger.info("Http response: %s %s" % (response.status, response.reason)) + response = perform_grafana_get_call(GRAFANA_DATASOURCE_URL, server) + create_datasource = True - if(response.status == 200): + if response.status == 200: datasources = response.read() datasources_json = json.loads(datasources) for i in xrange(0, len(datasources_json)): datasource_name = datasources_json[i]["name"] - if(datasource_name == METRICS_GRAFANA_DATASOURCE_NAME): - + if datasource_name == METRICS_GRAFANA_DATASOURCE_NAME: + create_datasource = False # datasource already exists Logger.info("Ambari Metrics Grafana datasource already present. Checking Metrics Collector URL") datasource_url = datasources_json[i]["url"] - if datasource_url == (params.metric_collector_protocol + "://" - + params.metric_collector_host + ":" - + params.metric_collector_port): - Logger.info("Metrics Collector URL validation succeeded. Skipping datasource creation") - GRAFANA_CONNECT_TRIES = 0 # No need to create datasource again - + if is_unchanged_datasource_url(datasource_url): + Logger.info("Metrics Collector URL validation succeeded.") + return else: # Metrics datasource present, but collector host is wrong. - - Logger.info("Metrics Collector URL validation failed.") datasource_id = datasources_json[i]["id"] - Logger.info("Deleting obselete Metrics datasource.") - conn = network.get_http_connection(params.hostname, - int(params.ams_grafana_port), - grafana_https_enabled) - conn.request("PUT", GRAFANA_URL + "/" + str(datasource_id), ams_datasource_json, headers) - response = conn.getresponse() - data = response.read() - Logger.info("Http data: %s" % data) - conn.close() + Logger.info("Metrics Collector URL validation failed. Updating " + "datasource, id = %s" % datasource_id) + + (response, data) = perform_grafana_put_call(GRAFANA_DATASOURCE_URL, datasource_id, + ams_datasource_json, server) if response.status == 200: Logger.info("Ambari Metrics Grafana data source updated.") - GRAFANA_CONNECT_TRIES = 0 # No need to create datasource again + elif response.status == 500: Logger.info("Ambari Metrics Grafana data source update failed. Not retrying.") raise Fail("Ambari Metrics Grafana data source update failed. PUT request status: %s %s \n%s" % (response.status, response.reason, data)) else: - Logger.info("Ambari Metrics Grafana data source update failed.") - if i < GRAFANA_CONNECT_TRIES - 1: - time.sleep(GRAFANA_CONNECT_TIMEOUT) - Logger.info("Next retry in %s seconds." - % (GRAFANA_CONNECT_TIMEOUT)) - else: - raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" % - (response.status, response.reason, data)) - - break + raise Fail("Ambari Metrics Grafana data source creation failed. " + "PUT request status: %s %s \n%s" % (response.status, response.reason, data)) + pass + pass + pass else: Logger.info("Error checking for Ambari Metrics Grafana datasource. Will attempt to create.") - if GRAFANA_CONNECT_TRIES > 0: - Logger.info("Attempting to create Ambari Metrics Grafana datasource") - - for i in xrange(0, GRAFANA_CONNECT_TRIES): - try: - - Logger.info("Generated datasource:\n%s" % ams_datasource_json) - - Logger.info("Connecting (POST) to %s:%s%s" % (params.hostname, - params.ams_grafana_port, - GRAFANA_URL)) - conn = network.get_http_connection(params.hostname, - int(params.ams_grafana_port), - grafana_https_enabled) - conn.request("POST", GRAFANA_URL, ams_datasource_json, headers) - - response = conn.getresponse() - Logger.info("Http response: %s %s" % (response.status, response.reason)) - except (httplib.HTTPException, socket.error) as ex: - if i < GRAFANA_CONNECT_TRIES - 1: - time.sleep(GRAFANA_CONNECT_TIMEOUT) - Logger.info("Connection to Grafana failed. Next retry in %s seconds." - % (GRAFANA_CONNECT_TIMEOUT)) - continue - else: - raise Fail("Ambari Metrics Grafana datasource not created") + if not create_datasource: + return + else: + Logger.info("Generating datasource:\n%s" % ams_datasource_json) - data = response.read() - Logger.info("Http data: %s" % data) - conn.close() + (response, data) = perform_grafana_post_call(GRAFANA_DATASOURCE_URL, ams_datasource_json, server) if response.status == 200: Logger.info("Ambari Metrics Grafana data source created.") - break elif response.status == 500: Logger.info("Ambari Metrics Grafana data source creation failed. Not retrying.") raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" % (response.status, response.reason, data)) else: Logger.info("Ambari Metrics Grafana data source creation failed.") - if i < GRAFANA_CONNECT_TRIES - 1: - time.sleep(GRAFANA_CONNECT_TIMEOUT) - Logger.info("Next retry in %s seconds." - % (GRAFANA_CONNECT_TIMEOUT)) - else: - raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" % + raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" % (response.status, response.reason, data)) pass + +def create_ams_dashboards(): + """ + Create dashboards in grafana from the json files + """ + import params + server = Server(protocol = params.ams_grafana_protocol.strip(), + host = params.ams_grafana_host.strip(), + port = params.ams_grafana_port) + + dashboard_files = params.get_grafana_dashboard_defs() + version = params.get_ambari_version() + Logger.info("Checking dashboards to update for Ambari version : %s" % version) + # Friendly representation of dashboard + Dashboard = namedtuple('Dashboard', ['uri', 'id', 'title', 'tags']) + + existing_dashboards = [] + response = perform_grafana_get_call(GRAFANA_SEARCH_BULTIN_DASHBOARDS, server) + if response and response.status == 200: + data = response.read() + try: + datasources = json.loads(data) + except: + Logger.error("Unable to parse JSON response from grafana request: %s" % + GRAFANA_SEARCH_BULTIN_DASHBOARDS) + Logger.info(data) + return + + for datasource in datasources: + existing_dashboards.append( + Dashboard(uri = datasource['uri'], id = datasource['id'], + title = datasource['title'], tags = datasource['tags']) + ) + pass + else: + Logger.error("Failed to execute search query on Grafana dashboards. " + "query = %s\n statuscode = %s\n reason = %s\n data = %s\n" % + (GRAFANA_SEARCH_BULTIN_DASHBOARDS, response.status, response.reason, response.read())) + return + + Logger.debug('Dashboard definitions found = %s' % str(dashboard_files)) + + if dashboard_files: + for dashboard_file in dashboard_files: + try: + with open(dashboard_file, 'r') as file: + dashboard_def = json.load(file) + except Exception, e: + Logger.error('Unable to load dashboard json file %s' % dashboard_file) + Logger.error(str(e)) + continue + + if dashboard_def: + update_def = True + # Make sure static json does not have id + if "id" in dashboard_def: + dashboard_def['id'] = None + # Set correct tags + dashboard_def['tags'] = [ 'builtin', version ] + dashboard_def['overwrite'] = True + + for dashboard in existing_dashboards: + if dashboard.title == dashboard_def['title']: + if version not in dashboard.tags: + # Found existing dashboard with wrong version - update dashboard + update_def = True + else: + update_def = False # Skip update + pass + + if update_def: + Logger.info("Updating dashboard definition for %s with tags: %s" % + (dashboard_def['title'], dashboard_def['tags'])) + + # Discrepancy in grafana export vs import format + dashboard_def_payload = { "dashboard" : dashboard_def } + paylaod = json.dumps(dashboard_def_payload).strip() + + (response, data) = perform_grafana_post_call(GRAFANA_DASHBOARDS_URL, paylaod, server) + + if response and response.status == 200: + Logger.info("Dashboard created successfully.\n %s" % str(data)) + else: + Logger.error("Failed creating dashboard: %s" % dashboard_def['title']) + pass + pass + pass + pass + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index 58ff71c..3bd98a2 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -21,11 +21,12 @@ limitations under the License. from functions import calc_xmn_from_xms from functions import check_append_heap_property from functions import trim_heap_property - +from resource_management.core.logger import Logger from resource_management import * import status_params from ambari_commons import OSCheck - +import ConfigParser +import os if OSCheck.is_windows_family(): from params_windows import * @@ -61,6 +62,41 @@ metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.trust metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "") metric_truststore_ca_certs='ca.pem' +agent_cache_dir = config['hostLevelParams']['agentCacheDir'] +service_package_folder = config['commandParams']['service_package_folder'] +dashboards_dir = os.path.join(agent_cache_dir, service_package_folder, 'files', 'grafana-dashboards') + +def get_grafana_dashboard_defs(): + dashboard_defs = [] + if os.path.exists(dashboards_dir): + for root, dirs, files in os.walk(dashboards_dir): + for file in files: + if 'grafana' in file: + dashboard_defs.append(os.path.join(root, file)) + return dashboard_defs + +# find ambari version for grafana dashboards +def get_ambari_version(): + ambari_version = None + AMBARI_AGENT_CONF = '/etc/ambari-agent/conf/ambari-agent.ini' + ambari_agent_config = ConfigParser.RawConfigParser() + if os.path.exists(AMBARI_AGENT_CONF): + try: + ambari_agent_config.read(AMBARI_AGENT_CONF) + data_dir = ambari_agent_config.get('agent', 'prefix') + ver_file = os.path.join(data_dir, 'version') + f = open(ver_file, "r") + ambari_version = f.read().strip() + f.close() + except Exception, e: + Logger.info('Unable to determine ambari version from version file.') + Logger.debug('Exception: %s' % str(e)) + # No hostname script identified in the ambari agent conf + pass + pass + return ambari_version + + if 'cluster-env' in config['configurations'] and \ 'metrics_collector_vip_host' in config['configurations']['cluster-env']: metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host'] @@ -92,6 +128,10 @@ ams_grafana_pid_dir = status_params.ams_grafana_pid_dir ams_grafana_conf_dir = '/etc/ambari-metrics-grafana/conf' ams_grafana_data_dir = default("/configurations/ams-grafana-env/metrics_grafana_data_dir", '/var/lib/ambari-metrics-grafana') +metrics_grafana_hosts = default('/clusterHostInfo/metrics_grafana_hosts', None) +ams_grafana_host = None +if metrics_grafana_hosts: + ams_grafana_host = metrics_grafana_hosts[0] ams_grafana_port = default("/configurations/ams-grafana-ini/port", 3000) ams_grafana_protocol = default("/configurations/ams-grafana-ini/protocol", 'http') ams_grafana_cert_file = default("/configurations/ams-grafana-ini/cert_file", '/etc/ambari-metrics/conf/ams-grafana.crt') http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index 4c21739..6ae54f0 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -3,6 +3,7 @@ "clusterName": "c1", "hostname": "c6401.ambari.apache.org", "hostLevelParams": { + "agentCacheDir": "/var/lib/ambari-agent/cache", "jdk_location": "http://c6401.ambari.apache.org:8080/resources/", "ambari_db_rca_password": "mapred", "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca", @@ -1131,6 +1132,9 @@ ], "metrics_collector_hosts": [ "c6401.ambari.apache.org" + ], + "metrics_grafana_hosts": [ + "c6401.ambari.apache.org" ] } } http://git-wip-us.apache.org/repos/asf/ambari/blob/ebee7108/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json index 95e931a..dd932d3 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json @@ -3,6 +3,7 @@ "clusterName": "c1", "hostname": "c6401.ambari.apache.org", "hostLevelParams": { + "agentCacheDir": "/var/lib/ambari-agent/cache", "jdk_location": "http://c6401.ambari.apache.org:8080/resources/", "ambari_db_rca_password": "mapred", "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",