little-cui closed pull request #335: SCB-508 Cache does not match the etcd store URL: https://github.com/apache/incubator-servicecomb-service-center/pull/335
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/.gitignore b/.gitignore index b6fa4196..855c6984 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ service-center.iml vendor/** !vendor/manifest -!vendor/manifest/github.com/didip/tollbooth !vendor/manifest/github.com/grpc-ecosystem/go-grpc-prometheus !vendor/manifest/github.com/grpc-ecosystem/grpc-gateway diff --git a/etc/conf/app.conf b/etc/conf/app.conf index d468dbe2..c2e7e880 100644 --- a/etc/conf/app.conf +++ b/etc/conf/app.conf @@ -17,6 +17,12 @@ component_name = service_center #run mode could be many options to specify the env like prod,dev runmode = dev +################################################################### +# Frontend Configurations +################################################################### +frontend_host_ip = 127.0.0.1 +frontend_host_port = 30103 + ################################################################### # sever options ################################################################### @@ -30,6 +36,8 @@ write_timeout = 60s max_header_bytes = 32768 # 32K max_body_bytes = 2097152 # 2M +enable_pprof = 0 + ################################################################### # plugin options ################################################################### @@ -53,13 +61,17 @@ manager_cluster = "127.0.0.1:2379" #<=0, use default 30s auto_sync_interval = 30s -#indicate how many revision you want to keep in etcd +# indicate how many revision you want to keep in etcd compact_index_delta = 100 compact_interval = 12h +# registry cache +enable_cache = 1 + +# pluggable cipher cipher_plugin = "" -#suppot buildin, unlimit +# suppot buildin, unlimit quota_plugin = "" #access control plugin @@ -108,21 +120,15 @@ log_format = text # whether enable record syslog log_sys = false -################################################################### -# Frontend Configurations -################################################################### -frontend_host_ip=127.0.0.1 -frontend_host_port=30103 - ################################################################### # above is the global configurations # you can overide above configuration in specific env ################################################################### [prod] -#DEBUG, INFO, WARN, ERROR, FATAL +# DEBUG, INFO, WARN, ERROR, FATAL loglevel = INFO logfile = ./service-center.log -##dev env, if you are a operator,ignore this env +## dev env, if you are a operator,ignore this env [dev] loglevel = DEBUG logfile = "" diff --git a/integration/health-metrics-grafana.json b/integration/health-metrics-grafana.json index 55e5c695..2404a611 100644 --- a/integration/health-metrics-grafana.json +++ b/integration/health-metrics-grafana.json @@ -1,4 +1,46 @@ { + "__inputs": [ + { + "name": "DS_LOCAL", + "label": "local", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.1.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "5.0.0" + } + ], "annotations": { "list": [ { @@ -15,8 +57,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "hideControls": false, - "id": 6, + "id": null, "links": [ { "icon": "external link", @@ -26,1039 +67,1095 @@ "url": "http://servicecomb.io" } ], - "refresh": false, - "rows": [ + "panels": [ { - "collapse": false, - "height": 234, - "panels": [ + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_LOCAL}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 0 + }, + "height": "", + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 4, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "height": "", - "id": 1, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "show": true, - "sort": null, - "sortDesc": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "minSpan": 2, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(max_over_time(service_center_http_request_durations_microseconds{job=\"service-center\",method=\"GET\"}[1m]))", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "read", - "refId": "A" - }, - { - "expr": "max(max_over_time(service_center_http_request_durations_microseconds{job=\"service-center\",method!=\"GET\"}[1m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "write", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "I/O Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "max(max_over_time(service_center_http_request_durations_microseconds{job=\"service-center\",method=\"GET\"}[1m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "read", + "refId": "A" }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": null, - "format": "percentunit", - "gauge": { - "maxValue": 1, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "minSpan": 2, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(service_center_http_success_total{job=\"service-center\"})/sum(service_center_http_request_total{job=\"service-center\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": ".5,.8", - "title": "Global Success Rate", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "expr": "max(max_over_time(service_center_http_request_durations_microseconds{job=\"service-center\",method!=\"GET\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "I/O Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "µs", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": null, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 13, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "minSpan": 2, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(service_center_http_request_total{job=\"service-center\"}[1m]))", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "", - "title": "Global Request Volume", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "${DS_LOCAL}", + "format": "percentunit", + "gauge": { + "maxValue": 1, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": null, - "format": "rps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 10, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "minSpan": 2, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(service_center_http_request_total{job=\"service-center\",code=~\"4.+\"}[1m]))", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "4XX Errors", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 4, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(service_center_http_success_total{job=\"service-center\"})/sum(service_center_http_request_total{job=\"service-center\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": ".5,.8", + "title": "Global Success Rate", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_LOCAL}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": null, - "format": "rps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 11, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "minSpan": 2, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(service_center_http_request_total{job=\"service-center\",code=~\"5.+\"}[1m]))", - "format": "time_series", - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "", - "title": "5XX Errors", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" + "name": "range to text", + "value": 2 } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "service metrics", - "titleSize": "h6" + "maxDataPoints": 100, + "minSpan": 4, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(service_center_http_request_total{job=\"service-center\"}[1m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "", + "title": "Global Request Volume", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" }, { - "collapse": false, - "height": "1px", - "panels": [ + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_LOCAL}", + "format": "rps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 10, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 4, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(service_center_http_request_total{job=\"service-center\",code=~\"4.+\"}[1m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "4XX Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_LOCAL}", + "format": "rps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 4, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ { - "content": "<div class=\"text-center dashboard-header\">\n <span>INSTANCE METRICS</span>\n</div>\n", - "height": "1px", - "id": 3, - "links": [], - "mode": "html", - "repeat": null, - "span": 12, - "title": "", - "transparent": true, - "type": "text" + "from": "null", + "text": "N/A", + "to": "null" } ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(service_center_http_request_total{job=\"service-center\",code=~\"5.+\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "", + "title": "5XX Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "content": "<div class=\"text-center dashboard-header\">\n <span>INSTANCE METRICS</span>\n</div>\n", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 6 + }, + "height": "1px", + "id": 3, + "links": [], + "mode": "html", "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "instance metrics", - "titleSize": "h6" + "title": "", + "transparent": true, + "type": "text" }, { - "collapse": false, - "height": 253, - "panels": [ + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_LOCAL}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "height": "", + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "height": "", - "id": 2, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{job=\"service-center\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "goroutines", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "go_goroutines{job=\"service-center\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "goroutines", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "height": "", - "id": 9, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(process_cpu_seconds_total{job=\"service-center\"}[1m])", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "cpu", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_LOCAL}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "height": "", + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(process_cpu_seconds_total{job=\"service-center\"}[1m])", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "cpu", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "height": "", - "id": 8, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(max_over_time(go_gc_duration_seconds{job=\"service-center\"}[1m]))", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "gc", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "GC Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "instances metrics", - "titleSize": "h6" + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_LOCAL}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "height": "", + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(max_over_time(go_gc_duration_seconds{job=\"service-center\"}[1m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "gc", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "GC Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "collapse": false, + "content": "<div class=\"text-center dashboard-header\">\n <span>RESOURCES</span>\n</div>\n", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 16 + }, "height": "1px", - "panels": [ + "id": 4, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_LOCAL}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "content": "<div class=\"text-center dashboard-header\">\n <span>RESOURCES</span>\n</div>\n", - "height": "1px", - "id": 4, - "links": [], - "mode": "html", - "span": 12, - "title": "", - "transparent": true, - "type": "text" + "expr": "go_threads{job=\"service-center\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "threads", + "refId": "A" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "resources", - "titleSize": "h6" + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Threads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "collapse": false, - "height": 250, - "panels": [ + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_LOCAL}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_open_fds{job=\"service-center\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fds", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "File Descriptions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_threads{job=\"service-center\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "threads", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Threads", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_open_fds{job=\"service-center\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "fds", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "File Descriptions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_LOCAL}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_heap_inuse_bytes{job=\"service-center\"} + go_memstats_heap_idle_bytes{job=\"service-center\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "heap", + "refId": "B" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_resident_memory_bytes{job=\"service-center\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "go_memstats_heap_inuse_bytes{job=\"service-center\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "inuse", + "refId": "C" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "id": 7, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "service_center_local_cache_size_bytes", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{resource}}.{{type}}", - "refId": "A" - }, - { - "expr": "sum(service_center_local_cache_size_bytes)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "ALL", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Cache", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "process_resident_memory_bytes{job=\"service-center\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "all", + "refId": "A" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_LOCAL}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(service_center_local_cache_size_bytes{job=\"service-center\"}) by (resource)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{resource}}", + "refId": "A" + }, + { + "expr": "sum(service_center_local_cache_size_bytes{job=\"service-center\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ALL", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cache", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], - "schemaVersion": 14, + "refresh": false, + "schemaVersion": 16, "style": "dark", "tags": [], "templating": { @@ -1095,5 +1192,6 @@ }, "timezone": "", "title": "ServiceCenter", - "version": 2 -} + "uid": "Zg6NoHGiz", + "version": 5 +} \ No newline at end of file diff --git a/integration/health_test.go b/integration/health_test.go index 62f89d5e..050a9258 100644 --- a/integration/health_test.go +++ b/integration/health_test.go @@ -37,7 +37,6 @@ var _ = Describe("Basic Api Test", func() { Expect(err).To(BeNil()) defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) - Expect(resp.StatusCode).To(Equal(http.StatusOK)) }) }) diff --git a/server/core/backend/store/async.go b/pkg/async/async.go similarity index 78% rename from server/core/backend/store/async.go rename to pkg/async/async.go index 47788549..87cc07b0 100644 --- a/server/core/backend/store/async.go +++ b/pkg/async/async.go @@ -14,17 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package store +package async -import "github.com/apache/incubator-servicecomb-service-center/pkg/async" +import "sync" -var service *async.AsyncTaskService +var ( + service *TaskService + once sync.Once +) func init() { - service = async.NewAsyncTaskService() - service.Run() + service = NewTaskService() } -func AsyncTaskService() *async.AsyncTaskService { +func Service() *TaskService { + once.Do(service.Run) return service } diff --git a/pkg/async/async_task.go b/pkg/async/async_task.go index 73ceb384..cda5d915 100644 --- a/pkg/async/async_task.go +++ b/pkg/async/async_task.go @@ -29,7 +29,7 @@ const ( DEFAULT_REMOVE_TASKS_INTERVAL = 30 * time.Second ) -type AsyncTask interface { +type Task interface { Key() string Do(ctx context.Context) error Err() error @@ -37,12 +37,12 @@ type AsyncTask interface { type scheduler struct { queue *util.UniQueue - latestTask AsyncTask + latestTask Task once sync.Once goroutine *util.GoRoutine } -func (s *scheduler) AddTask(ctx context.Context, task AsyncTask) (err error) { +func (s *scheduler) AddTask(ctx context.Context, task Task) (err error) { if task == nil || ctx == nil { return errors.New("invalid parameters") } @@ -67,7 +67,7 @@ func (s *scheduler) do(ctx context.Context) { if !ok { return } - at := task.(AsyncTask) + at := task.(Task) at.Do(ctx) s.latestTask = at } @@ -79,7 +79,7 @@ func (s *scheduler) Close() { s.goroutine.Close(true) } -func newScheduler(task AsyncTask) *scheduler { +func newScheduler(task Task) *scheduler { return &scheduler{ queue: util.NewUniQueue(), latestTask: task, @@ -87,7 +87,7 @@ func newScheduler(task AsyncTask) *scheduler { } } -type AsyncTaskService struct { +type TaskService struct { schedules map[string]*scheduler removeTasks map[string]struct{} goroutine *util.GoRoutine @@ -96,7 +96,7 @@ type AsyncTaskService struct { isClose bool } -func (lat *AsyncTaskService) getOrNewScheduler(task AsyncTask) (s *scheduler, isNew bool) { +func (lat *TaskService) getOrNewScheduler(task Task) (s *scheduler, isNew bool) { var ( ok bool key = task.Key() @@ -127,7 +127,7 @@ func (lat *AsyncTaskService) getOrNewScheduler(task AsyncTask) (s *scheduler, is return } -func (lat *AsyncTaskService) Add(ctx context.Context, task AsyncTask) error { +func (lat *TaskService) Add(ctx context.Context, task Task) error { if task == nil || ctx == nil { return errors.New("invalid parameters") } @@ -140,11 +140,11 @@ func (lat *AsyncTaskService) Add(ctx context.Context, task AsyncTask) error { return s.AddTask(ctx, task) } -func (lat *AsyncTaskService) DeferRemove(key string) error { +func (lat *TaskService) DeferRemove(key string) error { lat.lock.Lock() if lat.isClose { lat.lock.Unlock() - return errors.New("AsyncTaskService is stopped") + return errors.New("TaskService is stopped") } _, exist := lat.schedules[key] if !exist { @@ -156,7 +156,7 @@ func (lat *AsyncTaskService) DeferRemove(key string) error { return nil } -func (lat *AsyncTaskService) removeScheduler(key string) { +func (lat *TaskService) removeScheduler(key string) { if s, ok := lat.schedules[key]; ok { s.Close() delete(lat.schedules, key) @@ -165,7 +165,7 @@ func (lat *AsyncTaskService) removeScheduler(key string) { util.Logger().Debugf("remove scheduler, key is %s", key) } -func (lat *AsyncTaskService) LatestHandled(key string) (AsyncTask, error) { +func (lat *TaskService) LatestHandled(key string) (Task, error) { lat.lock.RLock() s, ok := lat.schedules[key] lat.lock.RUnlock() @@ -175,12 +175,12 @@ func (lat *AsyncTaskService) LatestHandled(key string) (AsyncTask, error) { return s.latestTask, nil } -func (lat *AsyncTaskService) daemon(ctx context.Context) { +func (lat *TaskService) daemon(ctx context.Context) { util.SafeCloseChan(lat.ready) for { select { case <-ctx.Done(): - util.Logger().Debugf("daemon thread exited for AsyncTaskService is stopped") + util.Logger().Debugf("daemon thread exited for TaskService is stopped") return case <-time.After(DEFAULT_REMOVE_TASKS_INTERVAL): if lat.isClose { @@ -191,6 +191,10 @@ func (lat *AsyncTaskService) daemon(ctx context.Context) { for key := range lat.removeTasks { lat.removeScheduler(key) } + + if l > DEFAULT_MAX_SCHEDULE_COUNT { + lat.renew() + } lat.lock.Unlock() if l > 0 { util.Logger().Infof("daemon thread completed, %d scheduler(s) removed", l) @@ -199,7 +203,7 @@ func (lat *AsyncTaskService) daemon(ctx context.Context) { } } -func (lat *AsyncTaskService) Run() { +func (lat *TaskService) Run() { lat.lock.Lock() if !lat.isClose { lat.lock.Unlock() @@ -210,7 +214,7 @@ func (lat *AsyncTaskService) Run() { lat.goroutine.Do(lat.daemon) } -func (lat *AsyncTaskService) Stop() { +func (lat *TaskService) Stop() { lat.lock.Lock() if lat.isClose { lat.lock.Unlock() @@ -229,16 +233,21 @@ func (lat *AsyncTaskService) Stop() { util.SafeCloseChan(lat.ready) } -func (lat *AsyncTaskService) Ready() <-chan struct{} { +func (lat *TaskService) Ready() <-chan struct{} { return lat.ready } -func NewAsyncTaskService() *AsyncTaskService { - return &AsyncTaskService{ - schedules: make(map[string]*scheduler, DEFAULT_MAX_SCHEDULE_COUNT), - removeTasks: make(map[string]struct{}, DEFAULT_MAX_SCHEDULE_COUNT), - goroutine: util.NewGo(context.Background()), - ready: make(chan struct{}), - isClose: true, +func (lat *TaskService) renew() { + lat.schedules = make(map[string]*scheduler, DEFAULT_MAX_SCHEDULE_COUNT) + lat.removeTasks = make(map[string]struct{}, DEFAULT_MAX_SCHEDULE_COUNT) +} + +func NewTaskService() (lat *TaskService) { + lat = &TaskService{ + goroutine: util.NewGo(context.Background()), + ready: make(chan struct{}), + isClose: true, } + lat.renew() + return } diff --git a/pkg/async/async_task_test.go b/pkg/async/async_task_test.go index 230637a4..080f3d79 100644 --- a/pkg/async/async_task_test.go +++ b/pkg/async/async_task_test.go @@ -57,7 +57,7 @@ func (tt *testTask) Do(ctx context.Context) error { } func TestBaseAsyncTasker_AddTask(t *testing.T) { - at := NewAsyncTaskService() + at := NewTaskService() at.Run() defer at.Stop() @@ -97,7 +97,7 @@ func TestBaseAsyncTasker_AddTask(t *testing.T) { } func TestBaseAsyncTasker_Stop(t *testing.T) { - at := NewAsyncTaskService() + at := NewTaskService() at.Stop() at.Run() @@ -132,7 +132,7 @@ func TestBaseAsyncTasker_Stop(t *testing.T) { } func TestBaseAsyncTasker_RemoveTask(t *testing.T) { - at := NewAsyncTaskService() + at := NewTaskService() at.Run() err := at.DeferRemove("test") diff --git a/pkg/chain/handler.go b/pkg/chain/handler.go index be482994..f5a62ac1 100644 --- a/pkg/chain/handler.go +++ b/pkg/chain/handler.go @@ -18,7 +18,6 @@ package chain import ( "github.com/apache/incubator-servicecomb-service-center/pkg/util" - "reflect" ) const CAP_SIZE = 10 @@ -37,7 +36,7 @@ func RegisterHandler(catalog string, h Handler) { handlers = append(handlers, h) handlersMap[catalog] = handlers - t := util.LoadStruct(reflect.ValueOf(h).Elem().Interface()) + t := util.ReflectObject(h) util.Logger().Infof("register handler[%s] %s/%s", catalog, t.Type.PkgPath(), t.Type.Name()) } diff --git a/pkg/util/reflect.go b/pkg/util/reflect.go index 6efa1ae5..695a7b91 100644 --- a/pkg/util/reflect.go +++ b/pkg/util/reflect.go @@ -30,21 +30,21 @@ var ( func init() { reflector = &Reflector{ - types: make(map[*uintptr]StructType, 100), + types: make(map[*uintptr]reflectObject, 100), } } -type StructType struct { +type reflectObject struct { Type reflect.Type Fields []reflect.StructField } type Reflector struct { - types map[*uintptr]StructType + types map[*uintptr]reflectObject mux sync.RWMutex } -func (r *Reflector) Load(obj interface{}) StructType { +func (r *Reflector) Load(obj interface{}) reflectObject { r.mux.RLock() itab := *(**uintptr)(unsafe.Pointer(&obj)) t, ok := r.types[itab] @@ -59,8 +59,31 @@ func (r *Reflector) Load(obj interface{}) StructType { r.mux.Unlock() return t } - t = StructType{ - Type: reflect.TypeOf(obj), + + v := reflect.ValueOf(obj) + if !v.IsValid() { + r.mux.Unlock() + return reflectObject{} + } + switch v.Kind() { + case reflect.Ptr: + if v.IsNil() { + r.mux.Unlock() + return reflectObject{} + } + fallthrough + case reflect.Interface: + r.mux.Unlock() + return r.Load(v.Elem().Interface()) + default: + t = reflectObject{ + Type: reflect.TypeOf(obj), + } + } + + if v.Kind() != reflect.Struct { + r.mux.Unlock() + return t } fl := t.Type.NumField() @@ -76,7 +99,7 @@ func (r *Reflector) Load(obj interface{}) StructType { return t } -func LoadStruct(obj interface{}) StructType { +func ReflectObject(obj interface{}) (s reflectObject) { return reflector.Load(obj) } diff --git a/pkg/util/reflect_test.go b/pkg/util/reflect_test.go index af102e0a..f5a4c514 100644 --- a/pkg/util/reflect_test.go +++ b/pkg/util/reflect_test.go @@ -34,7 +34,7 @@ type testField struct { func TestLoadStruct(t *testing.T) { obj1 := testStru{} - v := LoadStruct(obj1) + v := ReflectObject(obj1) if v.Type.String() != "util.testStru" { t.Fatalf("TestLoadStruct failed, %s != 'testStru'", v.Type.String()) } @@ -46,13 +46,15 @@ func TestLoadStruct(t *testing.T) { } obj2 := testStru{} - v = LoadStruct(obj2) + v = ReflectObject(obj2) + v = ReflectObject(&obj2) + v = ReflectObject(nil) } func BenchmarkLoadStruct(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - LoadStruct(testStru{}) + ReflectObject(testStru{}) } }) b.ReportAllocs() diff --git a/pkg/validate/validate.go b/pkg/validate/validate.go index e54cc557..1f6c3020 100644 --- a/pkg/validate/validate.go +++ b/pkg/validate/validate.go @@ -113,7 +113,7 @@ func (v *ValidateRule) Match(s interface{}) (ok bool, invalidValue interface{}) invalidValue = v break } - if ok, v := itemV.Match(sv.MapIndex(key).Interface()); !ok{ + if ok, v := itemV.Match(sv.MapIndex(key).Interface()); !ok { invalid = true invalidValue = v break @@ -124,7 +124,7 @@ func (v *ValidateRule) Match(s interface{}) (ok bool, invalidValue interface{}) Regexp: v.Regexp, } for i, l := 0, sv.Len(); i < l; i++ { - if ok, v := itemV.Match(sv.Index(i).Interface()) ; !ok{ + if ok, v := itemV.Match(sv.Index(i).Interface()); !ok { invalid = true invalidValue = v break @@ -213,7 +213,7 @@ func (v *Validator) Validate(s interface{}) error { return errors.New("not support validate type") } - st := util.LoadStruct(s) + st := util.ReflectObject(s) for i, l := 0, sv.NumField(); i < l; i++ { field := sv.Field(i) fieldName := st.Fields[i].Name @@ -242,7 +242,7 @@ func (v *Validator) Validate(s interface{}) error { } } // TODO null pointer如何校验 - if field.Kind() != reflect.Ptr{ + if field.Kind() != reflect.Ptr { ok, invalidValue := validate.Match(fi) if !ok { if filter(fieldName) { diff --git a/server/core/backend/store/cache_kv.go b/server/core/backend/store/cache_kv.go index 1b92e998..fcced8f2 100644 --- a/server/core/backend/store/cache_kv.go +++ b/server/core/backend/store/cache_kv.go @@ -46,8 +46,7 @@ func (c *KvCache) Data(k interface{}) interface{} { if !ok { return nil } - copied := *kv - return &copied + return kv } func (c *KvCache) Have(k interface{}) (ok bool) { @@ -79,14 +78,7 @@ func (c *KvCache) Unlock() { if c.size >= l && c.lastMaxSize > c.size*DEFAULT_COMPACT_TIMES && time.Now().Sub(c.lastRefresh) >= DEFAULT_COMPACT_TIMEOUT { - util.Logger().Infof("cache %s is not in use over %s, compact capacity to size %d->%d", - c.owner.Cfg.Key, DEFAULT_COMPACT_TIMEOUT, c.lastMaxSize, c.size) - // gc - newCache := make(map[string]*mvccpb.KeyValue, c.size) - for k, v := range c.store { - newCache[k] = v - } - c.store = newCache + c.compact() c.lastMaxSize = l c.lastRefresh = time.Now() } @@ -94,6 +86,19 @@ func (c *KvCache) Unlock() { c.rwMux.Unlock() } +func (c *KvCache) compact() { + // gc + newCache := make(map[string]*mvccpb.KeyValue, c.size) + for k, v := range c.store { + newCache[k] = v + } + c.store = newCache + + util.Logger().Infof("cache %s is not in use over %s, compact capacity to size %d->%d", + c.owner.Cfg.Key, DEFAULT_COMPACT_TIMEOUT, c.lastMaxSize, c.size) + +} + func (c *KvCache) Size() (l int) { c.rwMux.RLock() l = len(c.store) @@ -211,7 +216,7 @@ func (c *KvCacher) refresh(ctx context.Context) { start := time.Now() c.ListAndWatch(ctx) watchDuration := time.Since(start) - nextPeriod := c.Cfg.Period + nextPeriod := minWaitInterval if watchDuration > 0 && c.Cfg.Period > watchDuration { nextPeriod = c.Cfg.Period - watchDuration } @@ -220,8 +225,7 @@ func (c *KvCacher) refresh(ctx context.Context) { util.Logger().Debugf("stop to list and watch %s", c.Cfg) return case <-time.After(nextPeriod): - r := c.cache.RLock() - ReportCacheMetrics(c.Name(), "raw", r) + ReportCacheMetrics(c.Name(), "raw", c.cache.RLock()) c.cache.RUnlock() } } @@ -233,7 +237,7 @@ func (c *KvCacher) deferHandle(ctx context.Context) { } var ( - evts [event_block_size]KvEvent + evts [eventBlockSize]KvEvent i int ) for { @@ -246,7 +250,7 @@ func (c *KvCacher) deferHandle(ctx context.Context) { continue } - if i >= event_block_size { + if i >= eventBlockSize { c.onEvents(evts[:i]) i = 0 } @@ -295,7 +299,7 @@ func (c *KvCacher) filter(rev int64, items []*mvccpb.KeyValue) []KvEvent { newStore[util.BytesToStringWithNoCopy(kv.Key)] = kv } filterStopCh := make(chan struct{}) - eventsCh := make(chan [event_block_size]KvEvent, max/event_block_size+2) + eventsCh := make(chan [eventBlockSize]KvEvent, max/eventBlockSize+2) go c.filterDelete(store, newStore, rev, eventsCh, filterStopCh) @@ -314,8 +318,8 @@ func (c *KvCacher) filter(rev int64, items []*mvccpb.KeyValue) []KvEvent { } func (c *KvCacher) filterDelete(store map[string]*mvccpb.KeyValue, newStore map[string]*mvccpb.KeyValue, - rev int64, eventsCh chan [event_block_size]KvEvent, filterStopCh chan struct{}) { - var block [event_block_size]KvEvent + rev int64, eventsCh chan [eventBlockSize]KvEvent, filterStopCh chan struct{}) { + var block [eventBlockSize]KvEvent i := 0 for k, v := range store { _, ok := newStore[k] @@ -323,9 +327,9 @@ func (c *KvCacher) filterDelete(store map[string]*mvccpb.KeyValue, newStore map[ continue } - if i >= event_block_size { + if i >= eventBlockSize { eventsCh <- block - block = [event_block_size]KvEvent{} + block = [eventBlockSize]KvEvent{} i = 0 } @@ -346,15 +350,15 @@ func (c *KvCacher) filterDelete(store map[string]*mvccpb.KeyValue, newStore map[ } func (c *KvCacher) filterCreateOrUpdate(store map[string]*mvccpb.KeyValue, newStore map[string]*mvccpb.KeyValue, - rev int64, eventsCh chan [event_block_size]KvEvent, filterStopCh chan struct{}) { - var block [event_block_size]KvEvent + rev int64, eventsCh chan [eventBlockSize]KvEvent, filterStopCh chan struct{}) { + var block [eventBlockSize]KvEvent i := 0 for k, v := range newStore { ov, ok := store[k] if !ok { - if i >= event_block_size { + if i >= eventBlockSize { eventsCh <- block - block = [event_block_size]KvEvent{} + block = [eventBlockSize]KvEvent{} i = 0 } @@ -372,9 +376,9 @@ func (c *KvCacher) filterCreateOrUpdate(store map[string]*mvccpb.KeyValue, newSt continue } - if i >= event_block_size { + if i >= eventBlockSize { eventsCh <- block - block = [event_block_size]KvEvent{} + block = [eventBlockSize]KvEvent{} i = 0 } diff --git a/server/core/backend/store/common.go b/server/core/backend/store/common.go index 16ba78b9..fc04898e 100644 --- a/server/core/backend/store/common.go +++ b/server/core/backend/store/common.go @@ -118,7 +118,8 @@ const ( const ( DEFAULT_COMPACT_TIMES = 3 DEFAULT_COMPACT_TIMEOUT = 5 * time.Minute - event_block_size = 1000 + minWaitInterval = 100 * time.Millisecond + eventBlockSize = 1000 ) const ( diff --git a/server/core/backend/store/defer_instance.go b/server/core/backend/store/defer_instance.go index 519b773c..aa3734e9 100644 --- a/server/core/backend/store/defer_instance.go +++ b/server/core/backend/store/defer_instance.go @@ -52,9 +52,9 @@ func (iedh *InstanceEventDeferHandler) OnCondition(cache Cache, evts []KvEvent) iedh.once.Do(func() { iedh.cache = cache - iedh.items = make(map[string]deferItem, event_block_size) - iedh.pendingCh = make(chan []KvEvent, event_block_size) - iedh.deferCh = make(chan KvEvent, event_block_size) + iedh.items = make(map[string]deferItem, eventBlockSize) + iedh.pendingCh = make(chan []KvEvent, eventBlockSize) + iedh.deferCh = make(chan KvEvent, eventBlockSize) iedh.resetCh = make(chan struct{}) util.Go(iedh.check) }) @@ -142,12 +142,11 @@ func (iedh *InstanceEventDeferHandler) check(ctx context.Context) { } if iedh.enabled && len(iedh.items) == 0 { - iedh.enabled = false + iedh.renew() util.Logger().Warnf(nil, "self preservation is stopped") } case <-iedh.resetCh: - iedh.enabled = false - iedh.items = make(map[string]deferItem, event_block_size) + iedh.renew() util.Logger().Warnf(nil, "self preservation is reset") } } @@ -159,6 +158,11 @@ func (iedh *InstanceEventDeferHandler) recover(evt KvEvent) { iedh.deferCh <- evt } +func (iedh *InstanceEventDeferHandler) renew() { + iedh.enabled = false + iedh.items = make(map[string]deferItem, eventBlockSize) +} + func (iedh *InstanceEventDeferHandler) Reset() bool { if iedh.enabled { iedh.resetCh <- struct{}{} diff --git a/server/core/backend/store/indexer.go b/server/core/backend/store/indexer.go index 63c367ba..3f8cd7d7 100644 --- a/server/core/backend/store/indexer.go +++ b/server/core/backend/store/indexer.go @@ -182,6 +182,7 @@ func (i *Indexer) OnCacheEvent(evt KvEvent) { func (i *Indexer) buildIndex() { i.goroutine.Do(func(ctx context.Context) { util.SafeCloseChan(i.ready) + lastCompactTime := time.Now() for { select { case <-ctx.Done(): @@ -206,13 +207,40 @@ func (i *Indexer) buildIndex() { util.LogNilOrWarnf(t, "too long to rebuild(action: %s) index[%d], key is %s", evt.Type, key, len(i.prefixIndex)) case <-time.After(10 * time.Second): - i.prefixLock.RLock() + i.prefixLock.Lock() + if time.Now().Sub(lastCompactTime) >= DEFAULT_COMPACT_TIMEOUT { + i.compact() + lastCompactTime = time.Now() + } ReportCacheMetrics(i.cacher.Name(), "index", i.prefixIndex) - i.prefixLock.RUnlock() + i.prefixLock.Unlock() } } - util.Logger().Debugf("build %s index goroutine is stopped", i.cacher.Name()) + util.Logger().Debugf("the goroutine building index %s is stopped", i.cacher.Name()) }) + <-i.ready +} + +func (i *Indexer) compact() { + l := len(i.prefixIndex) + if l < DEFAULT_CACHE_INIT_SIZE { + l = DEFAULT_CACHE_INIT_SIZE + } + n := make(map[string]map[string]struct{}, l) + for k, v := range i.prefixIndex { + c, ok := n[k] + if !ok { + c = make(map[string]struct{}, len(v)) + n[k] = c + } + for ck, cv := range v { + c[ck] = cv + } + } + i.prefixIndex = n + + util.Logger().Infof("index %s(%s): compact root capacity to size %d", + i.cacher.Name(), DEFAULT_COMPACT_TIMEOUT, l) } func (i *Indexer) getPrefixKey(arr *[]string, prefix string) (count int) { @@ -245,6 +273,7 @@ func (i *Indexer) addPrefixKey(prefix, key string) { keys, ok := i.prefixIndex[prefix] if !ok { + // build parent index key and new child nodes keys = make(map[string]struct{}) i.prefixIndex[prefix] = keys } else if _, ok := keys[key]; ok { @@ -280,6 +309,11 @@ func (i *Indexer) Run() { i.isClose = false i.prefixLock.Unlock() + if _, ok := i.cacher.(*nullCacher); ok { + util.SafeCloseChan(i.ready) + return + } + i.buildIndex() i.cacher.Run() @@ -312,7 +346,7 @@ func NewCacheIndexer(cr Cacher) *Indexer { return &Indexer{ BuildTimeout: DEFAULT_ADD_QUEUE_TIMEOUT, cacher: cr, - prefixIndex: make(map[string]map[string]struct{}, DEFAULT_MAX_EVENT_COUNT), + prefixIndex: make(map[string]map[string]struct{}, DEFAULT_CACHE_INIT_SIZE), prefixBuildQueue: make(chan KvEvent, DEFAULT_MAX_EVENT_COUNT), goroutine: util.NewGo(context.Background()), ready: make(chan struct{}), diff --git a/server/core/backend/store/lease.go b/server/core/backend/store/lease.go index 192ae8c6..d6ea7029 100644 --- a/server/core/backend/store/lease.go +++ b/server/core/backend/store/lease.go @@ -25,7 +25,7 @@ import ( "time" ) -type LeaseAsyncTask struct { +type LeaseTask struct { key string LeaseID int64 TTL int64 @@ -35,11 +35,11 @@ type LeaseAsyncTask struct { err error } -func (lat *LeaseAsyncTask) Key() string { +func (lat *LeaseTask) Key() string { return lat.key } -func (lat *LeaseAsyncTask) Do(ctx context.Context) (err error) { +func (lat *LeaseTask) Do(ctx context.Context) (err error) { lat.StartTime = time.Now() lat.TTL, err = backend.Registry().LeaseRenew(ctx, lat.LeaseID) lat.EndTime = time.Now() @@ -66,12 +66,12 @@ func (lat *LeaseAsyncTask) Do(ctx context.Context) (err error) { return } -func (lat *LeaseAsyncTask) Err() error { +func (lat *LeaseTask) Err() error { return lat.err } -func NewLeaseAsyncTask(op registry.PluginOp) *LeaseAsyncTask { - return &LeaseAsyncTask{ +func NewLeaseAsyncTask(op registry.PluginOp) *LeaseTask { + return &LeaseTask{ key: ToLeaseAsyncTaskKey(util.BytesToStringWithNoCopy(op.Key)), LeaseID: op.Lease, CreateTime: time.Now(), diff --git a/server/core/backend/store/store.go b/server/core/backend/store/store.go index fe5a020a..e549cd41 100644 --- a/server/core/backend/store/store.go +++ b/server/core/backend/store/store.go @@ -19,6 +19,7 @@ package store import ( "github.com/apache/incubator-servicecomb-service-center/pkg/async" "github.com/apache/incubator-servicecomb-service-center/pkg/util" + "github.com/apache/incubator-servicecomb-service-center/server/core" pb "github.com/apache/incubator-servicecomb-service-center/server/core/proto" "github.com/apache/incubator-servicecomb-service-center/server/infra/registry" "github.com/coreos/etcd/mvcc/mvccpb" @@ -35,22 +36,22 @@ func init() { } type KvStore struct { - indexers map[StoreType]*Indexer - asyncTaskSvc *async.AsyncTaskService - lock sync.RWMutex - ready chan struct{} - goroutine *util.GoRoutine - isClose bool + indexers map[StoreType]*Indexer + taskService *async.TaskService + lock sync.RWMutex + ready chan struct{} + goroutine *util.GoRoutine + isClose bool } func (s *KvStore) Initialize() { s.indexers = make(map[StoreType]*Indexer) - s.asyncTaskSvc = async.NewAsyncTaskService() + s.taskService = async.NewTaskService() s.ready = make(chan struct{}) s.goroutine = util.NewGo(context.Background()) for i := StoreType(0); i != typeEnd; i++ { - store.newNullStore(i) + s.newIndexBuilder(i, NullCacher) } } @@ -59,14 +60,6 @@ func (s *KvStore) dispatchEvent(t StoreType, evt KvEvent) { EventProxy(t).OnEvent(evt) } -func (s *KvStore) newStore(t StoreType, opts ...KvCacherCfgOption) { - s.newIndexBuilder(t, NewKvCacher(t.String(), s.getKvCacherCfgOptions(t)...)) -} - -func (s *KvStore) newNullStore(t StoreType) { - s.newIndexBuilder(t, NullCacher) -} - func (s *KvStore) newIndexBuilder(t StoreType, cacher Cacher) { indexer := NewCacheIndexer(cacher) s.indexers[t] = indexer @@ -75,7 +68,7 @@ func (s *KvStore) newIndexBuilder(t StoreType, cacher Cacher) { func (s *KvStore) Run() { s.goroutine.Do(s.store) - s.asyncTaskSvc.Run() + s.taskService.Run() } func (s *KvStore) getKvCacherCfgOptions(t StoreType) (opts []KvCacherCfgOption) { @@ -98,16 +91,23 @@ func (s *KvStore) SelfPreservationHandler() DeferHandler { } func (s *KvStore) store(ctx context.Context) { + defer s.wait(ctx) + + if !core.ServerInfo.Config.EnableCache { + util.Logger().Warnf(nil, "registry cache mechanism is disabled") + return + } + for t := StoreType(0); t != typeEnd; t++ { switch t { case SCHEMA: + util.Logger().Infof("service center will not cache '%s'", SCHEMA) continue default: - s.newStore(t) + s.indexers[t].Stop() // release the exist indexer + s.newIndexBuilder(t, NewKvCacher(t.String(), s.getKvCacherCfgOptions(t)...)) } } - - s.wait(ctx) } func (s *KvStore) wait(ctx context.Context) { @@ -129,7 +129,7 @@ func (s *KvStore) onLeaseEvent(evt KvEvent) { } key := util.BytesToStringWithNoCopy(evt.Object.(*mvccpb.KeyValue).Key) - s.asyncTaskSvc.DeferRemove(ToLeaseAsyncTaskKey(key)) + s.taskService.DeferRemove(ToLeaseAsyncTaskKey(key)) } func (s *KvStore) closed() bool { return s.isClose @@ -145,7 +145,7 @@ func (s *KvStore) Stop() { i.Stop() } - s.asyncTaskSvc.Stop() + s.taskService.Stop() s.goroutine.Close(true) @@ -155,7 +155,7 @@ func (s *KvStore) Stop() { } func (s *KvStore) Ready() <-chan struct{} { - <-s.asyncTaskSvc.Ready() + <-s.taskService.Ready() return s.ready } @@ -234,15 +234,15 @@ func (s *KvStore) KeepAlive(ctx context.Context, opts ...registry.PluginOpOption return ttl, err } - err := s.asyncTaskSvc.Add(ctx, t) + err := s.taskService.Add(ctx, t) if err != nil { return 0, err } - itf, err := s.asyncTaskSvc.LatestHandled(t.Key()) + itf, err := s.taskService.LatestHandled(t.Key()) if err != nil { return 0, err } - pt := itf.(*LeaseAsyncTask) + pt := itf.(*LeaseTask) return pt.TTL, pt.Err() } diff --git a/server/core/info.go b/server/core/info.go index 74edc1b7..4a20dcc1 100644 --- a/server/core/info.go +++ b/server/core/info.go @@ -72,6 +72,9 @@ func newInfo() *pb.ServerInformation { LogSys: beego.AppConfig.DefaultBool("log_sys", false), PluginsDir: beego.AppConfig.DefaultString("plugins_dir", "./plugins"), + + EnablePProf: beego.AppConfig.DefaultInt("enable_pprof", 0) != 0, + EnableCache: beego.AppConfig.DefaultInt("enable_cache", 1) != 0, }, } } diff --git a/server/core/proto/services.go b/server/core/proto/services.go index 026b318b..ea27dd86 100644 --- a/server/core/proto/services.go +++ b/server/core/proto/services.go @@ -100,7 +100,8 @@ type ServerConfig struct { CompactIndexDelta int64 `json:"compactIndexDelta"` CompactInterval string `json:"compactInterval"` - EnablePProf bool `json:"enablePProf"` + EnablePProf bool `json:"-"` + EnableCache bool `json:"-"` LoggerName string `json:"-"` LogRotateSize int64 `json:"logRotateSize"` diff --git a/server/govern/service.go b/server/govern/service.go index 2ae05145..f841f9a4 100644 --- a/server/govern/service.go +++ b/server/govern/service.go @@ -240,9 +240,11 @@ func getServiceAllVersions(ctx context.Context, serviceKey *pb.MicroServiceKey) func getSchemaInfoUtil(ctx context.Context, domainProject string, serviceId string) ([]*pb.Schema, error) { key := apt.GenerateServiceSchemaKey(domainProject, serviceId, "") - resp, err := store.Store().Schema().Search(ctx, + opts := append(serviceUtil.FromContext(ctx), registry.WithStrKey(key), registry.WithPrefix()) + + resp, err := store.Store().Schema().Search(ctx, opts...) if err != nil { util.Logger().Errorf(err, "Get schema failed") return make([]*pb.Schema, 0), err diff --git a/server/handler/cache/cache.go b/server/handler/cache/cache.go index 60030fc5..f6674f31 100644 --- a/server/handler/cache/cache.go +++ b/server/handler/cache/cache.go @@ -35,25 +35,21 @@ func (l *CacheResponse) Handle(i *chain.Invocation) { scRev := store.Revision() w.Header().Set("X-Resource-Revision", fmt.Sprint(scRev)) - noCache := r.URL.Query().Get("noCache") == "1" rev, _ := strconv.ParseInt(r.URL.Query().Get("rev"), 10, 64) - - if rev == scRev && r.Method == http.MethodGet { + if rev != 0 && rev == scRev && r.Method == http.MethodGet { w.WriteHeader(http.StatusNotModified) i.Fail(nil) return } - if rev > scRev { - w.Header().Set("X-Resource-Revision", fmt.Sprint(rev)) - } + noCache := r.URL.Query().Get("noCache") == "1" + cacheOnly := r.URL.Query().Get("cacheOnly") == "1" if rev > scRev || noCache { i.WithContext("noCache", "1") } - cacheOnly := r.URL.Query().Get("cacheOnly") == "1" - if cacheOnly { + if cacheOnly && !noCache { i.WithContext("cacheOnly", "1") } diff --git a/server/plugin/infra/registry/etcd/etcd.go b/server/plugin/infra/registry/etcd/etcd.go index 5fbabd05..bfe7ea6c 100644 --- a/server/plugin/infra/registry/etcd/etcd.go +++ b/server/plugin/infra/registry/etcd/etcd.go @@ -29,6 +29,7 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" "github.com/coreos/etcd/mvcc/mvccpb" + "github.com/coreos/pkg/capnslog" "golang.org/x/net/context" "google.golang.org/grpc" "strings" @@ -45,6 +46,9 @@ var ( ) func init() { + l := &clientLogger{} + capnslog.SetFormatter(l) + clientv3.SetLogger(l) mgr.RegisterPlugin(mgr.Plugin{mgr.REGISTRY, "etcd", NewRegistry}) } @@ -528,7 +532,11 @@ func (c *EtcdClient) Watch(ctx context.Context, opts ...registry.PluginOpOption) key := util.BytesToStringWithNoCopy(op.Key) // 不能设置超时context,内部判断了连接超时和watch超时 - ws := client.Watch(context.Background(), key, c.toGetRequest(op)...) + wCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // #9103: must be not a context.TODO/Background, because the WatchChan will not closed when finish. + ws := client.Watch(wCtx, key, c.toGetRequest(op)...) var ok bool var resp clientv3.WatchResponse diff --git a/server/plugin/infra/registry/etcd/logger.go b/server/plugin/infra/registry/etcd/logger.go new file mode 100644 index 00000000..46fe7413 --- /dev/null +++ b/server/plugin/infra/registry/etcd/logger.go @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package etcd + +import ( + "fmt" + "github.com/apache/incubator-servicecomb-service-center/pkg/util" + "github.com/coreos/pkg/capnslog" +) + +// clientLogger implement from grcplog.LoggerV2s and capnslog.Formatter +type clientLogger struct { +} + +func (l *clientLogger) Format(pkg string, level capnslog.LogLevel, depth int, entries ...interface{}) { + switch level { + case capnslog.CRITICAL: + l.Fatalf(pkg+": %s", entries...) + case capnslog.ERROR: + l.Errorf(pkg+": %s", entries...) + case capnslog.WARNING: + l.Warningf(pkg+": %s", entries...) + case capnslog.NOTICE, capnslog.INFO: + l.Infof(pkg+": %s", entries...) + case capnslog.DEBUG, capnslog.TRACE: + return + default: + return + } + l.Flush() +} + +func (l *clientLogger) Flush() { +} + +func (l *clientLogger) Info(args ...interface{}) { + util.Logger().Info(fmt.Sprint(args...)) +} + +func (l *clientLogger) Infoln(args ...interface{}) { + util.Logger().Info(fmt.Sprint(args...)) +} + +func (l *clientLogger) Infof(format string, args ...interface{}) { + util.Logger().Infof(format, args...) +} + +func (l *clientLogger) Warning(args ...interface{}) { + util.Logger().Warn(fmt.Sprint(args...), nil) +} + +func (l *clientLogger) Warningln(args ...interface{}) { + util.Logger().Warn(fmt.Sprint(args...), nil) +} + +func (l *clientLogger) Warningf(format string, args ...interface{}) { + util.Logger().Warnf(nil, format, args...) +} + +func (l *clientLogger) Error(args ...interface{}) { + util.Logger().Error(fmt.Sprint(args...), nil) +} + +func (l *clientLogger) Errorln(args ...interface{}) { + util.Logger().Error(fmt.Sprint(args...), nil) +} + +func (l *clientLogger) Errorf(format string, args ...interface{}) { + util.Logger().Errorf(nil, format, args...) +} + +// V reports whether verbosity level l is at least the requested verbose level. +func (l *clientLogger) V(_ int) bool { + return true +} + +func (l *clientLogger) Fatal(args ...interface{}) { + util.Logger().Fatal(fmt.Sprint(args...), nil) +} + +func (l *clientLogger) Fatalf(format string, args ...interface{}) { + util.Logger().Fatalf(nil, format, args...) +} + +func (l *clientLogger) Fatalln(args ...interface{}) { + util.Logger().Fatal(fmt.Sprint(args...), nil) +} + +func (l *clientLogger) Print(args ...interface{}) { + util.Logger().Error(fmt.Sprint(args...), nil) +} + +func (l *clientLogger) Printf(format string, args ...interface{}) { + util.Logger().Errorf(nil, format, args...) +} + +func (l *clientLogger) Println(args ...interface{}) { + util.Logger().Error(fmt.Sprint(args...), nil) +} diff --git a/server/rest/pprof.go b/server/rest/pprof.go new file mode 100644 index 00000000..55dfbf28 --- /dev/null +++ b/server/rest/pprof.go @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package rest + +import ( + "github.com/apache/incubator-servicecomb-service-center/server/core" + "net/http" + "net/http/pprof" +) + +func init() { + if core.ServerInfo.Config.EnablePProf { + RegisterServerHandleFunc("/debug/pprof/", http.HandlerFunc(pprof.Index)) + RegisterServerHandleFunc("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) + RegisterServerHandleFunc("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) + RegisterServerHandleFunc("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) + RegisterServerHandleFunc("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) + RegisterServerHandler("/debug/pprof/heap", pprof.Handler("heap")) + RegisterServerHandler("/debug/pprof/goroutine", pprof.Handler("goroutine")) + RegisterServerHandler("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) + RegisterServerHandler("/debug/pprof/block", pprof.Handler("block")) + } +} diff --git a/server/rest/server_mux.go b/server/rest/server_mux.go index f55c59fd..5f1c6f80 100644 --- a/server/rest/server_mux.go +++ b/server/rest/server_mux.go @@ -16,14 +16,22 @@ */ package rest -import "net/http" +import ( + "github.com/apache/incubator-servicecomb-service-center/pkg/util" + "net/http" +) var DefaultServerMux = http.NewServeMux() func RegisterServerHandleFunc(pattern string, f http.HandlerFunc) { DefaultServerMux.HandleFunc(pattern, f) + + util.Logger().Infof("register server http handle function %s(), pattern %s", util.FuncName(f), pattern) } func RegisterServerHandler(pattern string, h http.Handler) { DefaultServerMux.Handle(pattern, h) + + t := util.ReflectObject(h).Type + util.Logger().Infof("register server http handler %s/%s, pattern %s", t.PkgPath(), t.Name(), pattern) } diff --git a/server/service/event/rule_event_handler.go b/server/service/event/rule_event_handler.go index bca8390c..07cc316f 100644 --- a/server/service/event/rule_event_handler.go +++ b/server/service/event/rule_event_handler.go @@ -19,6 +19,7 @@ package event import ( "encoding/json" "fmt" + "github.com/apache/incubator-servicecomb-service-center/pkg/async" "github.com/apache/incubator-servicecomb-service-center/pkg/util" "github.com/apache/incubator-servicecomb-service-center/server/core/backend/store" pb "github.com/apache/incubator-servicecomb-service-center/server/core/proto" @@ -28,7 +29,7 @@ import ( "golang.org/x/net/context" ) -type RulesChangedAsyncTask struct { +type RulesChangedTask struct { key string err error @@ -37,21 +38,21 @@ type RulesChangedAsyncTask struct { Rev int64 } -func (apt *RulesChangedAsyncTask) Key() string { +func (apt *RulesChangedTask) Key() string { return apt.key } -func (apt *RulesChangedAsyncTask) Do(ctx context.Context) error { - defer store.AsyncTaskService().DeferRemove(apt.Key()) +func (apt *RulesChangedTask) Do(ctx context.Context) error { + defer async.Service().DeferRemove(apt.Key()) apt.err = apt.publish(ctx, apt.DomainProject, apt.ProviderId, apt.Rev) return apt.err } -func (apt *RulesChangedAsyncTask) Err() error { +func (apt *RulesChangedTask) Err() error { return apt.err } -func (apt *RulesChangedAsyncTask) publish(ctx context.Context, domainProject, providerId string, rev int64) error { +func (apt *RulesChangedTask) publish(ctx context.Context, domainProject, providerId string, rev int64) error { provider, err := serviceUtil.GetService(ctx, domainProject, providerId) if err != nil { util.Logger().Errorf(err, "get provider %s service file failed", providerId) @@ -110,7 +111,7 @@ func (h *RuleEventHandler) OnEvent(evt store.KvEvent) { return } - store.AsyncTaskService().Add(context.Background(), + async.Service().Add(context.Background(), NewRulesChangedAsyncTask(domainProject, providerId, evt.Revision)) } @@ -118,8 +119,8 @@ func NewRuleEventHandler() *RuleEventHandler { return &RuleEventHandler{} } -func NewRulesChangedAsyncTask(domainProject, providerId string, rev int64) *RulesChangedAsyncTask { - return &RulesChangedAsyncTask{ +func NewRulesChangedAsyncTask(domainProject, providerId string, rev int64) *RulesChangedTask { + return &RulesChangedTask{ key: "RulesChangedAsyncTask_" + providerId, DomainProject: domainProject, ProviderId: providerId, diff --git a/server/service/event/tag_event_handler.go b/server/service/event/tag_event_handler.go index 2b14f29e..cf2afa70 100644 --- a/server/service/event/tag_event_handler.go +++ b/server/service/event/tag_event_handler.go @@ -19,6 +19,7 @@ package event import ( "encoding/json" "fmt" + "github.com/apache/incubator-servicecomb-service-center/pkg/async" "github.com/apache/incubator-servicecomb-service-center/pkg/util" "github.com/apache/incubator-servicecomb-service-center/server/core/backend/store" pb "github.com/apache/incubator-servicecomb-service-center/server/core/proto" @@ -28,7 +29,7 @@ import ( "golang.org/x/net/context" ) -type TagsChangedAsyncTask struct { +type TagsChangedTask struct { key string err error @@ -37,21 +38,21 @@ type TagsChangedAsyncTask struct { Rev int64 } -func (apt *TagsChangedAsyncTask) Key() string { +func (apt *TagsChangedTask) Key() string { return apt.key } -func (apt *TagsChangedAsyncTask) Do(ctx context.Context) error { - defer store.AsyncTaskService().DeferRemove(apt.Key()) +func (apt *TagsChangedTask) Do(ctx context.Context) error { + defer async.Service().DeferRemove(apt.Key()) apt.err = apt.publish(ctx, apt.DomainProject, apt.consumerId, apt.Rev) return apt.err } -func (apt *TagsChangedAsyncTask) Err() error { +func (apt *TagsChangedTask) Err() error { return apt.err } -func (apt *TagsChangedAsyncTask) publish(ctx context.Context, domainProject, consumerId string, rev int64) error { +func (apt *TagsChangedTask) publish(ctx context.Context, domainProject, consumerId string, rev int64) error { consumer, err := serviceUtil.GetService(ctx, domainProject, consumerId) if err != nil { util.Logger().Errorf(err, "get comsumer for publish event %s failed", consumerId) @@ -120,7 +121,7 @@ func (h *TagEventHandler) OnEvent(evt store.KvEvent) { return } - store.AsyncTaskService().Add(context.Background(), + async.Service().Add(context.Background(), NewTagsChangedAsyncTask(domainProject, consumerId, evt.Revision)) } @@ -128,8 +129,8 @@ func NewTagEventHandler() *TagEventHandler { return &TagEventHandler{} } -func NewTagsChangedAsyncTask(domainProject, consumerId string, rev int64) *TagsChangedAsyncTask { - return &TagsChangedAsyncTask{ +func NewTagsChangedAsyncTask(domainProject, consumerId string, rev int64) *TagsChangedTask { + return &TagsChangedTask{ key: "TagsChangedAsyncTask_" + consumerId, DomainProject: domainProject, consumerId: consumerId, ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services