Hello community, here is the log from the commit of package openSUSE-release-tools for openSUSE:Factory checked in at 2018-03-11 15:24:45 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old) and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openSUSE-release-tools" Sun Mar 11 15:24:45 2018 rev:68 rq:584760 version:20180309.7484722 Changes: -------- --- /work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes 2018-03-09 10:45:51.020289663 +0100 +++ /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new/openSUSE-release-tools.changes 2018-03-11 15:24:46.444242475 +0100 @@ -1,0 +2,42 @@ +Fri Mar 09 10:07:59 UTC 2018 - opensuse-releaset...@opensuse.org + +- Update to version 20180309.7484722: + * pkglistgen: do_dump_solv: flush after write before reading back. + * pkglistgen: do_dump_solv: verify primary repo gziped content sha256. + * pkglistgen: do_dump_solv: use BytesIo instead of StringIO. + +------------------------------------------------------------------- +Fri Mar 09 09:08:10 UTC 2018 - opensuse-releaset...@opensuse.org + +- Update to version 20180309.f20816d: + * pkglistgen: Do not run local service check when committing to release package + +------------------------------------------------------------------- +Fri Mar 09 07:42:54 UTC 2018 - opensuse-releaset...@opensuse.org + +- Update to version 20180309.f607883: + * leaper: fix can_accept_review for combined user/group usage + * create_test_Factory: No longer block kdelibs4-branding-upstream + +------------------------------------------------------------------- +Fri Mar 09 06:52:12 UTC 2018 - opensuse-releaset...@opensuse.org + +- Update to version 20180309.d9773c3: + * metrics: provide release metrics ingestion and dashboard. + +------------------------------------------------------------------- +Thu Mar 08 23:24:55 UTC 2018 - opensuse-releaset...@opensuse.org + +- Update to version 20180308.568be98: + * osclib/stagingapi: rename main_repo to cmain_repo. + * osclib/stagingapi: rename delreq_review to cdelreq_review. + * osclib/stagingapi: rename cstaging_nocleanup to cnocleanup_packages. + * osclib/staging-api: lazy-load all config values to allow for placement in remote config. + +------------------------------------------------------------------- +Thu Mar 08 15:26:23 UTC 2018 - opensuse-releaset...@opensuse.org + +- Update to version 20180308.17ebaaa: + * ReviewBot: allow both group and user review + +------------------------------------------------------------------- Old: ---- openSUSE-release-tools-20180308.defd995.obscpio New: ---- openSUSE-release-tools-20180309.7484722.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openSUSE-release-tools.spec ++++++ --- /var/tmp/diff_new_pack.ooRpp6/_old 2018-03-11 15:24:50.420099844 +0100 +++ /var/tmp/diff_new_pack.ooRpp6/_new 2018-03-11 15:24:50.424099701 +0100 @@ -20,7 +20,7 @@ %define source_dir osc-plugin-factory %define announcer_filename factory-package-news Name: openSUSE-release-tools -Version: 20180308.defd995 +Version: 20180309.7484722 Release: 0 Summary: Tools to aid in staging and release work for openSUSE/SUSE License: GPL-2.0-or-later AND MIT @@ -373,6 +373,7 @@ %exclude %{_datadir}/%{source_dir}/manager_42.py %exclude %{_datadir}/%{source_dir}/metrics %exclude %{_datadir}/%{source_dir}/metrics.py +%exclude %{_datadir}/%{source_dir}/metrics_release.py %exclude %{_datadir}/%{source_dir}/pkglistgen.py %exclude %{_datadir}/%{source_dir}/repo_checker.pl %exclude %{_datadir}/%{source_dir}/repo_checker.py @@ -446,12 +447,15 @@ %{_bindir}/osrt-metrics %{_datadir}/%{source_dir}/metrics %{_datadir}/%{source_dir}/metrics.py +%{_datadir}/%{source_dir}/metrics_release.py # To avoid adding grafana as BuildRequires since it does not live in same repo. %dir %attr(0750, grafana, grafana) %{_localstatedir}/lib/grafana %dir %{_localstatedir}/lib/grafana/dashboards %{_localstatedir}/lib/grafana/dashboards/%{name} %{_unitdir}/osrt-metrics@.service %{_unitdir}/osrt-metrics@.timer +%{_unitdir}/osrt-metrics-release@.service +%{_unitdir}/osrt-metrics-release@.timer %files repo-checker %defattr(-,root,root,-) ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.ooRpp6/_old 2018-03-11 15:24:50.476097835 +0100 +++ /var/tmp/diff_new_pack.ooRpp6/_new 2018-03-11 15:24:50.476097835 +0100 @@ -1,6 +1,6 @@ <servicedata> <service name="tar_scm"> <param name="url">https://github.com/openSUSE/osc-plugin-factory.git</param> - <param name="changesrevision">defd9954b5be1f8dc4f8c6a66869d9d1a98edb4b</param> + <param name="changesrevision">7484722b2abe685a03cf783661bd23282a7db52f</param> </service> </servicedata> ++++++ openSUSE-release-tools-20180308.defd995.obscpio -> openSUSE-release-tools-20180309.7484722.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180308.defd995/ReviewBot.py new/openSUSE-release-tools-20180309.7484722/ReviewBot.py --- old/openSUSE-release-tools-20180308.defd995/ReviewBot.py 2018-03-08 11:19:00.000000000 +0100 +++ new/openSUSE-release-tools-20180309.7484722/ReviewBot.py 2018-03-09 11:00:05.000000000 +0100 @@ -403,34 +403,34 @@ return (None, None) + def _has_open_review_by(self, root, by_what, reviewer): + states = set([review.get('state') for review in root.findall('review') if review.get(by_what) == reviewer]) + if not states: + return None + elif 'new' in states: + return True + return False + def can_accept_review(self, request_id): """return True if there is a new review for the specified reviewer""" states = set() url = osc.core.makeurl(self.apiurl, ('request', str(request_id))) try: root = ET.parse(osc.core.http_GET(url)).getroot() - if self.review_user: - by_what = 'by_user' - reviewer = self.review_user - elif self.review_group: - by_what = 'by_group' - reviewer = self.review_group - else: - return False - states = set([review.get('state') for review in root.findall('review') if review.get(by_what) == reviewer]) + if self.review_user and self._has_open_review_by(root, 'by_user', self.review_user): + return True + if self.review_group and self._has_open_review_by(root, 'by_group', self.review_group): + return True except urllib2.HTTPError as e: print('ERROR in URL %s [%s]' % (url, e)) - if not states: - return None - elif 'new' in states: - return True return False def set_request_ids_search_review(self): + review = None if self.review_user: review = "@by_user='%s' and @state='new'" % self.review_user - else: - review = "@by_group='%s' and @state='new'" % self.review_group + if self.review_group: + review = osc.core.xpath_join(review, "@by_group='%s' and @state='new'" % self.review_group) url = osc.core.makeurl(self.apiurl, ('search', 'request'), { 'match': "state/@name='review' and review[%s]" % review, 'withfullhistory': 1 } ) root = ET.parse(osc.core.http_GET(url)).getroot() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180308.defd995/create_test_Factory_dvd-2.testcase new/openSUSE-release-tools-20180309.7484722/create_test_Factory_dvd-2.testcase --- old/openSUSE-release-tools-20180308.defd995/create_test_Factory_dvd-2.testcase 2018-03-08 11:19:00.000000000 +0100 +++ new/openSUSE-release-tools-20180309.7484722/create_test_Factory_dvd-2.testcase 2018-03-09 11:00:05.000000000 +0100 @@ -55,7 +55,6 @@ job lock name kdebase4-workspace-branding-upstream job lock name kdm-branding-upstream job lock name kdebase4-runtime-branding-upstream -job lock name kdelibs4-branding-upstream job lock name sddm-branding-upstream job lock name plasma5-desktop-branding-upstream job lock name plasma5-workspace-branding-upstream diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180308.defd995/dist/package/openSUSE-release-tools.spec new/openSUSE-release-tools-20180309.7484722/dist/package/openSUSE-release-tools.spec --- old/openSUSE-release-tools-20180308.defd995/dist/package/openSUSE-release-tools.spec 2018-03-08 11:19:00.000000000 +0100 +++ new/openSUSE-release-tools-20180309.7484722/dist/package/openSUSE-release-tools.spec 2018-03-09 11:00:05.000000000 +0100 @@ -373,6 +373,7 @@ %exclude %{_datadir}/%{source_dir}/manager_42.py %exclude %{_datadir}/%{source_dir}/metrics %exclude %{_datadir}/%{source_dir}/metrics.py +%exclude %{_datadir}/%{source_dir}/metrics_release.py %exclude %{_datadir}/%{source_dir}/pkglistgen.py %exclude %{_datadir}/%{source_dir}/repo_checker.pl %exclude %{_datadir}/%{source_dir}/repo_checker.py @@ -446,12 +447,15 @@ %{_bindir}/osrt-metrics %{_datadir}/%{source_dir}/metrics %{_datadir}/%{source_dir}/metrics.py +%{_datadir}/%{source_dir}/metrics_release.py # To avoid adding grafana as BuildRequires since it does not live in same repo. %dir %attr(0750, grafana, grafana) %{_localstatedir}/lib/grafana %dir %{_localstatedir}/lib/grafana/dashboards %{_localstatedir}/lib/grafana/dashboards/%{name} %{_unitdir}/osrt-metrics@.service %{_unitdir}/osrt-metrics@.timer +%{_unitdir}/osrt-metrics-release@.service +%{_unitdir}/osrt-metrics-release@.timer %files repo-checker %defattr(-,root,root,-) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180308.defd995/metrics/grafana/release.json new/openSUSE-release-tools-20180309.7484722/metrics/grafana/release.json --- old/openSUSE-release-tools-20180308.defd995/metrics/grafana/release.json 1970-01-01 01:00:00.000000000 +0100 +++ new/openSUSE-release-tools-20180309.7484722/metrics/grafana/release.json 2018-03-09 11:00:05.000000000 +0100 @@ -0,0 +1,654 @@ +{ + "__inputs": [ + { + "name": "DS_TUMBLEWEED_SNAPSHOTS - AWS", + "label": "Tumbleweed Snapshots - AWS", + "description": "", + "type": "datasource", + "pluginId": "cloudwatch", + "pluginName": "CloudWatch" + } + ], + "__requires": [ + { + "type": "datasource", + "id": "cloudwatch", + "name": "CloudWatch", + "version": "5.0.0" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Metrics pertaining to release feedback.", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1520567435112, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$project", + "description": "Estimation of release stability based on feedback.", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$col", + "groupBy": [], + "measurement": "release_score", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "score" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": false, + "line": true, + "op": "lt", + "value": 70 + }, + { + "colorMode": "warning", + "fill": false, + "line": true, + "op": "lt", + "value": 90 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Stability Score", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$project", + "description": "Mailing list references to releases.", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$col", + "groupBy": [], + "measurement": "release_mail", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "reference_count" + ], + "type": "field" + }, + { + "params": [ + "reference count" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "thread_count" + ], + "type": "field" + }, + { + "params": [ + "thread count" + ], + "type": "alias" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Mail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$project", + "description": "Number of bugs filed after a release.", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$col", + "groupBy": [], + "measurement": "release_bug", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bug_count" + ], + "type": "field" + }, + { + "params": [ + "bug count" + ], + "type": "alias" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Bug", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$project", + "description": "Total count of binaries and the number changed in a release.", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$col", + "groupBy": [], + "measurement": "release_snapshot", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "binary_count" + ], + "type": "field" + }, + { + "params": [ + "binary count" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "binary_unique_count" + ], + "type": "field" + }, + { + "params": [ + "binary unique count" + ], + "type": "alias" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Binary", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TUMBLEWEED_SNAPSHOTS - AWS}", + "description": "The combined storage used by all Tumbleweed Snapshots stored as part of download.tumbleweed.boombatower.com.", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "count", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "size", + "dimensions": { + "BucketName": "download.tumbleweed.boombatower.com", + "StorageType": "StandardStorage" + }, + "metricName": "BucketSizeBytes", + "namespace": "AWS/S3", + "period": "", + "refId": "A", + "region": "default", + "statistics": [ + "Average" + ] + }, + { + "alias": "count", + "dimensions": { + "BucketName": "download.tumbleweed.boombatower.com", + "StorageType": "AllStorageTypes" + }, + "metricName": "NumberOfObjects", + "namespace": "AWS/S3", + "period": "", + "refId": "B", + "region": "default", + "statistics": [ + "Average" + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "S3 Storage Consumption", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "openSUSE:Factory", + "value": "openSUSE:Factory" + }, + "hide": 0, + "label": null, + "name": "project", + "options": [], + "query": "influxdb", + "refresh": 1, + "regex": "/openSUSE:.*/", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "OSRT: Release", + "uid": "3Hg0iSgiz", + "version": 14 +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180308.defd995/metrics.py new/openSUSE-release-tools-20180309.7484722/metrics.py --- old/openSUSE-release-tools-20180308.defd995/metrics.py 2018-03-08 11:19:00.000000000 +0100 +++ new/openSUSE-release-tools-20180309.7484722/metrics.py 2018-03-09 11:00:05.000000000 +0100 @@ -11,6 +11,7 @@ import sys import yaml +import metrics_release import osc.conf import osc.core import osclib.conf @@ -350,6 +351,10 @@ osc.conf.get_config(override_apiurl=args.apiurl) osc.conf.config['debug'] = args.debug + metrics_release.ingest(client) + if args.release_only: + return + # Use separate cache since it is persistent. Cache.CACHE_DIR = os.path.expanduser('~/.cache/osc-plugin-factory-metrics') if args.wipe_cache: @@ -384,6 +389,7 @@ parser.add_argument('--user', default='root', help='InfluxDB user') parser.add_argument('--password', default='root', help='InfluxDB password') parser.add_argument('--wipe-cache', action='store_true', help='wipe GET request cache before executing') + parser.add_argument('--release-only', action='store_true', help='ingest release metrics only') args = parser.parse_args() sys.exit(main(args)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180308.defd995/metrics_release.py new/openSUSE-release-tools-20180309.7484722/metrics_release.py --- old/openSUSE-release-tools-20180308.defd995/metrics_release.py 1970-01-01 01:00:00.000000000 +0100 +++ new/openSUSE-release-tools-20180309.7484722/metrics_release.py 2018-03-09 11:00:05.000000000 +0100 @@ -0,0 +1,55 @@ +from dateutil.parser import parse as date_parse +from metrics import timestamp +import requests +from urlparse import urljoin +import yaml + +BASEURL = 'http://review.tumbleweed.boombatower.com/data/' + +def data_load(name): + response = requests.get(urljoin(BASEURL, '{}.yaml'.format(name))) + return yaml.safe_load(response.text) + +def data_write(client, measurement, points): + client.drop_measurement(measurement) + client.write_points(points, 's') + +def ingest_data(client, name): + data = data_load(name) + + measurement = 'release_{}'.format(name) + map_func = globals()['map_{}'.format(name)] + points = [] + for release, details in data.items(): + points.append({ + 'measurement': measurement, + 'fields': map_func(details), + 'time': timestamp(date_parse(release)), + }) + + data_write(client, measurement, points) + print('wrote {} for {}'.format(len(points), name)) + +def map_bug(bugs): + return { + 'bug_count': len(bugs), + } + +def map_mail(details): + return { + 'reference_count': details['reference_count'], + 'thread_count': details['thread_count'], + } + +def map_score(details): + return details + +def map_snapshot(details): + return { + 'binary_count': details['binary_count'], + 'binary_unique_count': details['binary_unique_count'], + } + +def ingest(client): + for name in ['bug', 'mail', 'score', 'snapshot']: + ingest_data(client, name) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180308.defd995/osclib/accept_command.py new/openSUSE-release-tools-20180309.7484722/osclib/accept_command.py --- old/openSUSE-release-tools-20180308.defd995/osclib/accept_command.py 2018-03-08 11:19:00.000000000 +0100 +++ new/openSUSE-release-tools-20180309.7484722/osclib/accept_command.py 2018-03-09 11:00:05.000000000 +0100 @@ -48,7 +48,7 @@ return True def find_virtually_accepted_requests(self, project): - query = "match=state/@name='review'+and+(action/target/@project='{}'+and+action/@type='delete')+and+(review/@state='new'+and+review/@by_group='{}')".format(project, self.api.delreq_review) + query = "match=state/@name='review'+and+(action/target/@project='{}'+and+action/@type='delete')+and+(review/@state='new'+and+review/@by_group='{}')".format(project, self.api.cdelreq_review) url = self.api.makeurl(['search', 'request'], query) f = http_GET(url) @@ -86,7 +86,7 @@ http_PUT(url + '?comment=accept+command+update', data=content) def virtually_accept_delete(self, request_id, package): - self.api.add_review(request_id, by_group=self.api.delreq_review, msg='Request accepted. Cleanup in progress - DO NOT REVOKE!') + self.api.add_review(request_id, by_group=self.api.cdelreq_review, msg='Request accepted. Cleanup in progress - DO NOT REVOKE!') filelist = self.api.get_filelist_for_package(pkgname=package, project=self.api.project, expand='1', extension='spec') pkgs = self.api.extract_specfile_short(filelist) @@ -97,7 +97,7 @@ meta = ''.join(meta) # Update package meta to disable build self.api.create_package_container(self.api.project, pkg, meta=meta, disable_build=True) - wipebinaries(self.api.apiurl, self.api.project, package=pkg, repo=self.api.main_repo) + wipebinaries(self.api.apiurl, self.api.project, package=pkg, repo=self.api.cmain_repo) # Remove package from Rings if self.api.ring_packages.get(pkg): @@ -129,7 +129,7 @@ oldspecs = self.api.get_filelist_for_package(pkgname=req['package'], project=self.api.project, extension='spec') - if 'type' in req and req['type'] == 'delete' and self.api.delreq_review: + if 'type' in req and req['type'] == 'delete' and self.api.cdelreq_review: msg += ' and started handling of virtual accept process' print(msg) # Virtually accept the delete request @@ -154,7 +154,7 @@ return False pkglist = self.api.list_packages(project) - clean_list = set(pkglist) - set(self.api.cstaging_nocleanup) + clean_list = set(pkglist) - set(self.api.cnocleanup_packages) for package in clean_list: print "[cleanup] deleted %s/%s" % (project, package) @@ -189,13 +189,13 @@ def accept_other_new(self): changed = False - if self.api.delreq_review: + if self.api.cdelreq_review: rqlist = self.find_virtually_accepted_requests(self.api.project) for req in rqlist: if self.virtual_accept_request_has_no_binary(self.api.project, req['packages'][0]): # Accepting delreq-review review self.api.do_change_review_state(req['id'], 'accepted', - by_group=self.api.delreq_review, + by_group=self.api.cdelreq_review, message='Virtually accepted delete {}'.format(req['packages'][0])) rqlist = self.find_new_requests(self.api.project) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180308.defd995/osclib/stagingapi.py new/openSUSE-release-tools-20180309.7484722/osclib/stagingapi.py --- old/openSUSE-release-tools-20180308.defd995/osclib/stagingapi.py 2018-03-08 11:19:00.000000000 +0100 +++ new/openSUSE-release-tools-20180309.7484722/osclib/stagingapi.py 2018-03-09 11:00:05.000000000 +0100 @@ -67,19 +67,7 @@ self.project = project # Store some prefix / data used in the code. - self.cstaging = conf.config[project]['staging'] - self.cstaging_group = conf.config[project]['staging-group'] - self.cstaging_archs = conf.config[project]['staging-archs'].split() - self.cstaging_dvd_archs = conf.config[project]['staging-dvd-archs'].split() - self._cstaging_nocleanup = None - self.crings = conf.config[project]['rings'] - self.cnonfree = conf.config[project]['nonfree'] - self.crebuild = conf.config[project]['rebuild'] - self.cproduct = conf.config[project]['product'] - self.copenqa = conf.config[project]['openqa'] self.user = conf.get_apiurl_usr(apiurl) - self.delreq_review = conf.config[project]['delreq-review'] - self.main_repo = conf.config[project]['main-repo'] self._ring_packages = None self._ring_packages_for_links = None self._packages_staged = None @@ -99,13 +87,23 @@ Cache.init() - @property - def cstaging_nocleanup(self): - """Lazy-load value to allow for placement in remote config.""" - if self._cstaging_nocleanup is None: - self._cstaging_nocleanup = conf.config[self.project]['nocleanup-packages'].split() + def __getattr__(self, attr): + """Lazy-load all config values to allow for placement in remote config.""" + if attr.startswith('c'): + # Drop 'c' prefix and change to config key format. + key = attr[1:].replace('_', '-') + + # This will intentionally cause error if key does not exists. + value = conf.config[self.project][key] + if key.endswith('archs') or key == 'nocleanup-packages': + value = value.split() + + # This code will only be called for the first access. + setattr(self, attr, value) + return value - return self._cstaging_nocleanup + # Raise AttributeError like normal. + return self.__getattribute__(attr) @property def ring_packages(self): @@ -1285,7 +1283,7 @@ url = self.makeurl(['source', project, '_meta']) meta = ET.parse(http_GET(url)) - repository = meta.find('repository[@name="{}"]'.format(self.main_repo)) + repository = meta.find('repository[@name="{}"]'.format(self.cmain_repo)) changed = False for arch in self.cstaging_archs: if not repository.xpath('./arch[text()="{}"]'.format(arch)): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180308.defd995/pkglistgen.py new/openSUSE-release-tools-20180309.7484722/pkglistgen.py --- old/openSUSE-release-tools-20180308.defd995/pkglistgen.py 2018-03-08 11:19:00.000000000 +0100 +++ new/openSUSE-release-tools-20180309.7484722/pkglistgen.py 2018-03-09 11:00:05.000000000 +0100 @@ -45,6 +45,8 @@ from osclib.util import project_list_family_prior from xdg.BaseDirectory import save_cache_path import glob +import hashlib +import io import solv from pprint import pprint, pformat import os @@ -54,7 +56,6 @@ import yaml import requests import urlparse -from StringIO import StringIO import gzip import tempfile import random @@ -894,17 +895,24 @@ repomd = requests.get(url) ns = {'r': 'http://linux.duke.edu/metadata/repo'} root = ET.fromstring(repomd.content) - location = root.find('.//r:data[@type="primary"]/r:location', ns).get('href') + primary_element = root.find('.//r:data[@type="primary"]', ns) + location = primary_element.find('r:location', ns).get('href') f.write(repomd.content) + f.flush() os.lseek(f.fileno(), 0, os.SEEK_SET) repo.add_repomdxml(f, 0) url = urlparse.urljoin(baseurl, path_prefix + location) with requests.get(url, stream=True) as primary: - content = gzip.GzipFile(fileobj=StringIO(primary.content)) + sha256 = hashlib.sha256(primary.content).hexdigest() + sha256_expected = primary_element.find('r:checksum[@type="sha256"]', ns).text + if sha256 != sha256_expected: + raise Exception('checksums do not match {} != {}'.format(sha256, sha256_expected)) + + content = gzip.GzipFile(fileobj=io.BytesIO(primary.content)) os.lseek(f.fileno(), 0, os.SEEK_SET) f.write(content.read()) + f.flush() os.lseek(f.fileno(), 0, os.SEEK_SET) - # TODO: verify checksum repo.add_rpmmd(f, None, 0) repo.create_stubs() repo.write(ofh) @@ -1350,7 +1358,7 @@ # No proper API function to perform the same operation. print(subprocess.check_output( ' '.join(['cd', path, '&&', 'osc', 'addremove']), shell=True)) - package.commit(msg='Automatic update') + package.commit(msg='Automatic update', skip_local_service_run=True) if __name__ == "__main__": diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180308.defd995/systemd/osrt-metrics-release@.service new/openSUSE-release-tools-20180309.7484722/systemd/osrt-metrics-release@.service --- old/openSUSE-release-tools-20180308.defd995/systemd/osrt-metrics-release@.service 1970-01-01 01:00:00.000000000 +0100 +++ new/openSUSE-release-tools-20180309.7484722/systemd/osrt-metrics-release@.service 2018-03-09 11:00:05.000000000 +0100 @@ -0,0 +1,10 @@ +[Unit] +Description=openSUSE Release Tools: metrics release for %i + +[Service] +User=osrt-metrics +SyslogIdentifier=osrt-metrics +ExecStart=/usr/bin/osrt-metrics --debug --release-only -p "%i" + +[Install] +WantedBy=multi-user.target diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180308.defd995/systemd/osrt-metrics-release@.timer new/openSUSE-release-tools-20180309.7484722/systemd/osrt-metrics-release@.timer --- old/openSUSE-release-tools-20180308.defd995/systemd/osrt-metrics-release@.timer 1970-01-01 01:00:00.000000000 +0100 +++ new/openSUSE-release-tools-20180309.7484722/systemd/osrt-metrics-release@.timer 2018-03-09 11:00:05.000000000 +0100 @@ -0,0 +1,10 @@ +[Unit] +Description=openSUSE Release Tools: metrics release for %i + +[Timer] +OnBootSec=120 +OnCalendar=daily +Unit=osrt-metrics-release@%i.service + +[Install] +WantedBy=timers.target ++++++ openSUSE-release-tools.obsinfo ++++++ --- /var/tmp/diff_new_pack.ooRpp6/_old 2018-03-11 15:24:51.316067702 +0100 +++ /var/tmp/diff_new_pack.ooRpp6/_new 2018-03-11 15:24:51.316067702 +0100 @@ -1,5 +1,5 @@ name: openSUSE-release-tools -version: 20180308.defd995 -mtime: 1520504340 -commit: defd9954b5be1f8dc4f8c6a66869d9d1a98edb4b +version: 20180309.7484722 +mtime: 1520589605 +commit: 7484722b2abe685a03cf783661bd23282a7db52f