Hello community, here is the log from the commit of package openSUSE-release-tools for openSUSE:Factory checked in at 2019-11-10 22:44:15 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old) and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.2990 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openSUSE-release-tools" Sun Nov 10 22:44:15 2019 rev:244 rq:747076 version:20191108.6b1fd618 Changes: -------- --- /work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes 2019-11-09 23:42:38.873501388 +0100 +++ /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.2990/openSUSE-release-tools.changes 2019-11-10 22:44:19.393626690 +0100 @@ -1,0 +2,52 @@ +Fri Nov 08 23:15:03 UTC 2019 - opensuse-releaset...@opensuse.org + +- Update to version 20191108.6b1fd618: + * dist/obs/attribute-update: provide script for updating attribute definitions. + +------------------------------------------------------------------- +Fri Nov 08 22:15:04 UTC 2019 - opensuse-releaset...@opensuse.org + +- Update to version 20191108.4d09eda6: + * docs/origin-manager: provide comprehensive documentation. + +------------------------------------------------------------------- +Fri Nov 08 22:09:01 UTC 2019 - opensuse-releaset...@opensuse.org + +- Update to version 20191108.2a32702b: + * osclib/origin: policy_input_calculate(): s/unkown/unknown/. + * dist/obs: remove namespace from attribute names. + +------------------------------------------------------------------- +Fri Nov 08 21:45:33 UTC 2019 - opensuse-releaset...@opensuse.org + +- Update to version 20191108.03a954ae: + * osclib/origin: origin_update(): support blacklist patterns for initial updates. + * osc-origin: extract osrt_origin_update_packages(). + +------------------------------------------------------------------- +Fri Nov 08 21:21:31 UTC 2019 - opensuse-releaset...@opensuse.org + +- Update to version 20191108.a83e40e0: + * osclib/origin: origin_history(): include non-annotated requests. + +------------------------------------------------------------------- +Fri Nov 08 20:52:37 UTC 2019 - opensuse-releaset...@opensuse.org + +- Update to version 20191108.a0488c3f: + * osclib/origin: origin_annotation_dump(): include pending information. + +------------------------------------------------------------------- +Fri Nov 08 20:25:46 UTC 2019 - opensuse-releaset...@opensuse.org + +- Update to version 20191108.d271224d: + * osclib/core: request_action_list_maintenance_incident(): support another form. + * osclib/conf: configure maintenance staging review options for origin-manager. + * osclib/origin: add maintenance staging equivalent reviews to pending_submission_allowed_reviews. + * osclib/origin: drop repo-checker from pending_submission_allowed_reviews. + * osclib/origin: provide pending_submission_allowed_reviews_update option. + * osclib/origin: reviews_filter_allowed(): support negation rules. + * osclib/origin: provide reviews_filter_allowed(). + * osclib/origin: project_source_pending(): include psuedo incident review. + * osclib/core: reviews_remaining(): provide incident_psuedo option. + +------------------------------------------------------------------- Old: ---- openSUSE-release-tools-20191108.c7e1800d.obscpio New: ---- openSUSE-release-tools-20191108.6b1fd618.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openSUSE-release-tools.spec ++++++ --- /var/tmp/diff_new_pack.IgRhNR/_old 2019-11-10 22:44:20.469628087 +0100 +++ /var/tmp/diff_new_pack.IgRhNR/_new 2019-11-10 22:44:20.469628087 +0100 @@ -20,7 +20,7 @@ %define source_dir openSUSE-release-tools %define announcer_filename factory-package-news Name: openSUSE-release-tools -Version: 20191108.c7e1800d +Version: 20191108.6b1fd618 Release: 0 Summary: Tools to aid in staging and release work for openSUSE/SUSE License: GPL-2.0-or-later AND MIT ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.IgRhNR/_old 2019-11-10 22:44:20.497628123 +0100 +++ /var/tmp/diff_new_pack.IgRhNR/_new 2019-11-10 22:44:20.501628128 +0100 @@ -1,6 +1,6 @@ <servicedata> <service name="tar_scm"> <param name="url">https://github.com/openSUSE/openSUSE-release-tools.git</param> - <param name="changesrevision">c7e1800d3009933937ae104dbe226c23c1543a69</param> + <param name="changesrevision">45e0989b28e1ab5abb37e29d2f197cb58d638895</param> </service> </servicedata> ++++++ openSUSE-release-tools-20191108.c7e1800d.obscpio -> openSUSE-release-tools-20191108.6b1fd618.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateDelay.xml new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateDelay.xml --- old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateDelay.xml 2019-11-08 20:49:24.000000000 +0100 +++ new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateDelay.xml 2019-11-09 00:14:13.000000000 +0100 @@ -1,4 +1,4 @@ -<definition name="OSRT:OriginUpdateDelay" namespace="OSRT"> +<definition name="OriginUpdateDelay" namespace="OSRT"> <description>OriginManager update delay frequency in seconds (minimum time since source change)</description> <count>1</count> <modifiable_by role="maintainer"/> diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateFrequency.xml new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateFrequency.xml --- old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateFrequency.xml 2019-11-08 20:49:24.000000000 +0100 +++ new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateFrequency.xml 2019-11-09 00:14:13.000000000 +0100 @@ -1,4 +1,4 @@ -<definition name="OSRT:OriginUpdateFrequency" namespace="OSRT"> +<definition name="OriginUpdateFrequency" namespace="OSRT"> <description>OriginManager update frequency in seconds (minimum time since last request)</description> <count>1</count> <modifiable_by role="maintainer"/> diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateInitialBlacklist.xml new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateInitialBlacklist.xml --- old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateInitialBlacklist.xml 1970-01-01 01:00:00.000000000 +0100 +++ new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateInitialBlacklist.xml 2019-11-09 00:14:13.000000000 +0100 @@ -0,0 +1,5 @@ +<definition name="OriginUpdateInitialBlacklist" namespace="OSRT"> + <description>OriginManager update initial blacklist patterns (filters potential new package submissions)</description> + <count>1</count> + <modifiable_by role="maintainer"/> +</definition> diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateSupersede.xml new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateSupersede.xml --- old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateSupersede.xml 2019-11-08 20:49:24.000000000 +0100 +++ new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateSupersede.xml 2019-11-09 00:14:13.000000000 +0100 @@ -1,4 +1,4 @@ -<definition name="OSRT:OSRT:OriginUpdateSupersede" namespace="OSRT"> +<definition name="OriginUpdateSupersede" namespace="OSRT"> <description>OriginManager allowed to supersede existing requests (true or false)</description> <count>1</count> <modifiable_by role="maintainer"/> diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/attribute-update new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/attribute-update --- old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/attribute-update 1970-01-01 01:00:00.000000000 +0100 +++ new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/attribute-update 2019-11-09 00:14:13.000000000 +0100 @@ -0,0 +1,23 @@ +#!/bin/bash + +if [ $# -ne 2 ] ; then + echo "usage: $(basename "$0") <APIURL> <group>" + echo + echo " ./attribute-update obs factory-staging" + echo " ./attribute-update ibs sle-staging-managers" + exit 1 +fi + +for file in *.xml ; do + read -p "update $(basename "$file")? [y/n]: " response + if [ "$response" != "y" ] ; then + continue + fi + + temp="/tmp/$(basename "$file")" + sed "5i<modifiable_by group='$2'/>" "$file" > "$temp" + path="$(basename "$file" | sed 's|:|/|' | sed 's|.xml||')" + echo "PUT /attribute/$path/_meta" + osc -A "$1" api -X PUT -T "$temp" "/attribute/$path/_meta" + rm "$temp" +done diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20191108.c7e1800d/docs/origin-manager.md new/openSUSE-release-tools-20191108.6b1fd618/docs/origin-manager.md --- old/openSUSE-release-tools-20191108.c7e1800d/docs/origin-manager.md 1970-01-01 01:00:00.000000000 +0100 +++ new/openSUSE-release-tools-20191108.6b1fd618/docs/origin-manager.md 2019-11-09 00:14:13.000000000 +0100 @@ -0,0 +1,484 @@ +# Origin Manager + +The primary function of origin manager is, as the name implies, to _manage_ the _origin_ of packages. In other words to keep track of from what project a package originates, submit updates, review requests to detect origin changes, and enforce origin specific policies like adding appropriate reviews. + +The primary configuration for all parts of the tool is the `OSRT:OriginConfig` attribute to be placed on the target project. Once the config is created all relevant caches, listeners, and cron jobs will automatically include the project. The only exception is the `origin-manager` user should be added as a project _reviewer_ in order to activate the review portion of the tool. Conversely either removing the config or locking the project will disable all services. Locking the project will preserve the caches used by the web interface, but not update them which allows for archived viewing. + +Additionally, there are some other `OSRT:Origin*` attributes used to override the `OSRT:OriginConfig` outside of the target project and standard `OSRT:Config` relevant options. + +## Core concepts + +The following core concepts are important for understanding the operation of the tool and its configuration. + +### Origin + +The primary concept within origin manager is an _origin_ which is simply a project from which a package may originate. The origin config will contain a prioritized list of possible origins. In order to determine the origin for a package each origin is searched in priority order for the package and a matching source revision. The first origin containing a match is considered the origin of the package. + +An origin can change for a package without the source of the package changing in the target project in one of two ways. Either a devel project can be set on the target package in conjunction with an allowed `<devel>` origin or the package may change in a potential origin project. If the matching sources are accepted into a higher priority origin then the package will be consider as coming from the new origin. This automatic switching properly encapsulates the rather complicated interrelated product workflow. + +Alternatively, if the `OSRT:OriginConfig` is changed that may also change a package's origin. + +### Workaround + +A _workaround_ is a _pseudo origin_ used to track a package that used to match an origin, but had a source change accepted that matched no origin. The change to a workaround requires human review via the `fallback-group`. If the source later shows up in the origin project it will automatically switch back to not being considered a workaround. + +## Configuration + +As noted above all configuration for the tool is read from OBS attributes. + +### OSRT:OriginConfig + +Any target project that expects origin manager to manage it must contain this config. The config is expected to be valid _yaml_ of the following format. + +```yaml +origins: +- origin1: + policy_option1: policy_option1_value + +global_option1: global_option1_value +``` + +Note the origins are a list of single key-value pairs. The list allows the order to be maintained by _yaml_ parsers over a simple key-value set, but makes for a bit more cumbersome markup. The following demonstrates the default config. + +```yaml +# Wait instead of declining an initial source submission that cannot be found in +# any origin. +unknown_origin_wait: false + +# List of prioritized origins and their policy overrides. +origins: [] + +# User that will perform reviews (ie. user under which ./origin-manager.py will +# run). If not specified nor available from default reference 'origin-manager' +# will be used. +review-user: <config:origin-manager-review-user> + +# Group from which a review will be requested as a fallback whenever automated +# approval is not possible. +fallback-group: <config:origin-manager-fallback-group> + +# Fallback workaround project primarily used for migration to origin-manager. +fallback-workaround: {} +``` + +The following is the default policy for an origin. The policy of the new origin is applied to requests and when determining the current origin (especially relevant to pending* options). + +```yaml +# Additional by_group reviews to be added to requests from this origin. Special +# keys supported are: fallback and maintainer. The fallback key will be replaced +# by the fallback-group global configuration value and the maintainer key will +# be replaced by the maintainer from the origin package (by_package review) or +# the devel project for the origin package if configured. Maintainer reviews +# are skipped if the request creator is a maintainer. +additional_reviews: [] + +# Submit updates for packages from this origin and allow forward (based on +# revision order) submissions without fallback review. If disabled updates will +# not be submitted automatically and manually submitted updates will require +# fallback review. +automatic_updates: true + +# Submit new packages from this origin (highest priority, enabled, origin wins). +# New packages will only be submitted once. Any prior requests will block future +# submission of the package until it is accepted. +automatic_updates_initial: false + +# Submit updates even if an existing request is still pending. +automatic_updates_supersede: true + +# Delay update submission by a minimum of N seconds after a source change. +automatic_updates_delay: 0 + +# Cap updates at a minimum of N seconds since the last request. +automatic_updates_frequency: 0 + +# Always add a review for the package maintainer. +maintainer_review_always: false + +# Add a review for the package maintainer on the initial submission of package. +maintainer_review_initial: true + +# Approve source submissions that match pending submissions in the origin +# project of either the new state or with only allowed reviews +# (pending_submission_allowed_reviews). Otherwise, consider as originating from +# the project containing the pending request and wait for the conditions to be +# met. Disable to require submissions to be accepted before accepting review. +pending_submission_allow: false + +# Consider pending submissions for the purposes of determining origin, but wait +# for requests to be accepted before accepting review. +pending_submission_consider: false + +# List of reviews to allow for the purposes of accepting review when +# pending_submission_allow is enabled. +pending_submission_allowed_reviews: +# Non-maintenance projects: +- <config_source:staging>* +# Maintenance projects: +- <config_source:review-install-check> +- <config_source:review-openqa> + +# Control how early pending requests in an origin project are considered for +# submission to the target project. Submit pending requests with a set of +# allowed reviews, but still wait for the above reviews before being accepted. +# For example !<config:review-team> could be added to require the review team +# approval before mirroring a pending request. +'pending_submission_allowed_reviews_update': +- '!maintenance_incident' +``` + +The following _special_ origins are supported where `*` denotes all and `~` indicates a workaround (explained later). + +- `*`: applied to all origins above it +- `*~`: applied to all workaround origins above it and generates them if they do not exist +- `prefix*suffix`: describes a product family which is expanded into specific projects +- `origin1~`: describes a specific origin workaround +- `<devel>`: replaced with the devel project for a given package if set + +To add an origin with a completely default policy use `{}` as shown below. + +```yaml +origins: +- projectFoo: {} +``` + +#### Config references + +As one may have noticed, both the global options and policy defaults contain config references denoted by `<config:*>` or `<config_source:*>`. References are replaced with the corresponding value from the `OSRT:Config` for the relevant project. + +- `config`: read from the target project +- `config_source`: read from the source project (relative to a source change request) + +If no value is found in the relevant config the value is replaced by a blank which in the case of lists removes the entry. + +A reference may be used in conjunction with additional characters. This is useful for the `pending_submission_allowed_reviews` policy option which defaults to including `<config_source:staging>*`. This will allow any review that starts with `<config_source:staging>`. For example, when evaluated in the context of `openSUSE:Leap:15.2` for a request from `openSUSE:Factory` this will be evaluated as `openSUSE:Factory:Staging*` which will allow staging reviews. + +The reference abstraction is especially useful for cross product family origins. For example _SLE_ can use `openSUSE.org:openSUSE:Factory` as an origin and automatically have the appropriate staging reviews ignored based on the cross-OBS-instance config. + +#### Workarounds + +When a package is considered a workaround for an origin a `~` will be appended to the origin. Workarounds, just like any origin are only allowed if they are found in the list of origins from the config. Workarounds may be configured with different settings, such as `additional_reviews`, than the non-workaround origin. + +Workarounds of any origin may be allowed by adding `*~` origin at the end of the list. + +#### Family expansion + +Instead of defining specific origin projects for an entire family the config supports family expansion. Note that this has become less useful since layered projects such as maintenance projects have their history stacked on parents for origin finding purposes. As such it is typically sufficient to include just the previous product's `:Update` project instead of the whole family. + +Looking at the config from `openSUSE:Leap:15.1` helps explain this feature best. The config contained the following family expansions. + +- `SUSE:SLE-15*` +- `openSUSE:Leap:15*` + +These were expanded into the following. + +- SUSE:SLE-15-SP1:Update +- SUSE:SLE-15-SP1:GA +- SUSE:SLE-15:Update +- SUSE:SLE-15:GA +- openSUSE:Leap:15.0:Update +- openSUSE:Leap:15.0 + +Note that the _Leap_ expansion automatically did not include the target project nor its `:Update` project as they are considered above it. The `*~` origin in the config meant that each of these was also duplicated for a workaround origin. + +Family expansion can also be used with a suffix like what was done in `openSUSE:Leap:15.1:NonFree`. + +- openSUSE:Leap:15*:NonFree + +The above expanded into the following. + +- openSUSE:Leap:15.0:NonFree:Update +- openSUSE:Leap:15.0:NonFree + +#### Fallback workaround + +The `fallback-workaround` global option is used as a fallback project which will be considered a workaround for another origin. This was designed for the migration to origin manager before which a `:SLE-workarounds` subproject was used to house workaround sources. + +The option requires two keys to be set: `project` and `origin`. The `project` key is the project to search for matching sources and the `origin` is what origin of that will be used if matched. For example, `openSUSE:Leap:15.1` used the following. + +```yaml +fallback-workaround: + origin: 'SUSE:SLE-15-SP1:GA~' + project: 'openSUSE:Leap:15.1:SLE-workarounds' +``` + +The above config meant that if sources matched no origin, but matched those in `openSUSE:Leap:15.1:SLE-workarounds` the origin was considered `SUSE:SLE-15-SP1:GA~`. + +### Override attributes + +The following attributes are provided to override their corresponding policy option. These attributes are searched for in the following order of precedence. + +- origin package +- origin project +- target package +- target project + +If no attributes are found the `OSRT:OriginConfig` policy values for the matched origin are used. + +- `OSRT:OriginUpdateSkip`: `automatic_updates = false` +- `OSRT:OriginUpdateSupersede`: `automatic_updates_supersede` +- `OSRT:OriginUpdateDelay`: `automatic_updates_delay` +- `OSRT:OriginUpdateFrequency`: `automatic_updates_frequency` + +The `OSRT:OriginUpdateSkip` attribute can also be added to a target project to disable updates entirely, otherwise if the `OSRT:OriginConfig` includes at least one origin with `automatic_updates` then the project will be included in the update jobs. + +### Initial submission blacklist + +For target projects containing origins with `automatic_updates_initial` enabled the `OSRT:OriginUpdateInitialBlacklist` attribute can be utilized to blacklist new packages from being submitted. The blacklist does not stop those packages from being submitted manually or updated if they are included in the target project. The blacklist only limits the new packages considered for initial submission. + +The blacklist supports one entry per line where entries can be a python compatible regular expression and is placed on the target project to be applied to all origins. For example, the following config would be relevant for _Leap_ which has `automatic_updates_initial` enabled for _SLE_, but does not want _SLE_ specific packages. + +``` +.*SLE.* +.*SLED.* +.*SLES.* +.*suse-manager.* +.*spacewalk.* +kernel-livepatch-.* +system-role-.* +sca-.* +patterns-server-enterprise +unified-installer-release +``` + +### OSRT:Config + +A few options relevant to origin manager are standard options and thus available via the `OSRT:Config` attribute. + +```ini +# Group whose members are allowed to issue override commands. The staging-group +# is also included regardless of what value is set. +originmanager-override-group = '' + +# Minimum time in seconds to wait before reviewing source submissions. The +# default is 30 minutes with the intention of allowing parrallel submissions +# to origin projects and downstream projects that allow pending requests. +# Without the wait downstream projects can be reviewed before the upstream +# requests have been created. 30 minutes is also less than the 50 minutes that +# staging-bot waits which insure the quick strategy can still be utilized. +originmanager-request-age-min = 1800 +``` + +## Command line interface + +A CLI is provided for origin manager via the `osc-plugin-origin` package. See `osc origin --help` for complete reference. A few especially useful examples are included below. + +### config --origins-only + +The `config` command shows the expanded configuration for a given project. The `--origins-only` flag is useful for ensuring origin expansions are working as expected, but the full output is useful for ensuring policy overrides are working. + +``` +$ osc origin -p openSUSE:Leap:15.2 config --origins-only +``` + +``` +<devel> +<devel>~ +SUSE:SLE-15-SP2:GA +SUSE:SLE-15-SP2:GA~ +openSUSE:Leap:15.1:Update +openSUSE:Leap:15.1:Update~ +openSUSE:Leap:15.1 +openSUSE:Leap:15.1~ +openSUSE:Factory +openSUSE:Factory~ +``` + +### package --debug + +It can be helpful to understand the origin search path which can be done via the `--debug` flag on the `package` command. + +``` +$ osc origin -p openSUSE:Leap:15.2 package --debug adminer +``` + +``` +[D] origin_find: openSUSE:Leap:15.2/adminer with source 3bd0d78 (True, True, True) +[D] source_contain: openSUSE:Leap:15.1:Update 16c519a == 3bd0d78 +[D] source_contain: openSUSE:Leap:15.1:Update 440c7f2 == 3bd0d78 +[D] source_contain: openSUSE:Leap:15.1:Update 440c7f2 == 3bd0d78 +[D] source_contain: openSUSE:Leap:15.1:Update 4c26333 == 3bd0d78 +[D] source_contain: openSUSE:Leap:15.1:Update eb15f78 == 3bd0d78 +[D] source_contain: openSUSE:Leap:15.1 16c519a == 3bd0d78 +[D] source_contain: openSUSE:Leap:15.1 440c7f2 == 3bd0d78 +[D] source_contain: openSUSE:Leap:15.1 440c7f2 == 3bd0d78 +[D] source_contain: openSUSE:Leap:15.1 4c26333 == 3bd0d78 +[D] source_contain: openSUSE:Leap:15.1 eb15f78 == 3bd0d78 +[D] source_contain: openSUSE:Factory 3bd0d78 == 3bd0d78 (match) +openSUSE:Factory +``` + +Which shows that the latest revision from `openSUSE:Factory` is the first match and thus the origin. + +### potentials + +Understanding the _potential_ origins of a package is also useful. + +``` +$ osc origin -p openSUSE:Leap:15.2 potentials kernel-source +``` + +``` +origin version +Kernel:openSUSE-15.2 5.3.8 +openSUSE:Leap:15.1:Update unknown +openSUSE:Leap:15.1 4.12.14 +openSUSE:Factory 5.3.8 +``` + +### history + +Knowning the origin history of a package can also be enlightening. + +``` +$ osc origin -p openSUSE:Leap:15.2 history kernel-source +``` + +``` +origin state request +Kernel:openSUSE-15.2 declined 745644 +Kernel:openSUSE-15.2 review 745323 +Kernel:openSUSE-15.2 accepted 744385 +Kernel:openSUSE-15.2 declined 743969 +Kernel:openSUSE-15.2 declined 743419 +Kernel:openSUSE-15.2 declined 743414 +Kernel:openSUSE-15.2 accepted 742724 +Kernel:openSUSE-15.2 accepted 737241 +Kernel:openSUSE-15.2 accepted 733609 +Kernel:openSUSE-15.2 superseded 733094 +Kernel:openSUSE-15.2 superseded 733005 +Kernel:openSUSE-15.2 accepted 732312 +Kernel:openSUSE-15.2 accepted 731459 +openSUSE:Leap:15.1:Update~ accepted 728396 +openSUSE:Leap:15.1:Update~ accepted 725631 +openSUSE:Leap:15.1:Update accepted 724147 +openSUSE:Leap:15.1:Update accepted 721159 +``` + +## Review bot + +As with all `ReviewBot` based bots they can be debugged locally by using the `--debug` and `--dry` options. Origin manager provides detailed debug output for understanding both the origin search and policy evaluation. The following example demonstrates a request that is changing the origin from `openSUSE:Leap:15.1:Update` to `openSUSE:Factory`. + +``` +$ ./origin-manager.py --dry --debug id 746065 +``` + +``` +[I] checking 746065 +[D] origin_find: openSUSE:Leap:15.2/virtualbox with source d79bf01 (False, True, True) +[D] source_contain: openSUSE:Leap:15.1:Update 1655653 == d79bf01 +[D] source_contain: openSUSE:Leap:15.1:Update f642fe0 == d79bf01 +[D] source_contain: openSUSE:Leap:15.1:Update 22bd811 == d79bf01 +[D] source_contain: openSUSE:Leap:15.1:Update 48ab129 == d79bf01 +[D] source_contain: openSUSE:Leap:15.1:Update dd5c97b == d79bf01 +[D] source_contain: openSUSE:Leap:15.1 48ab129 == d79bf01 +[D] source_contain: openSUSE:Leap:15.1 dd5c97b == d79bf01 +[D] source_contain: openSUSE:Leap:15.1 da823e6 == d79bf01 +[D] source_contain: openSUSE:Leap:15.1 28ae8a7 == d79bf01 +[D] source_contain: openSUSE:Leap:15.1 0bc96d1 == d79bf01 +[D] source_contain: openSUSE:Factory d79bf01 == d79bf01 (match) +[D] origin_find: openSUSE:Leap:15.2/virtualbox with source 1655653 (True, True, True) +[D] source_contain: openSUSE:Leap:15.1:Update 1655653 == 1655653 (match) +[D] policy_evaluate: + +# policy +additional_reviews: [] +automatic_updates: true +automatic_updates_delay: 0 +automatic_updates_frequency: 0 +automatic_updates_initial: false +automatic_updates_supersede: true +maintainer_review_always: false +maintainer_review_initial: true +pending_submission_allow: false +pending_submission_allowed_reviews: +- openSUSE:Factory:Staging* +pending_submission_allowed_reviews_update: +- '!maintenance_incident' +pending_submission_consider: true + +# inputs +direction: unknown +higher_priority: false +new_package: false +origin_change: true +pending_submission: 'False' +same_family: false + +PolicyResult(wait=False, accept=True, reviews={'fallback': 'Changing to a lower priority origin.'}, comments=[]) +[D] skipped adding duplicate review for origin-reviewers +[D] broadening search to include any state on 746065 +[D] no previous comment to replace on 746065 +[I] 746065 accepted: origin: openSUSE:Factory +origin_old: openSUSE:Leap:15.1:Update + +[D] 746065 review not changed +``` + +From the `PolicyResult` it is clear that the review will be accepted with the addition of a review for the fallback group since `Changing to a lower priority origin.`. + +### Commands + +There are a two commands that can be used to control the review bot from an OBS request. The commands follow the standard reply syntax `@<user> <message>` and can be placed in the request description or a comment on the request. Commands issued by users not part of either the `staging-group` or `originmanager-override-group` will be ignored. In order to be accepted commands must match the following regular expression. + +```regex +^@(?P<user>[^ ,:]+)[,:]? (?P<args>.*)$ +``` + +The standard `override` command is available to force acceptance when the submission does not match any allowed origin, but this should be avoided if possible. The `unknown_origin_wait` global option can be enabled to make it easier to utilize the override feature. + +The origin manager specific command `change_devel` is also available with an expanded user pool to include the request creator. The command tells origin manager to consider the origin as a devel project from the request source project. Alternatively, a different origin project can be optionally specified. + +For example, a request originating from `projectDevel` would consider the origin for review purposes as if the devel on the package was set to `projectDevel` using the following command. + +``` +@origin-manager change_devel +``` + +The same request would consider the origin for review purposes as if the devel on the package was set to `projectFoo` using the following command. + +``` +@origin-manager change_devel projectFoo +``` + +## Automatic updates + +For projects with at least one origin having `automatic_updates` enabled source changes from enabled origins will be automatically submitted to the target project. There is both a cron job that runs once daily and an event listener that will make submissions immediately. The listener considers the same control options as the cron job so if something like `automatic_updates_delay` is configured the listener will not make updates for that project and instead they will fallback to the cron job. The cron job also handles configuration changes and backfilling newly managed projects. + +### Automatic change_devel + +If a source submission is accepted, but during update an origin cannot be matched the annotation on the most recent request will be checked to see if it was considered as devel project. If such an annotation exists a `change_devel` request will automatically be created to match the annotation. After the request is accept future updates will be submitted as appropriate. + +## Web interface + +A _web interface_ is provided using the _OBS operator_ server which wraps the _origin CLI plugin_. The web interface is backed by a cache which is updated trice-weekly and allows for quickly retrieving the full list of origins for all packages within a project. + +In conjunction to the primary web interface a [userscript](../userscript/README.md) is provided to automatically display origin information on the OBS web interface. Both the package view and request views are supplemented and provide links to the web interface. **Note that one must have an active OBS session for either to work.** + +### Layout + +The interface condenses a rather large amount of information into one screen and can be overwhelming at first glance. The following is a description of the columns from left the right. + +#### Package list + +A list of all _source_ packages within a target project are shown with their respective origin. The _revisions_ column denotes the type of the last 10 commits: green matches origin in target, red only in origin, and gray for target revisions that do not match origin. The _request_ column shows the highest open request for that package in the target project. + +#### Potentials and history + +The middle column shows the potential origins for the selected package along with the version within that origin. The two icons are external diff (via OBS) and submit to target from origin. + +The origin history shows source change requests for the selected package against the target project. If an annotation is present the origin from the annotation is extracted otherwise the source project is shown. + +#### Diff + +The last column shows a diff between the selected potential origin and the target project. When a request from the history is also selected it shows the diff between the potential origin and the request source. + +### Filtering and sorting + +One of the most useful features of the web interface is _filtering and sorting_ which allows for a number different package states to be identified. + +The _origin_ column can be sorted to see packages grouped by origin or filtered to show workarounds by entering `~`. Additionally, specific origins can be filtered or even the `None` origin. + +The _revisions_ column can be sorted to show the most _behind_ (most red revisions) packages first with the most _worked around_ (most gray revisions) packages next. This is useful for seeing if updates are behind and determining the reason (usually due to bot or human decline). + +The _request_ column can be sorted to place packages with requests at the top of the list which is useful when doing lots of reviews against open requests and thus flipping between requests and the origin manager web interface. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20191108.c7e1800d/osc-origin.py new/openSUSE-release-tools-20191108.6b1fd618/osc-origin.py --- old/openSUSE-release-tools-20191108.c7e1800d/osc-origin.py 2019-11-08 20:49:24.000000000 +0100 +++ new/openSUSE-release-tools-20191108.6b1fd618/osc-origin.py 2019-11-09 00:14:13.000000000 +0100 @@ -376,24 +376,7 @@ return if len(packages) == 0: - packages = set(package_list_kind_filtered(apiurl, opts.project)) - - # Include packages from origins with initial update enabled to allow for - # potential new package submissions. - for origin in origin_updatable_initial(apiurl, opts.project): - for package in package_list(apiurl, origin): - # Only add missing package if it does not exist in target - # project. If it exists in target then it is not a source - # package (since origin list is filtered to source) and should - # not be updated. This also properly avoids submitting a package - # that is a subpackage in target, but is a source package in an - # origin project. - if package in packages or entity_exists(apiurl, opts.project, package): - continue - - # No sense submitting a non-source package (most expensive). - if package_kind(apiurl, origin, package) == 'source': - packages.add(package) + packages = osrt_origin_update_packages(apiurl, opts.project) for package in packages: print('checking for updates to {}/{}...'.format(opts.project, package)) @@ -401,3 +384,25 @@ request_future = origin_update(apiurl, opts.project, package) if request_future: request_future.print_and_create(opts.dry) + +def osrt_origin_update_packages(apiurl, project): + packages = set(package_list_kind_filtered(apiurl, project)) + + # Include packages from origins with initial update enabled to allow for + # potential new package submissions. + for origin in origin_updatable_initial(apiurl, project): + for package in package_list(apiurl, origin): + # Only add missing package if it does not exist in target + # project. If it exists in target then it is not a source + # package (since origin list is filtered to source) and should + # not be updated. This also properly avoids submitting a package + # that is a subpackage in target, but is a source package in an + # origin project. + if package in packages or entity_exists(apiurl, project, package): + continue + + # No sense submitting a non-source package (most expensive). + if package_kind(apiurl, origin, package) == 'source': + packages.add(package) + + return packages diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20191108.c7e1800d/osclib/conf.py new/openSUSE-release-tools-20191108.6b1fd618/osclib/conf.py --- old/openSUSE-release-tools-20191108.c7e1800d/osclib/conf.py 2019-11-08 20:49:24.000000000 +0100 +++ new/openSUSE-release-tools-20191108.6b1fd618/osclib/conf.py 2019-11-09 00:14:13.000000000 +0100 @@ -120,6 +120,8 @@ 'repo_checker-arch-whitelist': 'x86_64', 'repo_checker-no-filter': 'True', 'repo_checker-package-comment-devel': 'True', + 'review-install-check': 'maintenance-installcheck', + 'review-openqa': 'qam-openqa', }, r'openSUSE:(?P<project>Backports:(?P<version>[^:]+))$': { 'staging': 'openSUSE:%(project)s:Staging', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20191108.c7e1800d/osclib/core.py new/openSUSE-release-tools-20191108.6b1fd618/osclib/core.py --- old/openSUSE-release-tools-20191108.c7e1800d/osclib/core.py 2019-11-08 20:49:24.000000000 +0100 +++ new/openSUSE-release-tools-20191108.6b1fd618/osclib/core.py 2019-11-09 00:14:13.000000000 +0100 @@ -759,12 +759,20 @@ return None -def reviews_remaining(request): +def reviews_remaining(request, incident_psuedo=False): reviews = [] for review in request.reviews: if review.state != 'accepted': reviews.append(review_short(review)) + if incident_psuedo: + # Add review in the same style as the staging review used for non + # maintenance projects to allow for the same wait on review. + for action in request.actions: + if action.type == 'maintenance_incident': + reviews.append('maintenance_incident') + break + return reviews def review_short(review): @@ -903,7 +911,15 @@ # Before being assigned to an incident. xpath_project = xpath_join(xpath_project, 'action/target/@project="{}"'.format( maintenance_project)) - xpath_project = xpath_join(xpath_project, 'action/source/@package="{}"'.format(package), op='and', inner=True) + + xpath_project_package = '' + xpath_project_package = xpath_join( + xpath_project_package, 'action/source/@package="{}"'.format(package)) + xpath_project_package = xpath_join( + xpath_project_package, 'action/source/@package="{}"'.format( + package_repository), op='or', inner=True) + + xpath_project = xpath_join(xpath_project, f'({xpath_project_package})', op='and', inner=True) xpath = xpath_join(xpath, xpath_project, op='or', nexpr_parentheses=True) xpath_project = '' @@ -933,7 +949,8 @@ for action in request.actions: if action.type == 'maintenance_incident' and action.tgt_releaseproject == project and ( - (action.tgt_package is None and action.src_package == package) or + (action.tgt_package is None and + (action.src_package == package or action.src_package == package_repository)) or (action.tgt_package == package_repository)): yield request, action break diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20191108.c7e1800d/osclib/origin.py new/openSUSE-release-tools-20191108.6b1fd618/osclib/origin.py --- old/openSUSE-release-tools-20191108.c7e1800d/osclib/origin.py 2019-11-08 20:49:24.000000000 +0100 +++ new/openSUSE-release-tools-20191108.6b1fd618/osclib/origin.py 2019-11-09 00:14:13.000000000 +0100 @@ -51,8 +51,16 @@ 'pending_submission_allow': False, 'pending_submission_consider': False, 'pending_submission_allowed_reviews': [ + # Non-maintenance projects: '<config_source:staging>*', - '<config_source:repo-checker>', + # Maintenance projects: + '<config_source:review-install-check>', + '<config_source:review-openqa>', + ], + # Submit pending requests with a set of allowed reviews, but still wait for + # the above reviews before being accepted. + 'pending_submission_allowed_reviews_update': [ + '!maintenance_incident', ], } @@ -277,7 +285,7 @@ if source_hash_consider == source_hash: return PendingRequestInfo( request_remote_identifier(apiurl, apiurl_remote, request.reqid), - reviews_remaining(request)) + reviews_remaining(request, True)) return False @@ -319,7 +327,14 @@ return None def origin_annotation_dump(origin_info_new, origin_info_old, override=False, raw=False): - data = {'origin': str(origin_info_new.project) if origin_info_new else 'None'} + data = {} + if origin_info_new is None: + data['origin'] = str(origin_info_new) + else: + data['origin'] = str(origin_info_new.project) + if origin_info_new.pending: + data['pending'] = origin_info_new.pending.identifier + if origin_info_old and origin_info_new.project != origin_info_old.project: data['origin_old'] = str(origin_info_old.project) @@ -403,9 +418,11 @@ def policy_get_preprocess(apiurl, origin, policy): project = origin.rstrip('~') config_project = Config.get(apiurl, project) - policy['pending_submission_allowed_reviews'] = list(filter(None, [ - config_resolve_variable(v, config_project, 'config_source') - for v in policy['pending_submission_allowed_reviews']])) + for suffix in ('', '_update'): + key = 'pending_submission_allowed_reviews{}'.format(suffix) + policy[key] = list(filter(None, [ + config_resolve_variable(v, config_project, 'config_source') + for v in policy[key]])) return policy @@ -462,7 +479,7 @@ else: inputs['direction'] = 'forward' if index_new < index_old else 'backward' except ValueError: - inputs['direction'] = 'unkown' + inputs['direction'] = 'unknown' return inputs @@ -519,18 +536,32 @@ return result def policy_input_evaluate_reviews_not_allowed(policy, inputs): + return reviews_filter_allowed(inputs['pending_submission'].reviews_remaining, + policy['pending_submission_allowed_reviews']) + +def reviews_filter_allowed(reviews_remaining, allowed_reviews): reviews_not_allowed = [] - for review_remaining in inputs['pending_submission'].reviews_remaining: - allowed = False - for review_allowed in policy['pending_submission_allowed_reviews']: + for review_remaining in reviews_remaining: + allowed = None + not_rule = False + for review_allowed in allowed_reviews: + result = True + if review_allowed.startswith('!'): + not_rule = True + review_allowed = review_allowed[1:] + result = False + if review_allowed.endswith('*') and review_remaining.startswith(review_allowed[:-1]): - allowed = True + allowed = result break if review_remaining == review_allowed: - allowed = True + allowed = result break - if not allowed: + # If not the allowed case then add the review to not allowed list. + # Allowed if either matches an allow rule or does not match anything + # when at least one not rule is present. + if not (allowed or (not_rule and allowed is None)): reviews_not_allowed.append(review_remaining) return reviews_not_allowed @@ -599,11 +630,10 @@ request_actions = request_action_list_source(apiurl, target_project, package, states=['all']) for request, action in sorted(request_actions, key=lambda i: i[0].reqid, reverse=True): annotation = origin_annotation_load(request, action, user) - if not annotation: - continue - history.append({ - 'origin': annotation.get('origin', 'None'), + # Not completely accurate, but more useful to have non-annotated + # entries than to exclude them. + 'origin': annotation['origin'] if annotation else action.src_project, 'request': request.reqid, 'state': request.state.name, 'source_project': action.src_project, @@ -655,6 +685,9 @@ return request_create_delete(apiurl, target_project, package, message) if not exists: + if origin_update_initial_blacklisted(apiurl, target_project, package): + return False + message = 'Submitting new package from highest potential origin.' return request_create_submit(apiurl, origin, package, target_project, message=message, ignore_if_any_request=True) @@ -691,7 +724,7 @@ if policy['pending_submission_allow']: request_id = origin_update_pending( - apiurl, origin_info.project, package, target_project, supersede, frequency) + apiurl, origin_info.project, package, target_project, policy, supersede, frequency) if request_id: return request_id @@ -699,11 +732,16 @@ return request_create_submit(apiurl, origin_info.project, package, target_project, message=message, supersede=supersede, frequency=frequency) -def origin_update_pending(apiurl, origin_project, package, target_project, supersede, frequency): +def origin_update_pending(apiurl, origin_project, package, target_project, policy, supersede, frequency): apiurl_remote, project_remote = project_remote_apiurl(apiurl, origin_project) request_actions = request_action_list_source( apiurl_remote, project_remote, package, include_release=True) for request, action in sorted(request_actions, key=lambda i: i[0].reqid, reverse=True): + reviews_not_allowed = reviews_filter_allowed( + reviews_remaining(request, True), policy['pending_submission_allowed_reviews_update']) + if len(reviews_not_allowed): + continue + identifier = request_remote_identifier(apiurl, apiurl_remote, request.reqid) message = 'Newer pending source available from package origin. See {}.'.format(identifier) src_project = project_remote_prefixed(apiurl, apiurl_remote, action.src_project) @@ -737,6 +775,28 @@ return values +def origin_update_initial_blacklisted(apiurl, target_project, package): + patterns = origin_update_initial_blacklisted_patterns(apiurl, target_project) + for pattern in patterns: + if pattern.match(package): + return True + + return False + +@memoize(session=True) +def origin_update_initial_blacklisted_patterns(apiurl, target_project): + patterns = [] + + blacklist = attribute_value_load(apiurl, target_project, 'OriginUpdateInitialBlacklist') + if blacklist: + for entry in blacklist.strip().splitlines(): + if not entry: + continue + + patterns.append(re.compile(entry, re.IGNORECASE)) + + return patterns + @memoize(session=True) def origin_updatable(apiurl): """ List of origin managed projects that can be updated. """ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20191108.c7e1800d/tests/origin_tests.py new/openSUSE-release-tools-20191108.6b1fd618/tests/origin_tests.py --- old/openSUSE-release-tools-20191108.c7e1800d/tests/origin_tests.py 2019-11-08 20:49:24.000000000 +0100 +++ new/openSUSE-release-tools-20191108.6b1fd618/tests/origin_tests.py 2019-11-09 00:14:13.000000000 +0100 @@ -363,6 +363,7 @@ upstream3_project = self.randomString('upstream3') package1 = self.randomString('package1') package2 = self.randomString('package2') + package3 = self.randomString('package3') target_package1 = self.wf.create_package(self.target_project, package1) upstream1_package1 = self.wf.create_package(upstream1_project, package1) @@ -375,6 +376,9 @@ upstream3_package2 = self.wf.create_package(upstream3_project, package2) upstream3_package2.create_commit() + upstream1_package3 = self.wf.create_package(upstream1_project, package3) + upstream1_package3.create_commit() + attribute_value_save(self.wf.apiurl, upstream1_project, 'ApprovedRequestSource', '', 'OBS') attribute_value_save(self.wf.apiurl, upstream2_project, 'ApprovedRequestSource', '', 'OBS') attribute_value_save(self.wf.apiurl, upstream3_project, 'ApprovedRequestSource', '', 'OBS') @@ -425,6 +429,14 @@ self.assertEqual(request_future, False) self.osc_user_pop() + # Ensure blacklist prevents initial package submission. + self.wf.create_attribute_type('OSRT', 'OriginUpdateInitialBlacklist', 1) + attribute_value_save(self.wf.apiurl, self.target_project, 'OriginUpdateInitialBlacklist', package3) + self.assertNoUpdate(package3) + + attribute_value_delete(self.wf.apiurl, self.target_project, 'OriginUpdateInitialBlacklist') + self.assertUpdate(package3) + def test_automatic_update_modes(self): self.remote_config_set_age_minimum() ++++++ openSUSE-release-tools.obsinfo ++++++ --- /var/tmp/diff_new_pack.IgRhNR/_old 2019-11-10 22:44:21.205629042 +0100 +++ /var/tmp/diff_new_pack.IgRhNR/_new 2019-11-10 22:44:21.205629042 +0100 @@ -1,5 +1,5 @@ name: openSUSE-release-tools -version: 20191108.c7e1800d -mtime: 1573242564 -commit: c7e1800d3009933937ae104dbe226c23c1543a69 +version: 20191108.6b1fd618 +mtime: 1573254853 +commit: 6b1fd6180011733457cc47bab1ff66e15e25b3fa