commit: 2dc7f260e99203e183d352601c6a479ae482697b Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Sat Feb 20 18:39:19 2021 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Sat Feb 20 18:39:19 2021 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2dc7f260
Add RunEmerge and CheckEmergeLogs Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org> buildbot_gentoo_ci/config/buildfactorys.py | 21 +--- buildbot_gentoo_ci/db/model.py | 11 ++ buildbot_gentoo_ci/db/projects.py | 23 +++++ buildbot_gentoo_ci/steps/builders.py | 155 ++++++++++++++++++++++++++++- buildbot_gentoo_ci/steps/update_db.py | 2 +- 5 files changed, 192 insertions(+), 20 deletions(-) diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index 2916847..3c77762 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -115,31 +115,20 @@ def run_build_request(): # run --regen if needed on repo # update packages before any tests # run pretend on packages update on worker - shell_commad_list = [ - 'emerge', - '-uDNv', - '--changed-deps', - '--changed-use', - '--pretend', - '@world' - ] - f.addStep(buildbot_steps.SetPropertyFromCommandNewStyle( - command=shell_commad_list, - strip=True, - extract_fn=builders.PersOutputOfEmerge, - workdir='/' - )) + f.addStep(builders.RunEmerge(step='pre-update')) # look at the log to see if we need to do stuff # run update package on worker + f.addStep(builders.RunEmerge(step='update')) # check log # run pretend @preserved-rebuild if needed - # look at the log to see if we need to do stuff - # run @preserved-rebuild + f.addStep(builders.RunEmerge(step='preserved-libs')) # check log # run depclean if set # depclean pretend + f.addStep(builders.RunEmerge(step='pre-depclean')) # look at the log to see if we need to do stuff # depclean + f.addStep(builders.RunEmerge(step='depclean')) # check log # setup make.conf if build id has changes make.conf as dict from SetMakeConf # setup package.* env if build id has changes diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py index 4c5e04c..0a2c145 100644 --- a/buildbot_gentoo_ci/db/model.py +++ b/buildbot_gentoo_ci/db/model.py @@ -161,6 +161,17 @@ class Model(base.DBConnectorComponent): sa.Column('value', sa.String(255), nullable=False), ) + projects_emerge_options = sautils.Table( + "projects_emerge_options", metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('project_uuid', sa.String(36), + sa.ForeignKey('projects.uuid', ondelete='CASCADE'), + nullable=False), + sa.Column('oneshot', sa.Boolean, default=True), + sa.Column('depclean', sa.Boolean, default=True), + sa.Column('preserved_libs', sa.Boolean, default=True), + ) + keywords = sautils.Table( "keywords", metadata, # unique uuid per keyword diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py index 1f19a00..eb8f727 100644 --- a/buildbot_gentoo_ci/db/projects.py +++ b/buildbot_gentoo_ci/db/projects.py @@ -130,6 +130,20 @@ class ProjectsConnectorComponent(base.DBConnectorComponent): res = yield self.db.pool.do(thd) return res + @defer.inlineCallbacks + def getProjectEmergeOptionsByUuid(self, uuid): + def thd(conn): + tbl = self.db.model.projects_emerge_options + q = tbl.select() + q = q.where(tbl.c.project_uuid == uuid) + res = conn.execute(q) + row = res.fetchone() + if not row: + return None + return self._row2dict_projects_emerge_options(conn, row) + res = yield self.db.pool.do(thd) + return res + def _row2dict(self, conn, row): return dict( uuid=row.uuid, @@ -170,3 +184,12 @@ class ProjectsConnectorComponent(base.DBConnectorComponent): value=row.value, build_id=0 ) + + def _row2dict_projects_emerge_options(self, conn, row): + return dict( + id=row.id, + project_uuid=row.project_uuid, + oneshot=row.oneshot, + depclean=row.depclean, + preserved_libs=row.preserved_libs + ) diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index 862f0d4..c0567c1 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -15,13 +15,15 @@ from buildbot.plugins import steps def PersOutputOfEmerge(rc, stdout, stderr): emerge_output = {} emerge_output['rc'] = rc + emerge_output['preserved_libs'] = False + emerge_output['depclean'] = False package_dict = {} print(stderr) + # split the lines for line in stdout.split('\n'): # package list subdict = {} - if line.startswith('[ebuild'): - # split the line + if line.startswith('[ebuild') or line.startswith('[binary'): # if binaries if line.startswith('[ebuild'): subdict['binary'] = False @@ -57,7 +59,18 @@ def PersOutputOfEmerge(rc, stdout, stderr): subdict['python_targets'] = None # CPU_FLAGS_X86 list package_dict[cpv] = subdict - #FIXME: Handling of stderr output + if line.startswith('>>>'): + #FIXME: Handling of >>> output + pass + if line.startswith('!!!'): + #FIXME: Handling of !!! output + if line.startswith('!!! existing preserved libs'): + pass + #FIXME: Handling of depclean output dict of packages that get removed or saved + # split the lines + #FIXME: Handling of stderr output + for line in stderr.split('\n'): + pass return { 'emerge_output' : emerge_output } @@ -154,6 +167,8 @@ class SetupPropertys(BuildStep): print(projectrepository_data) project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid']) self.setProperty('project_data', project_data, 'project_data') + self.setProperty('preserved_libs', False, 'preserved-libs') + self.setProperty('depclean', False, 'depclean') return SUCCESS class SetMakeProfile(BuildStep): @@ -311,6 +326,7 @@ class SetMakeConf(BuildStep): makeconf_variable_list.append('--rebuilt-binaries=y') makeconf_variable_list.append('--usepkg=y') makeconf_variable_list.append('--binpkg-respect-use=y') + makeconf_variable_list.append('--binpkg-changed-deps=y') makeconf_variable_list.append('--nospinner') makeconf_variable_list.append('--color=n') makeconf_variable_list.append('--ask=n') @@ -366,3 +382,136 @@ class SetMakeConf(BuildStep): workdir='/etc/portage/') ]) return SUCCESS + +class RunEmerge(BuildStep): + + name = 'RunEmerge' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, step=None,**kwargs): + self.step = step + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + project_data = self.getProperty('project_data') + projects_emerge_options = yield self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid']) + shell_commad_list = [ + 'emerge', + '-v' + ] + aftersteps_list = [] + if self.step == 'pre-update': + shell_commad_list.append('-uDN') + shell_commad_list.append('--changed-deps') + shell_commad_list.append('--changed-use') + shell_commad_list.append('--pretend') + shell_commad_list.append('@world') + aftersteps_list.append( + steps.SetPropertyFromCommandNewStyle( + command=shell_commad_list, + strip=True, + extract_fn=PersOutputOfEmerge, + workdir='/' + )) + aftersteps_list.append(CheckEmergeLogs('pre-update')) + + if self.step == 'update': + shell_commad_list.append('-uDNq') + shell_commad_list.append('--changed-deps') + shell_commad_list.append('--changed-use') + shell_commad_list.append('@world') + aftersteps_list.append( + steps.SetPropertyFromCommandNewStyle( + command=shell_commad_list, + strip=True, + extract_fn=PersOutputOfEmerge, + workdir='/', + timeout=None + )) + aftersteps_list.append(CheckEmergeLogs('update')) + + if self.step == 'preserved-libs' and self.getProperty('preserved_libs'): + shell_commad_list.append('-q') + shell_commad_list.append('@preserved-rebuild') + aftersteps_list.append( + steps.SetPropertyFromCommandNewStyle( + command=shell_commad_list, + strip=True, + extract_fn=PersOutputOfEmerge, + workdir='/', + timeout=None + )) + aftersteps_list.append(CheckEmergeLogs('preserved-libs')) + self.setProperty('preserved_libs', False, 'preserved-libs') + + if self.step == 'pre-depclean' and projects_emerge_options['depclean']: + shell_commad_list.append('--pretend') + shell_commad_list.append('--depclean') + aftersteps_list.append( + steps.SetPropertyFromCommandNewStyle( + command=shell_commad_list, + strip=True, + extract_fn=PersOutputOfEmerge, + workdir='/' + )) + aftersteps_list.append(CheckEmergeLogs('depclean')) + self.setProperty('depclean', False, 'depclean') + + if self.step == 'depclean' and self.getProperty('depclean') and projects_emerge_options['depclean']: + shell_commad_list.append('-q') + shell_commad_list.append('--depclean') + aftersteps_list.append( + steps.SetPropertyFromCommandNewStyle( + command=shell_commad_list, + strip=True, + extract_fn=PersOutputOfEmerge, + workdir='/' + )) + aftersteps_list.append(CheckEmergeLogs('depclean')) + if not self.step is None: + yield self.build.addStepsAfterCurrentStep(aftersteps_list) + return SUCCESS + +class CheckEmergeLogs(BuildStep): + + name = 'CheckLogs' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, step=None,**kwargs): + self.step = step + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + project_data = self.getProperty('project_data') + projects_emerge_options = yield self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid']) + emerge_output = self.getProperty('emerge_output') + shell_commad_list = [ + 'emerge', + '-v' + ] + aftersteps_list = [] + + #FIXME: Prosees the logs and do stuff + # preserved-libs + if emerge_output['preserved_libs'] and projects_emerge_options['preserved_libs']: + self.setProperty('preserved_libs', True, 'preserved-libs') + # depclean + # FIXME: check if don't remove needed stuff. + if emerge_output['depclean'] and projects_emerge_options['depclean']: + self.setProperty('depclean', True, 'depclean') + + if not self.step is None: + yield self.build.addStepsAfterCurrentStep(aftersteps_list) + return SUCCESS diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py index ee2ba88..3f20cb9 100644 --- a/buildbot_gentoo_ci/steps/update_db.py +++ b/buildbot_gentoo_ci/steps/update_db.py @@ -124,7 +124,7 @@ class CheckCPVGentooCiProject(BuildStep): def run(self): #self.cpv_changes = self.getProperty("cpv_changes") self.cpv_changes = [] - self.cpv_changes.append('dev-python/django-3.1.5') + self.cpv_changes.append('dev-python/django-3.1.7') self.cpv_changes.append('dev-python/scrypt-0.8.16') print(self.cpv_changes) print(self.getProperty("repository_data"))