commit: 3791e957fd2bb28f350f0ff7399c0c6e43b561d6
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Aug 5 18:32:11 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Aug 5 18:32:11 2022 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3791e957
Add haskell-updater for haskell packages
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 147 ++++++++++++++++++++++-------------
1 file changed, 91 insertions(+), 56 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index a54c68c..3b6a8d0 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -4,6 +4,7 @@
import os
import re
import json
+from pathlib import Path
from portage.versions import catpkgsplit, cpv_getversion
from portage.dep import dep_getcpv, dep_getslot, dep_getrepo
@@ -90,6 +91,8 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# split the lines
#FIXME: Handling of stderr output
stderr_line_list = []
+ if stderr != '' and not emerge_output['failed']:
+ emerge_output['failed'] = True
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
@@ -340,6 +343,7 @@ class SetupPropertys(BuildStep):
print(self.getProperty("project_build_data"))
self.masterdest = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'), str(self.getProperty("buildnumber")))
self.setProperty('masterdest', self.masterdest, 'masterdest')
+ self.setProperty('build_env', {}, 'build_env')
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for
project', self.getProperty('project_data')['name']])
return SUCCESS
@@ -355,7 +359,6 @@ class RunEmerge(BuildStep):
super().__init__(**kwargs)
self.descriptionSuffix = self.step
self.name = 'Setup emerge for ' + self.step + ' step'
- self.build_env = {}
self.build_timeout = 0
@defer.inlineCallbacks
@@ -374,25 +377,28 @@ class RunEmerge(BuildStep):
aftersteps_list = []
#FIXME: Set build timeout in config
self.build_timeout = 6600
- # set env
- # https://bugs.gentoo.org/683118
- # export TERM=linux
- # export TERMINFO=/etc/terminfo
- self.build_env['TERM'] = 'linux'
- self.build_env['TERMINFO'] = '/etc/terminfo'
- # Lang
- self.build_env['LANG'] = 'C.utf8'
- self.build_env['LC_MESSAGES'] = 'C'
- # no color
- self.build_env['CARGO_TERM_COLOR'] = 'never'
- self.build_env['GCC_COLORS'] = '0'
- self.build_env['OCAML_COLOR'] = 'never'
- self.build_env['PY_FORCE_COLOR'] = '0'
- self.build_env['PYTEST_ADDOPTS'] = '--color=no'
- self.build_env['NO_COLOR'] = '1'
- # not all terms support urls
- self.build_env['GCC_URLS'] = 'no'
- self.build_env['TERM_URLS'] = 'no'
+ if self.step == 'update':
+ # set env
+ build_env = {}
+ # https://bugs.gentoo.org/683118
+ # export TERM=linux
+ # export TERMINFO=/etc/terminfo
+ build_env['TERM'] = 'linux'
+ build_env['TERMINFO'] = '/etc/terminfo'
+ # Lang
+ build_env['LANG'] = 'C.utf8'
+ build_env['LC_MESSAGES'] = 'C'
+ # no color
+ build_env['CARGO_TERM_COLOR'] = 'never'
+ build_env['GCC_COLORS'] = '0'
+ build_env['OCAML_COLOR'] = 'never'
+ build_env['PY_FORCE_COLOR'] = '0'
+ build_env['PYTEST_ADDOPTS'] = '--color=no'
+ build_env['NO_COLOR'] = '1'
+ # not all terms support urls
+ build_env['GCC_URLS'] = 'no'
+ build_env['TERM_URLS'] = 'no'
+ self.setProperty('build_env', build_env, 'build_env')
if self.step == 'pre-update':
shell_commad_list.append('-uDN')
@@ -432,6 +438,7 @@ class RunEmerge(BuildStep):
strip=True,
extract_fn=PersOutputOfEmerge,
workdir='/',
+ env=self.getProperty("build_env"),
timeout=self.build_timeout
))
aftersteps_list.append(CheckEmergeLogs('update'))
@@ -548,7 +555,7 @@ class RunEmerge(BuildStep):
strip=True,
extract_fn=PersOutputOfEmerge,
workdir='/',
- env=self.build_env,
+ env=self.getProperty("build_env"),
timeout=self.build_timeout
))
aftersteps_list.append(CheckEmergeLogs('build'))
@@ -682,11 +689,9 @@ class CheckEmergeLogs(BuildStep):
@defer.inlineCallbacks
def createDistDir(self):
workdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'))
- self.aftersteps_list.append(steps.MasterShellCommand(
- name = 'Make directory for Uploaded files',
- command = ['mkdir', str(self.getProperty("buildnumber"))],
- workdir = workdir
- ))
+ check_dir = yield os.path.join(workdir,
str(self.getProperty("buildnumber")))
+ if not Path(check_dir).is_dir():
+ yield Path(check_dir).mkdir(parents=True)
def addFileUploade(self, sourcefile, destfile, name, url, urlText):
self.aftersteps_list.append(steps.FileUpload(
@@ -814,7 +819,7 @@ class CheckEmergeLogs(BuildStep):
print(emerge_output)
# this should be set in the config
retrays = 5
- if self.getProperty('rerun') <= retrays:
+ if self.getProperty('rerun') <= retrays and self.faild_cpv:
# when we need to change use. we could rerun pre-build with
# --autounmask-use=y --autounmask-write=y --autounmask-only=y
# but we use --binpkg--respect-use=y in EMERGE_DEFAULT_OPTS
@@ -895,7 +900,9 @@ class CheckEmergeLogs(BuildStep):
self.aftersteps_list.append(CheckEmergeLogs('extra-build'))
else:
# trigger parse_build_log with info about pre-build and it fail
- pass
+ if self.faild_cpv:
+ pass
+ self.setProperty('rerun', 0, 'rerun')
# Make Logfile dict
if self.step == 'extra-build' or self.step == 'build':
print(emerge_output)
@@ -922,7 +929,9 @@ class CheckEmergeLogs(BuildStep):
# trigger a logparser
# local_log_path dir set in config
# format
/var/cache/portage/logs/build/gui-libs/egl-wayland-1.1.6:20210321-173525.log.gz
+ rebuild = False
if self.step == 'build':
+ retrays = 1
# Find log for cpv that was requested or did failed
if not log_dict == {}:
# requested cpv
@@ -934,36 +943,62 @@ class CheckEmergeLogs(BuildStep):
self.log_data[cpv] = log_dict[cpv]
yield self.getLogFile(cpv, log_dict)
if self.faild_cpv:
- # failed and build requested cpv
- if cpv == self.faild_cpv:
- faild_version_data =
self.getProperty("version_data")
- else:
- # failed but not build requested cpv
- self.log_data[self.faild_cpv] =
log_dict[self.faild_cpv]
- yield self.getLogFile(self.faild_cpv, log_dict)
- faild_version_data = yield
self.getVersionData(self.faild_cpv)
- self.setProperty('faild_cpv', self.faild_cpv,
'faild_cpv')
- self.getEmergeFiles(self.faild_cpv)
- self.getBuildWorkDirs(self.faild_cpv)
+ c = yield catpkgsplit(self.faild_cpv)[0]
+ if c == 'dev-haskell':
+ rebuild = 'haskell'
+ if not rebuild or self.getProperty('rerun') >= retrays:
+ # failed and build requested cpv
+ if cpv == self.faild_cpv:
+ faild_version_data =
self.getProperty("version_data")
+ else:
+ # failed but not build requested cpv
+ self.log_data[self.faild_cpv] =
log_dict[self.faild_cpv]
+ yield self.getLogFile(self.faild_cpv, log_dict)
+ faild_version_data = yield
self.getVersionData(self.faild_cpv)
+ self.setProperty('faild_cpv', self.faild_cpv,
'faild_cpv')
+ self.getEmergeFiles(self.faild_cpv)
+ self.getBuildWorkDirs(self.faild_cpv)
+ rebuild = False
else:
self.getEmergeFiles(cpv)
- self.aftersteps_list.append(steps.Trigger(
- name = 'Setup properties for log parser and trigger
it',
- schedulerNames=['parse_build_log'],
- waitForFinish=False,
- updateSourceStamp=False,
- set_properties={
- 'cpv' : self.getProperty("cpv"),
- 'faild_version_data' : faild_version_data,
- 'project_build_data' :
self.getProperty('project_build_data'),
- 'log_build_data' : self.log_data,
- 'pkg_check_log_data' :
self.getProperty("pkg_check_log_data"),
- 'repository_data' :
self.getProperty('repository_data'),
- 'faild_cpv' : self.faild_cpv,
- 'step' : self.step,
- 'build_workername' : self.getProperty('workername')
- }
- ))
+ if not rebuild:
+ self.aftersteps_list.append(steps.Trigger(
+ name = 'Setup properties for log parser and
trigger it',
+ schedulerNames=['parse_build_log'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : self.getProperty("cpv"),
+ 'faild_version_data' : faild_version_data,
+ 'project_build_data' :
self.getProperty('project_build_data'),
+ 'log_build_data' : self.log_data,
+ 'pkg_check_log_data' :
self.getProperty("pkg_check_log_data"),
+ 'repository_data' :
self.getProperty('repository_data'),
+ 'faild_cpv' : self.faild_cpv,
+ 'step' : self.step,
+ 'build_workername' :
self.getProperty('workername')
+ }
+ ))
+ if rebuild:
+ #FIXME: Set build timeout in config
+ build_timeout = 6600
+ shell_commad_list = []
+ # rebuild broken haskell
+ if rebuild == 'haskell':
+ shell_commad_list.append('haskell-updater')
+ shell_commad_list.append('--')
+ shell_commad_list.append('--usepkg=n')
+ if shell_commad_list != []:
+ self.aftersteps_list.append(
+ steps.ShellCommand(
+ command=shell_commad_list,
+ workdir='/',
+ env=self.getProperty("build_env"),
+ timeout=build_timeout
+ ))
+ self.aftersteps_list.append(RunEmerge(step='build'))
+ self.aftersteps_list.append(CheckEmergeLogs('build'))
+ self.setProperty('rerun', self.getProperty('rerun') +
1, 'rerun')
if not self.step is None and self.aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
