commit: 550ae33d23ae93d7d779922a0b8e07710458c1a7 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Sat Apr 10 01:02:38 2021 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Sat Apr 10 01:02:38 2021 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=550ae33d
Add support for Failed to XX in emerge build Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org> buildbot_gentoo_ci/db/packages.py | 4 +- buildbot_gentoo_ci/db/versions.py | 3 +- buildbot_gentoo_ci/steps/builders.py | 90 ++++++++++++++++++++++++++++-------- buildbot_gentoo_ci/steps/logs.py | 7 ++- buildbot_gentoo_ci/steps/package.py | 4 +- buildbot_gentoo_ci/steps/version.py | 2 +- 6 files changed, 85 insertions(+), 25 deletions(-) diff --git a/buildbot_gentoo_ci/db/packages.py b/buildbot_gentoo_ci/db/packages.py index 13cc267..a22b9d8 100644 --- a/buildbot_gentoo_ci/db/packages.py +++ b/buildbot_gentoo_ci/db/packages.py @@ -27,11 +27,13 @@ from buildbot.db import base class PackagesConnectorComponent(base.DBConnectorComponent): @defer.inlineCallbacks - def getPackageByName(self, name): + def getPackageByName(self, name, c_uuid, repo_uuid): def thd(conn): tbl = self.db.model.packages q = tbl.select() q = q.where(tbl.c.name == name) + q = q.where(tbl.c.category_uuid == c_uuid) + q = q.where(tbl.c.repository_uuid == repo_uuid) res = conn.execute(q) row = res.fetchone() if not row: diff --git a/buildbot_gentoo_ci/db/versions.py b/buildbot_gentoo_ci/db/versions.py index 85bd105..768c1dc 100644 --- a/buildbot_gentoo_ci/db/versions.py +++ b/buildbot_gentoo_ci/db/versions.py @@ -28,11 +28,12 @@ from buildbot.util import datetime2epoch class VersionsConnectorComponent(base.DBConnectorComponent): @defer.inlineCallbacks - def getVersionByName(self, name): + def getVersionByName(self, name, p_uuid): def thd(conn): tbl = self.db.model.versions q = tbl.select() q = q.where(tbl.c.name == name) + q = q.where(tbl.c.package_uuid == p_uuid) res = conn.execute(q) row = res.fetchone() if not row: diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index af3be9d..e51576f 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -4,7 +4,7 @@ import os import re -from portage.versions import catpkgsplit +from portage.versions import catpkgsplit, cpv_getversion from twisted.internet import defer from twisted.python import log @@ -19,10 +19,10 @@ def PersOutputOfEmerge(rc, stdout, stderr): emerge_output['rc'] = rc emerge_output['preserved_libs'] = False emerge_output['change_use'] = False + emerge_output['failed'] = False package_dict = {} log_path_list = [] print(stderr) - emerge_output['stderr'] = stderr # split the lines for line in stdout.split('\n'): # package list @@ -67,6 +67,10 @@ def PersOutputOfEmerge(rc, stdout, stderr): # CPU_FLAGS_X86 list package_dict[cpv] = subdict if line.startswith('>>>'): + if line.startswith('>>> Failed to'): + emerge_output['failed'] = line.split(' ')[4][:-1] + if line.endswith('.log.gz') and emerge_output['failed']: + log_path_list.append(line.split(' ')[2]) #FIXME: Handling of >>> output pass if line.startswith('!!!'): @@ -78,9 +82,10 @@ def PersOutputOfEmerge(rc, stdout, stderr): log_path_list.append(line.split(' ')[4]) #FIXME: Handling of depclean output dict of packages that get removed or saved emerge_output['package'] = package_dict - emerge_output['log_paths'] = log_path_list + # split the lines #FIXME: Handling of stderr output + stderr_line_list = [] for line in stderr.split('\n'): if 'Change USE:' in line: line_list = line.split(' ') @@ -96,6 +101,16 @@ def PersOutputOfEmerge(rc, stdout, stderr): else: change_use_list = False emerge_output['change_use'] = change_use_list + err_line_list = [] + if line.startswith(' * '): + if line.endswith('.log.gz'): + log_path = line.split(' ')[3] + if log_path not in inlog_path_list: + log_path_list.append(log_path) + stderr_line_list.append(line) + emerge_output['stderr'] = stderr_line_list + emerge_output['log_paths'] = log_path_list + return { 'emerge_output' : emerge_output } @@ -484,6 +499,30 @@ class CheckEmergeLogs(BuildStep): self.step = step super().__init__(**kwargs) self.descriptionSuffix = self.step + self.aftersteps_list = [] + self.log_data = {} + + @defer.inlineCallbacks + def getVersionData(self, cpv): + c = yield catpkgsplit(cpv)[0] + p = yield catpkgsplit(cpv)[1] + category_data = yield self.gentooci.db.categorys.getCategoryByName(c) + package_data = yield self.gentooci.db.packages.getPackageByName(p, + category_data['uuid'], + self.getProperty('repository_data')['uuid']) + if package_data is None: + return None + version = yield cpv_getversion(cpv) + version_data = yield self.gentooci.db.versions.getVersionByName(version, package_data['uuid']) + return version_data + + @defer.inlineCallbacks + def getLogFile(self, cpv, log_dict): + masterdest = yield os.path.join(self.master.basedir, 'cpv_logs', log_dict[cpv]['full_logname']) + self.aftersteps_list.append(steps.FileUpload( + workersrc=log_dict[cpv]['log_path'], + masterdest=masterdest + )) @defer.inlineCallbacks def run(self): @@ -495,7 +534,6 @@ class CheckEmergeLogs(BuildStep): 'emerge', '-v' ] - aftersteps_list = [] #FIXME: Prosees the logs and do stuff # preserved-libs @@ -533,13 +571,13 @@ class CheckEmergeLogs(BuildStep): else: change_use_list.append(use_flag) change_use_string = separator2.join(change_use_list) - aftersteps_list.append( + self.aftersteps_list.append( steps.StringDownload(change_use_string + separator, workerdest='zz_autouse' + str(self.getProperty('rerun')), workdir='/etc/portage/package.use/') ) # rerun - aftersteps_list.append(RunEmerge(step='pre-build')) + self.aftersteps_list.append(RunEmerge(step='pre-build')) self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun') else: # trigger parse_build_log with info about pre-build and it fail @@ -564,31 +602,43 @@ class CheckEmergeLogs(BuildStep): full_logname = full_logname ) print(log_dict) - # Find log for cpv that was requested or did faild + # Find log for cpv that was requested or did failed if not log_dict == {}: # requested cpv - if self.getProperty('cpv') in log_dict: - log_data = log_dict[self.getProperty('cpv')] - masterdest = yield os.path.join(self.master.basedir, 'cpv_logs', log_data['full_logname']) - aftersteps_list.append(steps.FileUpload( - workersrc=log_data['log_path'], - masterdest=masterdest - )) - aftersteps_list.append(steps.Trigger( + print(log_dict) + cpv = self.getProperty('cpv') + faild_cpv = emerge_output['failed'] + if cpv in log_dict or faild_cpv in log_dict: + if cpv in log_dict: + self.log_data[cpv] = log_dict[cpv] + yield self.getLogFile(cpv, log_dict) + faild_version_data = False + if faild_cpv: + # failed and build requested cpv + if cpv == faild_cpv: + faild_version_data = self.getProperty("version_data") + else: + # failed but not build requested cpv + self.log_data[faild_cpv] = log_dict[faild_cpv] + yield self.getLogFile(faild_cpv, log_dict) + faild_version_data = yield self.getVersionData(faild_cpv) + self.aftersteps_list.append(steps.Trigger( schedulerNames=['parse_build_log'], waitForFinish=False, updateSourceStamp=False, set_properties={ 'cpv' : self.getProperty("cpv"), - 'faild_version_data' : self.getProperty('faild_version_data'), + 'faild_version_data' : faild_version_data, 'project_build_data' : self.getProperty('project_build_data'), - 'log_build_data' : log_data, + 'log_build_data' : self.log_data, 'pkg_check_log_data' : self.getProperty("pkg_check_log_data"), - 'repository_data' : self.getProperty('repository_data') + 'repository_data' : self.getProperty('repository_data'), + 'faild_cpv' : faild_cpv, + 'step' : self.step } )) - if not self.step is None and aftersteps_list != []: - yield self.build.addStepsAfterCurrentStep(aftersteps_list) + if not self.step is None and self.aftersteps_list != []: + yield self.build.addStepsAfterCurrentStep(self.aftersteps_list) return SUCCESS class CheckDepcleanLogs(BuildStep): diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py index 767fdd4..4c3bf01 100644 --- a/buildbot_gentoo_ci/steps/logs.py +++ b/buildbot_gentoo_ci/steps/logs.py @@ -157,7 +157,12 @@ class ParserBuildLog(BuildStep): # read it to a buffer # make a dict of the buffer # maby use mulitiprocces to speed up the search - file_path = yield os.path.join(self.master.basedir, 'cpv_logs', self.getProperty('log_build_data')['full_logname']) + print(self.getProperty('log_build_data')) + if self.getProperty('faild_cpv'): + log_cpv = self.getProperty('log_build_data')[self.getProperty('faild_cpv')] + else: + log_cpv = self.getProperty('log_build_data')[self.getProperty('cpv')] + file_path = yield os.path.join(self.master.basedir, 'cpv_logs', log_cpv['full_logname']) with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb'))) as f: for text_line in f: self.logfile_text_dict[self.index] = text_line.strip('\n') diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py index ea5ab23..b4e06b7 100644 --- a/buildbot_gentoo_ci/steps/package.py +++ b/buildbot_gentoo_ci/steps/package.py @@ -59,7 +59,9 @@ class CheckP(BuildStep): self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] self.package = yield catpkgsplit(self.getProperty("cpv"))[1] print(self.package) - self.package_data = yield self.gentooci.db.packages.getPackageByName(self.package) + self.package_data = yield self.gentooci.db.packages.getPackageByName(self.package, + self.getProperty("category_data")['uuid'], + self.getProperty("repository_data")['uuid']) print(self.package_data) if self.package_data is None: self.setProperty("package", self.package, 'package') diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py index 426ddd4..e6a927a 100644 --- a/buildbot_gentoo_ci/steps/version.py +++ b/buildbot_gentoo_ci/steps/version.py @@ -38,7 +38,7 @@ class GetVData(BuildStep): self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] self.version = yield cpv_getversion(self.getProperty("cpv")) print(self.version) - self.old_version_data = yield self.gentooci.db.versions.getVersionByName(self.version) + self.old_version_data = yield self.gentooci.db.versions.getVersionByName(self.version, self.getProperty("package_data")['uuid']) print(self.old_version_data) self.setProperty("old_version_data", self.old_version_data, 'old_version_data') self.setProperty("version", self.version, 'version')