commit: 550ae33d23ae93d7d779922a0b8e07710458c1a7
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 10 01:02:38 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Apr 10 01:02:38 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=550ae33d
Add support for Failed to XX in emerge build
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/packages.py | 4 +-
buildbot_gentoo_ci/db/versions.py | 3 +-
buildbot_gentoo_ci/steps/builders.py | 90 ++++++++++++++++++++++++++++--------
buildbot_gentoo_ci/steps/logs.py | 7 ++-
buildbot_gentoo_ci/steps/package.py | 4 +-
buildbot_gentoo_ci/steps/version.py | 2 +-
6 files changed, 85 insertions(+), 25 deletions(-)
diff --git a/buildbot_gentoo_ci/db/packages.py
b/buildbot_gentoo_ci/db/packages.py
index 13cc267..a22b9d8 100644
--- a/buildbot_gentoo_ci/db/packages.py
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -27,11 +27,13 @@ from buildbot.db import base
class PackagesConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
- def getPackageByName(self, name):
+ def getPackageByName(self, name, c_uuid, repo_uuid):
def thd(conn):
tbl = self.db.model.packages
q = tbl.select()
q = q.where(tbl.c.name == name)
+ q = q.where(tbl.c.category_uuid == c_uuid)
+ q = q.where(tbl.c.repository_uuid == repo_uuid)
res = conn.execute(q)
row = res.fetchone()
if not row:
diff --git a/buildbot_gentoo_ci/db/versions.py
b/buildbot_gentoo_ci/db/versions.py
index 85bd105..768c1dc 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -28,11 +28,12 @@ from buildbot.util import datetime2epoch
class VersionsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
- def getVersionByName(self, name):
+ def getVersionByName(self, name, p_uuid):
def thd(conn):
tbl = self.db.model.versions
q = tbl.select()
q = q.where(tbl.c.name == name)
+ q = q.where(tbl.c.package_uuid == p_uuid)
res = conn.execute(q)
row = res.fetchone()
if not row:
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index af3be9d..e51576f 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -4,7 +4,7 @@
import os
import re
-from portage.versions import catpkgsplit
+from portage.versions import catpkgsplit, cpv_getversion
from twisted.internet import defer
from twisted.python import log
@@ -19,10 +19,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['rc'] = rc
emerge_output['preserved_libs'] = False
emerge_output['change_use'] = False
+ emerge_output['failed'] = False
package_dict = {}
log_path_list = []
print(stderr)
- emerge_output['stderr'] = stderr
# split the lines
for line in stdout.split('\n'):
# package list
@@ -67,6 +67,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# CPU_FLAGS_X86 list
package_dict[cpv] = subdict
if line.startswith('>>>'):
+ if line.startswith('>>> Failed to'):
+ emerge_output['failed'] = line.split(' ')[4][:-1]
+ if line.endswith('.log.gz') and emerge_output['failed']:
+ log_path_list.append(line.split(' ')[2])
#FIXME: Handling of >>> output
pass
if line.startswith('!!!'):
@@ -78,9 +82,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
log_path_list.append(line.split(' ')[4])
#FIXME: Handling of depclean output dict of packages that get removed
or saved
emerge_output['package'] = package_dict
- emerge_output['log_paths'] = log_path_list
+
# split the lines
#FIXME: Handling of stderr output
+ stderr_line_list = []
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
@@ -96,6 +101,16 @@ def PersOutputOfEmerge(rc, stdout, stderr):
else:
change_use_list = False
emerge_output['change_use'] = change_use_list
+ err_line_list = []
+ if line.startswith(' * '):
+ if line.endswith('.log.gz'):
+ log_path = line.split(' ')[3]
+ if log_path not in inlog_path_list:
+ log_path_list.append(log_path)
+ stderr_line_list.append(line)
+ emerge_output['stderr'] = stderr_line_list
+ emerge_output['log_paths'] = log_path_list
+
return {
'emerge_output' : emerge_output
}
@@ -484,6 +499,30 @@ class CheckEmergeLogs(BuildStep):
self.step = step
super().__init__(**kwargs)
self.descriptionSuffix = self.step
+ self.aftersteps_list = []
+ self.log_data = {}
+
+ @defer.inlineCallbacks
+ def getVersionData(self, cpv):
+ c = yield catpkgsplit(cpv)[0]
+ p = yield catpkgsplit(cpv)[1]
+ category_data = yield self.gentooci.db.categorys.getCategoryByName(c)
+ package_data = yield self.gentooci.db.packages.getPackageByName(p,
+
category_data['uuid'],
+
self.getProperty('repository_data')['uuid'])
+ if package_data is None:
+ return None
+ version = yield cpv_getversion(cpv)
+ version_data = yield
self.gentooci.db.versions.getVersionByName(version, package_data['uuid'])
+ return version_data
+
+ @defer.inlineCallbacks
+ def getLogFile(self, cpv, log_dict):
+ masterdest = yield os.path.join(self.master.basedir, 'cpv_logs',
log_dict[cpv]['full_logname'])
+ self.aftersteps_list.append(steps.FileUpload(
+ workersrc=log_dict[cpv]['log_path'],
+ masterdest=masterdest
+ ))
@defer.inlineCallbacks
def run(self):
@@ -495,7 +534,6 @@ class CheckEmergeLogs(BuildStep):
'emerge',
'-v'
]
- aftersteps_list = []
#FIXME: Prosees the logs and do stuff
# preserved-libs
@@ -533,13 +571,13 @@ class CheckEmergeLogs(BuildStep):
else:
change_use_list.append(use_flag)
change_use_string = separator2.join(change_use_list)
- aftersteps_list.append(
+ self.aftersteps_list.append(
steps.StringDownload(change_use_string + separator,
workerdest='zz_autouse' +
str(self.getProperty('rerun')),
workdir='/etc/portage/package.use/')
)
# rerun
- aftersteps_list.append(RunEmerge(step='pre-build'))
+ self.aftersteps_list.append(RunEmerge(step='pre-build'))
self.setProperty('rerun', self.getProperty('rerun') + 1,
'rerun')
else:
# trigger parse_build_log with info about pre-build and it fail
@@ -564,31 +602,43 @@ class CheckEmergeLogs(BuildStep):
full_logname = full_logname
)
print(log_dict)
- # Find log for cpv that was requested or did faild
+ # Find log for cpv that was requested or did failed
if not log_dict == {}:
# requested cpv
- if self.getProperty('cpv') in log_dict:
- log_data = log_dict[self.getProperty('cpv')]
- masterdest = yield os.path.join(self.master.basedir,
'cpv_logs', log_data['full_logname'])
- aftersteps_list.append(steps.FileUpload(
- workersrc=log_data['log_path'],
- masterdest=masterdest
- ))
- aftersteps_list.append(steps.Trigger(
+ print(log_dict)
+ cpv = self.getProperty('cpv')
+ faild_cpv = emerge_output['failed']
+ if cpv in log_dict or faild_cpv in log_dict:
+ if cpv in log_dict:
+ self.log_data[cpv] = log_dict[cpv]
+ yield self.getLogFile(cpv, log_dict)
+ faild_version_data = False
+ if faild_cpv:
+ # failed and build requested cpv
+ if cpv == faild_cpv:
+ faild_version_data =
self.getProperty("version_data")
+ else:
+ # failed but not build requested cpv
+ self.log_data[faild_cpv] = log_dict[faild_cpv]
+ yield self.getLogFile(faild_cpv, log_dict)
+ faild_version_data = yield
self.getVersionData(faild_cpv)
+ self.aftersteps_list.append(steps.Trigger(
schedulerNames=['parse_build_log'],
waitForFinish=False,
updateSourceStamp=False,
set_properties={
'cpv' : self.getProperty("cpv"),
- 'faild_version_data' :
self.getProperty('faild_version_data'),
+ 'faild_version_data' : faild_version_data,
'project_build_data' :
self.getProperty('project_build_data'),
- 'log_build_data' : log_data,
+ 'log_build_data' : self.log_data,
'pkg_check_log_data' :
self.getProperty("pkg_check_log_data"),
- 'repository_data' :
self.getProperty('repository_data')
+ 'repository_data' :
self.getProperty('repository_data'),
+ 'faild_cpv' : faild_cpv,
+ 'step' : self.step
}
))
- if not self.step is None and aftersteps_list != []:
- yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ if not self.step is None and self.aftersteps_list != []:
+ yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
class CheckDepcleanLogs(BuildStep):
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 767fdd4..4c3bf01 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -157,7 +157,12 @@ class ParserBuildLog(BuildStep):
# read it to a buffer
# make a dict of the buffer
# maby use mulitiprocces to speed up the search
- file_path = yield os.path.join(self.master.basedir, 'cpv_logs',
self.getProperty('log_build_data')['full_logname'])
+ print(self.getProperty('log_build_data'))
+ if self.getProperty('faild_cpv'):
+ log_cpv =
self.getProperty('log_build_data')[self.getProperty('faild_cpv')]
+ else:
+ log_cpv =
self.getProperty('log_build_data')[self.getProperty('cpv')]
+ file_path = yield os.path.join(self.master.basedir, 'cpv_logs',
log_cpv['full_logname'])
with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb')))
as f:
for text_line in f:
self.logfile_text_dict[self.index] = text_line.strip('\n')
diff --git a/buildbot_gentoo_ci/steps/package.py
b/buildbot_gentoo_ci/steps/package.py
index ea5ab23..b4e06b7 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -59,7 +59,9 @@ class CheckP(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.package = yield catpkgsplit(self.getProperty("cpv"))[1]
print(self.package)
- self.package_data = yield
self.gentooci.db.packages.getPackageByName(self.package)
+ self.package_data = yield
self.gentooci.db.packages.getPackageByName(self.package,
+
self.getProperty("category_data")['uuid'],
+
self.getProperty("repository_data")['uuid'])
print(self.package_data)
if self.package_data is None:
self.setProperty("package", self.package, 'package')
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index 426ddd4..e6a927a 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -38,7 +38,7 @@ class GetVData(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.version = yield cpv_getversion(self.getProperty("cpv"))
print(self.version)
- self.old_version_data = yield
self.gentooci.db.versions.getVersionByName(self.version)
+ self.old_version_data = yield
self.gentooci.db.versions.getVersionByName(self.version,
self.getProperty("package_data")['uuid'])
print(self.old_version_data)
self.setProperty("old_version_data", self.old_version_data,
'old_version_data')
self.setProperty("version", self.version, 'version')
