[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 4707a4cb41619d8c35d2d9f7cb0cc5412f447492
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Mar 31 19:38:55 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Mar 31 19:38:55 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4707a4cb
Fix a typo on faild_dep
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 53f79ab..1db8cc2 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1084,7 +1084,7 @@ class CheckEmergeLogs(BuildStep):
'pkg_check_log_data' :
self.getProperty("pkg_check_log_data"),
'repository_data' :
self.getProperty('repository_data'),
'faild_cpv' : emerge_output['failed'],
-'faild_dep' : self.faild_dep
+'faild_dep' : self.faild_dep,
'build_workername' : self.getProperty('workername')
}
))
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: e8e5b67dc70b8b2b0222486574a20d6edb0577c2
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Mar 31 19:40:33 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Mar 31 19:40:33 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e8e5b67d
Add step logs to a log file
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 53 +---
1 file changed, 50 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 896778c..6b8eba5 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -7,6 +7,7 @@ import gzip
import io
import hashlib
import json
+import lzma
from portage.versions import catpkgsplit, cpv_getversion
@@ -74,6 +75,18 @@ def PersOutputOfEmergeInfo(rc, stdout, stderr):
'emerge_info_output' : emerge_info_output
}
+@defer.inlineCallbacks
+def WriteTextToFile(path, text_list, separator=False):
+separator2 = '\n'
+if separator:
+text_string = separator2.join(text_list)
+else:
+text_string = text_list
+text_string = text_string + separator2
+with lzma.open(path,"wt") as f:
+yield f.write(text_string)
+yield f.close
+
class SetupPropertys(BuildStep):
name = 'SetupPropertys'
@@ -109,14 +122,16 @@ class SetupPropertys(BuildStep):
# get steps/log id's from build
build_log_steps_data = {}
steps_data = yield
self.master.db.steps.getSteps(self.getProperty("project_build_data")['buildbot_build_id'])
-print(steps_data)
for step in steps_data:
+log_data = None
logs_data = yield self.master.db.logs.getLogs(step['id'])
-print(logs_data)
+for log_info in logs_data:
+if log_info['name'] != 'property changes':
+log_data = log_info
step_info = {}
step_info['name'] = step['name']
step_info['number'] = step['number']
-step_info['log_data'] = logs_data
+step_info['log_data'] = log_data
build_log_steps_data[step['id']] = step_info
self.setProperty("build_log_steps_data", build_log_steps_data,
'build_log_steps_data')
return SUCCESS
@@ -509,6 +524,38 @@ class ParserPkgCheckLog(BuildStep):
returnstatus = WARNINGS
return returnstatus
+class writeStepLogsToFile(BuildStep):
+
+name = 'writeStepLogsToFile'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = False
+flunkOnFailure = True
+warnOnWarnings = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+buildsteplogfile_path = yield
os.path.join(self.getProperty('logsdir'), 'buildsteps.log.xz')
+stepstologlist = ['Run emerge step build', 'Run emerge step build_1']
+logs_texts = []
+for k, v in self.getProperty('build_log_steps_data').items():
+if v['name'] in stepstologlist:
+logs_texts.append(f">>> BEGINING OF STEP: {v['name']}")
+logs_texts.append('')
+log_data = v['log_data']
+log_text = yield
self.master.db.logs.getLogLines(log_data['id'], 1, log_data['num_lines'])
+for line in log_text.split('\n'):
+if line.startswith('h BUILDMASTER='):
+line = 'h BUILDMASTER=XXX.XXX.XXX.XXX'
+logs_texts.append(line[1:])
+logs_texts.append(f">>> END OF STEP: {v['name']}")
+yield WriteTextToFile(buildsteplogfile_path, logs_texts,
separator=True)
+return SUCCESS
+
class SetupBugReportSteps(BuildStep):
name = 'SetupBugReportSteps'
description = 'Running'
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 16f9c0fc440bb4c15742b7e02827fad64d73db82
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Mar 23 14:13:36 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Mar 23 14:13:36 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=16f9c0fc
Redo Trigger for parse_build_log in CheckEmergeLogs
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 180 +--
1 file changed, 87 insertions(+), 93 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index c033196..73e82f5 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -895,8 +895,10 @@ class CheckEmergeLogs(BuildStep):
'-v'
]
emerge_output = self.getProperty('emerge_output')
-self.faild_cpv = emerge_output['failed']
+print(f"emerge_output: {emerge_output}")
+self.setProperty('faild_cpv', emerge_output['failed'], 'faild_cpv')
self.faild_version_data = False
+self.log_data = {}
package_dict = emerge_output['packages']
#FIXME: Prosees the logs and do stuff
@@ -916,7 +918,7 @@ class CheckEmergeLogs(BuildStep):
print(emerge_output)
# this should be set in the config
retrays = 5
-if self.getProperty('rerun') <= retrays and self.faild_cpv:
+if self.getProperty('rerun') <= retrays and
emerge_output['failed']:
# when we need to change use. we could rerun pre-build with
# --autounmask-use=y --autounmask-write=y --autounmask-only=y
# but we use --binpkg--respect-use=y in EMERGE_DEFAULT_OPTS
@@ -997,26 +999,24 @@ class CheckEmergeLogs(BuildStep):
self.aftersteps_list.append(CheckEmergeLogs('extra-build'))
else:
# trigger parse_build_log with info about pre-build and it fail
-if self.faild_cpv:
+if emerge_output['failed']:
pass
self.setProperty('rerun', 0, 'rerun')
# Make Logfile dict
if self.step == 'extra-build' or self.step == 'build':
-print(emerge_output)
log_dict = {}
# get cpv, logname and log path
for log_path in emerge_output['log_paths']:
c = log_path.split('/')[6]
full_logname = log_path.split('/')[7]
-print(full_logname)
+print(f"full_logname: {full_logname}")
pv = full_logname.split(':')[0]
cpv = c + '/' + pv
log_dict[cpv] = dict(
log_path = log_path,
full_logname = full_logname
)
-print('log_dict')
-print(log_dict)
+print(f"log_dict: {log_dict}")
if self.step == 'extra-build':
#FIXME: Check if extra build did work
self.aftersteps_list.append(RunEmerge(step='pre-build'))
@@ -1028,94 +1028,88 @@ class CheckEmergeLogs(BuildStep):
# local_log_path dir set in config
# format
/var/cache/portage/logs/build/gui-libs/egl-wayland-1.1.6:20210321-173525.log.gz
rebuild = False
-if self.step == 'build':
+if self.step == 'build' and log_dict != {}:
retrays = 1
-# Find log for cpv that was requested or did failed
-if not log_dict == {}:
-print('log_dict not emty')
-# requested cpv
-cpv = self.getProperty('cpv')
-if cpv in log_dict or self.faild_cpv in log_dict:
-print(cpv)
-print(self.faild_cpv)
+if isinstance(emerge_output['failed'], str):
+log_cpv = emerge_output['failed']
+if self.getProperty('rerun') < retrays:
+# set rebuild
+# haskell
+if c == 'dev-haskell':
+rebuild = 'haskell'
+# binary
+for cpv, v in package_dict.items():
+if cpv == emerge_output['failed']:
+if v['binary']:
+rebuild = 'binary'
+print(f"rebuild is: {rebuild}")
+else:
+log_cpv = self.getProperty('cpv')
+if not rebuild:
+if log_cpv in log_dict:
+print(f"Package {log_cpv} in log_dict")
+self.log_data[log_cpv] = log_dict[log_cpv]
yield self.createDistDir()
-if cpv in log_dict:
-self.log_data[cpv] = log_dict[cpv]
-yield self.getLogFile(cpv, log_dict)
-
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 6171182755e1b643be9205edae0a81d284bdab99
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Mar 21 16:56:34 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Mar 21 16:56:34 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=61711827
Add bb log id for build steps
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 9 +++--
buildbot_gentoo_ci/steps/logs.py | 15 ++-
2 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 0ac8526..c033196 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -347,13 +347,19 @@ class SetupPropertys(BuildStep):
self.setProperty('rerun', 0, 'rerun')
project_build_data = self.getProperty('project_build_data')
project_build_data['status'] = 'in-progress'
-project_build_data['buildbot_build_id'] =
self.getProperty("buildnumber")
yield self.gentooci.db.builds.setStatusBuilds(
project_build_data['id'],
project_build_data['status'])
+builder_id = yield
self.master.db.builders.findBuilderId(self.getProperty("buildername"),
autoCreate=False)
+print(builder_id)
+build_data = yield self.master.db.builds.getBuildByNumber(builder_id
,self.getProperty("buildnumber"))
+print(build_data)
+project_build_data['buildbot_build_id'] = build_data['id']
yield self.gentooci.db.builds.setBuildbotBuildIdBuilds(
project_build_data['id'],
project_build_data['buildbot_build_id'])
+project_build_data['builder_id'] = builder_id
+project_build_data['buildnumber'] = self.getProperty("buildnumber")
self.setProperty('project_build_data', project_build_data,
'project_build_data')
print(self.getProperty("project_build_data"))
self.masterdest = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'), str(self.getProperty("buildnumber")))
@@ -1082,7 +1088,6 @@ class CheckEmergeLogs(BuildStep):
'pkg_check_log_data' :
self.getProperty("pkg_check_log_data"),
'repository_data' :
self.getProperty('repository_data'),
'faild_cpv' : self.faild_cpv,
-'step' : self.step,
'build_workername' :
self.getProperty('workername')
}
))
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index b6c97b7..896778c 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -104,8 +104,21 @@ class SetupPropertys(BuildStep):
self.setProperty("log_cpv", log_cpv, 'log_cpv')
self.setProperty("bgo", False, 'bgo')
self.descriptionDone = 'Runing log checker on ' + log_cpv
-logsdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildbot_build_id']))
+logsdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildnumber']))
self.setProperty("logsdir", logsdir, 'logsdir')
+# get steps/log id's from build
+build_log_steps_data = {}
+steps_data = yield
self.master.db.steps.getSteps(self.getProperty("project_build_data")['buildbot_build_id'])
+print(steps_data)
+for step in steps_data:
+logs_data = yield self.master.db.logs.getLogs(step['id'])
+print(logs_data)
+step_info = {}
+step_info['name'] = step['name']
+step_info['number'] = step['number']
+step_info['log_data'] = logs_data
+build_log_steps_data[step['id']] = step_info
+self.setProperty("build_log_steps_data", build_log_steps_data,
'build_log_steps_data')
return SUCCESS
class SetupParserBuildLoger(BuildStep):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 33713a909627a0378845d9d0be4f689c3857667b
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Mar 19 17:18:49 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Mar 19 17:18:49 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=33713a90
Add compile to word_list as Ago use it as default
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 64bd3e7..b6c97b7 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -210,6 +210,8 @@ class MakeIssue(BuildStep):
cp = c + '/' + p
word_list.append(cp)
word_list.append(cpv_getversion(self.error_dict['cpv']))
+# add compile as Ago use that as default
+word_list.append('compile')
for word in self.error_dict['title_phase'].split(' '):
word_list.append(word)
for word in self.error_dict['title_issue_nice'].split(' '):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 6ca6f7fb23f0d63bbb53a457c1bd149a9cd40795
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Mar 17 20:40:25 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Mar 17 20:40:25 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6ca6f7fb
Add more support for change use, rebuild faild bin and clean some --change-*
emerge options
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 49
1 file changed, 39 insertions(+), 10 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 09739d7..0ac8526 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -31,6 +31,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['masked'] = False
emerge_output['failed'] = False
package_dict = {}
+change_use = {}
log_path_list = []
print('stderr')
print(stderr)
@@ -99,7 +100,6 @@ def PersOutputOfEmerge(rc, stdout, stderr):
change_use_list = []
# get cpv
cpv_split = line_list[1].split(':')
-change_use = {}
# add use flags
if line_list[4].startswith('+') or line_list[4].startswith('-'):
# we only support tre for now
@@ -120,7 +120,21 @@ def PersOutputOfEmerge(rc, stdout, stderr):
change_use_list = False
if change_use_list:
change_use[cpv_split[0]] = change_use_list
-emerge_output['change_use'] = change_use
+if 'USE changes are' in line:
+change_use_list = []
+change_use = {}
+for text in stderr.split('\n'):
+if text.startswith('>='):
+line_list = text.split(' ')
+for line2 in text.split(' '):
+if line2.startswith('>='):
+cpv = line2.split(':')[0].replace('>=', '')
+else:
+change_use_list.append(line2)
+if change_use_list:
+change_use[cpv] = change_use_list
+if change_use != {}:
+emerge_output['change_use'] = change_use
if line.startswith(' * '):
if line.endswith('.log'):
log_path = line.split(' ')[3]
@@ -355,8 +369,9 @@ class RunEmerge(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, step=None, **kwargs):
+def __init__(self, step=None, exclude=False, **kwargs):
self.step = step
+self.exclude = exclude
super().__init__(**kwargs)
self.descriptionSuffix = self.step
self.name = 'Setup emerge for ' + self.step + ' step'
@@ -367,6 +382,7 @@ class RunEmerge(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
projects_emerge_options = yield
self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
+#FIXME: Use settings from gentoo-ci-settings json file
self.stepname = 'Run emerge ' + 'step ' + self.step
shell_commad_list = [
'emerge',
@@ -402,9 +418,9 @@ class RunEmerge(BuildStep):
self.setProperty('build_env', build_env, 'build_env')
if self.step == 'pre-update':
-shell_commad_list.append('-uDN')
-shell_commad_list.append('--changed-deps')
-shell_commad_list.append('--changed-use')
+shell_commad_list.append('--update')
+shell_commad_list.append('--deep')
+shell_commad_list.append('--newuse')
shell_commad_list.append('--pretend')
shell_commad_list.append('@world')
# don't build bin for virtual and acct-*
@@ -423,9 +439,9 @@ class RunEmerge(BuildStep):
aftersteps_list.append(CheckEmergeLogs('pre-update'))
if self.step == 'update':
-shell_commad_list.append('-uDNq')
-shell_commad_list.append('--changed-deps')
-shell_commad_list.append('--changed-use')
+shell_commad_list.append('--update')
+shell_commad_list.append('--deep')
+shell_commad_list.append('--newuse')
shell_commad_list.append('@world')
# don't build bin for virtual and acct-*
shell_commad_list.append('--buildpkg-exclude')
@@ -600,6 +616,10 @@ class RunEmerge(BuildStep):
# we don't use the bin for the requsted cpv
shell_commad_list.append('--usepkg-exclude')
shell_commad_list.append(cp)
+if self.exclude:
+# we don't use the bin for the exclude cpv
+shell_commad_list.append('--usepkg-exclude')
+
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: c37dbde9c1e6688aa7bc9095fe64c91717d9c97d
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Mar 16 08:49:05 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Mar 16 08:49:05 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c37dbde9
Use faild_version_data when faild to get repo bug settings
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 16
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index f214ad6..a261dac 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -500,12 +500,20 @@ class SetupBugReportSteps(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
bug_config = self.gentooci.config.project['bug_config']
-bug_settings_data = yield
self.gentooci.db.projects.getBugsSettingsByProjectUuid(self.getProperty("project_data")['uuid'])
-project_repository_settings = yield
self.gentooci.db.projects.getRepositorySettingByProjUuidAndRepoUuid(self.getProperty("project_data")['uuid'],
self.getProperty('repository_data')['uuid'])
-# add bug enable profile and repo
make_bugreport = False
+print(bug_config)
if bug_config['enable'] and self.getProperty('status') == 'failed' and
self.getProperty('error_dict')['title_found'] and not self.getProperty('bgo'):
-if bug_settings_data['enable'] and
project_repository_settings['bug']:
+# add bug enable profile, repo and project
+bug_settings_data = yield
self.gentooci.db.projects.getBugsSettingsByProjectUuid(self.getProperty("project_data")['uuid'])
+if self.getProperty('failed_version_data'):
+version_data = self.getProperty('failed_version_data')
+else:
+version_data = self.getProperty("version_data")
+package_data = yield
self.gentooci.db.packages.getPackageByUuid(version_data['package_uuid'])
+project_repository_settings = yield
self.gentooci.db.projects.getRepositorySettingByProjUuidAndRepoUuid(self.getProperty("project_data")['uuid'],
package_data['repository_uuid'])
+print(bug_settings_data)
+print(project_repository_settings)
+if bug_settings_data['enabled'] and
project_repository_settings['bug']:
make_bugreport = True
if make_bugreport:
aftersteps_list = []
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, sql/, ...
commit: 4269430378d05d5161d87056caf6817890d44493
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Feb 29 20:52:42 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Feb 29 20:52:42 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=42694303
Make emails dict for bugreports
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/schedulers.py | 5 +++--
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/db/versions.py | 6 --
buildbot_gentoo_ci/steps/logs.py| 26 +++---
buildbot_gentoo_ci/steps/version.py | 4 +++-
sql/gentoo_ci_schema.sql| 1 +
6 files changed, 35 insertions(+), 8 deletions(-)
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index c34d714..a4180cc 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -40,19 +40,20 @@ def getGitChanges(props):
change_data['timestamp'] = k['when_timestamp']
change_data['branch'] = k['branch']
change_data['project'] = k['project']
+change_data['change_id'] = k['changeid']
return change_data
def bb_branch_fn(branch):
# check branch
print(f"Branch: {branch}")
-if branch.endswith('-mr') or branch.endswith('-pr') or branch == 'master':
+if branch.endswith('-mr') or branch.endswith('-pr') or
branch.endswith('master') or branch.endswith('main'):
return True
return False
def bb_category_fn(category):
# check event
print(f"Category: {category}")
-if category == 'push' or category == 'merge_request':
+if category == 'push' or category == 'merge_request' or category is None:
return True
return False
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index b6a0c43..dea3e15 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -348,6 +348,7 @@ class Model(base.DBConnectorComponent):
nullable=False),
sa.Column('file_hash', sa.String(255), nullable=False),
sa.Column('commit_id', sa.String(255), nullable=False),
+sa.Column('change_id', sa.Integer, nullable=True, default=0),
sa.Column('deleted', sa.Boolean, default=False),
sa.Column('deleted_at', sa.Integer, nullable=True),
)
diff --git a/buildbot_gentoo_ci/db/versions.py
b/buildbot_gentoo_ci/db/versions.py
index 0aaac1b..8d35602 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -55,7 +55,7 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
return res
@defer.inlineCallbacks
-def addVersion(self, name, package_uuid, file_hash, commit_id):
+def addVersion(self, name, package_uuid, file_hash, commit_id, change_id):
def thd(conn, no_recurse=False):
try:
tbl = self.db.model.versions
@@ -63,7 +63,8 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
r = conn.execute(q, dict(name=name,
package_uuid=package_uuid,
file_hash=file_hash,
- commit_id=commit_id))
+ commit_id=commit_id,
+ change_id=change_id))
except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
uuid = None
else:
@@ -178,6 +179,7 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
package_uuid=row.package_uuid,
file_hash=row.file_hash,
commit_id=row.commit_id,
+change_id = row.change_id,
deleted=row.deleted,
deleted_at=row.deleted_at
)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index aacb8a5..34aee29 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -250,6 +250,7 @@ class MakeIssue(BuildStep):
self.summary_log_list = []
self.error_dict = {}
self.aftersteps_list = []
+Maintainers = []
#self.error_dict['hash'] = hashlib.sha256()
for k, v in sorted(self.summary_log_dict.items()):
self.summary_log_list.append(v['text'])
@@ -263,11 +264,30 @@ class MakeIssue(BuildStep):
self.error_dict['phase'] = v['text'].split(' (')[1].split('
phase')[0]
error = True
if v['text'].startswith(' * Maintainer:'):
-Maintainers = []
for email in v['text'].split(':')[1].split(' '):
if email != '':
-Maintainers.append(email)
-self.setProperty("Maintainers", Maintainers, 'Maintainers')
+for e in email.split(','):
+
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, py/
commit: 92b796f7c2580f9bd93a225072d56cbf8f13deaa
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Feb 22 23:58:06 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Feb 22 23:58:06 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=92b796f7
Fix phase_error and pattern_infos
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 14 ++
py/log_parser.py | 4 ++--
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 50e3f3f..aacb8a5 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -46,9 +46,10 @@ def PersOutputOfLogParser(rc, stdout, stderr):
'status' :
s['status'],
'line' : s['line'],
'search_pattern' :
s['search_pattern'],
-'phase' :
s['phase'],
+'phase' :
s['phase'].lower(),
'uuid' : s['uuid'],
'description' :
s['description'],
+'url' : s['url'],
})
build_summery_output['summary_log_dict'] = summary_log_dict
#FIXME: Handling of stderr output
@@ -221,7 +222,7 @@ class MakeIssue(BuildStep):
for k, v in sorted(self.summary_log_dict.items()):
# get the issue error
for s in v['pattern_infos']:
-if s['search_type'] == self.error_dict['phase'] and
s['status'] == 'error':
+if s['phase'] == self.error_dict['phase'] and s['status'] ==
'error':
text_issue_list.append(v['text'])
# add the issue error
self.error_dict['cpv'] = self.getProperty('log_cpv')
@@ -259,9 +260,14 @@ class MakeIssue(BuildStep):
# check if the build did fail
if v['text'].startswith(' * ERROR:') and v['text'].endswith('
phase):'):
# get phase error
-phase_error = v['text'].split(' (')[1].split(' phase')[0]
-self.error_dict['phase'] = phase_error
+self.error_dict['phase'] = v['text'].split(' (')[1].split('
phase')[0]
error = True
+if v['text'].startswith(' * Maintainer:'):
+Maintainers = []
+for email in v['text'].split(':')[1].split(' '):
+if email != '':
+Maintainers.append(email)
+self.setProperty("Maintainers", Maintainers, 'Maintainers')
#FIXME: write summary_log_list to a file
# add issue/bug/pr report
if error:
diff --git a/py/log_parser.py b/py/log_parser.py
index e3725bb..e8f319b 100644
--- a/py/log_parser.py
+++ b/py/log_parser.py
@@ -28,8 +28,8 @@ def get_pattern_dict(catchissue, i, uuid):
patten_dict['end'] = catchissue[2]
patten_dict['status'] = catchissue[3]
patten_dict['search_type'] = catchissue[4]
-patten_dict['url'] = catchissue[5]
-patten_dict['description'] = catchissue[6]
+patten_dict['url'] = catchissue[6]
+patten_dict['description'] = catchissue[7]
return patten_dict
def addPatternToList(phaseList, log_search_patterns, uuid):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, py/
commit: 6d8986da642b5d3b5670be40d8fe8883086c7678
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Feb 21 21:15:49 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Feb 21 21:15:49 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6d8986da
Move search pattern db to settings repository
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 34 ---
py/log_parser.py | 206 ---
2 files changed, 127 insertions(+), 113 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 85017a5..50e3f3f 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -42,10 +42,13 @@ def PersOutputOfLogParser(rc, stdout, stderr):
}
for s in v['pattern_info']:
summary_log_dict[int(k)]['pattern_infos'].append({
-'type' : s['type'],
+'search_type' :
s['search_type'],
'status' :
s['status'],
-'id' : s['id'],
+'line' : s['line'],
'search_pattern' :
s['search_pattern'],
+'phase' :
s['phase'],
+'uuid' : s['uuid'],
+'description' :
s['description'],
})
build_summery_output['summary_log_dict'] = summary_log_dict
#FIXME: Handling of stderr output
@@ -118,14 +121,23 @@ class SetupParserBuildLoger(BuildStep):
@defer.inlineCallbacks
def run(self):
+self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.aftersteps_list = []
log_cpv =
self.getProperty('log_build_data')[self.getProperty('log_cpv')]
build_log_file_compressed = log_cpv['full_logname'] + '.xz'
mastersrc_log = yield os.path.join(self.getProperty('logsdir'),
build_log_file_compressed)
log_py = 'log_parser.py'
-config_log_py = 'logparser.json'
mastersrc_py = yield os.path.join(self.master.basedir, log_py)
-mastersrc_config = yield os.path.join(self.master.basedir,
config_log_py)
+repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty("project_data")['settings_repository_uuid'])
+# Git clone settings repo
+
self.aftersteps_list.append(steps.GitLab(repourl=repository_data['url'],
+name = 'RunGit',
+descriptionDone=repository_data['name'],
+mode=repository_data['mode'],
+method=repository_data['method'],
+submodules=True,
+alwaysUseLatest=repository_data['alwaysuselatest']
+))
# Upload logfile to worker
self.aftersteps_list.append(steps.FileDownload(
mastersrc=mastersrc_log,
@@ -136,11 +148,6 @@ class SetupParserBuildLoger(BuildStep):
mastersrc=mastersrc_py,
workerdest=log_py
))
-# Upload log parser py config
-self.aftersteps_list.append(steps.FileDownload(
-mastersrc=mastersrc_config,
-workerdest=config_log_py
-))
#Untar the log
shell_commad_list = []
shell_commad_list.append('xz')
@@ -159,8 +166,11 @@ class SetupParserBuildLoger(BuildStep):
command.append(log_cpv['full_logname'])
command.append('-u')
command.append(self.getProperty('project_data')['uuid'])
-command.append('-d')
-command.append(util.Secret("log_parser_database"))
+command.append('-c')
+#FIXME: set it by images/flavors
+command.append('8')
+#FIXME: debug
+#command.append('-d')
self.aftersteps_list.append(steps.SetPropertyFromCommand(
name =
'RunBuildLogParser',
haltOnFailure =
True,
@@ -211,7 +221,7 @@ class MakeIssue(BuildStep):
for k, v in sorted(self.summary_log_dict.items()):
# get the issue error
for s in v['pattern_infos']:
-if s['type'] == self.error_dict['phase'] and
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 7f25300ddcf94e1d5018e9bc343a22155e3b043c
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Feb 5 18:54:20 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Feb 5 18:54:20 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7f25300d
Use settings repository for dockerfiles
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/nodes.py | 34 ++
1 file changed, 14 insertions(+), 20 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/nodes.py
b/buildbot_gentoo_ci/steps/nodes.py
index 7a01cbb..c1928ba 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -77,30 +77,26 @@ class SetupBuildStepsForDocker(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
node_data = self.getProperty("node_data")
image_data = self.getProperty("image_data")
-dockerfile = self.getProperty("image_data")['dockerfile'] +
'.Dockerfile'
+dockerfile = os.path.join('Project',
self.getProperty("project_data")['uuid'],'docker', image_data['dockerfile'] +
'.Dockerfile')
+repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty("project_data")['settings_repository_uuid'])
dockerfile_path = os.path.join(self.master.basedir, 'docker')
#FIXME: Set build timeout in config
self.build_timeout = 6600
aftersteps_list = []
-aftersteps_list.append(steps.FileDownload(
-flunkOnFailure=True,
-name='Upload Dockerfile',
-mastersrc = os.path.join(dockerfile_path, dockerfile),
-workerdest = dockerfile
-))
+aftersteps_list.append(steps.GitLab(repourl=repository_data['url'],
+name = 'RunGit',
+descriptionDone=repository_data['name'],
+mode=repository_data['mode'],
+method=repository_data['method'],
+submodules=True,
+alwaysUseLatest=repository_data['alwaysuselatest']
+))
# build the gentoo docker buildbot-worker image
# set the build args
self.buildargs = {}
-# set PROJECTUUID
-self.buildargs['PROJECTUUID'] = self.getProperty("project_uuid")
-# set ACCEPT_KEYWORDS
-keyword_data = yield
self.gentooci.db.keywords.getKeywordById(self.getProperty('project_data')['keyword_id'])
-if self.getProperty('project_data')['status'] == 'unstable':
-self.buildargs['KEYWORDS'] = '~' + keyword_data['name']
-else:
-self.buildargs['KEYWORDS'] = keyword_data['name']
-# set ARCH
-self.buildargs['ARCH'] = keyword_data['name']
+# set PROJECTUUID and PROJECTNAME
+self.buildargs['PROJECTUUID'] =
self.getProperty("project_data")['uuid']
+self.buildargs['PROJECTNAME'] =
self.getProperty("project_data")['name']
# set MAKEOPTS
# set USE
# set CFLAGS
@@ -132,12 +128,10 @@ class SetupBuildStepsForDocker(BuildStep):
flunkOnFailure=True,
name='Build docker buildbot-worker image',
command=['docker',
- 'buildx',
'build',
'--file', dockerfile,
'--build-arg', 'PROJECTUUID='+
self.buildargs['PROJECTUUID'],
- '--build-arg', 'KEYWORDS=' +
self.buildargs['KEYWORDS'],
- '--build-arg', 'ARCH=' +
self.buildargs['ARCH'],
+ '--build-arg', 'PROJECTNAME='+
self.buildargs['PROJECTNAME'],
'--build-arg', 'BINHOSTURL=' +
self.buildargs['BINHOSTURL'],
'--build-arg', 'BOOTSTRAPTAG=' +
self.buildargs['BOOTSTRAPTAG'],
'--tag', image_data['tag'],
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 34ce6c4b4c8420b77870ef157c60984dc5ae46ce
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Feb 4 11:32:24 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Feb 4 11:32:24 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=34ce6c4b
Use project_data for profile link
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 775e076..f90da5a 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -92,12 +92,12 @@ class SetMakeProfile(BuildStep):
portage_repos_path = self.getProperty('portage_repos_path')
project_data = self.getProperty('project_data')
profile_repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
-makeprofiles_paths = []
+#makeprofiles_paths = []
#NOTE: pkgcheck don't support make.profile as a dir
# we only support one line in db
-makeprofiles_data = yield
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'],
'make.profile')
-for makeprofile in makeprofiles_data:
-makeprofile_path = yield os.path.join(portage_repos_path,
profile_repository_data['name'], 'profiles', makeprofile['value'], '')
+#makeprofiles_data = yield
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'],
'make.profile')
+#for makeprofile in makeprofiles_data:
+makeprofile_path = yield os.path.join(portage_repos_path,
profile_repository_data['name'], 'profiles', project_data['profile'], '')
#makeprofiles_paths.append('../../..' + makeprofile_path)
#separator = '\n'
#makeprofile_path_string = separator.join(makeprofiles_paths)
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: aebeb68573134f1f1879b7513d5051e562dd3b1c
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Feb 4 09:49:24 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Feb 4 09:49:24 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aebeb685
Use flavor to set MAKEOPTS -j*
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 74a8e37..775e076 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -228,7 +228,12 @@ class SetHostConf(BuildStep):
separator2 = ' '
makeconf_list = []
log = yield self.addLog('host.conf')
-makeconf_list.append('MAKEOPTS="-j14"')
+# set MAKEOPTS from flavor
+worker_data = yield
self.gentooci.db.workers.getWorkerByUuid(self.getProperty('workername'))
+print('worker_data')
+worker_flavor = yield
self.gentooci.db.workers.getFlavorById(worker_data['flavor_id'])
+print('worker_flavor')
+makeconf_list.append('MAKEOPTS="-j' + str(worker_flavor['cpu']) + '"')
makeconf_string = separator1.join(makeconf_list)
print(makeconf_string)
yield self.build.addStepsAfterCurrentStep([
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: eaf908437029fb291b327db680d550fbe36c86df Author: Magnus Granberg gentoo org> AuthorDate: Sat Feb 3 09:48:17 2024 + Commit: Magnus Granberg gentoo org> CommitDate: Sat Feb 3 09:48:17 2024 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=eaf90843 Remove minio Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/minio.py | 60 --- 1 file changed, 60 deletions(-) diff --git a/buildbot_gentoo_ci/steps/minio.py b/buildbot_gentoo_ci/steps/minio.py deleted file mode 100644 index ac9ac3d..000 --- a/buildbot_gentoo_ci/steps/minio.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -from minio import Minio -from minio.error import ResponseError, BucketAlreadyOwnedByYou, BucketAlreadyExists - -from twisted.internet import defer -from twisted.python import log - -from buildbot.process.buildstep import BuildStep -from buildbot.process.results import SUCCESS -from buildbot.process.results import FAILURE - -#FIXME: -# get url, user from config -# get password from secret -url = '' -user = '' -password = '' - -class putFileToMinio(BuildStep): - -name = 'putFileToMinio' -description = 'Running' -descriptionDone = 'Ran' -descriptionSuffix = None -haltOnFailure = False -flunkOnFailure = True -warnOnWarnings = True - -def __init__(self, filename, target, bucket, **kwargs): -self.filename = filename -self.bucket = bucket -self.target = target -super().__init__(**kwargs) - -def getMinioConnect(self, url, user, password): -minioclient = Minio( -url, -access_key = user, -secret_key = password, -secure = False -) -return minioclient - -@defer.inlineCallbacks -def pushFileToMinio(self): -try: -yield self.minio_connect.fput_object(self.bucket, self.target, self.filename) -except ResponseError as err: -print(err) -return FAILURE -return True - -@defer.inlineCallbacks -def run(self): -self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] -self.minio_connect = yield self.getMinioConnect(url, user, password) -success = yield self.pushFileToMinio() -return SUCCESS
[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/steps/
commit: 684420873cc8bd4319c4379a78a76be25abe1e15
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Oct 28 07:03:48 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Oct 28 07:03:48 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=68442087
Match min words
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 34 --
gentooci.cfg | 3 ++-
2 files changed, 18 insertions(+), 19 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 4d958b7..6a097d4 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -16,6 +16,8 @@ from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
from buildbot.plugins import util
+from buildbot_gentoo_ci.utils.regex import stripQuotesAndMore, finishTitle
+
from bugz.cli import check_bugz_token, login, list_bugs
from bugz.cli_argparser import make_arg_parser
from bugz.configfile import load_config
@@ -120,28 +122,23 @@ class GetBugs(BuildStep):
yield log.addStdout(f"Bug: {str(bug['id'])} Summary:
{bug['summary']}\n")
# we splite the lines to lists and try to match the words
matches = 0
-match_search_text =
list(self.getProperty('error_dict')['title_issue'].split())
-match_bug_text = list(bug['summary'].split())
-#FIXME: add check for cp
-for match_word in match_search_text:
-if match_word in match_bug_text:
-matches = matches + 1
-# try to match the nice words
-matches_nice = 0
-match_search_text_nice =
list(self.getProperty('error_dict')['title_issue_nice'].split())
-#FIXME: add check for cp
-for match_word in match_search_text_nice:
-if match_word in match_bug_text:
-matches_nice = matches_nice + 1
-print(f"Bug: {str(bug['id'])} Matched words: {str(matches)}
{str(matches_nice)} Summary: {bug['summary']}")
-#FIXME: set it in bug_config
-if matches >= 6 or matches_nice >= 6:
+bug_words = list(bug['summary'].split())
+print(bug_words)
+print(self.getProperty('error_dict')['words'])
+for word in bug_words:
+# clean the word
+word = finishTitle(stripQuotesAndMore(word))
+print(word)
+for match_word in self.getProperty('error_dict')['words']:
+if match_word in word:
+matches = matches + 1
+print(f"Bug: {str(bug['id'])} Matched words: {str(matches)}
Summary: {bug['summary']}")
+if matches >= len(self.getProperty('error_dict')['title_nice']) -
self.bug_config['min_match_word']:
match = {}
match['match'] = True
match['id'] = bug['id']
match['summary'] = bug['summary']
yield log.addStdout(f"Line to match:
{self.getProperty('error_dict')['title_issue']}\n")
-yield log.addStdout(f"Line to match:
{self.getProperty('error_dict')['title_issue_nice']}\n")
if match:
yield log.addStdout('Match bug: YES\n')
yield log.addStdout(f"Bug: {str(match['id'])} Summary:
{match['summary']}\n")
@@ -152,7 +149,8 @@ class GetBugs(BuildStep):
@defer.inlineCallbacks
def run(self):
-# self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+self.bug_config = self.gentooci.config.project['bug_config']
cpv = self.getProperty('error_dict')['cpv']
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
diff --git a/gentooci.cfg b/gentooci.cfg
index 474da76..4aecf35 100644
--- a/gentooci.cfg
+++ b/gentooci.cfg
@@ -49,9 +49,10 @@ c['worker_config'] = {
# This specifies bug configs
c['bug_config'] = {
'enable' : True,
-'aout_assignee' : False,
+'auto_assignee' : False,
'user' : 'larrythe...@cow.mu',
'extra_summery' : '',
+'min_match_word' : 2,
'buildbot_comment' : [
'This was build on a tinderbox that use Buildbot API[1].',
'We use docker images to build on.',
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/utils/
commit: d35e389caa0fe40ffaf6ec8e6e460148716ed092
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Sep 21 22:22:00 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Sep 21 22:22:00 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d35e389c
Move nice regex's to a file
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 38 +++---
buildbot_gentoo_ci/utils/regex.py | 22 ++
2 files changed, 45 insertions(+), 15 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index d71eb99..85017a5 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -25,6 +25,7 @@ from buildbot.plugins import util
#from buildbot_gentoo_ci.steps import minio
from buildbot_gentoo_ci.steps import master as master_steps
from buildbot_gentoo_ci.steps import bugs
+from buildbot_gentoo_ci.utils.regex import stripQuotesAndMore, finishTitle
def PersOutputOfLogParser(rc, stdout, stderr):
build_summery_output = {}
@@ -185,17 +186,23 @@ class MakeIssue(BuildStep):
super().__init__(**kwargs)
def getNiceErrorLine(self, full_line):
-# strip away hex addresses, loong path names, line and time numbers
and other stuff
-# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L469
-# FIXME: Add the needed line when needed
-new_line = []
-for line in full_line.split(' '):
-# Shorten the path
-if line.startswith('/usr/') or line.startswith('/var/') or
line.startswith('../'):
-split_path_line = os.path.split(line)
-line = line.replace(split_path_line[0], '...')
-new_line.append(line)
-return ' '.join(new_line)
+new_words = []
+for word in full_line.split(' '):
+new_words.append(finishTitle(stripQuotesAndMore(word)))
+return ' '.join(new_words)
+
+def BuildWordList(self):
+word_list = []
+word_list.append(self.error_dict['cpv'])
+c = catpkgsplit(self.error_dict['cpv'])[0]
+p = catpkgsplit(self.error_dict['cpv'])[1]
+cp = c + '/' + p
+word_list.append(cp)
+for word in self.error_dict['title_phase'].split(' '):
+word_list.append(word)
+for word in self.error_dict['title_issue_nice'].split(' '):
+word_list.append(word)
+return word_list
def ClassifyIssue(self):
# get the title for the issue
@@ -217,10 +224,11 @@ class MakeIssue(BuildStep):
self.error_dict['title_issue'] = 'title_issue : None'
self.error_dict['title_issue_nice'] = 'title_issue_nice : None'
self.error_dict['title_found'] = False
-self.error_dict['title_phase'] = 'fails to '+ self.error_dict['phase']
+ ':'
+self.error_dict['title_phase'] = 'fails to '+ self.error_dict['phase']
#set the error title
-self.error_dict['title'] = ' '.join([self.error_dict['title_phase'],
self.error_dict['title_issue']])
-self.error_dict['title_nice'] = '
'.join([self.error_dict['title_phase'], self.error_dict['title_issue_nice']])
+self.error_dict['title'] = ' '.join([self.error_dict['title_phase'] +
':', self.error_dict['title_issue']])
+self.error_dict['title_nice'] = '
'.join([self.error_dict['title_phase'] + ':',
self.error_dict['title_issue_nice']])
+self.error_dict['words'] = self.BuildWordList()
@defer.inlineCallbacks
def run(self):
@@ -247,7 +255,7 @@ class MakeIssue(BuildStep):
#FIXME: write summary_log_list to a file
# add issue/bug/pr report
if error:
-yield self.ClassifyIssue()
+self.ClassifyIssue()
print(self.error_dict)
self.setProperty("status", 'failed', 'status')
self.setProperty("error_dict", self.error_dict, 'error_dict')
diff --git a/buildbot_gentoo_ci/utils/regex.py
b/buildbot_gentoo_ci/utils/regex.py
new file mode 100644
index 000..29372e0
--- /dev/null
+++ b/buildbot_gentoo_ci/utils/regex.py
@@ -0,0 +1,22 @@
+# Copyright 2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import re
+
+# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L12
+def stripQuotesAndMore(word):
+word = re.sub(r"b'", "", word)
+word = re.sub(r"'", "", word)
+word = re.sub(r'`', '', word)
+word = re.sub(r'"', '', word)
+word = re.sub(r'\\', '', word)
+return word
+
+# strip away hex addresses, loong path names, line and time numbers and other
stuff
+# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L469
+# FIXME: Add the needed line when needed
+def finishTitle(word):
+if word.startswith('/'):
+word = word.split('/')[-1]
+word = re.sub(":\d+:\d+:", "", word)
+return word
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: db8e966eef3ede0212dcf85d6d5b82c28d3dfcd5
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 26 09:21:16 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 26 09:21:16 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=db8e966e
Check issue titel nice
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 13 +++--
buildbot_gentoo_ci/steps/logs.py | 31 +++
2 files changed, 26 insertions(+), 18 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index ad87be9..4d958b7 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -126,13 +126,22 @@ class GetBugs(BuildStep):
for match_word in match_search_text:
if match_word in match_bug_text:
matches = matches + 1
-print(f"Bug: {str(bug['id'])} Matched words: {str(matches)}
Summary: {bug['summary']}")
-if matches >= 5:
+# try to match the nice words
+matches_nice = 0
+match_search_text_nice =
list(self.getProperty('error_dict')['title_issue_nice'].split())
+#FIXME: add check for cp
+for match_word in match_search_text_nice:
+if match_word in match_bug_text:
+matches_nice = matches_nice + 1
+print(f"Bug: {str(bug['id'])} Matched words: {str(matches)}
{str(matches_nice)} Summary: {bug['summary']}")
+#FIXME: set it in bug_config
+if matches >= 6 or matches_nice >= 6:
match = {}
match['match'] = True
match['id'] = bug['id']
match['summary'] = bug['summary']
yield log.addStdout(f"Line to match:
{self.getProperty('error_dict')['title_issue']}\n")
+yield log.addStdout(f"Line to match:
{self.getProperty('error_dict')['title_issue_nice']}\n")
if match:
yield log.addStdout('Match bug: YES\n')
yield log.addStdout(f"Bug: {str(match['id'])} Summary:
{match['summary']}\n")
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 47112e0..d71eb99 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -184,18 +184,18 @@ class MakeIssue(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
-def getNiceErrorLine(self, line):
+def getNiceErrorLine(self, full_line):
# strip away hex addresses, loong path names, line and time numbers
and other stuff
-# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L467
+# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L469
# FIXME: Add the needed line when needed
-if re.search(': line', line):
-line = re.sub(r"\d", "", line)
-# Shorten the path
-if line.startswith('/usr/'):
-line = line.replace(os.path.split(line.split(' ', 1)[0])[0],
'/...')
-if re.search(': \d:\d: ', line):
-line = re.sub(r":\d:\d: ", ": ", line)
-return line
+new_line = []
+for line in full_line.split(' '):
+# Shorten the path
+if line.startswith('/usr/') or line.startswith('/var/') or
line.startswith('../'):
+split_path_line = os.path.split(line)
+line = line.replace(split_path_line[0], '...')
+new_line.append(line)
+return ' '.join(new_line)
def ClassifyIssue(self):
# get the title for the issue
@@ -215,11 +215,12 @@ class MakeIssue(BuildStep):
self.error_dict['title_found'] = True
else:
self.error_dict['title_issue'] = 'title_issue : None'
-self.error_dict['title_nice'] = 'title_issue : None'
+self.error_dict['title_issue_nice'] = 'title_issue_nice : None'
self.error_dict['title_found'] = False
-self.error_dict['title_phase'] = 'failed in '+ self.error_dict['phase']
+self.error_dict['title_phase'] = 'fails to '+ self.error_dict['phase']
+ ':'
#set the error title
-self.error_dict['title'] = self.error_dict['title_phase'] + ' - ' +
self.error_dict['title_issue']
+self.error_dict['title'] = ' '.join([self.error_dict['title_phase'],
self.error_dict['title_issue']])
+self.error_dict['title_nice'] = '
'.join([self.error_dict['title_phase'], self.error_dict['title_issue_nice']])
@defer.inlineCallbacks
def run(self):
@@ -249,8 +250,6 @@ class MakeIssue(BuildStep):
yield self.ClassifyIssue()
print(self.error_dict)
self.setProperty("status", 'failed', 'status')
-#FIXME: nice description for post bug
-self.error_dict['summary_log_nice'] = ''
self.setProperty("error_dict", self.error_dict,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: f664663c8bebcd6700c90e2ca445d82bdf9b8432
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Jun 10 09:53:48 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Jun 10 09:53:48 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f664663c
Add support for Post, Modify and Attach bugs
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 178 +--
1 file changed, 173 insertions(+), 5 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 6c85aac..ad87be9 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -1,7 +1,12 @@
-# Copyright 2022 Gentoo Authors
+# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+# Origins: bugz.cli
+# Modifyed by Gentoo Authors.
+
import re
+import os
+import xmlrpc
from twisted.internet import defer
@@ -9,6 +14,7 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
+from buildbot.plugins import util
from bugz.cli import check_bugz_token, login, list_bugs
from bugz.cli_argparser import make_arg_parser
@@ -19,8 +25,6 @@ from bugz.log import log_error, log_info
from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
-# Origins: bugz.cli
-# Modifyed by Gentoo Authors.
# main
def main_bugz(args):
ArgParser = make_arg_parser()
@@ -69,6 +73,32 @@ def search_bugz(args):
list_bugs(result, settings)
return result
+# post
+def post_bugs(args, params):
+"""Post a new bug"""
+settings = main_bugz(args)
+result = settings.call_bz(settings.bz.Bug.create, params)
+log_info('Bug %d submitted' % result['id'])
+return result
+
+# modify
+def modify_bugs(args, params):
+"""Modify an existing bug (eg. adding a comment or changing resolution.)"""
+settings = main_bugz(args)
+if len(params) < 2:
+raise BugzError('No changes were specified')
+result = settings.call_bz(settings.bz.Bug.update, params)
+return result
+
+def attach_bugs(args, params):
+""" Attach a file to a bug given a filename. """
+settings = main_bugz(args)
+fd = open(params['filename'], 'rb')
+params['data'] = xmlrpc.client.Binary(fd.read())
+fd.close()
+result = settings.call_bz(settings.bz.Bug.add_attachment, params)
+return result
+
class GetBugs(BuildStep):
name = 'GetBugs'
@@ -107,8 +137,9 @@ class GetBugs(BuildStep):
yield log.addStdout('Match bug: YES\n')
yield log.addStdout(f"Bug: {str(match['id'])} Summary:
{match['summary']}\n")
self.setProperty("bgo", match, 'bgo')
-return
+return match
yield log.addStdout('Match bug: NO\n')
+return match
@defer.inlineCallbacks
def run(self):
@@ -128,5 +159,142 @@ class GetBugs(BuildStep):
print(args)
buglist = search_bugz(args)
print(buglist)
-self.find_match(buglist)
+match = self.find_match(buglist)
+#FIXME: set bug id on build in db
+return SUCCESS
+
+class Post(BuildStep):
+
+name = 'Post bug'
+description = 'Running'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, bug_args, bugs_params, **kwargs):
+self.bug_args = bug_args
+self.bugs_params = bugs_params
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+#self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+#bug_config = self.gentooci.config.project['bug_config']
+args = []
+args.append('-u')
+args.append(self.bug_args['user'])
+args.append('-p')
+args.append(self.bug_args['passwd'])
+args.append('post')
+params = {}
+params['product'] = "Gentoo Linux"
+params['component'] = "Current packages"
+params['version'] = "unspecified"
+params['op_sys'] = "Linux"
+params['platform'] = "All"
+params['priority'] = "Normal"
+params['description'] = self.bugs_params['description']
+params['summary'] = self.bugs_params['summary']
+params['assigned_to'] = self.bugs_params['assigned_to']
+#params['cc'] = settings.cc
+params['url'] = self.bugs_params['url']
+bug_info = yield post_bugs(args, params)
+match = {}
+match['match'] = True
+match['id'] = bug_info['id']
+match['summary'] = self.bugs_params['summary']
+self.setProperty("bgo", match, 'bgo')
+print(match)
+self.descriptionDone = f"Bug: {bug_info['id']} submitted"
+#FIXME: update build with bug id
+return SUCCESS
+
+class
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 89be950457b1ffebe2ba2a27e0b6dc7f455e3115
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Jun 4 20:50:42 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Jun 4 20:50:42 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=89be9504
Add fetchonly step but not on yet
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 30 ++
1 file changed, 30 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 8692813..a5638b5 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -564,6 +564,34 @@ class RunEmerge(BuildStep):
))
aftersteps_list.append(CheckEmergeLogs('pre-build'))
+if self.step == 'fetchonly':
+if projects_emerge_options['oneshot']:
+shell_commad_list.append('-1')
+shell_commad_list.append('-f')
+#shell_commad_list.append('--quiet=n')
+shell_commad_list.append('=' + self.getProperty('cpv'))
+# we don't use the bin for the requsted cpv
+shell_commad_list.append('--usepkg-exclude')
+shell_commad_list.append(cp)
+# don't build bin for virtual and acct-*
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('virtual')
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('acct-*')
+aftersteps_list.append(
+steps.ShellCommand(
+warnOnWarnings = True,
+warnOnFailure = True,
+flunkOnFailure = False,
+flunkOnWarnings = False,
+name = self.stepname,
+command=shell_commad_list,
+#strip=True,
+#extract_fn=PersOutputOfEmerge,
+workdir='/',
+timeout=self.build_timeout
+))
+
if self.step == 'build':
if projects_emerge_options['oneshot']:
shell_commad_list.append('-1')
@@ -787,6 +815,7 @@ class CheckEmergeLogs(BuildStep):
self.addFileUploade(sourcefile, destfile, name, url, urlText)
# get elogs
self.getElogFiles(cpv)
+#FIXME: add etc/portage dir to upload
@defer.inlineCallbacks
def getBuildWorkDirs(self, cpv):
@@ -1223,6 +1252,7 @@ class RunBuild(BuildStep):
aftersteps_list = []
aftersteps_list.append(RunEmerge(step='pre-build'))
aftersteps_list.append(RunEmergeInfo())
+#aftersteps_list.append(RunEmerge(step='fetchonly'))
aftersteps_list.append(RunEmerge(step='build'))
aftersteps_list.append(RunEmerge(step='pre-depclean'))
aftersteps_list.append(RunEmerge(step='preserved-libs'))
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 33a7f172aa06d773f11d2916130a48d16760c7d1
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Jun 4 20:57:28 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Jun 4 20:57:28 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=33a7f172
Support splited lines in auxdb output
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/version.py | 53 +
1 file changed, 24 insertions(+), 29 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index c6f333e..98e5403 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -26,11 +26,9 @@ def PersOutputOfGetEapi(rc, stdout, stderr):
# split the lines
for line in stdout.split('\n'):
if line.startswith('EAPI'):
-print(line[-1])
m = _pms_eapi_re.match(line)
if m is not None:
eapi = m.group(2)
-print(eapi)
if eapi is None or not eapi_is_supported(eapi):
print('ERROR: invalid eapi or not found')
eapi = False
@@ -41,37 +39,34 @@ def PersOutputOfGetEapi(rc, stdout, stderr):
}
def PersOutputOfGetAuxdb(rc, stdout, stderr):
-metadata = None
NoSplit = ['DESCRIPTION']
ignore_list = ['SRC_URI']
-#make dict of the stout
-index = 1
-metadata_line_dict = {}
-for text_line in stdout.splitlines():
-metadata_line_dict[index] = text_line
-index = index + 1
+metadata = {}
+i = 0
if not stderr == '':
print('stderr')
-# should have 22 lines
-if len(auxdbkeys) != index -1:
-# number of lines is incorrect.
-print('ERROR: Number of lines is incorrect')
-print(metadata_line_dict)
-return {
-'auxdb' : metadata
-}
-# split all keys to list instead of speces
-metadata = {}
-i = 1
-for key in auxdbkeys:
-if metadata_line_dict[i][-1] == '=' or key in ignore_list:
-metadata[key] = False
-else:
-if ' ' in metadata_line_dict[i] and key not in NoSplit:
-metadata[key] = metadata_line_dict[i].replace(key + '=',
'').split(' ')
-else:
-metadata[key] = []
-metadata[key].append(metadata_line_dict[i].replace(key + '=',
''))
+# make a dict of metadata
+metadata_line_list = stdout.splitlines()
+index = len(metadata_line_list)
+for metadata_line in metadata_line_list:
+print(metadata_line)
+for key in auxdbkeys:
+if metadata_line.startswith(key) and key not in ignore_list:
+print(key)
+if metadata_line[-1] == '=':
+metadata[key] = False
+else:
+if ' ' in metadata_line and key not in NoSplit:
+if i+1 == index or
metadata_line_list[i+1].startswith(auxdbkeys):
+metadata[key] = metadata_line.replace(key + '=',
'').split(' ')
+else:
+# support one splited line
+print(metadata_line_list[i+1])
+metadata_line2 = metadata_line +
metadata_line_list[i+1]
+metadata[key] = metadata_line2.replace(key + '=',
'').split(' ')
+else:
+metadata[key] = []
+metadata[key].append(metadata_line.replace(key + '=',
''))
i = i + 1
return {
'auxdb' : metadata
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 6f1931fa4e868471cda59e88a1b1a8863753f9db
Author: Magnus Granberg gentoo org>
AuthorDate: Mon May 8 20:18:11 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon May 8 20:18:11 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6f1931fa
Use deep on eclean and check string on faild_cpv
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 30 +++---
1 file changed, 19 insertions(+), 11 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 64a3600..8692813 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
@@ -492,23 +492,23 @@ class RunEmerge(BuildStep):
# Run eclean on Packages and Distfiles
#FIXME: cleantime set in config
-cleantime_package = '1w'
+cleantime_package = '2d'
cleantime_distfiles = '2d'
if self.step == 'eclean':
aftersteps_list.append(
steps.ShellCommand(
flunkOnFailure=False,
name = 'Run eclean Packages',
-command=['eclean-pkg', '-Ct', cleantime_package],
+command=['eclean-pkg', '-C','-d', '-t' +
cleantime_package],
+workdir='/'
+))
+aftersteps_list.append(
+steps.ShellCommand(
+flunkOnFailure=False,
+name = 'Run eclean Distfiles',
+command=['eclean-dist', '-C', '-d', '-t' +
cleantime_distfiles],
workdir='/'
))
-#aftersteps_list.append(
-#steps.ShellCommand(
-#flunkOnFailure=False,
-#name = 'Run eclean Distfiles',
-#command=['eclean-dist', '-Ct', cleantime_distfiles],
-#workdir='/'
-#))
# Fix/Check Packages file
aftersteps_list.append(
steps.ShellCommand(
@@ -946,6 +946,7 @@ class CheckEmergeLogs(BuildStep):
log_path = log_path,
full_logname = full_logname
)
+print('log_dict')
print(log_dict)
if self.step == 'extra-build':
#FIXME: Check if extra build did work
@@ -962,15 +963,19 @@ class CheckEmergeLogs(BuildStep):
retrays = 1
# Find log for cpv that was requested or did failed
if not log_dict == {}:
+print('log_dict not emty')
# requested cpv
cpv = self.getProperty('cpv')
faild_version_data = False
if cpv in log_dict or self.faild_cpv in log_dict:
+print(cpv)
+print(self.faild_cpv)
yield self.createDistDir()
if cpv in log_dict:
self.log_data[cpv] = log_dict[cpv]
yield self.getLogFile(cpv, log_dict)
-if self.faild_cpv:
+if isinstance(self.faild_cpv, str):
+print('faild_cpv is string')
c = yield catpkgsplit(self.faild_cpv)[0]
if c == 'dev-haskell':
rebuild = 'haskell'
@@ -984,10 +989,13 @@ class CheckEmergeLogs(BuildStep):
yield self.getLogFile(self.faild_cpv, log_dict)
faild_version_data = yield
self.getVersionData(self.faild_cpv)
self.setProperty('faild_cpv', self.faild_cpv,
'faild_cpv')
+print(faild_version_data)
+print('getEmergeFiles')
self.getEmergeFiles(self.faild_cpv)
self.getBuildWorkDirs(self.faild_cpv)
rebuild = False
else:
+print('getEmergeFiles')
self.getEmergeFiles(cpv)
if not rebuild:
self.aftersteps_list.append(steps.Trigger(
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: fe31d987a48588a27ded9bb0c1f0b49c69f273a8
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Mar 10 22:35:27 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Mar 10 22:35:27 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=fe31d987
Add CheckEAPI
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/version.py | 32 +++-
1 file changed, 31 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index 59d19dc..c6f333e 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -409,6 +409,35 @@ class SetupPropertys(BuildStep):
self.setProperty("version", self.version, 'version')
return SUCCESS
+class CheckEAPI(BuildStep):
+
+name = 'CheckEAPI'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+#@defer.inlineCallbacks
+def run(self):
+# Run head 30 if we did't find it on head 10
+if self.getProperty("eapi"):
+return SKIPPED
+self.build.addStepsAfterCurrentStep([
+steps.SetPropertyFromCommand(
+name = 'RunGetEAPI',
+haltOnFailure = True,
+flunkOnFailure = True,
+command=['head', '-n','30',
self.getProperty("ebuild_file")],
+strip=False,
+extract_fn=PersOutputOfGetEapi
+)
+])
+return SUCCESS
+
class SetupStepsForCheckV(BuildStep):
name = 'Setup steps for Checking V'
@@ -441,10 +470,11 @@ class SetupStepsForCheckV(BuildStep):
name =
'RunGetEAPI',
haltOnFailure =
True,
flunkOnFailure =
True,
-command=['head',
'-n', '15', self.getProperty("ebuild_file")],
+command=['head',
'-n', '10', self.getProperty("ebuild_file")],
strip=False,
extract_fn=PersOutputOfGetEapi
))
+addStepVData.append(CheckEAPI())
addStepVData.append(steps.SetPropertyFromCommand(
name =
'GetPythonVersion',
haltOnFailure =
True,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: f9eed766d660ae99972201950f311c4cefc59be8 Author: Magnus Granberg gentoo org> AuthorDate: Fri Mar 10 22:33:41 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Fri Mar 10 22:33:41 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f9eed766 Download docker images befor build Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/nodes.py | 19 +++ 1 file changed, 19 insertions(+) diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py index f92727b..7a01cbb 100644 --- a/buildbot_gentoo_ci/steps/nodes.py +++ b/buildbot_gentoo_ci/steps/nodes.py @@ -108,6 +108,25 @@ class SetupBuildStepsForDocker(BuildStep): self.buildargs['BINHOSTURL'] = node_data['bin_host_url'] # set bootstrap image self.buildargs['BOOTSTRAPTAG'] = image_data['bootstrap_tag'] +# get the latest portage and bootstrap_tag images +aftersteps_list.append(steps.ShellCommand( +flunkOnFailure=True, +name='Pull ' + image_data['bootstrap_tag'], +command=['docker', + 'image', + 'pull', + image_data['bootstrap_tag'] + ] +)) +aftersteps_list.append(steps.ShellCommand( +flunkOnFailure=True, +name='Pull gentoo/portage:latest', +command=['docker', + 'image', + 'pull', + 'gentoo/portage:latest' + ] +)) # build the gentoo docker buildbot-worker image aftersteps_list.append(steps.ShellCommand( flunkOnFailure=True,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 1624134df50b603340e7e02d839f9d19476b52e8 Author: Magnus Granberg gentoo org> AuthorDate: Fri Mar 10 01:45:07 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Fri Mar 10 01:45:07 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1624134d Change cleantime_package to 1w and disable eclean-dist Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/builders.py | 16 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index a3f82a3..64a3600 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -492,7 +492,7 @@ class RunEmerge(BuildStep): # Run eclean on Packages and Distfiles #FIXME: cleantime set in config -cleantime_package = '3w' +cleantime_package = '1w' cleantime_distfiles = '2d' if self.step == 'eclean': aftersteps_list.append( @@ -502,13 +502,13 @@ class RunEmerge(BuildStep): command=['eclean-pkg', '-Ct', cleantime_package], workdir='/' )) -aftersteps_list.append( -steps.ShellCommand( -flunkOnFailure=False, -name = 'Run eclean Distfiles', -command=['eclean-dist', '-Ct', cleantime_distfiles], -workdir='/' -)) +#aftersteps_list.append( +#steps.ShellCommand( +#flunkOnFailure=False, +#name = 'Run eclean Distfiles', +#command=['eclean-dist', '-Ct', cleantime_distfiles], +#workdir='/' +#)) # Fix/Check Packages file aftersteps_list.append( steps.ShellCommand(
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: c71f1b00885a6bd3851182ebc27d5eea67351cbc
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Feb 28 21:41:07 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Feb 28 21:41:07 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c71f1b00
Add eclean to building steps
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 29 +
1 file changed, 29 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 1746122..a3f82a3 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -490,6 +490,34 @@ class RunEmerge(BuildStep):
))
aftersteps_list.append(CheckDepcleanLogs('depclean'))
+# Run eclean on Packages and Distfiles
+#FIXME: cleantime set in config
+cleantime_package = '3w'
+cleantime_distfiles = '2d'
+if self.step == 'eclean':
+aftersteps_list.append(
+steps.ShellCommand(
+flunkOnFailure=False,
+name = 'Run eclean Packages',
+command=['eclean-pkg', '-Ct', cleantime_package],
+workdir='/'
+))
+aftersteps_list.append(
+steps.ShellCommand(
+flunkOnFailure=False,
+name = 'Run eclean Distfiles',
+command=['eclean-dist', '-Ct', cleantime_distfiles],
+workdir='/'
+))
+# Fix/Check Packages file
+aftersteps_list.append(
+steps.ShellCommand(
+flunkOnFailure=False,
+name='Fix/Check binary Package file',
+command=['emaint', 'binhost', '--fix'],
+workdir='/'
+))
+
if self.step == 'match':
shell_commad_list.append('-pO')
# don't use bin for match
@@ -1191,6 +1219,7 @@ class RunBuild(BuildStep):
aftersteps_list.append(RunEmerge(step='pre-depclean'))
aftersteps_list.append(RunEmerge(step='preserved-libs'))
aftersteps_list.append(RunEmerge(step='depclean'))
+aftersteps_list.append(RunEmerge(step='eclean'))
self.setProperty('depclean', False, 'depclean')
self.setProperty('preserved_libs', False, 'preserved-libs')
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: a2d7f047652600dc548a7401e3cc6b9b90cd7f1c
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Feb 28 21:40:12 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Feb 28 21:40:12 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=a2d7f047
Change head to 15 for EAPI search
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index 14d1d97..59d19dc 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -441,7 +441,7 @@ class SetupStepsForCheckV(BuildStep):
name =
'RunGetEAPI',
haltOnFailure =
True,
flunkOnFailure =
True,
-command=['head',
'-n', '10', self.getProperty("ebuild_file")],
+command=['head',
'-n', '15', self.getProperty("ebuild_file")],
strip=False,
extract_fn=PersOutputOfGetEapi
))
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, py/
commit: 021a017b6e7ba89d28fac0e40db2ebb8baad9ce2
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Feb 21 01:39:54 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Feb 21 01:39:54 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=021a017b
Get all search pattern that match
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 4 +-
buildbot_gentoo_ci/steps/logs.py | 178 ---
py/log_parser.py | 53 +++-
3 files changed, 49 insertions(+), 186 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 801fc98..6c85aac 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -96,8 +96,8 @@ class GetBugs(BuildStep):
for match_word in match_search_text:
if match_word in match_bug_text:
matches = matches + 1
-if matches >= 10:
-print(f"Bug: {str(bug['id'])} Summary: {bug['summary']}")
+print(f"Bug: {str(bug['id'])} Matched words: {str(matches)}
Summary: {bug['summary']}")
+if matches >= 5:
match = {}
match['match'] = True
match['id'] = bug['id']
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 2a52308..e4fc951 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -19,6 +19,7 @@ from buildbot.process.results import FAILURE
from buildbot.process.results import WARNINGS
from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
+from buildbot.plugins import util
from buildbot_gentoo_ci.steps import minio
from buildbot_gentoo_ci.steps import master as master_steps
@@ -35,11 +36,15 @@ def PersOutputOfLogParser(rc, stdout, stderr):
for k, v in json.loads(line).items():
summary_log_dict[int(k)] = {
'text' : v['text'],
-'type' : v['type'],
-'status' : v['status'],
-'id' : v['id'],
-'search_pattern' : v['search_pattern']
+'pattern_infos' : [],
}
+for s in v['pattern_info']:
+summary_log_dict[int(k)]['pattern_infos'].append({
+'type' : s['type'],
+'status' :
s['status'],
+'id' : s['id'],
+'search_pattern' :
s['search_pattern'],
+})
build_summery_output['summary_log_dict'] = summary_log_dict
#FIXME: Handling of stderr output
return {
@@ -140,6 +145,8 @@ class SetupParserBuildLoger(BuildStep):
command.append(log_cpv['full_logname'])
command.append('-u')
command.append(self.getProperty('project_data')['uuid'])
+command.append('-d')
+command.append(util.Secret("log_parser_database"))
self.aftersteps_list.append(steps.SetPropertyFromCommand(
name =
'RunBuildLogParser',
haltOnFailure =
True,
@@ -152,159 +159,6 @@ class SetupParserBuildLoger(BuildStep):
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
-class ParserBuildLog(BuildStep):
-
-name = 'ParserBuildLog'
-description = 'Running'
-descriptionDone = 'Ran'
-descriptionSuffix = None
-haltOnFailure = True
-flunkOnFailure = True
-
-def __init__(self, **kwargs):
-self.logfile_text_dict = {}
-self.summery_dict = {}
-self.index = 1
-self.log_search_pattern_list = []
-self.max_text_lines = 0
-super().__init__(**kwargs)
-
-#FIXME: ansifilter
-def ansiFilter(self, text):
-return text
-
-@defer.inlineCallbacks
-def get_log_search_pattern(self):
-# get pattern from the projects
-# add that to log_search_pattern_list
-for project_pattern in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
-# check if the search pattern is vaild
-try:
-re.compile(project_pattern['search'])
-except re.error:
-print("Non valid regex pattern")
-print(project_pattern)
-else:
-self.log_search_pattern_list.append(project_pattern)
-# get the default project pattern
-# add if
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: ea8cb4a47f01964a09a7bd6f863d871a01c8b76d Author: Magnus Granberg gentoo org> AuthorDate: Wed Nov 9 21:05:15 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Wed Nov 9 21:05:15 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ea8cb4a4 Rename setBuildbotLog to Summarylog Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/buildfactorys.py | 2 +- buildbot_gentoo_ci/steps/logs.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index 7f2dff9..d07a143 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -180,7 +180,7 @@ def parse_build_log(): # set it SUCCESS/FAILURE/WARNINGS f.addStep(logs.MakeIssue()) # add sum log to buildbot log -f.addStep(logs.setBuildbotLog()) +f.addStep(logs.Summarylog()) # pers the emerge info f.addStep(logs.ReadEmergeInfoLog()) # add emerge info to log and db diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py index 67845e7..2a52308 100644 --- a/buildbot_gentoo_ci/steps/logs.py +++ b/buildbot_gentoo_ci/steps/logs.py @@ -404,9 +404,9 @@ class MakeIssue(BuildStep): yield self.build.addStepsAfterCurrentStep(self.aftersteps_list) return SUCCESS -class setBuildbotLog(BuildStep): +class Summarylog(BuildStep): -name = 'setBuildbotLog' +name = 'Summarylog' description = 'Running' descriptionDone = 'Ran' descriptionSuffix = None
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 1a48ccb70732fa8367c36c842861c080c03eb1a2
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Sep 25 11:42:02 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Sep 25 11:42:02 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1a48ccb7
Add support for test on merge_request
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 10 ++
buildbot_gentoo_ci/steps/portage.py | 5 +
2 files changed, 15 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 3b6a8d0..00980e3 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1296,6 +1296,16 @@ class SetupStepts(BuildStep):
if build:
yield log.addStdout('Runing Build: YES\n')
aftersteps_list.append(RunBuild())
+# run tests
+test = False
+if self.getProperty('projectrepository_data')['test']:
+test = True
+if self.getProperty('projectrepository_data')['test_mr'] and
self.getProperty("event") == 'merge_request':
+test = True
+if test:
+yield log.addStdout('Runing Test: YES\n')
+else:
+yield log.addStdout('Runing Test: NO\n')
# run eclean pkg and dist
#if build:
#f.addStep(builders.RunEclean(step='pkg')
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index ff06c58..6d838d7 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -295,7 +295,12 @@ class SetPackageDefault(BuildStep):
yield log.addStdout(line + separator1)
# for test we need to add env and use
#FIXME: check restrictions, test use mask and required use
+test = False
if self.getProperty('projectrepository_data')['test']:
+test = True
+if self.getProperty('projectrepository_data')['test_mr'] and
self.getProperty("event") == 'merge_request':
+test = True
+if test:
auxdb_iuses = yield
self.gentooci.db.versions.getMetadataByUuidAndMatadata(self.getProperty("version_data")['uuid'],
'iuse')
for auxdb_iuse in auxdb_iuses:
iuse, status = getIUseValue(auxdb_iuse['value'])
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
commit: 3e54616897711b1958da178ca0578da08dd55cb3
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Sep 25 11:46:08 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Sep 25 11:46:08 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3e546168
Fix a typo on setStausBuilds
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/builds.py | 2 +-
buildbot_gentoo_ci/steps/builders.py | 2 +-
buildbot_gentoo_ci/steps/logs.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/db/builds.py b/buildbot_gentoo_ci/db/builds.py
index 6639ed5..57302ef 100644
--- a/buildbot_gentoo_ci/db/builds.py
+++ b/buildbot_gentoo_ci/db/builds.py
@@ -54,7 +54,7 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
return self.db.pool.do(thd)
@defer.inlineCallbacks
-def setSatusBuilds(self, id, status):
+def setStausBuilds(self, id, status):
updated_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 00980e3..aed9edb 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -333,7 +333,7 @@ class SetupPropertys(BuildStep):
project_build_data = self.getProperty('project_build_data')
project_build_data['status'] = 'in-progress'
project_build_data['buildbot_build_id'] =
self.getProperty("buildnumber")
-yield self.gentooci.db.builds.setSatusBuilds(
+yield self.gentooci.db.builds.setStausBuilds(
project_build_data['id'],
project_build_data['status'])
yield self.gentooci.db.builds.setBuildbotBuildIdBuilds(
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 57f9394..becf7d1 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -596,7 +596,7 @@ class setBuildStatus(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
project_build_data = self.getProperty('project_build_data')
-yield self.gentooci.db.builds.setSatusBuilds(
+yield self.gentooci.db.builds.setStausBuilds(
project_build_data['id'],
self.getProperty('status')
)
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: 0d19b4627eba83ffccadc753105b5cbdf53a68e7
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Sep 1 14:56:42 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Sep 1 14:56:42 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0d19b462
Add Bug ID for Irc reporter and nice titel
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/reporters.py | 7 ---
buildbot_gentoo_ci/steps/bugs.py | 24
buildbot_gentoo_ci/steps/logs.py | 16
3 files changed, 36 insertions(+), 11 deletions(-)
diff --git a/buildbot_gentoo_ci/config/reporters.py
b/buildbot_gentoo_ci/config/reporters.py
index 9832305..bd55037 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -8,10 +8,11 @@ from buildbot.reporters.message import MessageFormatter
from buildbot_gentoo_ci.reporters import irc
irc_template = '''{% set resultsList = ["\x0303SUCCESS", "\x0308WARNINGS",
"\x0304FAILURE"] %}\
-{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303"
}}repo/{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
+{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303"
}}Repo:{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
{{ build['properties']['revision'][0]|truncate(10, True) }} {{ "\x0302" }}{{
build['properties']['owners'][0][0] }}{{ "\x03" }} \
-{{ build['properties']['event'][0] }} {{ projects }}:{{
build['properties']['project_data'][0]['name'] }} \
-{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }}\
+{{ "\x0306" }}{{ build['properties']['event'][0] }}{{ "\x03" }} {{ projects
}}:{{ build['properties']['project_data'][0]['name'] }} \
+{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }} \
+{% if build['properties']['bgo'][0]['match'] is true %}{{ "\x0311" }}Bugid:
{{build['properties']['bgo'][0]['id']}}{{ "\x03" }}{% endif %}\
'''
def ircGenerators():
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 27b1388..801fc98 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -87,20 +87,28 @@ class GetBugs(BuildStep):
yield log.addStdout('Open Bugs\n')
match = False
for bug in buglist:
-yield log.addStdout('Bug: ' + str(bug['id']) + ' Summary: ' +
bug['summary'] +'\n')
-if re.search(self.getProperty('error_dict')['title_issue'][:20],
bug['summary']):
-print('Bug found')
-print(bug)
+yield log.addStdout(f"Bug: {str(bug['id'])} Summary:
{bug['summary']}\n")
+# we splite the lines to lists and try to match the words
+matches = 0
+match_search_text =
list(self.getProperty('error_dict')['title_issue'].split())
+match_bug_text = list(bug['summary'].split())
+#FIXME: add check for cp
+for match_word in match_search_text:
+if match_word in match_bug_text:
+matches = matches + 1
+if matches >= 10:
+print(f"Bug: {str(bug['id'])} Summary: {bug['summary']}")
match = {}
+match['match'] = True
match['id'] = bug['id']
match['summary'] = bug['summary']
+yield log.addStdout(f"Line to match:
{self.getProperty('error_dict')['title_issue']}\n")
if match:
-yield log.addStdout('Match bug found\n')
-yield log.addStdout('Bug: ' + str(match['id']) + ' Summary: ' +
match['summary'] +'\n')
+yield log.addStdout('Match bug: YES\n')
+yield log.addStdout(f"Bug: {str(match['id'])} Summary:
{match['summary']}\n")
self.setProperty("bgo", match, 'bgo')
return
-yield log.addStdout('NO Match bug found\n')
-self.setProperty("bgo", False, 'bgo')
+yield log.addStdout('Match bug: NO\n')
@defer.inlineCallbacks
def run(self):
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 164bb24..57f9394 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -91,6 +91,7 @@ class SetupPropertys(BuildStep):
else:
log_cpv = self.getProperty('cpv')
self.setProperty("log_cpv", log_cpv, 'log_cpv')
+self.setProperty("bgo", dict( match=False), 'bgo')
self.descriptionDone = 'Runing log checker on ' + log_cpv
return SUCCESS
@@ -333,6 +334,19 @@ class MakeIssue(BuildStep):
yield log.addStdout(log_cpv['full_logname'] + '\n')
yield log.addStdout('world.log' + '\n')
+def getNiceErrorLine(self, line):
+# strip away hex addresses, loong
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 25877447fec0f718c22bdc7cd6f31423aae75f7e
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Aug 10 21:51:13 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Aug 10 21:51:13 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=25877447
Set head to 10 for EAPI search
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index dbeaf14..26cf9fc 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -436,7 +436,7 @@ class SetupStepsForCheckV(BuildStep):
name =
'RunGetEAPI',
haltOnFailure =
True,
flunkOnFailure =
True,
-command=['head',
'-n', '8', self.getProperty("ebuild_file")],
+command=['head',
'-n', '10', self.getProperty("ebuild_file")],
strip=False,
extract_fn=PersOutputOfGetEapi
))
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 3791e957fd2bb28f350f0ff7399c0c6e43b561d6
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Aug 5 18:32:11 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Aug 5 18:32:11 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3791e957
Add haskell-updater for haskell packages
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 147 ++-
1 file changed, 91 insertions(+), 56 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index a54c68c..3b6a8d0 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -4,6 +4,7 @@
import os
import re
import json
+from pathlib import Path
from portage.versions import catpkgsplit, cpv_getversion
from portage.dep import dep_getcpv, dep_getslot, dep_getrepo
@@ -90,6 +91,8 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# split the lines
#FIXME: Handling of stderr output
stderr_line_list = []
+if stderr != '' and not emerge_output['failed']:
+emerge_output['failed'] = True
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
@@ -340,6 +343,7 @@ class SetupPropertys(BuildStep):
print(self.getProperty("project_build_data"))
self.masterdest = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'), str(self.getProperty("buildnumber")))
self.setProperty('masterdest', self.masterdest, 'masterdest')
+self.setProperty('build_env', {}, 'build_env')
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for
project', self.getProperty('project_data')['name']])
return SUCCESS
@@ -355,7 +359,6 @@ class RunEmerge(BuildStep):
super().__init__(**kwargs)
self.descriptionSuffix = self.step
self.name = 'Setup emerge for ' + self.step + ' step'
-self.build_env = {}
self.build_timeout = 0
@defer.inlineCallbacks
@@ -374,25 +377,28 @@ class RunEmerge(BuildStep):
aftersteps_list = []
#FIXME: Set build timeout in config
self.build_timeout = 6600
-# set env
-# https://bugs.gentoo.org/683118
-# export TERM=linux
-# export TERMINFO=/etc/terminfo
-self.build_env['TERM'] = 'linux'
-self.build_env['TERMINFO'] = '/etc/terminfo'
-# Lang
-self.build_env['LANG'] = 'C.utf8'
-self.build_env['LC_MESSAGES'] = 'C'
-# no color
-self.build_env['CARGO_TERM_COLOR'] = 'never'
-self.build_env['GCC_COLORS'] = '0'
-self.build_env['OCAML_COLOR'] = 'never'
-self.build_env['PY_FORCE_COLOR'] = '0'
-self.build_env['PYTEST_ADDOPTS'] = '--color=no'
-self.build_env['NO_COLOR'] = '1'
-# not all terms support urls
-self.build_env['GCC_URLS'] = 'no'
-self.build_env['TERM_URLS'] = 'no'
+if self.step == 'update':
+# set env
+build_env = {}
+# https://bugs.gentoo.org/683118
+# export TERM=linux
+# export TERMINFO=/etc/terminfo
+build_env['TERM'] = 'linux'
+build_env['TERMINFO'] = '/etc/terminfo'
+# Lang
+build_env['LANG'] = 'C.utf8'
+build_env['LC_MESSAGES'] = 'C'
+# no color
+build_env['CARGO_TERM_COLOR'] = 'never'
+build_env['GCC_COLORS'] = '0'
+build_env['OCAML_COLOR'] = 'never'
+build_env['PY_FORCE_COLOR'] = '0'
+build_env['PYTEST_ADDOPTS'] = '--color=no'
+build_env['NO_COLOR'] = '1'
+# not all terms support urls
+build_env['GCC_URLS'] = 'no'
+build_env['TERM_URLS'] = 'no'
+self.setProperty('build_env', build_env, 'build_env')
if self.step == 'pre-update':
shell_commad_list.append('-uDN')
@@ -432,6 +438,7 @@ class RunEmerge(BuildStep):
strip=True,
extract_fn=PersOutputOfEmerge,
workdir='/',
+env=self.getProperty("build_env"),
timeout=self.build_timeout
))
aftersteps_list.append(CheckEmergeLogs('update'))
@@ -548,7 +555,7 @@ class RunEmerge(BuildStep):
strip=True,
extract_fn=PersOutputOfEmerge,
workdir='/',
-env=self.build_env,
+env=self.getProperty("build_env"),
timeout=self.build_timeout
))
aftersteps_list.append(CheckEmergeLogs('build'))
@@ -682,11 +689,9 @@ class CheckEmergeLogs(BuildStep):
@defer.inlineCallbacks
def createDistDir(self):
workdir = yield
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 3f7cd0136414f731cb97d50c486c291776d16830 Author: Magnus Granberg gentoo org> AuthorDate: Sat Jul 30 22:40:02 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Sat Jul 30 22:40:02 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3f7cd013 Fix UpdateRepos step in nodes Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/nodes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py index 3e15e5f..902f7a8 100644 --- a/buildbot_gentoo_ci/steps/nodes.py +++ b/buildbot_gentoo_ci/steps/nodes.py @@ -22,7 +22,7 @@ from buildbot.process.results import SKIPPED from buildbot.plugins import steps, util from buildbot_gentoo_ci.steps import portage as portage_steps -from buildbot_gentoo_ci.steps import builders as builders_steps +from buildbot_gentoo_ci.steps import repos as repos_steps class SetupPropertys(BuildStep): name = 'Setup propertys for stage4 image' @@ -87,7 +87,7 @@ class SetupStage4Steps(BuildStep): # download stage3 aftersteps_list.append(GetSteg3()) # setup portage -aftersteps_list.append(builders_steps.UpdateRepos()) +aftersteps_list.append(repos_steps.UpdateRepos()) aftersteps_list.append(portage_steps.SetReposConf()) aftersteps_list.append(portage_steps.SetMakeConf()) # add localegen
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
commit: aba0962b8f901301302a2fefacac14f8cd34a51b
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 27 19:32:09 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 27 19:32:09 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aba0962b
Move UpdateRepos to repos.py and support gitlab stage
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 4 +-
buildbot_gentoo_ci/db/model.py | 12 -
buildbot_gentoo_ci/db/repositorys.py | 19 +++-
buildbot_gentoo_ci/steps/builders.py | 40
buildbot_gentoo_ci/steps/repos.py | 73 +-
5 files changed, 102 insertions(+), 46 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 67447aa..7f2dff9 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -46,7 +46,7 @@ def update_db_cpv():
# set needed Propertys
f.addStep(package.SetupPropertys())
# update the repositorys listed in project_repository
-f.addStep(builders.UpdateRepos())
+f.addStep(repos.UpdateRepos())
# add repo.conf
#f.addStep(portage.SetReposConf())
# if categorys in db
@@ -99,7 +99,7 @@ def run_build_request():
# set needed Propertys
f.addStep(builders.SetupPropertys())
# update the repositorys listed in project_repository
-f.addStep(builders.UpdateRepos())
+f.addStep(repos.UpdateRepos())
# Clean and add new /etc/portage
#NOTE: remove the symlink befor the dir
#f.addStep(buildbot_steps.ShellCommand(
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 908ebb5..d6e9860 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -71,10 +71,18 @@ class Model(base.DBConnectorComponent):
# description of the repository
sa.Column('description', sa.Text, nullable=True),
sa.Column('url', sa.String(255), nullable=True),
-sa.Column('type', sa.Enum('gitpuller'), nullable=False,
default='gitpuller'),
+sa.Column('branch', sa.String(255), nullable=True),
+sa.Column('type', sa.Enum('git', 'gitlab'), nullable=False,
default='git'),
+sa.Column('mode', sa.Enum('full', 'incremental'), nullable=False,
default='incremental'),
+sa.Column('method', sa.Enum('clobber', 'fresh', 'clean', 'copy'),
nullable=False, default='fresh'),
+sa.Column('alwaysuselatest', sa.Boolean, default=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
-sa.Column('ebuild', sa.Boolean, default=False),
+sa.Column('ebuild', sa.Boolean, default=True),
+sa.Column('merge', sa.Boolean, default=False),
+# filenames for SecretInAFile
+sa.Column('sshprivatekey', sa.String(50), nullable=True),
+sa.Column('sshhostkey', sa.String(50), nullable=True),
)
# Use by GitPoller
diff --git a/buildbot_gentoo_ci/db/repositorys.py
b/buildbot_gentoo_ci/db/repositorys.py
index 8bac1a1..4d5d048 100644
--- a/buildbot_gentoo_ci/db/repositorys.py
+++ b/buildbot_gentoo_ci/db/repositorys.py
@@ -91,15 +91,32 @@ class
RepositorysConnectorComponent(base.DBConnectorComponent):
yield self.db.pool.do(thd)
def _row2dict(self, conn, row):
+if row.branch == 'none':
+branch = False
+else:
+branch = row.branch
+if row.sshprivatekey == 'none':
+sshprivatekey = False
+sshhostkey = False
+else:
+sshprivatekey = row.sshprivatekey
+sshhostkey = row.sshhostkey
return dict(
uuid=row.uuid,
name=row.name,
description=row.description,
url=row.url,
+type=row.type,
+branch=branch,
+mode=row.mode,
+method=row.method,
+alwaysuselatest=row.alwaysuselatest,
auto=row.auto,
enabled=row.enabled,
ebuild=row.ebuild,
-type=row.type
+merge=row.merge,
+sshprivatekey=sshprivatekey,
+sshhostkey=sshhostkey
)
def _row2dict_gitpuller(self, conn, row):
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 383ae60..a54c68c 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -343,46 +343,6 @@ class SetupPropertys(BuildStep):
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for
project', self.getProperty('project_data')['name']])
return SUCCESS
-# Should be moved to repos.py
-class UpdateRepos(BuildStep):
-
-name = 'UpdateRepos'
-description = 'Running'
-descriptionDone = 'Ran'
-descriptionSuffix = None
-
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 843b4272bf91ac8cf135664cab48c515f23d6cad
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jul 26 12:47:11 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jul 26 12:47:11 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=843b4272
Use rootworkdir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/nodes.py | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/nodes.py
b/buildbot_gentoo_ci/steps/nodes.py
index a77c878..3e15e5f 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -76,6 +76,7 @@ class SetupStage4Steps(BuildStep):
workerdest = yield
os.path.join(self.getProperty("workerbase"), self.getProperty('project_uuid'))
workerdest_etc = yield os.path.join(workerdest, 'etc')
print(workerdest_etc)
+self.setProperty('rootworkdir', workerdest, 'rootworkdir')
# create dir
aftersteps_list.append(steps.ShellCommand(
flunkOnFailure=True,
@@ -86,9 +87,9 @@ class SetupStage4Steps(BuildStep):
# download stage3
aftersteps_list.append(GetSteg3())
# setup portage
-
aftersteps_list.append(builders_steps.UpdateRepos(workdir=workerdest))
-
aftersteps_list.append(portage_steps.SetReposConf(workdir=workerdest))
-
aftersteps_list.append(portage_steps.SetMakeConf(workdir=workerdest))
+aftersteps_list.append(builders_steps.UpdateRepos())
+aftersteps_list.append(portage_steps.SetReposConf())
+aftersteps_list.append(portage_steps.SetMakeConf())
# add localegen
#FIXME: set that in config
locale_conf = []
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 63ea1fb9235a4585754d5b8acc41c281596ab4fc
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jul 26 12:57:35 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jul 26 12:57:35 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=63ea1fb9
Clean up and add use of rootworkdir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 204
1 file changed, 21 insertions(+), 183 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 431fe9a..ff06c58 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -20,6 +20,7 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
+from buildbot.process import remotecommand
from buildbot.plugins import steps
from buildbot_gentoo_ci.steps import master as master_steps
@@ -34,39 +35,6 @@ def WriteTextToFile(path, text_list):
yield f.write(separator)
yield f.close
-def PersOutputOfEbuildSH(rc, stdout, stderr):
-metadata = None
-NoSplit = []
-NoSplit.append('DESCRIPTION')
-#make dict of the stout
-index = 1
-metadata_line_dict = {}
-for text_line in stdout.splitlines():
-metadata_line_dict[index] = text_line
-index = index + 1
-# should have 22 lines
-if len(auxdbkeys) != index -1:
-# number of lines is incorrect.
-return {
-'auxdb' : metadata
-}
-# split all keys to list instead of speces
-metadata = {}
-i = 1
-for key in auxdbkeys:
-if metadata_line_dict[i] == '':
-metadata[key] = None
-else:
-if ' ' in metadata_line_dict[i] and key not in NoSplit:
-metadata[key] = metadata_line_dict[i].split(' ')
-else:
-metadata[key] = []
-metadata[key].append(metadata_line_dict[i])
-i = i + 1
-return {
-'auxdb' : metadata
-}
-
class SetMakeProfile(BuildStep):
name = 'SetMakeProfile'
@@ -76,8 +44,7 @@ class SetMakeProfile(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, workdir=False, **kwargs):
-self.rootworkdir = workdir
+def __init__(self, **kwargs):
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -105,8 +72,8 @@ class SetMakeProfile(BuildStep):
'ln',
'-s'
]
-if self.rootworkdir:
-symlink_makeprofile_path = yield os.path.join(self.rootworkdir,
'etc/portage/make.profile')
+if self.getProperty('rootworkdir'):
+symlink_makeprofile_path = yield
os.path.join(self.getProperty('rootworkdir'), 'etc/portage/make.profile')
else:
symlink_makeprofile_path = '/etc/portage/make.profile'
shell_commad_list.append(makeprofile_path)
@@ -130,8 +97,7 @@ class SetReposConf(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, workdir=False, **kwargs):
-self.rootworkdir = workdir
+def __init__(self, **kwargs):
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -139,8 +105,8 @@ class SetReposConf(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
portage_repos_path = self.getProperty('portage_repos_path')
project_data = self.getProperty('project_data')
-if self.rootworkdir:
-portage_etc_path = yield os.path.join(self.rootworkdir,
'etc/portage/')
+if self.getProperty('rootworkdir'):
+portage_etc_path = yield
os.path.join(self.getProperty('rootworkdir'), 'etc/portage/')
else:
portage_etc_path = '/etc/portage/'
# setup the default.conf
@@ -148,6 +114,13 @@ class SetReposConf(BuildStep):
if repos_conf_data is None:
print('Default repo is not set in repos.conf')
return FAILURE
+self.aftersteps_list = []
+# check if we have the dir
+cmd = remotecommand.RemoteCommand('stat', {'file': portage_etc_path +
'repos.conf'})
+yield self.runCommand(cmd)
+if cmd.didFail():
+self.aftersteps_list.append(steps.MakeDirectory(dir="repos.conf",
+workdir=portage_etc_path))
log = yield self.addLog('repos.conf')
# check if repos_conf_data['value'] is vaild repo name
separator = '\n'
@@ -156,11 +129,11 @@ class SetReposConf(BuildStep):
default_conf.append('main-repo = ' + repos_conf_data['value'])
default_conf.append('auto-sync = no')
default_conf_string = separator.join(default_conf)
-yield
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 8ab6bf127155a3bddca43767fb7aa208a3ea4fe7
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jul 26 12:59:19 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jul 26 12:59:19 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8ab6bf12
Add support for use of rootworkdir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 9 +
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 38dfa79..383ae60 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -314,6 +314,7 @@ class SetupPropertys(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
print('build this %s' % self.getProperty("cpv"))
self.setProperty('portage_repos_path',
self.gentooci.config.project['project']['worker_portage_repos_path'],
'portage_repos_path')
+self.setProperty('rootworkdir', False, 'rootworkdir')
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
project_data = yield
self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
@@ -342,6 +343,7 @@ class SetupPropertys(BuildStep):
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for
project', self.getProperty('project_data')['name']])
return SUCCESS
+# Should be moved to repos.py
class UpdateRepos(BuildStep):
name = 'UpdateRepos'
@@ -351,8 +353,7 @@ class UpdateRepos(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, workdir=False, **kwargs):
-self.rootworkdir = workdir
+def __init__(self, **kwargs):
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -364,8 +365,8 @@ class UpdateRepos(BuildStep):
projects_repositorys_data = yield
self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
for project_repository_data in projects_repositorys_data:
repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
-if self.rootworkdir:
-repository_path = os.path.join(self.rootworkdir,
portage_repos_path[1:], repository_data['name'])
+if self.getProperty('rootworkdir'):
+repository_path =
os.path.join(self.getProperty('rootworkdir'), portage_repos_path[1:],
repository_data['name'])
else:
repository_path = os.path.join(portage_repos_path,
repository_data['name'], '')
yield self.build.addStepsAfterCurrentStep([
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 59e77c4f9f2ca0d05aa4945e3e765353734791b1
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jul 26 12:53:07 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jul 26 12:53:07 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=59e77c4f
Add SetupPropertys for use of rootworkdir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/package.py | 18 ++
1 file changed, 18 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/package.py
b/buildbot_gentoo_ci/steps/package.py
index 76c90cc..81b400d 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -16,6 +16,23 @@ from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.plugins import steps
+class SetupPropertys(BuildStep):
+name = 'Setup propertys for CPV check'
+description = 'Running'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+# set this in config
+super().__init__(**kwargs)
+
+#@defer.inlineCallbacks
+def run(self):
+self.setProperty('portage_repos_path', '/repositorys',
'portage_repos_path')
+self.setProperty('rootworkdir', '/var/lib/buildbot_worker',
'rootworkdir')
+return SUCCESS
+
class AddPackage(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
@@ -100,6 +117,7 @@ class TriggerCheckForV(BuildStep):
'category_data' : self.getProperty("category_data"),
'change_data' : self.getProperty("change_data"),
'project_data' : self.getProperty("project_data"),
+'cp_worker' : self.getProperty('workername'),
}
)
)
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: ae8716fd198fb5502781e4e6147f59bb19361b21 Author: Magnus Granberg gentoo org> AuthorDate: Tue Jul 26 11:53:52 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Tue Jul 26 11:53:52 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ae8716fd Set 1H for timeout Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/logs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py index e139cf9..164bb24 100644 --- a/buildbot_gentoo_ci/steps/logs.py +++ b/buildbot_gentoo_ci/steps/logs.py @@ -120,7 +120,7 @@ class SetupParserBuildLoger(BuildStep): self.aftersteps_list.append(steps.FileDownload( mastersrc=mastersrc_log, workerdest=log_cpv['full_logname'] -)) +)) # Upload log parser py code self.aftersteps_list.append(steps.FileDownload( mastersrc=mastersrc_py, @@ -145,7 +145,8 @@ class SetupParserBuildLoger(BuildStep): flunkOnFailure = True, command=command, strip=False, - extract_fn=PersOutputOfLogParser + extract_fn=PersOutputOfLogParser, +timeout=3600 )) yield self.build.addStepsAfterCurrentStep(self.aftersteps_list) return SUCCESS
[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: 07a0c6eb02b632e88bc8d288c33015c8b43e93e0
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Jul 22 22:18:44 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Jul 22 22:18:44 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=07a0c6eb
Use Webhook for Gitlab and use git@ for repo's
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/schedulers.py | 3 ++-
buildbot_gentoo_ci/steps/builders.py| 12
master.cfg | 12 ++--
3 files changed, 20 insertions(+), 7 deletions(-)
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index 7c0aa6b..2e5379c 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -37,7 +37,8 @@ def getGitChanges(props):
change_data['committer'] = k['committer']
change_data['comments'] = k['comments']
change_data['revision'] = k['revision']
-change_data['timestamp'] =k['when_timestamp']
+change_data['timestamp'] = k['when_timestamp']
+change_data['branch'] = k['branch']
return change_data
def gentoo_schedulers():
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index ccb28e1..38dfa79 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -16,7 +16,7 @@ from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
from buildbot.process.results import WARNINGS
-from buildbot.plugins import steps
+from buildbot.plugins import steps, util
#FIXME: should be set in config
hosturl = 'http://90.231.13.235:8000'
@@ -374,7 +374,11 @@ class UpdateRepos(BuildStep):
mode='full',
submodules=True,
alwaysUseLatest=True,
-workdir=repository_path)
+workdir=repository_path,
+#FIXME: set filenames in repositorys db
+sshPrivateKey =
util.Secret("gitlab.gentoo.org_gentoo-ci.priv"),
+sshHostKey = util.Secret("gitlab.gentoo.org.host")
+)
])
return SUCCESS
@@ -1272,7 +1276,7 @@ class SetupStepts(BuildStep):
else:
for cpv, v in package_dict.items():
if re.search(cp, cpv):
-yield log.addStdout('Got' + cpv + '\n')
+yield log.addStdout('Got' + cpv + '\n')
yield log.addStdout('Match: NO\n')
# check for error
if stderr != []:
@@ -1284,7 +1288,7 @@ class SetupStepts(BuildStep):
yield log.addStdout('Error: NO\n')
return SKIPPED
build = True
-yield log.addStdout('Got' + self.getProperty("cpv") + '\n')
+yield log.addStdout('Got ' + self.getProperty("cpv") + '\n')
yield log.addStdout('Match: YES\n')
# update packages before any tests
if build:
diff --git a/master.cfg b/master.cfg
index 7d5af61..af50ec2 100644
--- a/master.cfg
+++ b/master.cfg
@@ -48,7 +48,7 @@ c['protocols'] = {'pb': {'port': 9989}}
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Here we point to the buildbot version of a
python hello-world project.
-c['change_source'] = change_source.gentoo_change_source()
+#c['change_source'] = change_source.gentoo_change_source()
### SCHEDULERS
@@ -91,7 +91,15 @@ c['titleURL'] = "https://gentoo-ci.gentoo.org;
c['buildbotURL'] = "http://90.231.13.235:8010/;
# minimalistic config to activate new web UI
-c['www'] = dict(port=8010, plugins=dict(waterfall_view={}, console_view={},
grid_view={}))
+c['www'] = dict(
+port=8010,
+plugins=dict(waterfall_view={}, console_view={}, grid_view={}),
+change_hook_dialects={
+'gitlab' : {
+'secret': util.Secret("WWWHookGitlabToken"),
+},
+},
+)
c['www']['ui_default_config'] = {
'Waterfall.lazy_limit_waterfall': 408,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 8d734b15cfcdc72915a0f2c63f186f4a5a4073b6
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 13 20:01:00 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 13 20:01:00 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8d734b15
Don't remove the repo dir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/nodes.py | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/nodes.py
b/buildbot_gentoo_ci/steps/nodes.py
index 2a25562..a77c878 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -172,12 +172,12 @@ class SetupStage4Steps(BuildStep):
workdir=self.getProperty("workerbase")
))
# remove the gentoo repo
-aftersteps_list.append(steps.ShellCommand(
-flunkOnFailure=True,
-name='Remove the repo dir',
-command=['rm', '-R', self.getProperty('project_uuid') +
'/var/db/repos/gentoo'],
-workdir=self.getProperty("workerbase")
-))
+#aftersteps_list.append(steps.ShellCommand(
+#flunkOnFailure=True,
+#name='Remove the repo dir',
+#command=['rm', '-R', self.getProperty('project_uuid') +
'/var/db/repos/gentoo'],
+#workdir=self.getProperty("workerbase")
+#))
# compress it
aftersteps_list.append(steps.ShellCommand(
flunkOnFailure=True,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 26b8a9034b0ce05535fb8730a4a43dcf4f70aa14
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Jul 8 21:53:50 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Jul 8 21:53:50 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=26b8a903
Fix a typo for Title
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index e5e61a5..6bd3279 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -301,7 +301,7 @@ class MakeIssue(BuildStep):
else:
cpv = self.getProperty('cpv')
self.error_dict['cpv'] = cpv
-yield log.addStdout('Titel:' + '\n')
+yield log.addStdout('Title:' + '\n')
yield log.addStdout(separator2.join([cpv, '-',
self.error_dict['title']]) + separator1)
yield log.addStdout('Summary:' + '\n')
for line in self.summary_log_list:
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 86dae64f04e20bf74405496289564cd993429c03
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 6 18:44:08 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 6 18:44:08 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=86dae64f
Change title_issue and title_phase
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 2 +-
buildbot_gentoo_ci/steps/logs.py | 29 +++--
2 files changed, 4 insertions(+), 27 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index e43c926..27b1388 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -88,7 +88,7 @@ class GetBugs(BuildStep):
match = False
for bug in buglist:
yield log.addStdout('Bug: ' + str(bug['id']) + ' Summary: ' +
bug['summary'] +'\n')
-if re.search(self.getProperty('error_dict')['title_phase'][:20],
bug['summary']):
+if re.search(self.getProperty('error_dict')['title_issue'][:20],
bug['summary']):
print('Bug found')
print(bug)
match = {}
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 7a63658..e5e61a5 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -318,39 +318,16 @@ class MakeIssue(BuildStep):
text_phase_list = []
for k, v in sorted(self.summary_log_dict.items()):
# get the issue error
-if v['text'].startswith(' * ERROR:') and v['text'].endswith('
phase):'):
-issue_text = self.summary_log_dict[k + 1]['text']
-if issue_text.startswith(' * ninja -v -j'):
-issue_text = 'ninja failed'
-if issue_text.startswith(' * (no error'):
-issue_text = False
-if issue_text:
-text_issue_list.append(issue_text)
-# get the phase error
if v['type'] == self.error_dict['phase'] and v['status'] ==
'error':
-text_phase_list.append(v['text'])
-# if not get the first issue
-if text_issue_list == []:
-for k, v in self.summary_log_dict.items():
-if v['type'] == 'issues':
-text_issue_list.append(v['text'])
-# if not get the first error
-if text_phase_list == []:
-for k, v in self.summary_log_dict.items():
-if v['status'] == 'error':
-text_phase_list.append(v['text'])
+text_issue_list.append(v['text'])
# add the issue error
if text_issue_list != []:
self.error_dict['title_issue'] = text_issue_list[0].replace('*',
'').strip()
else:
self.error_dict['title_issue'] = 'title_issue : None'
-# add the error line
-if text_phase_list != []:
-self.error_dict['title_phase'] = text_phase_list[0].replace('*',
'').strip()
-else:
-self.error_dict['title_phase'] = 'title_phase : None'
+self.error_dict['title_phase'] = 'failed in '+ self.error_dict['phase']
#set the error title
-self.error_dict['title'] = self.error_dict['title_issue'] + ' (' +
self.error_dict['title_phase'] + ')'
+self.error_dict['title'] = self.error_dict['title_phase'] + ' - ' +
self.error_dict['title_issue']
@defer.inlineCallbacks
def run(self):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 0486f29cc0842870d85029553f2c229799896fa5
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Jul 4 17:43:38 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Jul 4 17:43:38 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0486f29c
Add support for search bugs on b.g.o
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 124 +++
buildbot_gentoo_ci/steps/logs.py | 14 -
2 files changed, 135 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
new file mode 100644
index 000..e43c926
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -0,0 +1,124 @@
+# Copyright 2022 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import re
+
+from twisted.internet import defer
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
+
+from bugz.cli import check_bugz_token, login, list_bugs
+from bugz.cli_argparser import make_arg_parser
+from bugz.configfile import load_config
+from bugz.settings import Settings
+from bugz.exceptions import BugzError
+from bugz.log import log_error, log_info
+
+from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
+
+# Origins: bugz.cli
+# Modifyed by Gentoo Authors.
+# main
+def main_bugz(args):
+ArgParser = make_arg_parser()
+opt = ArgParser.parse_args(args)
+
+ConfigParser = load_config(getattr(opt, 'config_file', None))
+
+check_bugz_token()
+settings = Settings(opt, ConfigParser)
+return settings
+
+# search
+def search_bugz(args):
+settings = main_bugz(args)
+valid_keys = ['alias', 'assigned_to', 'component', 'creator',
+'limit', 'offset', 'op_sys', 'platform',
+'priority', 'product', 'resolution', 'severity',
+'version', 'whiteboard', 'cc']
+
+params = {}
+d = vars(settings)
+for key in d:
+if key in valid_keys:
+params[key] = d[key]
+if 'search_statuses' in d:
+if 'all' not in d['search_statuses']:
+params['status'] = d['search_statuses']
+if 'terms' in d:
+params['summary'] = d['terms']
+
+if not params:
+raise BugzError('Please give search terms or options.')
+
+log_info('Searching for bugs meeting the following criteria:')
+for key in params:
+log_info(' {0:<20} = {1}'.format(key, params[key]))
+
+login(settings)
+
+result = settings.call_bz(settings.bz.Bug.search, params)['bugs']
+
+if not len(result):
+log_info('No bugs found.')
+return []
+else:
+list_bugs(result, settings)
+return result
+
+class GetBugs(BuildStep):
+
+name = 'GetBugs'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def find_match(self, buglist):
+log = yield self.addLog('Bugs')
+yield log.addStdout('Open Bugs\n')
+match = False
+for bug in buglist:
+yield log.addStdout('Bug: ' + str(bug['id']) + ' Summary: ' +
bug['summary'] +'\n')
+if re.search(self.getProperty('error_dict')['title_phase'][:20],
bug['summary']):
+print('Bug found')
+print(bug)
+match = {}
+match['id'] = bug['id']
+match['summary'] = bug['summary']
+if match:
+yield log.addStdout('Match bug found\n')
+yield log.addStdout('Bug: ' + str(match['id']) + ' Summary: ' +
match['summary'] +'\n')
+self.setProperty("bgo", match, 'bgo')
+return
+yield log.addStdout('NO Match bug found\n')
+self.setProperty("bgo", False, 'bgo')
+
+@defer.inlineCallbacks
+def run(self):
+# self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+cpv = self.getProperty('error_dict')['cpv']
+c = yield catpkgsplit(cpv)[0]
+p = yield catpkgsplit(cpv)[1]
+cp = c + '/' + p
+# search for open bugs
+args = []
+args.append('--skip-auth')
+args.append('search')
+# set limit
+# set date last 30 days
+# search for cp
+args.append(cp)
+print(args)
+buglist = search_bugz(args)
+print(buglist)
+self.find_match(buglist)
+return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index d6ee705..7a63658 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -22,6 +22,7 @@ from buildbot.plugins import steps
from
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: c556e56ad98d3e81fac06417661fd7edf2c1c9fb
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jun 28 23:26:22 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jun 28 23:26:22 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c556e56a
Change emerge retrys to 5 from 3
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index fa11290..dcd54c2 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -18,7 +18,7 @@ from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
#FIXME: should be set in config
-hosturl = 'http://77.110.8.67:8000'
+hosturl = 'http://90.231.13.235:8000'
def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output = {}
@@ -856,7 +856,8 @@ class CheckEmergeLogs(BuildStep):
if self.step == 'pre-build':
print(emerge_output)
# this should be set in the config
-if self.getProperty('rerun') <= 3:
+retrays = 5
+if self.getProperty('rerun') <= retrays:
# when we need to change use. we could rerun pre-build with
# --autounmask-use=y --autounmask-write=y --autounmask-only=y
# but we use --binpkg--respect-use=y in EMERGE_DEFAULT_OPTS
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: e15a995fa6e1a649f34ac98d446be3c4db686a9d
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jun 1 00:14:18 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jun 1 00:14:18 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e15a995f
set build timeout to 6600
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 1d4f52d..fa11290 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -385,8 +385,7 @@ class RunEmerge(BuildStep):
self.descriptionSuffix = self.step
self.name = 'Setup emerge for ' + self.step + ' step'
self.build_env = {}
-#FIXME: Set build timeout in config
-self.build_timeout = 1800
+self.build_timeout = 0
@defer.inlineCallbacks
def run(self):
@@ -399,6 +398,8 @@ class RunEmerge(BuildStep):
'-v'
]
aftersteps_list = []
+#FIXME: Set build timeout in config
+self.build_timeout = 6600
# set env
# https://bugs.gentoo.org/683118
# export TERM=linux
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: 1f57c498694cbdce80ad43b2b5a3c0c44cdee588
Author: Magnus Granberg gentoo org>
AuthorDate: Sat May 28 14:54:50 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat May 28 14:54:50 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1f57c498
Add support for docker latent workers
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/workers.py | 39 ++--
buildbot_gentoo_ci/steps/builders.py | 15 --
2 files changed, 46 insertions(+), 8 deletions(-)
diff --git a/buildbot_gentoo_ci/config/workers.py
b/buildbot_gentoo_ci/config/workers.py
index fbdc2a9..d4c20f7 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -1,7 +1,7 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2022 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-from buildbot.plugins import worker
+from buildbot.plugins import worker, util
class gentoo_ci_workers():
def __init__(self, worker_data, **kwargs):
@@ -47,17 +47,52 @@ class gentoo_ci_workers():
print(node_worker)
return node_worker
+@util.renderer
+def docker_images(props):
+return 'bb-worker-' + props.getProperty('project_uuid') + ':latest'
+
+@util.renderer
+def docker_volumes(props):
+volumes_list = []
+#FIXME: set in master.cfg
+src_dir = '/srv/gentoo/portage/' + props.getProperty('project_uuid')
+dest_dir = '/var/cache/portage'
+#add distdir
+volumes_list.append(src_dir + '/distfiles' + ':' + dest_dir + '/distfiles')
+#add bindir
+volumes_list.append(src_dir + '/packages' + ':' + dest_dir + '/packages')
+return volumes_list
+
def gentoo_workers(worker_data):
w = []
g_ci_w = gentoo_ci_workers(worker_data)
LocalWorkers = g_ci_w.getLocalWorkersUuid()
BuildWorkers = g_ci_w.getBuildWorkersAllData()
NodeWorkers = g_ci_w.getNodedWorkersAllData()
+docker_hostconfig = {}
+# For use of sandbox stuff
+# FEATURES="ipc-sandbox network-sandbox pid-sandbox"
+docker_hostconfig['cap_add'] = ['SYS_ADMIN', 'NET_ADMIN', 'SYS_PTRACE']
+# libseccomp overhead
+# https://github.com/seccomp/libseccomp/issues/153
+docker_hostconfig['security_opt'] = ['seccomp=unconfined']
for local_worker in LocalWorkers:
w.append(worker.LocalWorker(local_worker))
for build_worker in BuildWorkers:
if build_worker['type'] == 'default':
w.append(worker.Worker(build_worker['uuid'],
build_worker['password']))
+#FIXME: set settings in master.cfg
+if build_worker['type'] == 'docker':
+w.append(worker.DockerLatentWorker(build_worker['uuid'],
+build_worker['password'],
+docker_host='tcp://192.168.1.3:2375',
+image=docker_images,
+volumes=docker_volumes,
+hostconfig=docker_hostconfig,
+followStartupLogs=True,
+masterFQDN='192.168.1.5',
+build_wait_timeout=3600
+))
for node_worker in NodeWorkers:
if node_worker['type'] == 'node':
w.append(worker.Worker(node_worker['uuid'],
node_worker['password']))
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 8ce6545..1d4f52d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -236,7 +236,8 @@ class TriggerRunBuildRequest(BuildStep):
'projectrepository_data' :
self.projectrepository_data,
'use_data' : self.use_data,
'fullcheck' : self.getProperty("fullcheck"),
-'project_build_data' : project_build_data
+'project_build_data' : project_build_data,
+'project_uuid' : self.project_data['uuid']
}
)])
return SUCCESS
@@ -384,6 +385,8 @@ class RunEmerge(BuildStep):
self.descriptionSuffix = self.step
self.name = 'Setup emerge for ' + self.step + ' step'
self.build_env = {}
+#FIXME: Set build timeout in config
+self.build_timeout = 1800
@defer.inlineCallbacks
def run(self):
@@ -454,7 +457,7 @@ class RunEmerge(BuildStep):
strip=True,
extract_fn=PersOutputOfEmerge,
workdir='/',
-timeout=None
+timeout=self.build_timeout
))
aftersteps_list.append(CheckEmergeLogs('update'))
if projects_emerge_options['preserved_libs']:
@@ -470,7 +473,7 @@ class RunEmerge(BuildStep):
strip=True,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/
commit: 2fac94e594b5f50b00780361788df91800a59ea7
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Apr 22 12:37:35 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Apr 22 12:37:35 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2fac94e5
Add support stage4 build
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 9 +
buildbot_gentoo_ci/config/schedulers.py| 28 ++-
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/steps/nodes.py | 322 +
4 files changed, 344 insertions(+), 16 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 86fa162..a714e47 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -13,6 +13,7 @@ from buildbot_gentoo_ci.steps import builders
from buildbot_gentoo_ci.steps import portage
from buildbot_gentoo_ci.steps import logs
from buildbot_gentoo_ci.steps import repos
+from buildbot_gentoo_ci.steps import nodes
def update_db_check():
f = util.BuildFactory()
@@ -216,3 +217,11 @@ def parse_build_log():
# setup things for the irc bot
#f.addStep(logs.SetIrcInfo())
return f
+
+def run_build_stage4_request():
+f = util.BuildFactory()
+# set needed Propertys
+f.addStep(nodes.SetupPropertys())
+# set the needed steps for making the stage4
+f.addStep(nodes.SetupStage4Steps())
+return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index e520076..8c08006 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -46,24 +46,18 @@ def gentoo_schedulers():
builderNames = builderUpdateDbNames,
change_filter=util.ChangeFilter(branch='master'),
)
-test_updatedb = schedulers.ForceScheduler(
-name="force",
-buttonName="pushMe!",
-label="My nice Force form",
-builderNames=['update_db_check'],
+create_stage4 = schedulers.ForceScheduler(
+name="create_stage4",
+buttonName="Create stage4",
+label="Create stage4 form",
+builderNames=['run_build_stage4_request'],
# A completely customized property list. The name of the
# property is the name of the parameter
properties=[
-util.NestedParameter(name="options", label="Build Options",
-layout="vertical", fields=[
-util.StringParameter(name="cpv_changes",
-label="Package to check",
-default="dev-lang/python-3.8", size=80),
-util.StringParameter(name="repository",
-label="repo",
-default="gentoo", size=80),
-])
-])
+util.StringParameter(name="project_uuid",
+label="Project uuid",
+default="e89c2c1a-46e0-4ded-81dd-c51afeb7fcfd", size=36),
+])
update_cpv_data = schedulers.Triggerable(name="update_cpv_data",
builderNames=["update_cpv_data"])
update_repo_check = schedulers.Triggerable(name="update_repo_check",
@@ -76,8 +70,10 @@ def gentoo_schedulers():
builderNames=["run_build_request"])
parse_build_log = schedulers.Triggerable(name="parse_build_log",
builderNames=["parse_build_log"])
+run_build_stage4_request =
schedulers.Triggerable(name="run_build_stage4_request",
+ builderNames=["run_build_stage4_request"])
s = []
-s.append(test_updatedb)
+s.append(create_stage4)
s.append(scheduler_update_db)
s.append(update_repo_check)
s.append(update_cpv_data)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 18fb7ff..908ebb5 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -110,6 +110,7 @@ class Model(base.DBConnectorComponent):
sa.Column('keyword_id', sa.Integer,
sa.ForeignKey('keywords.id', ondelete='CASCADE'),
nullable=False),
+sa.Column('image', sa.String(255), nullable=False),
sa.Column('status', sa.Enum('stable','unstable','all'),
nullable=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
diff --git a/buildbot_gentoo_ci/steps/nodes.py
b/buildbot_gentoo_ci/steps/nodes.py
new file mode 100644
index 000..37cf917
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -0,0 +1,322 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import re
+import json
+import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
+
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: e09c9ab4e45d4a884ea6d2da386f5ea3c8f13c2c
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Apr 22 12:34:35 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Apr 22 12:34:35 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e09c9ab4
Add support rootworkdir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 14 +-
buildbot_gentoo_ci/steps/portage.py | 29 ++---
2 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 4ade33d..8ce6545 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -344,7 +344,8 @@ class UpdateRepos(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, **kwargs):
+def __init__(self, workdir=False, **kwargs):
+self.rootworkdir = workdir
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -356,14 +357,17 @@ class UpdateRepos(BuildStep):
projects_repositorys_data = yield
self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
for project_repository_data in projects_repositorys_data:
repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
-repository_path = yield os.path.join(portage_repos_path,
repository_data['name'])
+if self.rootworkdir:
+repository_path = os.path.join(self.rootworkdir,
portage_repos_path[1:], repository_data['name'])
+else:
+repository_path = os.path.join(portage_repos_path,
repository_data['name'], '')
yield self.build.addStepsAfterCurrentStep([
steps.Git(repourl=repository_data['url'],
name = 'Git pull ' + repository_data['name'],
mode='full',
submodules=True,
alwaysUseLatest=True,
-workdir=os.path.join(repository_path, ''))
+workdir=repository_path)
])
return SUCCESS
@@ -798,8 +802,8 @@ class CheckEmergeLogs(BuildStep):
print(cpv_build_dir)
self.setProperty('cpv_build_dir', cpv_build_dir, 'cpv_build_dir')
cpv_build_work_dir = yield os.path.join(cpv_build_dir, 'work')
-#FIXME: take find pattern from db or config
-find_pattern_list = ['meson-log.txt', 'CMakeCache.txt']
+#FIXME: take find pattern from db
+find_pattern_list = ['meson-log.txt', 'CMakeCache.txt', 'testlog.txt',
'*.out', 'project-config.jam', 'testlog-x11.txt']
shell_commad_list = []
# we have *.log as default
shell_commad_list.append('find')
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 2332626..431fe9a 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -76,7 +76,8 @@ class SetMakeProfile(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, **kwargs):
+def __init__(self, workdir=False, **kwargs):
+self.rootworkdir = workdir
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -104,8 +105,12 @@ class SetMakeProfile(BuildStep):
'ln',
'-s'
]
+if self.rootworkdir:
+symlink_makeprofile_path = yield os.path.join(self.rootworkdir,
'etc/portage/make.profile')
+else:
+symlink_makeprofile_path = '/etc/portage/make.profile'
shell_commad_list.append(makeprofile_path)
-shell_commad_list.append('/etc/portage/make.profile')
+shell_commad_list.append(symlink_makeprofile_path)
yield self.build.addStepsAfterCurrentStep([
steps.ShellCommand(
command=shell_commad_list,
@@ -125,7 +130,8 @@ class SetReposConf(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, **kwargs):
+def __init__(self, workdir=False, **kwargs):
+self.rootworkdir = workdir
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -133,6 +139,10 @@ class SetReposConf(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
portage_repos_path = self.getProperty('portage_repos_path')
project_data = self.getProperty('project_data')
+if self.rootworkdir:
+portage_etc_path = yield os.path.join(self.rootworkdir,
'etc/portage/')
+else:
+portage_etc_path = '/etc/portage/'
# setup the default.conf
repos_conf_data = yield
self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'],
'repos.conf')
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: adea4206026bfb0ef1cc9a5f2995a90e58fd9fa5
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Jan 28 10:09:12 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Jan 28 10:09:12 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=adea4206
Change use of repository_basedir and use pathlib for symlink
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 31 ++-
buildbot_gentoo_ci/steps/repos.py | 21 ++---
buildbot_gentoo_ci/steps/version.py | 5 ++---
3 files changed, 34 insertions(+), 23 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index e5d0a25..2332626 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -3,6 +3,7 @@
import os
import io
+from pathlib import Path
from portage import config as portage_config
from portage import auxdbkeys
@@ -403,24 +404,29 @@ class CheckPathLocal(BuildStep):
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
-self.repository_linkname =
self.gentooci.config.project['repository_basedir']
-self.repository_basedir2 = '/home/repos2/'
self.portage_path = yield os.path.join('etc', 'portage')
self.profile_path = yield os.path.join(self.portage_path,
'make.profile')
self.repos_path = yield os.path.join(self.portage_path, 'repos.conf')
-print(os.getcwd())
+self.repository_basedir_db = yield os.path.join(self.master.basedir,
'repositorys')
+self.build_repository_basedir_db = yield
os.path.join(self.getProperty("builddir"), 'repositorys')
+#print(self.repository_basedir_db)
+log = yield self.addLog('CheckPathLocal')
+#print(os.getcwd())
print(self.getProperty("builddir"))
-yield os.chdir(self.getProperty("builddir"))
-print(os.getcwd())
+#yield os.chdir(self.getProperty("builddir"))
+#print(os.getcwd())
for x in [
self.portage_path,
self.profile_path,
self.repos_path,
]:
-if not os.path.isdir(x):
-os.makedirs(x)
-if not os.path.islink(self.repository_linkname):
-os.symlink(self.repository_basedir2, self.repository_linkname)
+check_dir = yield os.path.join(self.getProperty("builddir"), x)
+if not Path(check_dir).is_dir():
+yield Path(check_dir).mkdir(parents=True)
+yield log.addStdout(' '.join(['Makeing missing dir', x]))
+if not Path(self.build_repository_basedir_db).is_dir():
+yield
Path(self.build_repository_basedir_db).symlink_to(self.repository_basedir_db)
+yield log.addStdout(' '.join(['Makeing missing link',
'repositorys', 'to', self.repository_basedir_db]))
return SUCCESS
class SetMakeProfileLocal(BuildStep):
@@ -442,7 +448,7 @@ class SetMakeProfileLocal(BuildStep):
return SKIPPED
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.profile_repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty('project_data')['profile_repository_uuid'])
-self.repository_basedir =
self.gentooci.config.project['repository_basedir']
+self.repository_basedir = 'repositorys'
makeprofiles_paths = []
makeprofiles_data = yield
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'],
'make.profile')
for makeprofile in makeprofiles_data:
@@ -468,8 +474,7 @@ class SetReposConfLocal(BuildStep):
repos_conf_path = yield os.path.join('etc', 'portage', 'repos.conf')
repos_conf_default_path = yield os.path.join(repos_conf_path,
'default.conf')
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
-# the path should be set in the confg
-self.repository_basedir2 = '/home/repos2/'
+self.repository_basedir_db = yield os.path.join(self.master.basedir,
'repositorys')
if not os.path.isfile(repos_conf_default_path):
# setup the default.conf
repos_conf_data = yield
self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'],
'repos.conf')
@@ -483,7 +488,7 @@ class SetReposConfLocal(BuildStep):
yield WriteTextToFile(repos_conf_default_path, default_conf)
repos_conf_repository_path = yield os.path.join(repos_conf_path,
self.getProperty("repository_data")['name'] + '.conf')
if not os.path.isfile(repos_conf_repository_path):
-repository_path = yield os.path.join(self.repository_basedir2,
self.getProperty("repository_data")['name'])
+
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: d3fb5e15c10d25fa06c6862fbbcd996a2d6182df
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Jan 24 08:41:34 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Jan 24 08:41:34 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d3fb5e15
set ACCEPT_LICENSE by project
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 08ab559..e5d0a25 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -224,8 +224,9 @@ class SetMakeConf(BuildStep):
for v in makeconf_variables_values_data:
if v['build_id'] == 0:
makeconf_variable_list.append(v['value'])
-if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list !=
[]:
-makeconf_variable_list.append('ACCEPT_LICENSE="*"')
+#NOTE: set it by project
+#if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list
!= []:
+#makeconf_variable_list.append('ACCEPT_LICENSE="*"')
if makeconf_variable_list != []:
makeconf_variable_string = k['variable'] + '="' +
separator2.join(makeconf_variable_list) + '"'
makeconf_list.append(makeconf_variable_string)
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 49658ae9a6842348fbad6d0674a7a7daf7aa0a37
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Jan 24 00:38:42 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Jan 24 00:38:42 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=49658ae9
add url for logs downloads
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 69
1 file changed, 46 insertions(+), 23 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 527c2b3..4ade33d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -17,6 +17,9 @@ from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
+#FIXME: should be set in config
+hosturl = 'http://77.110.8.67:8000'
+
def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output = {}
emerge_output['rc'] = rc
@@ -603,12 +606,14 @@ class CheckElogLogs(BuildStep):
super().__init__(**kwargs)
self.aftersteps_list = []
-def addFileUploade(self, sourcefile, destfile, name):
+def addFileUploade(self, sourcefile, destfile, name, url, urlText):
self.aftersteps_list.append(steps.FileUpload(
name = name,
mode = 0o644,
-workersrc=sourcefile,
-masterdest=destfile
+workersrc = sourcefile,
+masterdest = destfile,
+url = url,
+urlText = urlText
))
@defer.inlineCallbacks
@@ -625,7 +630,9 @@ class CheckElogLogs(BuildStep):
destfile = yield os.path.join(self.getProperty('masterdest'),
elogfile.replace('.log', '.elog'))
sourcefile = yield os.path.join(workdir, elogfile)
name = 'Upload Elogs'
-self.addFileUploade(sourcefile, destfile, name)
+url = '/'.join([hosturl, self.getProperty('workername'),
str(self.getProperty("buildnumber")), elogfile.replace('.log', '.elog')])
+urlText = elogfile
+self.addFileUploade(sourcefile, destfile, name, url, urlText)
if self.aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
@@ -669,7 +676,9 @@ class CheckBuildWorkDirs(BuildStep):
mode = 0o644,
workersrc = compressed_log_file,
masterdest = masterdest_file,
-workdir = cpv_build_dir
+workdir = cpv_build_dir,
+url = '/'.join([hosturl, self.getProperty('workername'),
str(self.getProperty("buildnumber")), compressed_log_file]),
+urlText = 'Compresed file for all finds'
))
if self.aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
@@ -710,24 +719,29 @@ class CheckEmergeLogs(BuildStep):
workdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'))
self.aftersteps_list.append(steps.MasterShellCommand(
name = 'Make directory for Uploaded files',
-command=['mkdir', str(self.getProperty("buildnumber"))],
-workdir=workdir
+command = ['mkdir', str(self.getProperty("buildnumber"))],
+workdir = workdir
))
-def addFileUploade(self, sourcefile, destfile, name):
+def addFileUploade(self, sourcefile, destfile, name, url, urlText):
self.aftersteps_list.append(steps.FileUpload(
name = name,
mode = 0o644,
-workersrc=sourcefile,
-masterdest=destfile
+workersrc = sourcefile,
+masterdest = destfile,
+url=url,
+urlText=urlText
))
@defer.inlineCallbacks
def getLogFile(self, cpv, log_dict):
-destfile = yield os.path.join(self.getProperty('masterdest'),
log_dict[cpv]['full_logname'])
+file = log_dict[cpv]['full_logname']
+destfile = yield os.path.join(self.getProperty('masterdest'), file)
sourcefile = log_dict[cpv]['log_path']
name = 'Upload build log'
-self.addFileUploade(sourcefile, destfile, name)
+url = '/'.join([hosturl, self.getProperty('workername'),
str(self.getProperty("buildnumber")), file])
+urlText = file
+self.addFileUploade(sourcefile, destfile, name, url, urlText)
@defer.inlineCallbacks
def getElogFiles(self, cpv):
@@ -750,20 +764,29 @@ class CheckEmergeLogs(BuildStep):
@defer.inlineCallbacks
def getEmergeFiles(self, cpv):
# get emerge info
-destfile = yield os.path.join(self.getProperty('masterdest'),
'emerge_info.txt')
-sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
+file = 'emerge_info.txt'
+destfile =
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 2d640d0aa09a6d662d30ee83914276d5d7d08556
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Jan 22 20:49:55 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Jan 22 20:49:55 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2d640d0a
Fix naming on alot of steps in run_build_request and warning for pre-build
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 70
1 file changed, 48 insertions(+), 22 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index f2fe16c..527c2b3 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -289,10 +289,8 @@ class GetProjectRepositoryData(BuildStep):
return SUCCESS
class SetupPropertys(BuildStep):
-
-name = 'SetupPropertys'
+name = 'Setup propertys for building'
description = 'Running'
-#descriptionDone = 'Ran'
descriptionSuffix = None
haltOnFailure = True
flunkOnFailure = True
@@ -305,7 +303,6 @@ class SetupPropertys(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
print('build this %s' % self.getProperty("cpv"))
-self.descriptionDone = 'Building this %s' % self.getProperty("cpv")
self.setProperty('portage_repos_path',
self.gentooci.config.project['project']['worker_portage_repos_path'],
'portage_repos_path')
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
@@ -332,7 +329,7 @@ class SetupPropertys(BuildStep):
print(self.getProperty("project_build_data"))
self.masterdest = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'), str(self.getProperty("buildnumber")))
self.setProperty('masterdest', self.masterdest, 'masterdest')
-
+self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for
project', self.getProperty('project_data')['name']])
return SUCCESS
class UpdateRepos(BuildStep):
@@ -359,6 +356,7 @@ class UpdateRepos(BuildStep):
repository_path = yield os.path.join(portage_repos_path,
repository_data['name'])
yield self.build.addStepsAfterCurrentStep([
steps.Git(repourl=repository_data['url'],
+name = 'Git pull ' + repository_data['name'],
mode='full',
submodules=True,
alwaysUseLatest=True,
@@ -368,7 +366,6 @@ class UpdateRepos(BuildStep):
class RunEmerge(BuildStep):
-name = 'RunEmerge'
description = 'Running'
descriptionDone = 'Ran'
haltOnFailure = True
@@ -378,6 +375,7 @@ class RunEmerge(BuildStep):
self.step = step
super().__init__(**kwargs)
self.descriptionSuffix = self.step
+self.name = 'Setup emerge for ' + self.step + ' step'
self.build_env = {}
@defer.inlineCallbacks
@@ -385,6 +383,7 @@ class RunEmerge(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
projects_emerge_options = yield
self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
+self.stepname = 'Run emerge ' + 'step ' + self.step
shell_commad_list = [
'emerge',
'-v'
@@ -423,6 +422,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommand(
+name = self.stepname,
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -442,6 +442,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommand(
+name = self.stepname,
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -457,6 +458,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('@preserved-rebuild')
aftersteps_list.append(
steps.SetPropertyFromCommand(
+name = self.stepname,
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -471,6 +473,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('--depclean')
aftersteps_list.append(
steps.SetPropertyFromCommand(
+name = self.stepname,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: d3d07a68aa2c1bcba40c613f68867054bd71942d
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jan 18 20:20:13 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jan 18 20:20:13 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d3d07a68
Use tee instead of >> on emerge -qpv0 cpv
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 5088c4a..f2fe16c 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1109,7 +1109,9 @@ class RunEmergeInfo(BuildStep):
shell_commad_list.append('=' + self.getProperty('cpv'))
shell_commad_list.append('--usepkg-exclude')
shell_commad_list.append(c + '/' + p)
-shell_commad_list.append('>>')
+shell_commad_list.append('|')
+shell_commad_list.append('tee')
+shell_commad_list.append('-a')
shell_commad_list.append('/tmp/emerge_info.txt')
aftersteps_list.append(
steps.ShellCommand(
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: d319ffb16f4efcf62a7e3cee640f6af28f616021
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jan 18 19:38:31 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jan 18 19:38:31 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d319ffb1
Use tee instead of > on emerge info
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index dd2fbc2..5088c4a 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1089,7 +1089,8 @@ class RunEmergeInfo(BuildStep):
'emerge',
]
shell_commad_list.append('--info')
-shell_commad_list.append('>')
+shell_commad_list.append('|')
+shell_commad_list.append('tee')
shell_commad_list.append('/tmp/emerge_info.txt')
aftersteps_list.append(
steps.ShellCommand(
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 1fc28f8dc77b5ef0c9d2d6eac054d91b455d9048
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jan 18 19:16:44 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jan 18 19:16:44 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1fc28f8d
Add support for find logfiles
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 110 ++-
1 file changed, 97 insertions(+), 13 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index d8314e4..dd2fbc2 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -181,6 +181,19 @@ def PersOutputOfElogLs(rc, stdout, stderr):
'elog_ls_output' : elog_ls_output
}
+def PersOutputOfBuildWorkdir(rc, stdout, stderr):
+build_workdir_find_output = {}
+build_workdir_find_output['rc'] = rc
+build_workdir_find_list = []
+for line in stdout.split('\n'):
+find_line = line.replace('./', '')
+if find_line != '':
+build_workdir_find_list.append(find_line)
+build_workdir_find_output['build_workdir_find'] = build_workdir_find_list
+return {
+'build_workdir_find_output' : build_workdir_find_output
+}
+
class TriggerRunBuildRequest(BuildStep):
name = 'TriggerRunBuildRequest'
@@ -317,6 +330,9 @@ class SetupPropertys(BuildStep):
project_build_data['buildbot_build_id'])
self.setProperty('project_build_data', project_build_data,
'project_build_data')
print(self.getProperty("project_build_data"))
+self.masterdest = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'), str(self.getProperty("buildnumber")))
+self.setProperty('masterdest', self.masterdest, 'masterdest')
+
return SUCCESS
class UpdateRepos(BuildStep):
@@ -578,6 +594,7 @@ class CheckElogLogs(BuildStep):
def addFileUploade(self, sourcefile, destfile):
self.aftersteps_list.append(steps.FileUpload(
+mode = 0o644,
workersrc=sourcefile,
masterdest=destfile
))
@@ -600,6 +617,51 @@ class CheckElogLogs(BuildStep):
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
+class CheckBuildWorkDirs(BuildStep):
+
+name = 'CheckBuildWorkdir'
+description = 'Running'
+descriptionDone = 'Ran'
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+self.aftersteps_list = []
+
+@defer.inlineCallbacks
+def run(self):
+cpv = self.getProperty('faild_cpv')
+cpv_build_dir = yield os.path.join('/', 'var', 'tmp', 'portage',
self.getProperty('cpv_build_dir'))
+compressed_log_file = cpv.replace('/', '_') + '.' +
str(self.getProperty("buildnumber")) + '.logs.tar.bz2'
+masterdest_file = yield os.path.join(self.getProperty('masterdest'),
compressed_log_file)
+# cpv_build_work_dir = yield os.path.join(cpv_build_dir, 'work')
+if self.getProperty('build_workdir_find_output')['build_workdir_find']
!= []:
+shell_commad_list = []
+shell_commad_list.append('tar')
+shell_commad_list.append('-cjpf')
+shell_commad_list.append(compressed_log_file)
+for find_line in
sorted(self.getProperty('build_workdir_find_output')['build_workdir_find']):
+print(find_line)
+filename = yield os.path.join('work', find_line)
+shell_commad_list.append(filename)
+self.aftersteps_list.append(
+steps.ShellCommand(
+name = 'Tarlogs',
+command = shell_commad_list,
+workdir = cpv_build_dir
+))
+self.aftersteps_list.append(steps.FileUpload(
+name = 'UploadFindlogs',
+mode = 0o644,
+workersrc = compressed_log_file,
+masterdest = masterdest_file,
+workdir = cpv_build_dir
+))
+if self.aftersteps_list != []:
+yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
+return SUCCESS
+
class CheckEmergeLogs(BuildStep):
name = 'CheckEmergeLogs'
@@ -640,13 +702,14 @@ class CheckEmergeLogs(BuildStep):
def addFileUploade(self, sourcefile, destfile):
self.aftersteps_list.append(steps.FileUpload(
+mode = 0o644,
workersrc=sourcefile,
masterdest=destfile
))
@defer.inlineCallbacks
def getLogFile(self, cpv, log_dict):
-destfile = yield os.path.join(self.masterdest,
log_dict[cpv]['full_logname'])
+destfile = yield os.path.join(self.getProperty('masterdest'),
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 5e9aea3e883a20bc0adedceeeca2767230dd0ea3
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Jan 8 17:24:59 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Jan 8 17:24:59 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5e9aea3e
Upload emerge and elogs
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 126 +--
1 file changed, 91 insertions(+), 35 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index e76fe3d..d8314e4 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -170,6 +170,17 @@ def PersOutputOfEmergeInfo(rc, stdout, stderr):
'emerge_info_output' : emerge_info_output
}
+def PersOutputOfElogLs(rc, stdout, stderr):
+elog_ls_output = {}
+elog_ls_output['rc'] = rc
+elog_ls_list = []
+for line in stdout.split('\n'):
+elog_ls_list.append(line)
+elog_ls_output['elog_ls'] = elog_ls_list
+return {
+'elog_ls_output' : elog_ls_output
+}
+
class TriggerRunBuildRequest(BuildStep):
name = 'TriggerRunBuildRequest'
@@ -553,6 +564,42 @@ class RunEmerge(BuildStep):
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
+class CheckElogLogs(BuildStep):
+
+name = 'CheckElogLogs'
+description = 'Running'
+descriptionDone = 'Ran'
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+self.aftersteps_list = []
+
+def addFileUploade(self, sourcefile, destfile):
+self.aftersteps_list.append(steps.FileUpload(
+workersrc=sourcefile,
+masterdest=destfile
+))
+
+@defer.inlineCallbacks
+def run(self):
+elog_ls_output = self.getProperty('elog_ls_output')
+workdir = yield os.path.join('/', 'var', 'cache', 'portage', 'logs',
'elog')
+for elogfile in elog_ls_output['elog_ls']:
+if self.getProperty('faild_cpv'):
+cpv = self.getProperty('faild_cpv')
+else:
+cpv = self.getProperty('cpv')
+if elogfile.replace(':', '/').startswith(cpv):
+print(elogfile)
+destfile = yield os.path.join(self.getProperty('masterdest'),
elogfile.replace('.log', '.elog'))
+sourcefile = yield os.path.join(workdir, elogfile)
+self.addFileUploade(sourcefile, destfile)
+if self.aftersteps_list != []:
+yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
+return SUCCESS
+
class CheckEmergeLogs(BuildStep):
name = 'CheckEmergeLogs'
@@ -567,6 +614,7 @@ class CheckEmergeLogs(BuildStep):
self.descriptionSuffix = self.step
self.aftersteps_list = []
self.log_data = {}
+self.faild_cpv = False
@defer.inlineCallbacks
def getVersionData(self, cpv):
@@ -603,16 +651,38 @@ class CheckEmergeLogs(BuildStep):
self.addFileUploade(sourcefile, destfile)
@defer.inlineCallbacks
-def getEmergeFiles(self, faild_version_data):
+def getElogFiles(self, cpv):
+workdir = yield os.path.join('/', 'var', 'cache', 'portage', 'logs',
'elog')
+elog_cpv = cpv.replace('/', ':')
+shell_commad_list = []
+shell_commad_list.append('ls')
+#shell_commad_list.append(elog_cpv + '*')
+self.aftersteps_list.append(
+steps.SetPropertyFromCommand(
+command=shell_commad_list,
+strip=True,
+extract_fn=PersOutputOfElogLs,
+workdir=workdir,
+timeout=None
+))
+self.aftersteps_list.append(CheckElogLogs())
+
+@defer.inlineCallbacks
+def getEmergeFiles(self, cpv):
# get emerge info
destfile = yield os.path.join(self.masterdest, 'emerge_info.txt')
sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
self.addFileUploade(sourcefile, destfile)
-#FIXME:
-# if faild_version_data:
# get emerge.log
-# get elogs
+destfile2 = yield os.path.join(self.masterdest, 'emerge.log')
+sourcefile2 = yield os.path.join('/', 'var', 'log', 'emerge.log')
+self.addFileUploade(sourcefile2, destfile2)
# world file
+destfile3 = yield os.path.join(self.masterdest, 'world')
+sourcefile3 = yield os.path.join('/', 'var', 'lib', 'portage', 'world')
+self.addFileUploade(sourcefile3, destfile3)
+# get elogs
+self.getElogFiles(cpv)
def getBuildWorkdirFiles(self):
#FIXME:
@@ -625,6 +695,8 @@ class CheckEmergeLogs(BuildStep):
project_data = self.getProperty('project_data')
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: b3edf64aaff42659b3593dfe24a7e0a68e1881a0
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Dec 24 00:33:19 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Dec 24 00:33:19 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b3edf64a
put emerge and package info to a file insted
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 4 ++
buildbot_gentoo_ci/steps/builders.py | 74 +++--
buildbot_gentoo_ci/steps/logs.py | 75 +-
3 files changed, 137 insertions(+), 16 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 27f0076..6203a1f 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -172,8 +172,12 @@ def parse_build_log():
f.addStep(logs.MakeIssue())
# add sum log to buildbot log
f.addStep(logs.setBuildbotLog())
+# pers the emerge info
+f.addStep(logs.SetupParserEmergeInfoLog())
# add emerge info to log and db
f.addStep(logs.setEmergeInfoLog())
+# add package info to log and db
+f.addStep(logs.setPackageInfoLog())
# set BuildStatus
f.addStep(logs.setBuildStatus())
# setup things for the irc bot
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 7247abd..e76fe3d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -590,14 +590,35 @@ class CheckEmergeLogs(BuildStep):
workdir=workdir
))
-@defer.inlineCallbacks
-def getLogFile(self, cpv, log_dict):
-masterdest = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'), str(self.getProperty("buildnumber"))
,log_dict[cpv]['full_logname'])
+def addFileUploade(self, sourcefile, destfile):
self.aftersteps_list.append(steps.FileUpload(
-workersrc=log_dict[cpv]['log_path'],
-masterdest=masterdest
+workersrc=sourcefile,
+masterdest=destfile
))
+@defer.inlineCallbacks
+def getLogFile(self, cpv, log_dict):
+destfile = yield os.path.join(self.masterdest,
log_dict[cpv]['full_logname'])
+sourcefile = log_dict[cpv]['log_path']
+self.addFileUploade(sourcefile, destfile)
+
+@defer.inlineCallbacks
+def getEmergeFiles(self, faild_version_data):
+# get emerge info
+destfile = yield os.path.join(self.masterdest, 'emerge_info.txt')
+sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
+self.addFileUploade(sourcefile, destfile)
+#FIXME:
+# if faild_version_data:
+# get emerge.log
+# get elogs
+# world file
+
+def getBuildWorkdirFiles(self):
+#FIXME:
+# get files from the build workdir
+pass
+
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
@@ -610,6 +631,8 @@ class CheckEmergeLogs(BuildStep):
]
package_dict = emerge_output['packages']
+self.masterdest = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'), str(self.getProperty("buildnumber")))
+
#FIXME: Prosees the logs and do stuff
# preserved-libs
if emerge_output['preserved_libs'] and
projects_emerge_options['preserved_libs']:
@@ -757,12 +780,12 @@ class CheckEmergeLogs(BuildStep):
print(log_dict)
cpv = self.getProperty('cpv')
faild_cpv = emerge_output['failed']
+faild_version_data = False
if cpv in log_dict or faild_cpv in log_dict:
+yield self.createDistDir()
if cpv in log_dict:
self.log_data[cpv] = log_dict[cpv]
-yield self.createDistDir()
yield self.getLogFile(cpv, log_dict)
-faild_version_data = False
if faild_cpv:
# failed and build requested cpv
if cpv == faild_cpv:
@@ -772,6 +795,8 @@ class CheckEmergeLogs(BuildStep):
self.log_data[faild_cpv] = log_dict[faild_cpv]
yield self.getLogFile(faild_cpv, log_dict)
faild_version_data = yield
self.getVersionData(faild_cpv)
+self.getBuildWorkdirFiles()
+self.getEmergeFiles(faild_version_data)
self.aftersteps_list.append(steps.Trigger(
schedulerNames=['parse_build_log'],
waitForFinish=False,
@@ -785,7 +810,6 @@ class CheckEmergeLogs(BuildStep):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/logs/
commit: d80a7a0551d4d0dd4ce5431dc66377418ee199e7
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Dec 12 10:57:54 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Dec 12 10:57:54 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d80a7a05
Add MakeIssue code in parse_build_log
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/model.py| 2 +-
buildbot_gentoo_ci/logs/log_parser.py | 75 +---
buildbot_gentoo_ci/steps/logs.py | 106 +-
3 files changed, 133 insertions(+), 50 deletions(-)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 75ba873..18fb7ff 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -230,7 +230,7 @@ class Model(base.DBConnectorComponent):
sa.Column('start', sa.Integer, default=0),
sa.Column('end', sa.Integer, default=0),
sa.Column('status', sa.Enum('info', 'warning', 'ignore', 'error'),
default='info'),
-sa.Column('type', sa.Enum('info', 'qa', 'compile', 'configure',
'install', 'postinst', 'prepare', 'setup', 'test', 'unpack', 'ignore',
'issues', 'misc', 'elog'), default='info'),
+sa.Column('type', sa.Enum('info', 'qa', 'compile', 'configure',
'install', 'postinst', 'prepare', 'pretend', 'setup', 'test', 'unpack',
'ignore', 'issues', 'misc', 'elog'), default='info'),
sa.Column('search_type', sa.Enum('in', 'startswith', 'endswith',
'search'), default='in'),
)
diff --git a/buildbot_gentoo_ci/logs/log_parser.py
b/buildbot_gentoo_ci/logs/log_parser.py
index 550573d..11faf48 100644
--- a/buildbot_gentoo_ci/logs/log_parser.py
+++ b/buildbot_gentoo_ci/logs/log_parser.py
@@ -21,7 +21,7 @@ class ProjectsPattern(Base):
start = sa.Column(sa.Integer, default=0)
end = sa.Column(sa.Integer, default=0)
status = sa.Column(sa.Enum('info', 'warning', 'ignore', 'error'),
default='info')
-type = sa.Column(sa.Enum('info', 'qa', 'compile', 'configure', 'install',
'postinst', 'prepare', 'setup', 'test', 'unpack', 'ignore', 'issues', 'misc',
'elog'), default='info')
+type = sa.Column(sa.Enum('info', 'qa', 'compile', 'configure', 'install',
'postinst', 'prepare', 'pretend', 'setup', 'test', 'unpack', 'ignore',
'issues', 'misc', 'elog'), default='info')
search_type = sa.Column(sa.Enum('in', 'startswith', 'endswith', 'search'),
default='in')
def getDBSession(config):
@@ -31,26 +31,23 @@ def getDBSession(config):
return Session()
def getMultiprocessingPool(config):
-# Use cores/4 when multiprocessing
return Pool(processes = int(config['core']))
-# multi_pool = getMultiprocessingPool()
-# use multi_pool.apply_async(function, (args list)
-# multi_pool.close()
-# multi_pool.join()
def addPatternToList(Session, pattern_list, uuid):
for project_pattern in
Session.query(ProjectsPattern).filter_by(project_uuid=uuid).all():
# check if the search pattern is vaild
+project_pattern_search = project_pattern.search
try:
-re.compile(project_pattern.search)
+re.compile(project_pattern_search)
except re.error:
print("Non valid regex pattern")
-print(project_pattern)
+print(project_pattern.search)
+print(project_pattern.id)
else:
patten_dict = {}
patten_dict['id'] = project_pattern.id
patten_dict['project_uuid'] = project_pattern.project_uuid
-patten_dict['search'] = project_pattern.search
+patten_dict['search'] = project_pattern_search
patten_dict['start'] = project_pattern.start
patten_dict['end'] = project_pattern.end
patten_dict['status'] = project_pattern.status
@@ -74,25 +71,34 @@ def search_buildlog(log_search_pattern_list,
logfile_text_dict, tmp_index, max_t
# loop true the pattern list for match
for search_pattern in log_search_pattern_list:
search_hit = False
-if search_pattern['search_type'] == 'in':
-if search_pattern['search'] in text_line:
-search_hit = True
-if search_pattern['search_type'] == 'startswith':
-if text_line.startswith(search_pattern['search']):
-search_hit = True
-if search_pattern['search_type'] == 'endswith':
-if text_line.endswith(search_pattern['search']):
-search_hit = True
-if search_pattern['search_type'] == 'search':
-if re.search(search_pattern['search'], text_line):
-search_hit = True
+# check if should ignore the line
+#FIXME take the ignore line pattern from db
+if text_line.startswith('>>> /'):
+pass
+if else re.search('./\w+/'):
+pass
+else:
+# search for match
+if
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, /
commit: 91b8e50411c464e6026d37d5e0b2bba577e539c8
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Nov 6 17:54:20 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Nov 6 17:54:20 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=91b8e504
Add environment to the build step
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 22 ++
gentooci.cfg | 4
2 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 93e3529..4c9444a 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -352,6 +352,7 @@ class RunEmerge(BuildStep):
self.step = step
super().__init__(**kwargs)
self.descriptionSuffix = self.step
+self.build_env = {}
@defer.inlineCallbacks
def run(self):
@@ -363,6 +364,26 @@ class RunEmerge(BuildStep):
'-v'
]
aftersteps_list = []
+# set env
+# https://bugs.gentoo.org/683118
+# export TERM=linux
+# export TERMINFO=/etc/terminfo
+self.build_env['TERM'] = 'linux'
+self.build_env['TERMINFO'] = '/etc/terminfo'
+# Lang
+self.build_env['LANG'] = 'C.utf8'
+self.build_env['LC_MESSAGES'] = 'C'
+# no color
+self.build_env['CARGO_TERM_COLOR'] = 'never'
+self.build_env['GCC_COLORS'] = '0'
+self.build_env['OCAML_COLOR'] = 'never'
+self.build_env['PY_FORCE_COLOR'] = '0'
+self.build_env['PYTEST_ADDOPTS'] = '--color=no'
+self.build_env['NO_COLOR'] = '1'
+# not all terms support urls
+self.build_env['GCC_URLS'] = 'no'
+self.build_env['TERM_URLS'] = 'no'
+
if self.step == 'pre-update':
shell_commad_list.append('-uDN')
shell_commad_list.append('--changed-deps')
@@ -522,6 +543,7 @@ class RunEmerge(BuildStep):
strip=True,
extract_fn=PersOutputOfEmerge,
workdir='/',
+env=self.build_env,
timeout=None
))
aftersteps_list.append(CheckEmergeLogs('build'))
diff --git a/gentooci.cfg b/gentooci.cfg
index 3701878..ca3afe3 100644
--- a/gentooci.cfg
+++ b/gentooci.cfg
@@ -16,12 +16,8 @@ c = BuildmasterConfig = {}
c['db_url'] = "postgresql://buildbot:password@ip/gentoo-ci"
makeconf_list = []
-makeconf_list.append('LC_MESSAGES=C')
-makeconf_list.append('NOCOLOR="true"')
-makeconf_list.append('GCC_COLORS=""')
makeconf_list.append('PORTAGE_TMPFS="/dev/shm"')
makeconf_list.append('CLEAN_DELAY=0')
-makeconf_list.append('NOCOLOR=true')
makeconf_list.append('PORT_LOGDIR="/var/cache/portage/logs"')
makeconf_list.append('PKGDIR="/var/cache/portage/packages"')
makeconf_list.append('DISTDIR="/var/cache/portage/distfiles"')
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 1b1e91118c31d997289c3ed9b00638a71ae94588
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Nov 1 02:23:12 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Nov 1 02:23:12 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1b1e9111
Add more logging and fix use test
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 3 ++
buildbot_gentoo_ci/steps/portage.py | 59 ++--
2 files changed, 46 insertions(+), 16 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 79be77f..93e3529 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -607,6 +607,7 @@ class CheckEmergeLogs(BuildStep):
separator = '\n'
separator2 = ' '
change_use_list = []
+log = yield self.addLog('change_use')
for cpv, v in emerge_output['change_use'].items():
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
@@ -623,6 +624,8 @@ class CheckEmergeLogs(BuildStep):
workdir='/etc/portage/package.use/'
)
)
+yield log.addStdout('File: ' + 'zz_autouse' +
str(self.getProperty('rerun')) + '\n')
+yield log.addStdout(change_use_string + '\n')
# rerun
self.aftersteps_list.append(RunEmerge(step='pre-build'))
self.setProperty('rerun', self.getProperty('rerun') + 1,
'rerun')
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 408c1e9..08ab559 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -111,6 +111,8 @@ class SetMakeProfile(BuildStep):
workdir='/'
)
])
+log = yield self.addLog('make.profile')
+yield log.addStdout('Profile path: ' + makeprofile_path + '\n')
return SUCCESS
class SetReposConf(BuildStep):
@@ -135,6 +137,7 @@ class SetReposConf(BuildStep):
if repos_conf_data is None:
print('Default repo is not set in repos.conf')
return FAILURE
+log = yield self.addLog('repos.conf')
# check if repos_conf_data['value'] is vaild repo name
separator = '\n'
default_conf = []
@@ -147,6 +150,10 @@ class SetReposConf(BuildStep):
workerdest="repos.conf/default.conf",
workdir='/etc/portage/')
])
+# display the default.conf
+yield log.addStdout('File: ' + 'default.conf' + '\n')
+for line in default_conf:
+yield log.addStdout(line + '\n')
# add all repos that project have in projects_repositorys to
repos.conf/reponame.conf
projects_repositorys_data = yield
self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
for project_repository_data in projects_repositorys_data:
@@ -159,11 +166,15 @@ class SetReposConf(BuildStep):
repository_conf.append('sync-type = git')
repository_conf.append('auto-sync = no')
repository_conf_string = separator.join(repository_conf)
+filename = repository_data['name'] + '.conf'
yield self.build.addStepsAfterCurrentStep([
steps.StringDownload(repository_conf_string + separator,
-workerdest='repos.conf/' +
repository_data['name'] + '.conf',
+workerdest='repos.conf/' + filename,
workdir='/etc/portage/')
])
+yield log.addStdout('File: ' + filename + '\n')
+for line in repository_conf:
+yield log.addStdout(line + '\n')
return SUCCESS
class SetMakeConf(BuildStep):
@@ -187,6 +198,7 @@ class SetMakeConf(BuildStep):
separator1 = '\n'
separator2 = ' '
makeconf_list = []
+log = yield self.addLog('make.conf')
for k in makeconf_variables_data:
makeconf_variables_values_data = yield
self.gentooci.db.projects.getProjectMakeConfById(project_data['uuid'], k['id'])
makeconf_variable_list = []
@@ -236,6 +248,9 @@ class SetMakeConf(BuildStep):
workerdest="make.conf",
workdir='/etc/portage/')
])
+# display the make.conf
+for line in makeconf_list:
+yield log.addStdout(line + '\n')
return SUCCESS
class SetPackageDefault(BuildStep):
@@ -255,18 +270,28 @@ class SetPackageDefault(BuildStep):
self.gentooci =
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 800a64ca521fef9d400b5bbe7fa4f9719934b2dd
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Oct 23 22:27:09 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Oct 23 22:27:09 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=800a64ca
Use commmit id instead of time to check if repo need to be updated
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/repos.py | 29 +
1 file changed, 17 insertions(+), 12 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/repos.py
b/buildbot_gentoo_ci/steps/repos.py
index a2d46ce..f5c12d2 100644
--- a/buildbot_gentoo_ci/steps/repos.py
+++ b/buildbot_gentoo_ci/steps/repos.py
@@ -117,16 +117,25 @@ class CheckRepository(BuildStep):
repo = git.Repo(repository_path)
success = True
else:
+print('repo commit id to check')
+print(self.getProperty("revision"))
try:
-yield repo.git.pull()
+commits =
list(repo.iter_commits(rev=self.getProperty("revision"), max_count=2))
except:
-pass
+try:
+yield repo.git.pull()
+except:
+pass
+else:
+success = True
else:
-success = True
+print('repo allready has commit id')
+print(commits[0].hexsha)
+return None
if success:
headcommit = repo.head.commit
+print('repo updated to commit id')
print(headcommit.hexsha)
-print(headcommit.committed_date)
# chmod needed for ebuilds metadata portage.GetAuxMetadata step
# yield self.setchmod(repository_path)
return success
@@ -144,15 +153,11 @@ class CheckRepository(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(repository_uuid)
self.descriptionSuffix = repository_data['name']
-if repository_data['type'] == 'gitpuller':
-Poller_data = yield
self.gentooci.db.repositorys.getGitPollerByUuid(repository_uuid)
-print(Poller_data['updated_at'])
-print(self.getProperty("commit_time"))
-if Poller_data['updated_at'] > self.getProperty("commit_time"):
-return SKIPPED
+#self.Poller_data = yield
self.gentooci.db.repositorys.getGitPollerByUuid(repository_uuid)
success = yield self.checkRepos(repository_data)
+if success is None:
+return SKIPPED
if not success:
return FAILURE
-if repository_data['type'] == 'gitpuller':
-yield
self.gentooci.db.repositorys.updateGitPollerTime(repository_uuid)
+#yield
self.gentooci.db.repositorys.updateGitPollerTime(repository_uuid)
return SUCCESS
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 0bff8de0077754ac80083c432210bdeb15ae4383
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Oct 23 22:24:59 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Oct 23 22:24:59 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0bff8de0
Add ParserPkgCheckLog to parse_build_log
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 51
1 file changed, 51 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index d86e208..08ec6ce 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -16,6 +16,7 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import WARNINGS
+from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
from buildbot_gentoo_ci.steps import minio
@@ -305,6 +306,56 @@ class Upload(BuildStep):
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
+class ParserPkgCheckLog(BuildStep):
+
+name = 'ParserPkgCheckLog'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = False
+flunkOnWarnings = False
+flunkOnFailure = False
+warnOnWarnings = False
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+if self.getProperty("pkg_check_log_data") is None:
+return SKIPPED
+returnstatus = SUCCESS
+error = False
+warning = False
+log = yield self.addLog('Pkgcheck')
+print(self.getProperty("pkg_check_log_data"))
+for a in self.getProperty("pkg_check_log_data"):
+status = ''
+print(a)
+if isinstance(a, dict):
+for k, i in a.items():
+if k.startswith('_'):
+if k == '_info':
+status = 'INFO: '
+if k == '_error':
+status = 'ERROR: '
+error = True
+if k == '_warning':
+status = 'WARNING: '
+warning = True
+if k == '_style':
+status = 'STYLE: '
+if isinstance(i, dict):
+for b, c in i.items():
+yield log.addStdout(status + b + c + '\n')
+else:
+yield log.addStdout(i + '\n')
+if error:
+returnstatus = FAILURE
+if warning and not error:
+returnstatus = WARNINGS
+return returnstatus
+
class setBuildStatus(BuildStep):
name = 'setBuildStatus'
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: 66b8e3dc0f0d56d3a154353b4363efa65850e4d1
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Oct 7 09:18:11 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Oct 7 09:18:11 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=66b8e3dc
Fix support for bb 3.3.0
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 2 +-
buildbot_gentoo_ci/steps/builders.py | 22 +++---
buildbot_gentoo_ci/steps/portage.py| 2 +-
3 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 41c3171..45f8f48 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -101,7 +101,7 @@ def run_build_request():
f.addStep(builders.SetupPropertys())
# Clean and add new /etc/portage
#NOTE: remove the symlink befor the dir
-f.addStep(buildbot_steps.ShellCommandNewStyle(
+f.addStep(buildbot_steps.ShellCommand(
command=['rm', 'make.profile'],
workdir='/etc/portage/'
))
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 80fe52f..79be77f 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -375,7 +375,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('--buildpkg-exclude')
shell_commad_list.append('acct-*')
aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -394,7 +394,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('--buildpkg-exclude')
shell_commad_list.append('acct-*')
aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -409,7 +409,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('-q')
shell_commad_list.append('@preserved-rebuild')
aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -423,7 +423,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('--pretend')
shell_commad_list.append('--depclean')
aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfDepclean,
@@ -439,7 +439,7 @@ class RunEmerge(BuildStep):
if self.getProperty('depclean'):
pass
aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfDepclean,
@@ -468,7 +468,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('--usepkg=n')
shell_commad_list.append(c + '/' + p)
aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -492,7 +492,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('acct-*')
shell_commad_list.append('-p')
aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -517,7 +517,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('--buildpkg-exclude')
shell_commad_list.append('acct-*')
aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -663,7 +663,7 @@ class CheckEmergeLogs(BuildStep):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 0e344fe1aa9e845d8bdaa704ee3d15064771ec49
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Oct 6 01:36:22 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Oct 6 01:36:22 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0e344fe1
Use JsonReporter instead of XmlReporter for pkgcheck
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 39
1 file changed, 22 insertions(+), 17 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 8f0aadb..80fe52f 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -3,6 +3,7 @@
import os
import re
+import json
from portage.versions import catpkgsplit, cpv_getversion
from portage.dep import dep_getcpv, dep_getslot, dep_getrepo
@@ -131,18 +132,13 @@ def PersOutputOfEmerge(rc, stdout, stderr):
def PersOutputOfPkgCheck(rc, stdout, stderr):
pkgcheck_output = {}
pkgcheck_output['rc'] = rc
-#FIXME: Handling of stdout output
-pkgcheck_xml_list = []
+print(stdout)
+pkgcheck_json_list = []
# split the lines
for line in stdout.split('\n'):
-# pkgcheck output list
-if line.startswith('
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 9a4cbee8916bebb51ad771a8e05e675b3b3a68b7
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Oct 2 20:53:19 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Oct 2 20:53:19 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=9a4cbee8
Use gitpython instead of pygit2
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/repos.py | 37 +++--
1 file changed, 23 insertions(+), 14 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/repos.py
b/buildbot_gentoo_ci/steps/repos.py
index 5b6b621..a2d46ce 100644
--- a/buildbot_gentoo_ci/steps/repos.py
+++ b/buildbot_gentoo_ci/steps/repos.py
@@ -2,7 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
import os
-import pygit2
+import git
from twisted.internet import defer
@@ -104,20 +104,31 @@ class CheckRepository(BuildStep):
@defer.inlineCallbacks
def checkRepos(self, repository_data):
+success = False
repository_path = yield
os.path.join(self.getProperty("repository_basedir"), repository_data['name'])
-repo_path = yield pygit2.discover_repository(repository_path)
-print(repo_path)
-if repo_path is None:
-yield pygit2.clone_repository(repository_data['url'],
repository_path)
-success = True
+try:
+repo = git.Repo(repository_path)
+except:
+try:
+yield git.Repo.clone_from(repository_data['url'],
repository_path)
+except:
+pass
+else:
+repo = git.Repo(repository_path)
+success = True
else:
-repo = yield pygit2.Repository(repo_path)
-commit = repo.get(repo.head.target)
-success = yield self.gitPull(repo)
-print(commit.hex)
-print(commit.commit_time)
+try:
+yield repo.git.pull()
+except:
+pass
+else:
+success = True
+if success:
+headcommit = repo.head.commit
+print(headcommit.hexsha)
+print(headcommit.committed_date)
# chmod needed for ebuilds metadata portage.GetAuxMetadata step
-yield self.setchmod(repository_path)
+# yield self.setchmod(repository_path)
return success
@defer.inlineCallbacks
@@ -140,8 +151,6 @@ class CheckRepository(BuildStep):
if Poller_data['updated_at'] > self.getProperty("commit_time"):
return SKIPPED
success = yield self.checkRepos(repository_data)
-if success is None:
-return SKIPPED
if not success:
return FAILURE
if repository_data['type'] == 'gitpuller':
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: c7aed08bcefcd3a3c4b4941f968416ba6d2a530e
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Sep 28 22:16:42 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Sep 28 22:16:42 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c7aed08b
Fix package exclude code missing package_exclude['packages']
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 4f2b1b6..8f0aadb 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -458,12 +458,14 @@ class RunEmerge(BuildStep):
p = yield catpkgsplit(cpv)[1]
# Check if package is on the exclude list
if packages_excludes != []:
+print(packages_excludes)
+print(cpv)
for package_exclude in packages_excludes:
-if '/' not in package_exclude:
-if package_exclude == p:
+if '/' in package_exclude['package']:
+if package_exclude['package'] == c + '/' + p:
return SKIPPED
else:
-if package_exclude == c + '/' + p:
+if package_exclude['package'] == p:
return SKIPPED
shell_commad_list.append('-pO')
# don't use bin for match
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 00dbecbf9d20ff77a2dace764940341c6a9a9657 Author: Magnus Granberg gentoo org> AuthorDate: Wed Sep 22 00:04:56 2021 + Commit: Magnus Granberg gentoo org> CommitDate: Wed Sep 22 00:04:56 2021 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=00dbecbf Use umask in bb config intead of chmodin step Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/builders.py | 4 1 file changed, 4 deletions(-) diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index 4073672..7732f39 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -334,10 +334,6 @@ class UpdateRepos(BuildStep): for project_repository_data in projects_repositorys_data: repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid']) repository_path = yield os.path.join(portage_repos_path, repository_data['name']) -yield self.build.addStepsAfterCurrentStep([ -steps.ShellCommand(command=["chmod", "-R", "o+rX", repository_data['name']], - workdir=portage_repos_path) -]) yield self.build.addStepsAfterCurrentStep([ steps.Git(repourl=repository_data['url'], mode='full',
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: ee1e1227138eb6d1bff5c0685f03878240c77ac5 Author: Magnus Granberg gentoo org> AuthorDate: Tue Sep 21 01:28:32 2021 + Commit: Magnus Granberg gentoo org> CommitDate: Tue Sep 21 01:28:32 2021 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ee1e1227 Chmod on the repository so portage can read it after the git step Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/builders.py | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index ad66aec..4073672 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -335,7 +335,11 @@ class UpdateRepos(BuildStep): repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid']) repository_path = yield os.path.join(portage_repos_path, repository_data['name']) yield self.build.addStepsAfterCurrentStep([ -steps.Git(repourl=repository_data['url'], +steps.ShellCommand(command=["chmod", "-R", "o+rX", repository_data['name']], + workdir=portage_repos_path) +]) +yield self.build.addStepsAfterCurrentStep([ +steps.Git(repourl=repository_data['url'], mode='full', submodules=True, alwaysUseLatest=True,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 5b97bafbb4228887c54bd835586ee6b8c006cc85
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Sep 20 23:27:24 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Sep 20 23:27:24 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5b97bafb
Support for circular deps media-libs/harfbuzz
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 110 ---
1 file changed, 88 insertions(+), 22 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 9918938..ad66aec 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -5,6 +5,7 @@ import os
import re
from portage.versions import catpkgsplit, cpv_getversion
+from portage.dep import dep_getcpv, dep_getslot, dep_getrepo
from twisted.internet import defer
from twisted.python import log
@@ -19,15 +20,15 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['rc'] = rc
emerge_output['preserved_libs'] = False
emerge_output['change_use'] = False
+emerge_output['circular_deps'] = False
emerge_output['failed'] = False
package_dict = {}
log_path_list = []
print(stderr)
# split the lines
for line in stdout.split('\n'):
-# package list
+# package dict
subdict = {}
-subdict2 = {}
if line.startswith('[ebuild') or line.startswith('[binary') or
line.startswith('[nomerge'):
# if binaries
if line.startswith('[ebuild') or line.startswith('[nomerge'):
@@ -36,21 +37,14 @@ def PersOutputOfEmerge(rc, stdout, stderr):
subdict['binary'] = True
# action [ N ] stuff
subdict['action'] = line[8:15].replace(' ', '')
-# cpv
# We my have more then one spece betvine ] and cpv
-cpv_line = re.sub(' +', ' ', line)
-# get cpv
-cpv_split = re.search('] (.+?) ', cpv_line).group(1).split(':')
-print(cpv_split)
-cpv = cpv_split[0]
+pkg_line = re.sub(' +', ' ', line)
+# get pkg
+pkg = '=' + re.search('] (.+?) ', pkg_line).group(1)
# repository
+subdict['repository'] = dep_getrepo(pkg)
# slot
-if cpv_split[1] == '':
-subdict['slot'] = None
-subdict['repository'] = cpv_split[2]
-else:
-subdict['slot'] = cpv_split[1]
-subdict['repository'] = cpv_split[3]
+subdict['slot'] = dep_getslot(pkg)
# if action U version cpv
if 'U' in subdict['action']:
subdict['old_version'] = re.search(' \[(.+?)] ',
line).group(1).split(':')
@@ -66,8 +60,8 @@ def PersOutputOfEmerge(rc, stdout, stderr):
subdict['python_targets'] = re.search('PYTHON_TARGETS="(.+?)"
', line).group(1).split(' ')
else:
subdict['python_targets'] = None
-# CPU_FLAGS_X86 list
-package_dict[cpv] = subdict
+# FIXME: CPU_FLAGS_X86 list
+package_dict[dep_getcpv(pkg)] = subdict
if line.startswith('>>>'):
if line.startswith('>>> Failed to'):
emerge_output['failed'] = line.split(' ')[4][:-1]
@@ -83,12 +77,11 @@ def PersOutputOfEmerge(rc, stdout, stderr):
if line.endswith('.log.gz'):
log_path_list.append(line.split(' ')[4])
#FIXME: Handling of depclean output dict of packages that get removed
or saved
-emerge_output['package'] = package_dict
+emerge_output['packages'] = package_dict
# split the lines
#FIXME: Handling of stderr output
stderr_line_list = []
-emerge_output['change_use'] = False
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
@@ -123,6 +116,9 @@ def PersOutputOfEmerge(rc, stdout, stderr):
log_path = line.split(' ')[3]
if log_path not in inlog_path_list:
log_path_list.append(log_path)
+#FIXME: make dict of cpv listed in the circular dependencies
+if line.endswith('circular dependencies:'):
+emerge_output['circular_deps'] = True
stderr_line_list.append(line)
emerge_output['stderr'] = stderr_line_list
emerge_output['log_paths'] = log_path_list
@@ -282,7 +278,6 @@ class SetupPropertys(BuildStep):
def __init__(self, **kwargs):
# set this in config
-self.portage_repos_path = '/var/db/repos/'
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -290,7 +285,7 @@ class SetupPropertys(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
print('build this %s' % self.getProperty("cpv"))
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
commit: a54c33a5119cb5528bb83ab73b86068dd749b970
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Sep 20 23:29:24 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Sep 20 23:29:24 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=a54c33a5
Support fore more make.conf options from the db
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/db/projects.py | 1 +
buildbot_gentoo_ci/steps/portage.py | 59 +
3 files changed, 16 insertions(+), 45 deletions(-)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index d9a3972..e2cbc8d 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -113,6 +113,7 @@ class Model(base.DBConnectorComponent):
sa.Column('status', sa.Enum('stable','unstable','all'),
nullable=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
+sa.Column('use_default', sa.Boolean, default=True),
sa.Column('created_by', sa.Integer,
sa.ForeignKey('users.uid', ondelete='CASCADE'),
nullable=False),
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index fbef435..408450e 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -211,6 +211,7 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
status=row.status,
auto=row.auto,
enabled=row.enabled,
+use_default=row.use_default,
created_by=row.created_by
)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 0a131d7..03552b6 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -179,7 +179,7 @@ class SetMakeConf(BuildStep):
@defer.inlineCallbacks
def run(self):
-#FIXME: Make a dict before we pass it to the make.conf
+#FIXME: Make a dict before we pass it to the log
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
makeconf_variables_data = yield
self.gentooci.db.portages.getVariables()
@@ -189,46 +189,25 @@ class SetMakeConf(BuildStep):
for k in makeconf_variables_data:
makeconf_variables_values_data = yield
self.gentooci.db.projects.getProjectMakeConfById(project_data['uuid'], k['id'])
makeconf_variable_list = []
-# we add some default values
-#FIXME:
-# we could set them in a config variables
-# FEATURES
-if k['variable'] == 'FEATURES':
-makeconf_variable_list.append('xattr')
-makeconf_variable_list.append('cgroup')
-makeconf_variable_list.append('-news')
-makeconf_variable_list.append('-collision-protect')
-makeconf_variable_list.append('split-log')
-makeconf_variable_list.append('compress-build-logs')
-# EMERGE_DEFAULT_OPTS
-if k['variable'] == 'EMERGE_DEFAULT_OPTS':
-makeconf_variable_list.append('--buildpkg=y')
-makeconf_variable_list.append('--rebuild-if-new-rev=y')
-makeconf_variable_list.append('--rebuilt-binaries=y')
-makeconf_variable_list.append('--usepkg=y')
-makeconf_variable_list.append('--binpkg-respect-use=y')
-makeconf_variable_list.append('--binpkg-changed-deps=y')
-makeconf_variable_list.append('--nospinner')
-makeconf_variable_list.append('--color=n')
-makeconf_variable_list.append('--ask=n')
-makeconf_variable_list.append('--quiet-build=y')
-makeconf_variable_list.append('--quiet-fail=y')
# CFLAGS
if k['variable'] == 'CFLAGS' or k['variable'] == 'FCFLAGS':
makeconf_variable_list.append('-O2')
makeconf_variable_list.append('-pipe')
-makeconf_variable_list.append('-march=native')
makeconf_variable_list.append('-fno-diagnostics-color')
#FIXME:
# Depend on worker we may have to add a diffrent march
+makeconf_variable_list.append('-march=native')
if k['variable'] == 'CXXFLAGS':
makeconf_variable_list.append('${CFLAGS}')
if k['variable'] == 'FFLAGS':
makeconf_variable_list.append('${FCFLAGS}')
-if k['variable'] == 'ACCEPT_PROPERTIES':
-makeconf_variable_list.append('-interactive')
-if k['variable'] == 'ACCEPT_RESTRICT':
-makeconf_variable_list.append('-fetch')
+# Add
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 1f2f01cb4febebe5d47b4ef6c9591b73cbdfc5a5
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Sep 7 00:46:53 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Sep 7 00:46:53 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1f2f01cb
Support more options in project dict in config file
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 2 +-
buildbot_gentoo_ci/steps/portage.py | 2 +-
buildbot_gentoo_ci/steps/update_db.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index d69c447..d86e208 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -37,7 +37,7 @@ class SetupPropertys(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
project_data = yield
self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_build_data')['project_uuid'])
-default_project_data = yield
self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project'])
+default_project_data = yield
self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
version_data = yield
self.gentooci.db.versions.getVersionByUuid(self.getProperty('project_build_data')['version_uuid'])
self.setProperty("project_data", project_data, 'project_data')
self.setProperty("default_project_data", default_project_data,
'default_project_data')
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 37f7342..0a131d7 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -326,7 +326,7 @@ class SetEnvDefault(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
-default_project_data = yield
self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project'])
+default_project_data = yield
self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
aftersteps_list = []
separator1 = '\n'
separator2 = ' '
diff --git a/buildbot_gentoo_ci/steps/update_db.py
b/buildbot_gentoo_ci/steps/update_db.py
index 8e7f8ee..0ed4be1 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -19,7 +19,7 @@ class GetDataGentooCiProject(BuildStep):
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
-self.project_data = yield
self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project'])
+self.project_data = yield
self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
if self.project_data is None:
log.err('No data for project in the database')
return FAILURE
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 41b470c43fd84b66454576d1ec0d84433f0508b0
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Jul 17 08:14:35 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Jul 17 08:14:35 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=41b470c4
Fix bug in PersOutputOfEmerge for cpv line with more then one spec
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 63a1ae9..5b53018 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -37,8 +37,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# action [ N ] stuff
subdict['action'] = line[8:15].replace(' ', '')
# cpv
-#FIXME: We my have more then one spece betvine ] and cpv
-cpv_split = re.search('] (.+?) ', line).group(1).split(':')
+# We my have more then one spece betvine ] and cpv
+cpv_line = re.sub(' +', ' ', line)
+# get cpv
+cpv_split = re.search('] (.+?) ', cpv_line).group(1).split(':')
print(cpv_split)
cpv = cpv_split[0]
# repository
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
commit: c479075032be16a23ebbecf06d5f17bd8881e938
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Jul 9 15:49:39 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Jul 9 15:49:39 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c4790750
Move setting of project_build_data
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/builds.py | 1 -
buildbot_gentoo_ci/steps/builders.py | 30 --
2 files changed, 16 insertions(+), 15 deletions(-)
diff --git a/buildbot_gentoo_ci/db/builds.py b/buildbot_gentoo_ci/db/builds.py
index 2ad67b8..2cd52bc 100644
--- a/buildbot_gentoo_ci/db/builds.py
+++ b/buildbot_gentoo_ci/db/builds.py
@@ -40,7 +40,6 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
q = tbl.insert()
r = conn.execute(q,
dict(project_uuid=project_build_data['project_uuid'],
version_uuid=project_build_data['version_uuid'],
-
buildbot_build_id=project_build_data['buildbot_build_id'],
status=project_build_data['status'],
requested=project_build_data['requested'],
created_at=created_at,
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 0bed647..63a1ae9 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -189,6 +189,17 @@ class TriggerRunBuildRequest(BuildStep):
@defer.inlineCallbacks
def run(self):
+self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+if self.getProperty('project_build_data') is None:
+project_build_data = {}
+project_build_data['project_uuid'] =
self.getProperty('project_data')['uuid']
+project_build_data['version_uuid'] =
self.getProperty("version_data")['uuid']
+project_build_data['status'] = 'waiting'
+project_build_data['requested'] = False
+project_build_data['id'], project_build_data['build_id'] = yield
self.gentooci.db.builds.addBuild(
+
project_build_data)
+else:
+project_build_data = self.getProperty('project_build_data')
yield self.build.addStepsAfterCurrentStep([
steps.Trigger(
schedulerNames=['run_build_request'],
@@ -200,7 +211,7 @@ class TriggerRunBuildRequest(BuildStep):
'projectrepository_data' :
self.getProperty('projectrepository_data'),
'use_data' : self.getProperty("use_data"),
'fullcheck' : self.getProperty("fullcheck"),
-'project_build_data' : None
+'project_build_data' : project_build_data
}
)])
return SUCCESS
@@ -233,6 +244,7 @@ class GetProjectRepositoryData(BuildStep):
# set Property projectrepository_data so we can use it in the
trigger
self.setProperty('projectrepository_data',
projectrepository_data, 'projectrepository_data')
self.setProperty('use_data', None, 'use_data')
+self.setProperty('project_data', project_data, 'project_data')
# get name o project keyword
project_keyword_data = yield
self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
# if not * (all keywords)
@@ -280,22 +292,12 @@ class SetupPropertys(BuildStep):
self.setProperty('faild_version_data', None, 'faild_version_data')
self.setProperty('rerun', 0, 'rerun')
print(self.getProperty("buildnumber"))
-if self.getProperty('project_build_data') is None:
-project_build_data = {}
-project_build_data['project_uuid'] = project_data['uuid']
-project_build_data['version_uuid'] =
self.getProperty("version_data")['uuid']
-project_build_data['status'] = 'in-progress'
-project_build_data['requested'] = False
-project_build_data['buildbot_build_id'] =
self.getProperty("buildnumber")
-project_build_data['id'], project_build_data['build_id'] = yield
self.gentooci.db.builds.addBuild(
-
project_build_data)
-else:
-project_build_data = self.getProperty('project_build_data')
-yield self.gentooci.db.builds.setSatusBuilds(
+project_build_data = self.getProperty('project_build_data')
+yield self.gentooci.db.builds.setSatusBuilds(
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 001f737e5d8cbc62272415580e34b53cee6d81b5 Author: Magnus Granberg gentoo org> AuthorDate: Tue May 18 08:55:41 2021 + Commit: Magnus Granberg gentoo org> CommitDate: Tue May 18 08:55:41 2021 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=001f737e Make git use the latest revision Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/builders.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index 3a5de47..0bed647 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -327,8 +327,9 @@ class UpdateRepos(BuildStep): repository_path = yield os.path.join(portage_repos_path, repository_data['name']) yield self.build.addStepsAfterCurrentStep([ steps.Git(repourl=repository_data['url'], -mode='incremental', +mode='full', submodules=True, +alwaysUseLatest=True, workdir=os.path.join(repository_path, '')) ]) return SUCCESS
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: c5da0d10e98b62390d47b5b13d66a135706177ed
Author: Magnus Granberg gentoo org>
AuthorDate: Sun May 2 15:24:05 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun May 2 15:24:05 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c5da0d10
Add support for emerge --info
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 2 ++
buildbot_gentoo_ci/steps/builders.py | 44 +-
buildbot_gentoo_ci/steps/logs.py | 24 +++-
3 files changed, 68 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index e08127b..417e385 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -177,6 +177,8 @@ def parse_build_log():
f.addStep(logs.MakeIssue())
# add sum log to buildbot log
f.addStep(logs.setBuildbotLog())
+# add emerge info to log and db
+f.addStep(logs.setEmergeInfoLog())
# set BuildStatus
f.addStep(logs.setBuildStatus())
# setup things for the irc bot
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 50f924d..3a5de47 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -164,6 +164,17 @@ def PersOutputOfDepclean(rc, stdout, stderr):
'depclean_output' : depclean_output
}
+def PersOutputOfEmergeInfo(rc, stdout, stderr):
+emerge_info_output = {}
+emerge_info_output['rc'] = rc
+emerge_info_list = []
+for line in stdout.split('\n'):
+emerge_info_list.append(line)
+emerge_info_output['emerge_info'] = emerge_info_list
+return {
+'emerge_info_output' : emerge_info_output
+}
+
class TriggerRunBuildRequest(BuildStep):
name = 'TriggerRunBuildRequest'
@@ -650,7 +661,8 @@ class CheckEmergeLogs(BuildStep):
'pkg_check_log_data' :
self.getProperty("pkg_check_log_data"),
'repository_data' :
self.getProperty('repository_data'),
'faild_cpv' : faild_cpv,
-'step' : self.step
+'step' : self.step,
+'emerge_info' :
self.getProperty('emerge_info_output')['emerge_info']
}
))
if not self.step is None and self.aftersteps_list != []:
@@ -761,6 +773,35 @@ class CheckPkgCheckLogs(BuildStep):
self.setProperty('pkg_check_log_data', None, 'pkg_check_log_data')
return SUCCESS
+class RunEmergeInfo(BuildStep):
+
+name = 'RunEmergeInfo'
+description = 'Running'
+descriptionDone = 'Ran'
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+aftersteps_list = []
+shell_commad_list = [
+'emerge',
+]
+shell_commad_list.append('--info')
+aftersteps_list.append(
+steps.SetPropertyFromCommandNewStyle(
+command=shell_commad_list,
+strip=True,
+extract_fn=PersOutputOfEmergeInfo,
+workdir='/',
+timeout=None
+))
+yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+return SUCCESS
+
class RunBuild(BuildStep):
name = 'RunBuild'
@@ -780,6 +821,7 @@ class RunBuild(BuildStep):
# trigger pars_build_log if we have any logs to check
return SUCCESS
aftersteps_list = []
+aftersteps_list.append(RunEmergeInfo())
aftersteps_list.append(RunEmerge(step='pre-build'))
aftersteps_list.append(RunEmerge(step='build'))
self.setProperty('depclean', False, 'depclean')
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 0960b9b..3c0ef8d 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -252,7 +252,29 @@ class setBuildbotLog(BuildStep):
# add line for line
for line in self.getProperty('summary_log_list'):
yield log.addStdout(line + '\n')
-# add emerge info log
+return SUCCESS
+
+class setEmergeInfoLog(BuildStep):
+
+name = 'setEmergeInfoLog'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = False
+flunkOnFailure = True
+warnOnWarnings = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+#setup the log
+log = yield self.addLog('emerge_info')
+#FIXME: add emerge info to db
+# add line for line
+
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: b2f1c115420e05c27fb7a57cf45734cb7f4a7731
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Apr 26 20:26:04 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Apr 26 20:26:04 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b2f1c115
Add more checks and move * and >>> check to last
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 121 ++-
1 file changed, 70 insertions(+), 51 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 178d71e..0960b9b 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -57,39 +57,46 @@ class ParserBuildLog(BuildStep):
self.summery_dict = {}
self.index = 1
self.log_search_pattern_list = []
-self.max_text_lines = self.index -1
+self.max_text_lines = 0
super().__init__(**kwargs)
+#FIXME: ansifilter
+def ansiFilter(self, text):
+return text
+
@defer.inlineCallbacks
def get_log_search_pattern(self):
# get pattern from the projects
# add that to log_search_pattern_list
for project_pattern in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
-self.log_search_pattern_list.append(project_pattern)
-# get the default profile pattern
+# check if the search pattern is vaild
+try:
+re.compile(project_pattern['search'])
+except re.error:
+print("Non valid regex pattern")
+print(project_pattern)
+else:
+self.log_search_pattern_list.append(project_pattern)
+# get the default project pattern
# add if not pattern is in project ignore
+self.project_pattern_ignore = yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('project_data')['uuid'])
for project_pattern in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('default_project_data')['uuid'])):
-match = True
-for project_pattern_ignore in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('default_project_data')['uuid'])):
-if project_pattern['search'] ==
project_pattern_ignore['search']:
-match = False
-if match:
-self.log_search_pattern_list.append(project_pattern)
+if not project_pattern['search'] in self.project_pattern_ignore:
+# check if the search pattern is vaild
+try:
+re.compile(project_pattern['search'])
+except re.error:
+print("Non valid regex pattern")
+print(project_pattern)
+else:
+self.log_search_pattern_list.append(project_pattern)
def search_buildlog(self, tmp_index):
# get text line to search
-text_line = self.logfile_text_dict[tmp_index]
+text_line = self.ansiFilter(self.logfile_text_dict[tmp_index])
# loop true the pattern list for match
for search_pattern in self.log_search_pattern_list:
search_hit = False
-# we add all line that start with ' * ' as info
-# we add all line that start with '>>>' but not '>>> /' as info
-if text_line.startswith(' * ') or (text_line.startswith('>>>') and
not text_line.startswith('>>> /')):
-self.summery_dict[tmp_index] = {}
-self.summery_dict[tmp_index]['text'] = text_line
-self.summery_dict[tmp_index]['type'] = 'info'
-self.summery_dict[tmp_index]['status'] = 'info'
-self.summery_dict[tmp_index]['search_pattern_id'] = 0
if search_pattern['search_type'] == 'in':
if search_pattern['search'] in text_line:
search_hit = True
@@ -100,63 +107,74 @@ class ParserBuildLog(BuildStep):
if text_line.endswith(search_pattern['search']):
search_hit = True
if search_pattern['search_type'] == 'search':
-if search_pattern['search'] in text_line:
+if re.search(search_pattern['search'], text_line):
search_hit = True
+# add the line if the pattern match
if search_hit:
print(text_line)
-print(search_pattern['search'])
+print(search_pattern)
+print(tmp_index)
self.summery_dict[tmp_index] = {}
self.summery_dict[tmp_index]['text'] = text_line
self.summery_dict[tmp_index]['type'] = search_pattern['type']
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: c059e969ff0e106823e0bed4729d4c04db02eee8
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Apr 20 07:05:33 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Apr 20 07:05:33 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c059e969
Support more then one Chaange USE
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 48
1 file changed, 32 insertions(+), 16 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index d9817d8..50f924d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -86,21 +86,35 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# split the lines
#FIXME: Handling of stderr output
stderr_line_list = []
+emerge_output['change_use'] = False
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
change_use_list = []
# get cpv
cpv_split = line_list[1].split(':')
-change_use_list.append(cpv_split[0])
+change_use = {}
# add use flags
if line_list[4].startswith('+') or line_list[4].startswith('-'):
-# we only support one for now
+# we only support tre for now
if line_list[4].endswith(')'):
change_use_list.append(line_list[4].replace(')', ''))
+elif line_list[5].endswith(')'):
+change_use_list.append(line_list[4])
+change_use_list.append(line_list[5].replace(')', ''))
+elif line_list[6].endswith(')'):
+change_use_list.append(line_list[4])
+change_use_list.append(line_list[5])
+change_use_list.append(line_list[6].replace(')', ''))
+elif not line_list[6].endswith(')'):
+change_use_list.append(line_list[4])
+change_use_list.append(line_list[5])
+change_use_list.append(line_list[6])
else:
change_use_list = False
-emerge_output['change_use'] = change_use_list
+if change_use_list:
+change_use[cpv_split[0]] = change_use_list
+emerge_output['change_use'] = change_use
err_line_list = []
if line.startswith(' * '):
if line.endswith('.log.gz'):
@@ -226,7 +240,7 @@ class SetupPropertys(BuildStep):
name = 'SetupPropertys'
description = 'Running'
-descriptionDone = 'Ran'
+#descriptionDone = 'Ran'
descriptionSuffix = None
haltOnFailure = True
flunkOnFailure = True
@@ -240,6 +254,7 @@ class SetupPropertys(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
print('build this %s' % self.getProperty("cpv"))
+self.descriptionDone = 'Building this %s' % self.getProperty("cpv")
self.setProperty('portage_repos_path', self.portage_repos_path,
'portage_repos_path')
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
@@ -550,6 +565,7 @@ class CheckEmergeLogs(BuildStep):
# update package.* if needed and rerun pre-build max 3 times
if self.step == 'pre-build':
print(emerge_output)
+# this should be set in the config
if self.getProperty('rerun') <= 3:
# when we need to change use. we could rerun pre-build with
# --autounmask-use=y --autounmask-write=y --autounmask-only=y
@@ -560,21 +576,21 @@ class CheckEmergeLogs(BuildStep):
separator = '\n'
separator2 = ' '
change_use_list = []
-cpv = emerge_output['change_use'][0]
-c = yield catpkgsplit(cpv)[0]
-p = yield catpkgsplit(cpv)[1]
-change_use_list.append(c + '/' + p)
-# we only support one use
-use_flag = emerge_output['change_use'][1]
-if use_flag.startswith('+'):
-change_use_list.append(use_flag.replace('+', ''))
-else:
-change_use_list.append(use_flag)
+for cpv, v in emerge_output['change_use'].items():
+c = yield catpkgsplit(cpv)[0]
+p = yield catpkgsplit(cpv)[1]
+change_use_list.append(c + '/' + p)
+for use_flag in v:
+if use_flag.startswith('+'):
+change_use_list.append(use_flag.replace('+',
''))
+else:
+
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 482017e7c399b4e97502617f0aa70ff50df54b7b
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Apr 13 19:03:04 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Apr 13 19:03:04 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=482017e7
check repository_linkname symalink in LocalWorkers
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index c1a4d90..c7e82ce 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -18,6 +18,7 @@ from twisted.python import log
from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
from buildbot_gentoo_ci.steps import master as master_steps
@@ -367,6 +368,9 @@ class CheckPathLocal(BuildStep):
@defer.inlineCallbacks
def run(self):
+self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+self.repository_linkname =
self.gentooci.config.project['repository_basedir']
+self.repository_basedir2 = '/home/repos2/'
self.portage_path = yield os.path.join('etc', 'portage')
self.profile_path = yield os.path.join(self.portage_path,
'make.profile')
self.repos_path = yield os.path.join(self.portage_path, 'repos.conf')
@@ -381,6 +385,8 @@ class CheckPathLocal(BuildStep):
]:
if not os.path.isdir(x):
os.makedirs(x)
+if not os.path.islink(self.repository_linkname):
+os.symlink(self.repository_basedir2, self.repository_linkname)
return SUCCESS
class SetMakeProfileLocal(BuildStep):
@@ -399,7 +405,7 @@ class SetMakeProfileLocal(BuildStep):
def run(self):
parent_path = yield os.path.join('etc','portage', 'make.profile',
'parent')
if os.path.isfile(parent_path):
-return SUCCESS
+return SKIPPED
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.repository_basedir =
self.gentooci.config.project['repository_basedir']
makeprofiles_paths = []
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
commit: 409167eb42b9fbacf05f00a36621e897361138bd
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Apr 13 14:24:15 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Apr 13 14:24:15 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=409167eb
Remove project_repository_uuid from db. use int for datetime in db. change
mirror_url to url
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/builds.py | 7 +++
buildbot_gentoo_ci/db/model.py | 32
buildbot_gentoo_ci/db/projects.py| 1 -
buildbot_gentoo_ci/db/repositorys.py | 2 +-
buildbot_gentoo_ci/db/versions.py| 6 ++
buildbot_gentoo_ci/steps/builders.py | 2 +-
buildbot_gentoo_ci/steps/portage.py | 4 ++--
buildbot_gentoo_ci/steps/repos.py| 2 +-
8 files changed, 22 insertions(+), 34 deletions(-)
diff --git a/buildbot_gentoo_ci/db/builds.py b/buildbot_gentoo_ci/db/builds.py
index 79814b7..2ad67b8 100644
--- a/buildbot_gentoo_ci/db/builds.py
+++ b/buildbot_gentoo_ci/db/builds.py
@@ -23,13 +23,12 @@ import sqlalchemy as sa
from twisted.internet import defer
from buildbot.db import base
-from buildbot.util import epoch2datetime
class BuildsConnectorComponent(base.DBConnectorComponent):
#@defer.inlineCallbacks
def addBuild(self, project_build_data):
-created_at = epoch2datetime(int(self.master.reactor.seconds()))
+created_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
tbl = self.db.model.projects_builds
# get the highest current number
@@ -56,7 +55,7 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
def setSatusBuilds(self, build_id, project_uuid, status):
-updated_at = epoch2datetime(int(self.master.reactor.seconds()))
+updated_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
tbl = self.db.model.projects_builds
@@ -69,7 +68,7 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
def setBuildbotBuildIdBuilds(self, build_id, project_uuid,
buildbot_build_id):
-updated_at = epoch2datetime(int(self.master.reactor.seconds()))
+updated_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
tbl = self.db.model.projects_builds
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 54c964f..ede5cb7 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -70,7 +70,7 @@ class Model(base.DBConnectorComponent):
sa.Column('name', sa.String(255), nullable=False),
# description of the repository
sa.Column('description', sa.Text, nullable=True),
-sa.Column('mirror_url', sa.String(255), nullable=True),
+sa.Column('url', sa.String(255), nullable=True),
sa.Column('type', sa.Enum('gitpuller'), nullable=False,
default='gitpuller'),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
@@ -98,15 +98,11 @@ class Model(base.DBConnectorComponent):
"projects", metadata,
# unique id per project
sa.Column('uuid', sa.String(36), primary_key=True,
- default=lambda: str(uuid.uuid4()),
- ),
+ default=lambda: str(uuid.uuid4())),
# project's name
sa.Column('name', sa.String(255), nullable=False),
# description of the project
sa.Column('description', sa.Text, nullable=True),
-sa.Column('project_repository_uuid', sa.String(36),
- sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
- nullable=False),
sa.Column('profile', sa.String(255), nullable=False),
sa.Column('profile_repository_uuid', sa.String(36),
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
@@ -216,10 +212,10 @@ class Model(base.DBConnectorComponent):
sa.Column('buildbot_build_id', sa.Integer),
sa.Column('status',
sa.Enum('failed','completed','in-progress','waiting', 'warning'),
nullable=False),
sa.Column('requested', sa.Boolean, default=False),
-sa.Column('created_at', sa.DateTime, nullable=True),
-sa.Column('updated_at', sa.DateTime, nullable=True),
+sa.Column('created_at', sa.Integer, nullable=True),
+sa.Column('updated_at', sa.Integer, nullable=True),
sa.Column('deleted', sa.Boolean, default=False),
-sa.Column('deleted_at', sa.DateTime, nullable=True),
+sa.Column('deleted_at', sa.Integer, nullable=True),
)
projects_pattern = sautils.Table(
@@ -247,16 +243,14 @@ class Model(base.DBConnectorComponent):
categorys = sautils.Table(
"categorys", metadata,
sa.Column('uuid',
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
commit: 550ae33d23ae93d7d779922a0b8e07710458c1a7
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Apr 10 01:02:38 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Apr 10 01:02:38 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=550ae33d
Add support for Failed to XX in emerge build
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/packages.py| 4 +-
buildbot_gentoo_ci/db/versions.py| 3 +-
buildbot_gentoo_ci/steps/builders.py | 90
buildbot_gentoo_ci/steps/logs.py | 7 ++-
buildbot_gentoo_ci/steps/package.py | 4 +-
buildbot_gentoo_ci/steps/version.py | 2 +-
6 files changed, 85 insertions(+), 25 deletions(-)
diff --git a/buildbot_gentoo_ci/db/packages.py
b/buildbot_gentoo_ci/db/packages.py
index 13cc267..a22b9d8 100644
--- a/buildbot_gentoo_ci/db/packages.py
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -27,11 +27,13 @@ from buildbot.db import base
class PackagesConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
-def getPackageByName(self, name):
+def getPackageByName(self, name, c_uuid, repo_uuid):
def thd(conn):
tbl = self.db.model.packages
q = tbl.select()
q = q.where(tbl.c.name == name)
+q = q.where(tbl.c.category_uuid == c_uuid)
+q = q.where(tbl.c.repository_uuid == repo_uuid)
res = conn.execute(q)
row = res.fetchone()
if not row:
diff --git a/buildbot_gentoo_ci/db/versions.py
b/buildbot_gentoo_ci/db/versions.py
index 85bd105..768c1dc 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -28,11 +28,12 @@ from buildbot.util import datetime2epoch
class VersionsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
-def getVersionByName(self, name):
+def getVersionByName(self, name, p_uuid):
def thd(conn):
tbl = self.db.model.versions
q = tbl.select()
q = q.where(tbl.c.name == name)
+q = q.where(tbl.c.package_uuid == p_uuid)
res = conn.execute(q)
row = res.fetchone()
if not row:
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index af3be9d..e51576f 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -4,7 +4,7 @@
import os
import re
-from portage.versions import catpkgsplit
+from portage.versions import catpkgsplit, cpv_getversion
from twisted.internet import defer
from twisted.python import log
@@ -19,10 +19,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['rc'] = rc
emerge_output['preserved_libs'] = False
emerge_output['change_use'] = False
+emerge_output['failed'] = False
package_dict = {}
log_path_list = []
print(stderr)
-emerge_output['stderr'] = stderr
# split the lines
for line in stdout.split('\n'):
# package list
@@ -67,6 +67,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# CPU_FLAGS_X86 list
package_dict[cpv] = subdict
if line.startswith('>>>'):
+if line.startswith('>>> Failed to'):
+emerge_output['failed'] = line.split(' ')[4][:-1]
+if line.endswith('.log.gz') and emerge_output['failed']:
+log_path_list.append(line.split(' ')[2])
#FIXME: Handling of >>> output
pass
if line.startswith('!!!'):
@@ -78,9 +82,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
log_path_list.append(line.split(' ')[4])
#FIXME: Handling of depclean output dict of packages that get removed
or saved
emerge_output['package'] = package_dict
-emerge_output['log_paths'] = log_path_list
+
# split the lines
#FIXME: Handling of stderr output
+stderr_line_list = []
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
@@ -96,6 +101,16 @@ def PersOutputOfEmerge(rc, stdout, stderr):
else:
change_use_list = False
emerge_output['change_use'] = change_use_list
+err_line_list = []
+if line.startswith(' * '):
+if line.endswith('.log.gz'):
+log_path = line.split(' ')[3]
+if log_path not in inlog_path_list:
+log_path_list.append(log_path)
+stderr_line_list.append(line)
+emerge_output['stderr'] = stderr_line_list
+emerge_output['log_paths'] = log_path_list
+
return {
'emerge_output' : emerge_output
}
@@ -484,6 +499,30 @@ class CheckEmergeLogs(BuildStep):
self.step = step
super().__init__(**kwargs)
self.descriptionSuffix = self.step
+self.aftersteps_list = []
+self.log_data = {}
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: 81e7f0bdd56bf11fdbc0990191273ad59c1a3b01
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Apr 4 20:29:38 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Apr 4 20:29:38 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=81e7f0bd
Add support for Change USE, one use only
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 2 +-
buildbot_gentoo_ci/steps/builders.py | 58 ++
buildbot_gentoo_ci/steps/portage.py| 1 +
3 files changed, 54 insertions(+), 7 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 6e8baf6..fddfa2b 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -120,7 +120,7 @@ def run_build_request():
# setup env
f.addStep(portage.SetEnvDefault())
# setup package.*
-#f.addStep(portage.SetPackageDefault())
+f.addStep(portage.SetPackageDefault())
# setup files in /etc if needed
# run --regen if needed on repo
# update packages before any tests
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index b28d7e5..96572c0 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -19,6 +19,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['rc'] = rc
emerge_output['preserved_libs'] = False
emerge_output['depclean'] = False
+emerge_output['change_use'] = False
package_dict = {}
log_path_list = []
print(stderr)
@@ -82,7 +83,20 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# split the lines
#FIXME: Handling of stderr output
for line in stderr.split('\n'):
-pass
+if 'Change USE:' in line:
+line_list = line.split(' ')
+change_use_list = []
+# get cpv
+cpv_split = line_list[1].split(':')
+change_use_list.append(cpv_split[0])
+# add use flags
+if line_list[4].startswith('+') or line_list[4].startswith('-'):
+# we only support one for now
+if line_list[4].endswith(')'):
+change_use_list.append(line_list[4].replace(')', ''))
+else:
+change_use_list = False
+emerge_output['change_use'] = change_use_list
return {
'emerge_output' : emerge_output
}
@@ -209,6 +223,7 @@ class SetupPropertys(BuildStep):
self.setProperty('cpv_build', False, 'cpv_build')
self.setProperty('pkg_check_log_data', None, 'pkg_check_log_data')
self.setProperty('faild_version_data', None, 'faild_version_data')
+self.setProperty('rerun', 0, 'rerun')
print(self.getProperty("buildnumber"))
if self.getProperty('project_build_data') is None:
project_build_data = {}
@@ -268,13 +283,13 @@ class RunEmerge(BuildStep):
name = 'RunEmerge'
description = 'Running'
descriptionDone = 'Ran'
-descriptionSuffix = None
haltOnFailure = True
flunkOnFailure = True
def __init__(self, step=None,**kwargs):
self.step = step
super().__init__(**kwargs)
+self.descriptionSuffix = self.step
@defer.inlineCallbacks
def run(self):
@@ -388,7 +403,6 @@ class RunEmerge(BuildStep):
cpv = self.getProperty("cpv")
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
-shell_commad_list.append('-p')
shell_commad_list.append('=' + self.getProperty('cpv'))
# we don't use the bin for the requsted cpv
shell_commad_list.append('--usepkg-exclude')
@@ -398,6 +412,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('virtual')
shell_commad_list.append('--buildpkg-exclude')
shell_commad_list.append('acct-*')
+shell_commad_list.append('-p')
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
command=shell_commad_list,
@@ -444,13 +459,13 @@ class CheckEmergeLogs(BuildStep):
name = 'CheckLogs'
description = 'Running'
descriptionDone = 'Ran'
-descriptionSuffix = None
haltOnFailure = True
flunkOnFailure = True
def __init__(self, step=None,**kwargs):
self.step = step
super().__init__(**kwargs)
+self.descriptionSuffix = self.step
@defer.inlineCallbacks
def run(self):
@@ -480,10 +495,41 @@ class CheckEmergeLogs(BuildStep):
print(self.getProperty('cpv_build'))
#FIXME:
-# update package.* if needed and rerun pre-build max X times
+# update package.* if needed and rerun pre-build max 3 times
if self.step == 'pre-build':
print(emerge_output)
-
+
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: 915abba59f59767d341bb190de5bba5ae03097d0
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Apr 4 20:26:01 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Apr 4 20:26:01 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=915abba5
Add build log sammary
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 2 +-
buildbot_gentoo_ci/steps/logs.py | 35 +-
2 files changed, 30 insertions(+), 7 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index d8db8f8..6e8baf6 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -177,7 +177,7 @@ def parse_build_log():
# set it SUCCESS/FAILURE/WARNINGS
f.addStep(logs.MakeIssue())
# add sum log to buildbot log
-#f.addStep(logs.setBuildbotLog)
+f.addStep(logs.setBuildbotLog())
# set BuildStatus
f.addStep(logs.setBuildStatus())
# setup things for the irc bot
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 5fe248f..767fdd4 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -175,7 +175,7 @@ class ParserBuildLog(BuildStep):
print(self.summery_dict)
# remove all lines with ignore in the dict
# setProperty summery_dict
-self.setProperty("summery_log_dict", self.summery_dict,
'summery_log_dict')
+self.setProperty("summary_log_dict", self.summery_dict,
'summary_log_dict')
return SUCCESS
class MakeIssue(BuildStep):
@@ -193,27 +193,50 @@ class MakeIssue(BuildStep):
#@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
-summery_log_dict = self.getProperty('summery_log_dict')
+summary_log_dict = self.getProperty('summary_log_dict')
error = False
warning = False
-self.summery_log_list = []
+self.summary_log_list = []
log_hash = hashlib.sha256()
-for k, v in sorted(summery_log_dict.items()):
+for k, v in sorted(summary_log_dict.items()):
if v['status'] == 'error':
error = True
if v['status'] == 'warning':
warning = True
-self.summery_log_list.append(v['text'])
+self.summary_log_list.append(v['text'])
log_hash.update(v['text'].encode('utf-8'))
# add build log
# add issue/bug/pr report
-self.setProperty("summery_log_list", self.summery_log_list,
'summery_log_list')
+self.setProperty("summary_log_list", self.summary_log_list,
'summary_log_list')
if error:
self.setProperty("status", 'failed', 'status')
if warning:
self.setProperty("status", 'warning', 'status')
return SUCCESS
+class setBuildbotLog(BuildStep):
+
+name = 'setBuildbotLog'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = False
+flunkOnFailure = True
+warnOnWarnings = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+#setup the log
+log = yield self.addLog('summary')
+# add line for line
+for line in self.getProperty('summary_log_list'):
+yield log.addStdout(line + '\n')
+# add emerge info log
+return SUCCESS
+
class setBuildStatus(BuildStep):
name = 'setBuildStatus'
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 489eb63a368e9bbfa73fcbda96cae8a1222e1287
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Apr 2 22:06:40 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Apr 2 22:06:40 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=489eb63a
Don't run emerge build quiet
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index dce633f..b28d7e5 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -412,7 +412,6 @@ class RunEmerge(BuildStep):
cpv = self.getProperty("cpv")
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
-shell_commad_list.append('-q')
if projects_emerge_options['oneshot']:
shell_commad_list.append('-1')
shell_commad_list.append('=' + self.getProperty('cpv'))
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 8d863f91968ff02e963ac59a75265e7e3af2cfcd
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Apr 2 22:03:07 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Apr 2 22:03:07 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8d863f91
Check so keyword is not None
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 11 ++-
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index ff6dd94..dce633f 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -171,11 +171,12 @@ class GetProjectRepositoryData(BuildStep):
if project_keyword_data['name'] != '*' or
project_data['status'] == 'all':
self.setProperty('fullcheck', False, 'fullcheck')
# get status of the keyword on cpv
-if project_keyword_data['name'] in
self.getProperty("version_keyword_dict"):
-version_keywords_data =
self.getProperty("version_keyword_dict")[project_keyword_data['name']]
-# if match trigger BuildRequest on cpv
-if project_data['status'] ==
version_keywords_data['status']:
-yield
self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
+if self.getProperty("version_keyword_dict") is not None:
+if project_keyword_data['name'] in
self.getProperty("version_keyword_dict"):
+version_keywords_data =
self.getProperty("version_keyword_dict")[project_keyword_data['name']]
+# if match trigger BuildRequest on cpv
+if project_data['status'] ==
version_keywords_data['status']:
+yield
self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
return SUCCESS
class SetupPropertys(BuildStep):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 1560eecc8a0c32e219acfeda461dd6cb2d015f30
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Apr 2 22:09:09 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Apr 2 22:09:09 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1560eecc
Check '>>>' add pattern type
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 30 ++
1 file changed, 22 insertions(+), 8 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index ae41fe5..5fe248f 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -62,7 +62,7 @@ class ParserBuildLog(BuildStep):
@defer.inlineCallbacks
def get_log_search_pattern(self):
-# get pattern from the profile
+# get pattern from the projects
# add that to log_search_pattern_list
for project_pattern in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
self.log_search_pattern_list.append(project_pattern)
@@ -82,13 +82,21 @@ class ParserBuildLog(BuildStep):
text_line = self.logfile_text_dict[tmp_index]
# loop true the pattern list for match
for search_pattern in self.log_search_pattern_list:
-if re.search(search_pattern['search'], text_line) and not
search_pattern['status'] == 'ignore':
+# we add all line that start with ' * ' as info
+# we add all line that start with '>>>' but not '>>> /' as info
+if text_line.startswith(' * ') or (text_line.startswith('>>>') and
not text_line.startswith('>>> /')):
self.summery_dict[tmp_index] = {}
self.summery_dict[tmp_index]['text'] = text_line
+self.summery_dict[tmp_index]['type'] = 'info'
+self.summery_dict[tmp_index]['status'] = 'info'
+if re.search(search_pattern['search'], text_line):
+self.summery_dict[tmp_index] = {}
+self.summery_dict[tmp_index]['text'] = text_line
+self.summery_dict[tmp_index]['type'] = search_pattern['type']
self.summery_dict[tmp_index]['status'] =
search_pattern['status']
# add upper text lines if requested
# max 10
-if search_pattern['start'] is not 0:
+if search_pattern['start'] != 0:
i = tmp_index
i_start = i - search_pattern['start']
match = True
@@ -99,10 +107,11 @@ class ParserBuildLog(BuildStep):
else:
self.summery_dict[i] = {}
self.summery_dict[i]['text'] =
self.logfile_text_dict[i]
+self.summery_dict[i]['type'] =
search_pattern['type']
self.summery_dict[i]['status'] = 'info'
# add lower text lines if requested
# max 10
-if search_pattern['end'] is not 0:
+if search_pattern['end'] != 0:
i = tmp_index
i_end = i + search_pattern['end']
match = True
@@ -113,6 +122,7 @@ class ParserBuildLog(BuildStep):
else:
self.summery_dict[i] = {}
self.summery_dict[i]['text'] =
self.logfile_text_dict[i]
+self.summery_dict[i]['type'] =
search_pattern['type']
self.summery_dict[i]['status'] = 'info'
# add text lines if requested that we need to search for the
end
# max 10
@@ -127,13 +137,15 @@ class ParserBuildLog(BuildStep):
if not i + 1 > self.max_text_lines or not
re.search(search_pattern['search_end'], self.logfile_text_dict[i + 1]):
self.summery_dict[i] = {}
self.summery_dict[i]['text'] =
self.logfile_text_dict[i]
+self.summery_dict[i]['type'] =
search_pattern['type']
self.summery_dict[i]['status'] = 'info'
else:
match = False
else:
self.summery_dict[i] = {}
self.summery_dict[i]['text'] =
self.logfile_text_dict[i]
-self.summery_dict[i]['status'] = 'info'
+self.summery_dict[i]['type'] =
search_pattern['type']
+self.summery_dict[i]['status'] = 'info'
@defer.inlineCallbacks
def run(self):
@@ -148,9 +160,9 @@ class ParserBuildLog(BuildStep):
file_path = yield os.path.join(self.master.basedir, 'cpv_logs',
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 8bdcac12a5c3a41d8321a55b8fef36355dc0f6e7
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Apr 2 22:11:49 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Apr 2 22:11:49 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8bdcac12
Add GetAuxMetadata and use that to get aux metadata
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 63 ---
buildbot_gentoo_ci/steps/update_db.py | 17 +-
buildbot_gentoo_ci/steps/version.py | 12 ---
3 files changed, 83 insertions(+), 9 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 293b862..d66c952 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -10,6 +10,7 @@ from portage import _encodings
from portage import _unicode_encode
from portage import _parse_eapi_ebuild_head, eapi_is_supported
from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
+from portage import portdbapi
from twisted.internet import defer
from twisted.python import log
@@ -249,7 +250,7 @@ class SetMakeConf(BuildStep):
makeconf_list.append('DISTDIR="/var/cache/portage/distfiles"')
makeconf_list.append('PORTAGE_ELOG_CLASSES="*"')
# We need echo:info to get the logfile name
-makeconf_list.append('PORTAGE_ELOG_SYSTEM="save echo:info"')
+makeconf_list.append('PORTAGE_ELOG_SYSTEM="save:* echo:info"')
# add ACCEPT_KEYWORDS from the project_data info
keyword_data = yield
self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
if project_data['status'] == 'unstable':
@@ -395,7 +396,8 @@ class SetReposConfLocal(BuildStep):
repos_conf_path = yield os.path.join('etc', 'portage', 'repos.conf')
repos_conf_default_path = yield os.path.join(repos_conf_path,
'default.conf')
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
-self.repository_basedir =
self.gentooci.config.project['repository_basedir']
+# the path should be set in the confg
+self.repository_basedir2 = '/home/repos2/'
if not os.path.isfile(repos_conf_default_path):
# setup the default.conf
repos_conf_data = yield
self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'],
'repos.conf')
@@ -409,7 +411,7 @@ class SetReposConfLocal(BuildStep):
yield WriteTextToFile(repos_conf_default_path, default_conf)
repos_conf_repository_path = yield os.path.join(repos_conf_path,
self.getProperty("repository_data")['name'] + '.conf')
if not os.path.isfile(repos_conf_repository_path):
-repository_path = yield os.path.join(self.getProperty("builddir"),
self.repository_basedir, self.getProperty("repository_data")['name'])
+repository_path = yield os.path.join(self.repository_basedir2,
self.getProperty("repository_data")['name'])
repository_conf = []
repository_conf.append('[' +
self.getProperty("repository_data")['name'] + ']')
repository_conf.append('location = ' + repository_path)
@@ -444,7 +446,7 @@ class SetMakeConfLocal(BuildStep):
makeconf_list.append('ACCEPT_KEYWORDS="~amd64 amd64"')
makeconf_list.append('EMERGE_DEFAULT_OPTS=""')
makeconf_list.append('ABI_X86="32 64"')
-makeconf_list.append('FEATURES=""')
+makeconf_list.append('FEATURES="sandbox"')
yield WriteTextToFile(make_conf_path, makeconf_list)
return SUCCESS
@@ -530,3 +532,56 @@ class SetEnvForEbuildSH(BuildStep):
))
yield self.build.addStepsAfterCurrentStep(addStepEbuildSH)
return SUCCESS
+
+class GetAuxMetadata(BuildStep):
+
+name = 'GetAuxMetadata'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+#@defer.inlineCallbacks
+def thd_getAuxDbKeys(self):
+auxdbs = self.myportdb.aux_get(self.getProperty("cpv"), auxdbkeys,
myrepo=self.getProperty("repository_data")['name'])
+return auxdbs
+
+@defer.inlineCallbacks
+def run(self):
+config_root = yield os.path.join(self.getProperty("builddir"), '')
+# setup mysettings and myportdb
+mysettings = yield portage_config(config_root = config_root)
+self.myportdb = yield portdbapi(mysettings=mysettings)
+auxdbs = yield self.thd_getAuxDbKeys()
+metadata = None
+NoSplit = []
+NoSplit.append('DESCRIPTION')
+# should have 22 lines
+if len(auxdbkeys) != len(auxdbs) or not isinstance(auxdbs, list):
+# number of lines is incorrect or
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: f4ee7748cdd77534ace36b6bd957f27229fbd748
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Mar 29 00:24:46 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Mar 29 00:24:46 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f4ee7748
Change code for the PersOutputOfEbuildSH so we may fix *deps to long lines
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 32 +++-
buildbot_gentoo_ci/steps/version.py | 10 ++
2 files changed, 25 insertions(+), 17 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index e2315cb..293b862 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -32,27 +32,33 @@ def WriteTextToFile(path, text_list):
def PersOutputOfEbuildSH(rc, stdout, stderr):
metadata = None
-metadata_lines = stdout.splitlines()
-metadata_valid = True
NoSplit = []
NoSplit.append('DESCRIPTION')
-if len(auxdbkeys) != len(metadata_lines):
-# Don't trust bash's returncode if the
+#make dict of the stout
+index = 1
+metadata_line_dict = {}
+for text_line in stdout.splitlines():
+metadata_line_dict[index] = text_line
+index = index + 1
+# should have 22 lines
+if len(auxdbkeys) != index -1:
# number of lines is incorrect.
return {
'auxdb' : metadata
}
-else:
-metadata_tmp = dict(zip(auxdbkeys, metadata_lines))
+# split all keys to list instead of speces
metadata = {}
-for k, v in metadata_tmp.items():
-if v == '':
-metadata[k] = None
+i = 1
+for key in auxdbkeys:
+if metadata_line_dict[i] == '':
+metadata[key] = None
else:
-if ' ' in v and k not in NoSplit:
-metadata[k] = v.split(' ')
+if ' ' in metadata_line_dict[i] and key not in NoSplit:
+metadata[key] = metadata_line_dict[i].split(' ')
else:
-metadata[k] = v
+metadata[key] = []
+metadata[key].append(metadata_line_dict[i])
+i = i + 1
return {
'auxdb' : metadata
}
@@ -519,7 +525,7 @@ class SetEnvForEbuildSH(BuildStep):
command=ebuild_commands,
env=ebuild_env,
workdir=self.getProperty("builddir"),
-strip=True,
+strip=False,
extract_fn=PersOutputOfEbuildSH
))
yield self.build.addStepsAfterCurrentStep(addStepEbuildSH)
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index 143a758..4c35914 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -125,9 +125,13 @@ class AddVersionKeyword(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.version_keyword_dict = {}
auxdb = self.getProperty("auxdb")['KEYWORDS']
-if auxdb is None:
-auxdb = []
+if auxdb is None or not isinstance(auxdb, list):
+self.version_keyword_dict = None
+self.setProperty('version_keyword_dict',
self.version_keyword_dict, 'version_keyword_dict')
+return SUCCESS
+print(auxdb)
for keyword in auxdb:
+print(keyword)
status = 'stable'
if keyword[0] in ["~"]:
keyword = keyword[1:]
@@ -147,8 +151,6 @@ class AddVersionKeyword(BuildStep):
version_keyword_data['keyword_id'],
version_keyword_data['status'])
self.version_keyword_dict[keyword] = version_keyword_data
-if self.version_keyword_dict == {}:
-self.version_keyword_dict = None
self.setProperty('version_keyword_dict', self.version_keyword_dict,
'version_keyword_dict')
return SUCCESS
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: e01a8423d8cc9371320ea0016107d13c3a7dd991
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Mar 29 00:33:07 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Mar 29 00:33:07 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e01a8423
Change code in TriggerCheckForCPV for trigging
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/update_db.py | 50 +--
1 file changed, 25 insertions(+), 25 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/update_db.py
b/buildbot_gentoo_ci/steps/update_db.py
index d35a967..b84e256 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -181,40 +181,40 @@ class TriggerCheckForCPV(BuildStep):
# check if git_change is a string or a list
if not isinstance(self.git_changes, list):
return FAILURE
-self.success = True
addStepUpdateCPVData = []
for change_data in self.git_changes:
# make a trigger for all cpv in the list
for cpv in change_data['cpvs']:
+self.success = True
+if change_data['repository'] !=
self.getProperty("repository_data")['name']:
+log.msg("%s don't match" % change_data['repository'])
+self.success = False
+# Trigger cpv builds and update db if we are working with
ebuilds
# check that cpv is valied
if catpkgsplit(cpv) is None:
log.msg("%s is not vaild package name" % cpv)
self.success = False
-else:
-if change_data['repository'] !=
self.getProperty("repository_data")['name']:
-log.msg("%s don't match" % change_data['repository'])
-self.success = False
-else:
-revision_data = {}
-revision_data['author'] = change_data['author']
-revision_data['committer'] = change_data['committer']
-revision_data['comments'] = change_data['comments']
-revision_data['revision'] = change_data['revision']
-# call update_cpv_data
-addStepUpdateCPVData.append(
-steps.Trigger(
-schedulerNames=['update_cpv_data'],
-waitForFinish=False,
-updateSourceStamp=False,
-set_properties={
-'cpv' : cpv,
-'project_data' :
self.getProperty("project_data"),
-'repository_data' :
self.getProperty("repository_data"),
-'revision_data' : revision_data,
-}
-)
+if self.success:
+revision_data = {}
+revision_data['author'] = change_data['author']
+revision_data['committer'] = change_data['committer']
+revision_data['comments'] = change_data['comments']
+revision_data['revision'] = change_data['revision']
+addStepUpdateCPVData.append(
+steps.Trigger(
+schedulerNames=['update_cpv_data'],
+waitForFinish=False,
+updateSourceStamp=False,
+set_properties={
+'cpv' : cpv,
+'project_data' :
self.getProperty("project_data"),
+'repository_data' :
self.getProperty("repository_data"),
+'revision_data' : revision_data,
+}
)
-yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData)
+)
+print(addStepUpdateCPVData)
+yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData)
if self.success is False:
return FAILURE
return SUCCESS
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: b720c6bbcb79211a276d06a1eaf1543b89644767
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Mar 24 23:14:47 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Mar 24 23:14:47 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b720c6bb
Add the log parser BuildFactory steps
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/builders.py | 8 +++
buildbot_gentoo_ci/config/buildfactorys.py | 22 +++-
buildbot_gentoo_ci/config/schedulers.py| 3 +
buildbot_gentoo_ci/steps/builders.py | 90 --
buildbot_gentoo_ci/steps/portage.py| 10 +++-
5 files changed, 101 insertions(+), 32 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index c7b4469..0235753 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -49,4 +49,12 @@ def gentoo_builders(b=[]):
factory=buildfactorys.run_build_request()
)
)
+# FIXME: get workers from db
+# Use multiplay workers
+b.append(util.BuilderConfig(
+name='parse_build_log',
+workername='updatedb_1',
+factory=buildfactorys.parse_build_log()
+)
+)
return b
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index db48130..df7baef 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -10,6 +10,7 @@ from buildbot_gentoo_ci.steps import package
from buildbot_gentoo_ci.steps import version
from buildbot_gentoo_ci.steps import builders
from buildbot_gentoo_ci.steps import portage
+from buildbot_gentoo_ci.steps import logs
def update_db_check():
f = util.BuildFactory()
@@ -159,5 +160,24 @@ def run_build_request():
f.addStep(builders.RunEmerge(step='depclean'))
f.addStep(builders.RunEmerge(step='preserved-libs'))
f.addStep(builders.RunEmerge(step='depclean'))
-f.addStep(builders.setBuildStatus())
+return f
+
+def parse_build_log():
+f = util.BuildFactory()
+# FIXME: 6
+# set needed Propertys
+f.addStep(logs.SetupPropertys())
+# pers the build log for info qa errors
+#f.addStep(logs.ParserBuildLog())
+# pers the log from pkg check
+#f.addStep(logs.ParserPkgCheckLog())
+# Upload the log to the cloud and remove the log
+#f.addStep(logs.Upload())
+# check the sum log if we need to make a issue/bug/pr report
+# set it SUCCESS/FAILURE/WARNINGS
+#f.addStep(logs.MakeIssue())
+# set BuildStatus
+#f.addStep(logs.setBuildStatus())
+# setup things for the irc bot
+#f.addStep(logs.SetIrcInfo())
return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index 0cbec96..efdac75 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -74,6 +74,8 @@ def gentoo_schedulers():
builderNames=["build_request_data"])
run_build_request = schedulers.Triggerable(name="run_build_request",
builderNames=["run_build_request"])
+pers_build_log = schedulers.Triggerable(name="parse_build_log",
+ builderNames=["parse_build_log"])
s = []
s.append(test_updatedb)
s.append(scheduler_update_db)
@@ -81,4 +83,5 @@ def gentoo_schedulers():
s.append(update_v_data)
s.append(build_request_data)
s.append(run_build_request)
+s.append(parse_build_log)
return s
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 22608e3..ff6dd94 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -20,22 +20,26 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['preserved_libs'] = False
emerge_output['depclean'] = False
package_dict = {}
+log_path_list = []
print(stderr)
emerge_output['stderr'] = stderr
# split the lines
for line in stdout.split('\n'):
# package list
subdict = {}
-if line.startswith('[ebuild') or line.startswith('[binary'):
+subdict2 = {}
+if line.startswith('[ebuild') or line.startswith('[binary') or
line.startswith('[nomerge'):
# if binaries
-if line.startswith('[ebuild'):
+if line.startswith('[ebuild') or line.startswith('[nomerge'):
subdict['binary'] = False
else:
subdict['binary'] = True
# action [ N ] stuff
subdict['action'] = line[8:15].replace(' ', '')
# cpv
+#FIXME: We my have more then one spece betvine ] and cpv
cpv_split = re.search('] (.+?) ', line).group(1).split(':')
+print(cpv_split)
cpv = cpv_split[0]
#
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: c852e4cb0089da63776d0b5939eadd1180da7080
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Mar 20 13:41:42 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Mar 20 13:41:42 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c852e4cb
Fix gitpull back to origin
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/update_db.py | 45 +++
1 file changed, 25 insertions(+), 20 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/update_db.py
b/buildbot_gentoo_ci/steps/update_db.py
index bfcd2ef..d35a967 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -65,7 +65,7 @@ class CheckPath(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.repository_basedir =
self.gentooci.config.project['repository_basedir']
-self.portage_path = 'portage'
+self.portage_path = yield os.path.join('etc', 'portage')
self.profile_path = yield os.path.join(self.portage_path,
'make.profile')
self.repos_path = yield os.path.join(self.portage_path, 'repos.conf')
print(os.getcwd())
@@ -74,10 +74,11 @@ class CheckPath(BuildStep):
success = True
print(os.getcwd())
for x in [
- self.profile_path,
- self.repos_path,
- self.repository_basedir
- ]:
+self.portage_path,
+self.profile_path,
+self.repos_path,
+self.repository_basedir
+]:
if not os.path.isdir(x):
os.makedirs(x)
return SUCCESS
@@ -96,48 +97,52 @@ class UpdateRepos(BuildStep):
# Origin:
https://github.com/MichaelBoselowitz/pygit2-examples/blob/master/examples.py#L54
# Modifyed by Gentoo Authors.
-@defer.inlineCallbacks
def gitPull(self, repo, remote_name='origin', branch='master'):
for remote in repo.remotes:
if remote.name == remote_name:
-yield remote.fetch()
-remote_master_id = yield
repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target
-merge_result, _ = yield repo.merge_analysis(remote_master_id)
+remote.fetch()
+remote_master_id =
repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target
+print(remote_master_id)
+merge_result, _ = repo.merge_analysis(remote_master_id)
+print(merge_result)
# Up to date, do nothing
if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
+print('UP_TO_DATE')
return
# We can just fastforward
elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
-yield repo.checkout_tree(repo.get(remote_master_id))
+print('FASTFORWARD')
+repo.checkout_tree(repo.get(remote_master_id))
try:
-master_ref = yield
repo.lookup_reference('refs/heads/%s' % (branch))
-yield master_ref.set_target(remote_master_id)
+master_ref = repo.lookup_reference('refs/heads/%s' %
(branch))
+master_ref.set_target(remote_master_id)
except KeyError:
-yield repo.create_branch(branch,
repo.get(remote_master_id))
-yield repo.head.set_target(remote_master_id)
+repo.create_branch(branch, repo.get(remote_master_id))
+repo.head.set_target(remote_master_id)
elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
-yield repo.merge(remote_master_id)
-
+print('NORMAL')
+repo.merge(remote_master_id)
if repo.index.conflicts is not None:
for conflict in repo.index.conflicts:
print('Conflicts found in:', conflict[0].path)
raise AssertionError('Conflicts, ah!!')
-user = yield repo.default_signature
-tree = yield repo.index.write_tree()
-commit = yield repo.create_commit('HEAD',
+user = repo.default_signature
+tree = repo.index.write_tree()
+commit = repo.create_commit('HEAD',
user,
user,
'Merge!',
tree,
[repo.head.target,
remote_master_id])
# We need to do this or git CLI will think
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: ee9e8401620340982d6e922ca982f50fee1ec6fa
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Mar 20 13:34:03 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Mar 20 13:34:03 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ee9e8401
Change GetCommitdata to use revision_data. Use SetEnvForEbuildSH to get auxdb
data.
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/master.py | 235
buildbot_gentoo_ci/steps/portage.py | 130 +++-
buildbot_gentoo_ci/steps/version.py | 66 +++---
3 files changed, 380 insertions(+), 51 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/master.py
b/buildbot_gentoo_ci/steps/master.py
new file mode 100644
index 000..dac3830
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/master.py
@@ -0,0 +1,235 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.steps.master.py
+# buildbot.steps.shell.py
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import os
+import pprint
+import re
+
+from twisted.internet import defer
+from twisted.internet import error
+from twisted.internet import reactor
+from twisted.internet.protocol import ProcessProtocol
+from twisted.python import runtime
+from twisted.python.versions import Version
+
+from buildbot.process import buildstep
+from buildbot.process import logobserver
+from buildbot.process.results import FAILURE
+from buildbot.process.results import SUCCESS
+from buildbot.util import deferwaiter
+
+
+class MasterSetPropertyFromCommand(buildstep.ShellMixin, buildstep.BuildStep):
+
+"""
+Run a shell command locally - on the buildmaster. The shell command
+COMMAND is specified just as for a RemoteShellCommand. Note that extra
+logfiles are not supported.
+"""
+name = "Mastersetproperty"
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+renderables = ['command', 'env', 'property']
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, command, property=None, extract_fn=None, strip=True,
+ includeStdout=True, includeStderr=False, **kwargs):
+self.env = kwargs.pop('env', None)
+self.usePTY = kwargs.pop('usePTY', 0)
+self.interruptSignal = kwargs.pop('interruptSignal', 'KILL')
+self.logEnviron = kwargs.pop('logEnviron', True)
+
+self.property = property
+self.extract_fn = extract_fn
+self.strip = strip
+self.includeStdout = includeStdout
+self.includeStderr = includeStderr
+
+if not ((property is not None) ^ (extract_fn is not None)):
+config.error(
+"Exactly one of property and extract_fn must be set")
+
+super().__init__(**kwargs)
+
+self.command = command
+self.masterWorkdir = self.workdir
+self._deferwaiter = deferwaiter.DeferWaiter()
+self._status_object = None
+self.success = True
+
+if self.extract_fn:
+self.includeStderr = True
+
+self.observer = logobserver.BufferLogObserver(
+wantStdout=self.includeStdout,
+wantStderr=self.includeStderr)
+self.addLogObserver('stdio', self.observer)
+
+class LocalPP(ProcessProtocol):
+
+def __init__(self, step):
+self.step = step
+self._finish_d = defer.Deferred()
+self.step._deferwaiter.add(self._finish_d)
+
+def outReceived(self, data):
+self.step._deferwaiter.add(self.step.stdio_log.addStdout(data))
+
+def errReceived(self, data):
+self.step._deferwaiter.add(self.step.stdio_log.addStderr(data))
+
+def processEnded(self, status_object):
+if status_object.value.exitCode is not None:
+msg = "exit status {}\n".format(status_object.value.exitCode)
+self.step._deferwaiter.add(self.step.stdio_log.addHeader(msg))
+
+if status_object.value.signal is not None:
+msg = "signal {}\n".format(status_object.value.signal)
+self.step._deferwaiter.add(self.step.stdio_log.addHeader(msg))
+
+
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: f2a3a0fdedff108964c0ffee79cd2757edcb6f32
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Mar 20 13:40:11 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Mar 20 13:40:11 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f2a3a0fd
Check that keyword is in version_keyword_dict
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 9 +
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index d09c4b6..22608e3 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -163,10 +163,11 @@ class GetProjectRepositoryData(BuildStep):
if project_keyword_data['name'] != '*' or
project_data['status'] == 'all':
self.setProperty('fullcheck', False, 'fullcheck')
# get status of the keyword on cpv
-version_keywords_data =
self.getProperty("version_keyword_dict")[project_keyword_data['name']]
-# if unstable trigger BuildRequest on cpv
-if project_data['status'] ==
version_keywords_data['status']:
-yield
self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
+if project_keyword_data['name'] in
self.getProperty("version_keyword_dict"):
+version_keywords_data =
self.getProperty("version_keyword_dict")[project_keyword_data['name']]
+# if match trigger BuildRequest on cpv
+if project_data['status'] ==
version_keywords_data['status']:
+yield
self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
return SUCCESS
class SetupPropertys(BuildStep):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: 04836c8c78369e1a5acbc460343b003c3a337f20
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Mar 18 23:16:08 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Mar 18 23:16:08 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=04836c8c
Clean up Package and Category code
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 8 +++---
buildbot_gentoo_ci/steps/category.py | 25 ++---
buildbot_gentoo_ci/steps/package.py| 44 +-
3 files changed, 44 insertions(+), 33 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index f5a3b35..db48130 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -41,17 +41,17 @@ def update_db_cp():
# else
# add category to db step
# return category_data
-f.addStep(category.CheckCGentooCiProject())
+f.addStep(category.CheckC())
# if package in db
# return package_data
# add check package path step at end
# else
# add package to db step
# return package_data
-f.addStep(package.CheckPGentooCiProject())
+f.addStep(package.CheckP())
# Trigger new builders with v from cpv
-# return package_data, cpv, repository_data, project_data, config_root
-f.addStep(package.TriggerCheckVGentooCiProject())
+# return package_data, cpv, repository_data, project_data
+f.addStep(package.TriggerCheckForV())
return f
def update_db_v():
diff --git a/buildbot_gentoo_ci/steps/category.py
b/buildbot_gentoo_ci/steps/category.py
index 1aca99a..81061e6 100644
--- a/buildbot_gentoo_ci/steps/category.py
+++ b/buildbot_gentoo_ci/steps/category.py
@@ -18,6 +18,13 @@ class AddCategory(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
+name = 'AddCategory'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
@@ -26,27 +33,25 @@ class AddCategory(BuildStep):
self.category_data['uuid'] = yield
self.gentooci.db.categorys.addCategory(self.category_data['name'])
print(self.category_data)
self.setProperty("category_data", self.category_data, 'category_data')
-self.setProperty("config_root", self.getProperty("config_root"),
'config_root')
-self.setProperty("project_data", self.getProperty("project_data"),
'project_data')
-self.setProperty("repository_data",
self.getProperty("repository_data"), 'repository_data')
-self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
return SUCCESS
-class CheckCGentooCiProject(BuildStep):
+class CheckC(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
+name = 'CheckC'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.category = yield self.getProperty("cpv").split('/')[0]
print(self.category)
-print(self.getProperty("repository"))
self.category_data = yield
self.gentooci.db.categorys.getCategoryByName(self.category)
-self.setProperty("config_root", self.getProperty("config_root"),
'config_root')
-self.setProperty("project_data", self.getProperty("project_data"),
'project_data')
-self.setProperty("repository_data",
self.getProperty("repository_data"), 'repository_data')
-self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
print(self.category_data)
if self.category_data is None:
self.setProperty("category", self.category, 'category')
diff --git a/buildbot_gentoo_ci/steps/package.py
b/buildbot_gentoo_ci/steps/package.py
index 3a28da2..ea5ab23 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -20,6 +20,13 @@ class AddPackage(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
+name = 'AddPackage'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
@@ -34,28 +41,25 @@ class AddPackage(BuildStep):
)
print(self.package_data)
self.setProperty("package_data", self.package_data, 'package_data')
-self.setProperty("config_root",
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
commit: 5176761ad56557e45359729d7b2989dadeace7b5
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Mar 11 01:14:02 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Mar 11 01:14:02 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5176761a
Add GitPuller to change_source and change cpv_changes to git_changes
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/change_source.py | 18 ++
buildbot_gentoo_ci/config/schedulers.py| 40 +
buildbot_gentoo_ci/db/model.py | 10 +++-
buildbot_gentoo_ci/db/repositorys.py | 48 ++-
buildbot_gentoo_ci/steps/update_db.py | 95 +++---
5 files changed, 134 insertions(+), 77 deletions(-)
diff --git a/buildbot_gentoo_ci/config/change_source.py
b/buildbot_gentoo_ci/config/change_source.py
new file mode 100644
index 000..454ed4a
--- /dev/null
+++ b/buildbot_gentoo_ci/config/change_source.py
@@ -0,0 +1,18 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from buildbot.plugins import changes, util
+
+#FIXME:
+# Get the repositorys info from the gentoo-ci db
+def gentoo_change_source(cs=[]):
+cs.append(changes.GitPoller(
+repourl='https://github.com/gentoo/gentoo.git',
+branches=True,
+workdir= 'repositorys' + '/gentoo.git/',
+pollInterval=240,
+pollRandomDelayMin=20,
+pollRandomDelayMax=60,
+project='gentoo'
+))
+return cs
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index 7cfb122..0cbec96 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -4,7 +4,7 @@
from buildbot.plugins import schedulers, util
@util.renderer
-def builderUpdateDbNames(self, props):
+def builderUpdateDbNames(props):
builders = set()
for f in props.files:
if f.endswith('.ebuild'):
@@ -12,22 +12,38 @@ def builderUpdateDbNames(self, props):
return list(builders)
@util.renderer
-def cpvUpdateDb(props):
-cpv_changes = []
-for f in props.files:
-if f.endswith('.ebuild'):
-cppv = f.split('.eb', 0)
-cpv = cppv.split('/', 0) + '/' + cppv.split('/', 2)
-if not cpv in cpv_changes:
-cpv_changes.append(cpv)
-return cpv_changes
+def gitUpdateDb(props):
+git_changes = []
+print(props.changes)
+for k in props.changes:
+change_data = {}
+print(k)
+change_data['cpvs'] = []
+for v in k['files']:
+if v.endswith('.ebuild'):
+c = v.split('/')[0]
+pv = v.split('/')[2][:-7]
+cpv = c + '/' + pv
+print(cpv)
+change_data['cpvs'].append(cpv)
+if k['repository'].endswith('.git'):
+for v in k['repository'].split('/'):
+if v.endswith('.git'):
+change_data['repository'] = v[:-4]
+change_data['author'] = k['author']
+change_data['committer'] = k['committer']
+change_data['comments'] = k['comments']
+change_data['revision'] = k['revision']
+git_changes.append(change_data)
+print(git_changes)
+return git_changes
def gentoo_schedulers():
scheduler_update_db = schedulers.SingleBranchScheduler(
name='scheduler_update_db',
treeStableTimer=60,
properties = {
-'cpv_changes' : cpvUpdateDb,
+'git_changes' : gitUpdateDb,
},
builderNames = builderUpdateDbNames,
change_filter=util.ChangeFilter(branch='master'),
@@ -60,7 +76,7 @@ def gentoo_schedulers():
builderNames=["run_build_request"])
s = []
s.append(test_updatedb)
-#s.append(scheduler_update_db)
+s.append(scheduler_update_db)
s.append(update_cpv_data)
s.append(update_v_data)
s.append(build_request_data)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 2221ba4..939059c 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -71,14 +71,15 @@ class Model(base.DBConnectorComponent):
# description of the repository
sa.Column('description', sa.Text, nullable=True),
sa.Column('mirror_url', sa.String(255), nullable=True),
+sa.Column('type', sa.Enum('gitpuller'), nullable=False,
default='gitpuller'),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
sa.Column('ebuild', sa.Boolean, default=False),
)
# Use by GitPoller
-repository_gitpuller = sautils.Table(
-"repository_gitpuller", metadata,
+repositorys_gitpullers = sautils.Table(
+"repositorys_gitpullers",
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 95436406f0010275ce5fdb6142b916ca66f91af0
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Feb 28 18:47:19 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Feb 28 18:47:19 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=95436406
Don't use binary or build some packages
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 32
1 file changed, 32 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 81d6e95..21472c7 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -255,6 +255,11 @@ class RunEmerge(BuildStep):
shell_commad_list.append('--changed-use')
shell_commad_list.append('--pretend')
shell_commad_list.append('@world')
+# don't build bin for virtual and acct-*
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('virtual')
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
command=shell_commad_list,
@@ -269,6 +274,11 @@ class RunEmerge(BuildStep):
shell_commad_list.append('--changed-deps')
shell_commad_list.append('--changed-use')
shell_commad_list.append('@world')
+# don't build bin for virtual and acct-*
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('virtual')
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
command=shell_commad_list,
@@ -338,8 +348,19 @@ class RunEmerge(BuildStep):
aftersteps_list.append(CheckEmergeLogs('match'))
if self.step == 'pre-build':
+cpv = self.getProperty("cpv")
+c = yield catpkgsplit(cpv)[0]
+p = yield catpkgsplit(cpv)[1]
shell_commad_list.append('-p')
shell_commad_list.append('=' + self.getProperty('cpv'))
+# we don't use the bin for the requsted cpv
+shell_commad_list.append('--usepkg-exclude')
+shell_commad_list.append(c + '/' + p)
+# don't build bin for virtual and acct-*
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('virtual')
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
command=shell_commad_list,
@@ -351,10 +372,21 @@ class RunEmerge(BuildStep):
aftersteps_list.append(CheckEmergeLogs('pre-build'))
if self.step == 'build':
+cpv = self.getProperty("cpv")
+c = yield catpkgsplit(cpv)[0]
+p = yield catpkgsplit(cpv)[1]
shell_commad_list.append('-q')
if projects_emerge_options['oneshot']:
shell_commad_list.append('-1')
shell_commad_list.append('=' + self.getProperty('cpv'))
+# we don't use the bin for the requsted cpv
+shell_commad_list.append('--usepkg-exclude')
+shell_commad_list.append(c + '/' + p)
+# don't build bin for virtual and acct-*
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('virtual')
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
command=shell_commad_list,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
commit: 22ad0178cca549026ab51f9f26e3fb9ad28f6374
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Feb 27 22:45:35 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Feb 27 22:45:35 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=22ad0178
Add SetPackageDefault and SetEnvDefault to steps
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 6 +-
buildbot_gentoo_ci/db/model.py | 27 -
buildbot_gentoo_ci/db/portages.py | 14 +
buildbot_gentoo_ci/db/projects.py | 44 ++
buildbot_gentoo_ci/steps/portage.py| 97 ++
5 files changed, 185 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index ac8bac1..acafed8 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -9,6 +9,7 @@ from buildbot_gentoo_ci.steps import category
from buildbot_gentoo_ci.steps import package
from buildbot_gentoo_ci.steps import version
from buildbot_gentoo_ci.steps import builders
+from buildbot_gentoo_ci.steps import portage
def update_db_check():
f = util.BuildFactory()
@@ -114,9 +115,10 @@ def run_build_request():
f.addStep(builders.UpdateRepos())
# setup make.conf
f.addStep(builders.SetMakeConf())
-# setup package.*
-#f.addStep(portages.SetPackageUse())
# setup env
+f.addStep(portage.SetEnvDefault())
+# setup package.*
+#f.addStep(portage.SetPackageDefault())
# setup files in /etc if needed
# run --regen if needed on repo
# update packages before any tests
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 2153b79..b91f158 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -139,7 +139,6 @@ class Model(base.DBConnectorComponent):
sa.Column('project_uuid', sa.String(36),
sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
nullable=False),
-# FIXME: directorys should be moved to own table
sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'),
nullable=False),
sa.Column('value', sa.String(255), nullable=False),
)
@@ -162,6 +161,32 @@ class Model(base.DBConnectorComponent):
sa.Column('value', sa.String(255), nullable=False),
)
+# projects etc/portage/env settings
+projects_portages_env = sautils.Table(
+"projects_portages_env", metadata,
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+sa.Column('makeconf_id', sa.String(255),
+ sa.ForeignKey('portages_makeconf.id', ondelete='CASCADE'),
+ nullable=False),
+sa.Column('name', sa.String(255), nullable=False),
+sa.Column('value', sa.String(255), nullable=False),
+)
+
+# projects etc/portage/package.* settings
+projects_portage_package = sautils.Table(
+"projects_portage_package", metadata,
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+sa.Column('directorys', sa.Enum('use', 'accept_keywords', 'env'),
nullable=False),
+sa.Column('value1', sa.String(255), nullable=False),
+sa.Column('value2', sa.String(255), nullable=True),
+)
+
projects_emerge_options = sautils.Table(
"projects_emerge_options", metadata,
sa.Column('id', sa.Integer, primary_key=True),
diff --git a/buildbot_gentoo_ci/db/portages.py
b/buildbot_gentoo_ci/db/portages.py
index 428fb84..86321e3 100644
--- a/buildbot_gentoo_ci/db/portages.py
+++ b/buildbot_gentoo_ci/db/portages.py
@@ -36,6 +36,20 @@ class PortagesConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+@defer.inlineCallbacks
+def getVariableById(self, id):
+def thd(conn):
+tbl = self.db.model.portages_makeconf
+q = tbl.select()
+q = q.where(tbl.c.id == id)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
def _row2dict(self, conn, row):
return dict(
id=row.id,
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 37d401c..37ae2f1 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -130,6 +130,28 @@ class
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 2da4defbcae32889b60c852179d1d718bcccbf57
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Feb 24 21:26:06 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Feb 24 21:26:06 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2da4defb
Don't use binary in the match and fix a bug in the match logic
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 16
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 3ecb78b..3c19345 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -475,6 +475,8 @@ class RunEmerge(BuildStep):
timeout=None
))
aftersteps_list.append(CheckEmergeLogs('update'))
+if projects_emerge_options['preserved_libs']:
+self.setProperty('preserved_libs', True, 'preserved-libs')
if self.step == 'preserved-libs' and
self.getProperty('preserved_libs'):
shell_commad_list.append('-q')
@@ -501,9 +503,8 @@ class RunEmerge(BuildStep):
workdir='/'
))
aftersteps_list.append(CheckEmergeLogs('depclean'))
-self.setProperty('depclean', False, 'depclean')
-if self.step == 'depclean' and self.getProperty('depclean') and
projects_emerge_options['depclean']:
+if self.step == 'depclean' and self.getProperty('depclean'):
shell_commad_list.append('-q')
shell_commad_list.append('--depclean')
aftersteps_list.append(
@@ -520,6 +521,8 @@ class RunEmerge(BuildStep):
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
shell_commad_list.append('-pO')
+# don't use bin for match
+shell_commad_list.append('--usepkg=n')
shell_commad_list.append(c + '/' + p)
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
@@ -546,7 +549,8 @@ class RunEmerge(BuildStep):
if self.step == 'build':
shell_commad_list.append('-q')
-shell_commad_list.append('-1')
+if projects_emerge_options['oneshot']:
+shell_commad_list.append('-1')
shell_commad_list.append('=' + self.getProperty('cpv'))
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
@@ -557,6 +561,8 @@ class RunEmerge(BuildStep):
timeout=None
))
aftersteps_list.append(CheckEmergeLogs('build'))
+if projects_emerge_options['preserved_libs']:
+self.setProperty('preserved_libs', True, 'preserved-libs')
if not self.step is None and aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
@@ -598,7 +604,7 @@ class CheckEmergeLogs(BuildStep):
# FIXME: check if cpv match
if self.step == 'match'and
self.getProperty('projectrepository_data')['build']:
-if emerge_output['package'][self.getProperty('cpv')]:
+if self.getProperty('cpv') in emerge_output['package']:
self.setProperty('cpv_build', True, 'cpv_build')
print(self.getProperty('cpv_build'))
@@ -701,5 +707,7 @@ class RunBuild(BuildStep):
aftersteps_list = []
aftersteps_list.append(RunEmerge(step='pre-build'))
aftersteps_list.append(RunEmerge(step='build'))
+self.setProperty('depclean', False, 'depclean')
+self.setProperty('preserved_libs', False, 'preserved-libs')
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/
commit: 4324b82edd9d69a27b1db9f4272564ddc51a75dd
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Feb 22 20:27:14 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Feb 22 20:27:14 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4324b82e
Add RunPkgCheck and CheckPkgCheckLogs
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 11 ++-
buildbot_gentoo_ci/db/model.py | 2 +-
buildbot_gentoo_ci/db/projects.py | 6 +-
buildbot_gentoo_ci/steps/builders.py | 120 +++--
4 files changed, 129 insertions(+), 10 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 3c77762..cfe4131 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -92,13 +92,15 @@ def run_build_request():
# set needed Propertys
f.addStep(builders.SetupPropertys())
# Clean and add new /etc/portage
+#FIXME: Is don't like symlinks
f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
workdir='/etc/'))
f.addStep(buildbot_steps.MakeDirectory(dir="portage",
workdir='/etc/'))
# setup the profile
-f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
-workdir='/etc/portage/'))
+#NOTE: pkgcheck do not support it as a dir
+#f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
+#workdir='/etc/portage/'))
f.addStep(builders.SetMakeProfile())
# setup repos.conf dir
f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
@@ -132,4 +134,9 @@ def run_build_request():
# check log
# setup make.conf if build id has changes make.conf as dict from
SetMakeConf
# setup package.* env if build id has changes
+# setup pkgcheck.conf if needed
+#f.addStep(builders.SetPkgCheckConf())
+# run pkgcheck if wanted
+# check log
+f.addStep(builders.RunPkgCheck())
return f
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 0a2c145..4d691ea 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -128,7 +128,7 @@ class Model(base.DBConnectorComponent):
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('auto', sa.Boolean, default=False),
-sa.Column('pkgcheck', sa.Boolean, default=False),
+sa.Column('pkgcheck', sa.Enum('package','full','none'),
default='none'),
)
# projects etc/portage settings
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index eb8f727..9aeca50 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -160,12 +160,16 @@ class
ProjectsConnectorComponent(base.DBConnectorComponent):
)
def _row2dict_projects_repositorys(self, conn, row):
+if row.pkgcheck == 'none':
+pkgcheck = False
+else:
+pkgcheck=row.pkgcheck
return dict(
id=row.id,
project_uuid=row.project_uuid,
repository_uuid=row.repository_uuid,
auto=row.auto,
-pkgcheck=row.pkgcheck
+pkgcheck=pkgcheck
)
def _row2dict_projects_portage(self, conn, row):
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index c0567c1..6d46ae6 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -4,6 +4,8 @@
import os
import re
+from portage.versions import catpkgsplit
+
from twisted.internet import defer
from twisted.python import log
@@ -75,6 +77,26 @@ def PersOutputOfEmerge(rc, stdout, stderr):
'emerge_output' : emerge_output
}
+def PersOutputOfPkgCheck(rc, stdout, stderr):
+pkgcheck_output = {}
+pkgcheck_output['rc'] = rc
+#FIXME: Handling of stdout output
+pkgcheck_xml_list = []
+# split the lines
+for line in stdout.split('\n'):
+# pkgcheck output list
+if line.startswith('
