subject:"\[gentoo\-commits\] proj\/tinderbox\-cluster\:master commit in\: buildbot_gentoo

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/

2024-03-30 Thread Magnus Granberg

commit: 99dcb588c27184023d01ae6a55308e999ce25c20
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Mar 30 09:16:33 2024 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Mar 30 09:16:33 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=99dcb588

Add build dep irc report

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/reporters.py | 3 ++-
 buildbot_gentoo_ci/steps/builders.py   | 5 -
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/buildbot_gentoo_ci/config/reporters.py 
b/buildbot_gentoo_ci/config/reporters.py
index ede9178..c1f0aef 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -13,7 +13,8 @@ irc_template = '''{% set resultsList = ["\x0303SUCCESS", 
"\x0308WARNINGS", "\x03
 {{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303" 
}}Repo:{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
 {{ build['properties']['revision'][0]|truncate(10, True) }} {{ "\x0302" }}{{ 
build['properties']['owners'][0][0] }}{{ "\x03" }} \
 {{ "\x0306" }}{{ build['properties']['event'][0] }}{{ "\x03" }} {{ projects 
}}:{{ build['properties']['project_data'][0]['name'] }} \
-{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{ 
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }} \
+{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{ 
"\x02" }} Dep: {{ resultsList[build['properties']['faild_dep'][0]] }} \
+{{ "\x0312" }}{{ build_url }}{{ "\x03" }} \
 {% if build['properties']['bgo'][0]['match'] is true %}{{ "\x0311" }}Bug: {{ 
"\x03" }}https://bugs.gentoo.org/{{build['properties']['bgo'][0]['id']}}{% 
endif %}\
 '''
 

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index 73e82f5..53f79ab 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -899,6 +899,7 @@ class CheckEmergeLogs(BuildStep):
 self.setProperty('faild_cpv', emerge_output['failed'], 'faild_cpv')
 self.faild_version_data = False
 self.log_data = {}
+self.faild_dep = 0
 package_dict = emerge_output['packages']
 
 #FIXME: Prosees the logs and do stuff
@@ -1060,6 +1061,7 @@ class CheckEmergeLogs(BuildStep):
 self.faild_version_data = 
self.getProperty("version_data")
 else:
 repository = False
+self.faild_dep = 2
 for cpv, v in package_dict.items():
 if cpv == emerge_output['failed']:
 repository = v['repository']
@@ -1082,10 +1084,11 @@ class CheckEmergeLogs(BuildStep):
 'pkg_check_log_data' : 
self.getProperty("pkg_check_log_data"),
 'repository_data' : 
self.getProperty('repository_data'),
 'faild_cpv' : emerge_output['failed'],
+'faild_dep' : self.faild_dep
 'build_workername' : self.getProperty('workername')
 }
 ))
-if rebuild:
+if isinstance(rebuild, str):
 #FIXME: Set build timeout in config
 build_timeout = 6600
 shell_commad_list = []

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2024-01-06 Thread Magnus Granberg

commit: eb2f05efcf0392a17894233de29b5c3ede8e1b43
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Jan  6 22:27:24 2024 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Jan  6 22:27:24 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=eb2f05ef

Add getGitlabContext to Gitlab status

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/reporters.py | 21 +++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/buildbot_gentoo_ci/config/reporters.py 
b/buildbot_gentoo_ci/config/reporters.py
index 54472b0..ede9178 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -1,6 +1,8 @@
 # Copyright 2022 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+from twisted.internet import defer
+
 from buildbot.plugins import util
 from buildbot.reporters.gitlab import GitLabStatusPush
 from buildbot.reporters.generators.build import BuildStatusGenerator, 
BuildStartEndStatusGenerator
@@ -42,9 +44,24 @@ irc_reporter = irc.IRCStatusPush("irc.libera.chat", 
"gci_test",
  noticeOnChannel=True
  )
 #gitlab
+@util.renderer
+@defer.inlineCallbacks
+def getGitlabContext(props):
+context = None
+gentooci = props.master.namedServices['services'].namedServices['gentooci']
+if props.getProperty('buildername') == 'update_v_data':
+context = 'Update database and check ebuild'
+if props.getProperty('buildername') == 'run_build_request':
+project_data = yield 
gentooci.db.projects.getProjectByUuid(props.getProperty('project_uuid'))
+context = f"Project/{project_data['description']}/Build"
+if props.getProperty('buildername') == 'parse_build_log':
+project_data = yield 
gentooci.db.projects.getProjectByUuid(props.getProperty('project_build_data')['project_uuid'])
+context = f"Project/{project_data['description']}/Check Log"
+return context
 def gitlabGenerators():
 builders = [
-#'run_build_request',
+'update_v_data',
+'run_build_request',
 'parse_build_log'
 ]
 return [
@@ -53,7 +70,7 @@ def gitlabGenerators():
 )
 ]
 gitlab_gentoo_org = GitLabStatusPush(token=util.Secret("gitlabToken"),
-#context= util.Interpolate('Buildbot 
%(prop:buildername)s'),
+context= getGitlabContext,
 baseURL='https://gitlab.gentoo.org',
 generators=gitlabGenerators(),
 #debug=True,

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/

2023-06-26 Thread Magnus Granberg

commit: f95dce7cfc33e1d5a0e606c5562cac59c2dc0fdd
Author: Magnus Granberg  gentoo  org>
AuthorDate: Mon Jun 26 21:52:10 2023 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Mon Jun 26 21:52:10 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f95dce7c

Add run_clean_db_request to clean db

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py  | 29 +
 buildbot_gentoo_ci/config/buildfactorys.py |  9 +++
 buildbot_gentoo_ci/config/schedulers.py|  3 +
 buildbot_gentoo_ci/steps/clean.py  | 97 ++
 buildbot_gentoo_ci/steps/package.py| 45 ++
 5 files changed, 183 insertions(+)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index b3ddb16..584dfdf 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -20,6 +20,24 @@ def getWorkers(worker_type, workers):
 worker_list.append(worker['uuid'])
 return worker_list
 
+# check if we have same package_data
+@defer.inlineCallbacks
+def checkPackageData(master, builder, req1, req2):
+canBeCollapsed = yield buildrequest.BuildRequest.canBeCollapsed(master, 
req1, req2)
+selfBuildset , otherBuildset = yield defer.gatherResults([
+master.data.get(('buildsets', req1['buildsetid'])),
+master.data.get(('buildsets', req2['buildsetid']))
+])
+print(f"TestCollapsed")
+print(f"canBeCollapsed: {canBeCollapsed}")
+print(f"selfBuildset: {selfBuildset}")
+print(f"otherBuildset: {otherBuildset}")
+if canBeCollapsed and selfBuildset['parent_buildid'] != None and \
+otherBuildset['parent_buildid'] != None:
+return True
+else:
+return False
+
 @defer.inlineCallbacks
 def CanWorkerBuildProject(builder, wfb, request):
 gentooci = 
builder.master.namedServices['services'].namedServices['gentooci']
@@ -118,4 +136,15 @@ def gentoo_builders(worker_data):
 factory=buildfactorys.run_build_images_request()
 )
 )
+# Use multiplay workers
+b.append(util.BuilderConfig(
+name='run_clean_db_request',
+workernames=getWorkersLocal('local', worker_data['local']),
+# look builder so we only do one time
+# look=
+# if we allready runnin with same package_data properties then 
skip/collapse
+#collapseRequests=checkPackageData,
+factory=buildfactorys.run_clean_db_request()
+)
+)
 return b

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index b47a209..6755308 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -14,6 +14,7 @@ from buildbot_gentoo_ci.steps import portage
 from buildbot_gentoo_ci.steps import logs
 from buildbot_gentoo_ci.steps import repos
 from buildbot_gentoo_ci.steps import nodes
+from buildbot_gentoo_ci.steps import clean
 
 def update_db_check():
 f = util.BuildFactory()
@@ -66,6 +67,8 @@ def update_db_cpv():
 f.addStep(package.TriggerCheckForV())
 # update metadata if needed
 f.addStep(package.CheckMetadataPackagePath())
+# clean package db if needed (clean up ebuilds in db)
+f.addStep(package.TriggerCleanPackageDb())
 return f
 
 def update_db_v():
@@ -201,3 +204,9 @@ def run_build_images_request():
 # set the needed steps for making the image
 f.addStep(nodes.SetupBuildTypeAndSteps())
 return f
+
+def run_clean_db_request():
+f = util.BuildFactory()
+# set needed Propertys
+f.addStep(clean.SetupPropertys())
+return f

diff --git a/buildbot_gentoo_ci/config/schedulers.py 
b/buildbot_gentoo_ci/config/schedulers.py
index 7833233..c34d714 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -92,6 +92,8 @@ def gentoo_schedulers():
builderNames=["parse_build_log"])
 run_build_images_request = 
schedulers.Triggerable(name="run_build_images_request",
builderNames=["run_build_images_request"])
+run_clean_db_request = schedulers.Triggerable(name="run_clean_db_request",
+   builderNames=["run_clean_db_request"])
 s = []
 s.append(create_images)
 s.append(scheduler_update_db)
@@ -101,4 +103,5 @@ def gentoo_schedulers():
 s.append(build_request_data)
 s.append(run_build_request)
 s.append(parse_build_log)
+s.append(run_clean_db_request)
 return s

diff --git a/buildbot_gentoo_ci/steps/clean.py 
b/buildbot_gentoo_ci/steps/clean.py
new file mode 100644
index 000..73733f1
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/clean.py
@@ -0,0 +1,97 @@
+# Copyright 2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from twisted.internet import defer
+from twisted.python import log
+

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2023-06-10 Thread Magnus Granberg

commit: e75fa8c7184989508dc3c445cf8b32fd99634d99
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Jun 10 10:06:11 2023 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Jun 10 10:06:11 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e75fa8c7

Add SetupBugReportSteps for bugreporting

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 2ca2ca9..b47a209 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -188,10 +188,10 @@ def parse_build_log():
 f.addStep(logs.setEmergeInfoLog())
 # add package info to log and db
 f.addStep(logs.setPackageInfoLog())
+# make the bugreport
+f.addStep(logs.SetupBugReportSteps())
 # set BuildStatus
 f.addStep(logs.setBuildStatus())
-# setup things for the irc bot
-#f.addStep(logs.SetIrcInfo())
 return f
 
 def run_build_images_request():

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, /, buildbot_gentoo_ci/steps/

2023-06-10 Thread Magnus Granberg

commit: c2cd0bb34537771d61dd8d7c577cecf25c446200
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Jun 10 10:02:20 2023 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Jun 10 10:02:20 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c2cd0bb3

Add support for SetupBugReportSteps and compression with xz

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/config.py |   3 +
 buildbot_gentoo_ci/steps/logs.py| 141 
 gentooci.cfg|  20 +
 3 files changed, 132 insertions(+), 32 deletions(-)

diff --git a/buildbot_gentoo_ci/config/config.py 
b/buildbot_gentoo_ci/config/config.py
index e2f33be..696ab0f 100644
--- a/buildbot_gentoo_ci/config/config.py
+++ b/buildbot_gentoo_ci/config/config.py
@@ -93,6 +93,7 @@ class GentooCiConfig(util.ComparableMixin):
 "db_url",
 "project",
 "worker_config",
+"bug_config",
 "repository_basedir"
 ])
 
@@ -152,3 +153,5 @@ class GentooCiConfig(util.ComparableMixin):
 self.project['repository_basedir'] = DEFAULT_REPOSITORY_BASEDIR
 if 'worker_config' in config_dict:
 self.project['worker_config'] = config_dict['worker_config']
+if 'bug_config' in config_dict:
+self.project['bug_config'] = config_dict['bug_config']

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index e4fc951..ee11166 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import os
@@ -14,6 +14,7 @@ from twisted.internet import defer
 from twisted.python import log
 
 from buildbot.process.buildstep import BuildStep
+from buildbot.process.properties import Properties
 from buildbot.process.results import SUCCESS
 from buildbot.process.results import FAILURE
 from buildbot.process.results import WARNINGS
@@ -21,7 +22,7 @@ from buildbot.process.results import SKIPPED
 from buildbot.plugins import steps
 from buildbot.plugins import util
 
-from buildbot_gentoo_ci.steps import minio
+#from buildbot_gentoo_ci.steps import minio
 from buildbot_gentoo_ci.steps import master as master_steps
 from buildbot_gentoo_ci.steps import bugs
 
@@ -91,13 +92,15 @@ class SetupPropertys(BuildStep):
 self.setProperty("default_project_data", default_project_data, 
'default_project_data')
 self.setProperty("version_data", version_data, 'version_data')
 self.setProperty("status", 'completed', 'status')
-if self.getProperty('faild_cpv'):
+if isinstance(self.getProperty('faild_cpv'), str):
 log_cpv = self.getProperty('faild_cpv')
 else:
 log_cpv = self.getProperty('cpv')
 self.setProperty("log_cpv", log_cpv, 'log_cpv')
-self.setProperty("bgo", dict( match=False), 'bgo')
+self.setProperty("bgo", False, 'bgo')
 self.descriptionDone = 'Runing log checker on ' + log_cpv
+logsdir = yield os.path.join(self.master.basedir, 'workers', 
self.getProperty('build_workername'), 
str(self.getProperty("project_build_data")['buildbot_build_id']))
+self.setProperty("logsdir", logsdir, 'logsdir')
 return SUCCESS
 
 class SetupParserBuildLoger(BuildStep):
@@ -115,9 +118,9 @@ class SetupParserBuildLoger(BuildStep):
 @defer.inlineCallbacks
 def run(self):
 self.aftersteps_list = []
-workdir = yield os.path.join(self.master.basedir, 'workers', 
self.getProperty('build_workername'), 
str(self.getProperty("project_build_data")['buildbot_build_id']))
 log_cpv = 
self.getProperty('log_build_data')[self.getProperty('log_cpv')]
-mastersrc_log = yield os.path.join(workdir, log_cpv['full_logname'])
+build_log_file_compressed = log_cpv['full_logname'] + '.xz'
+mastersrc_log = yield os.path.join(self.getProperty('logsdir'), 
build_log_file_compressed)
 log_py = 'log_parser.py'
 config_log_py = 'logparser.json'
 mastersrc_py = yield os.path.join(self.master.basedir, log_py)
@@ -125,7 +128,7 @@ class SetupParserBuildLoger(BuildStep):
 # Upload logfile to worker
 self.aftersteps_list.append(steps.FileDownload(
 mastersrc=mastersrc_log,
-
workerdest=log_cpv['full_logname']
+
workerdest=build_log_file_compressed
 ))
 # Upload log parser py code
 self.aftersteps_list.append(steps.FileDownload(
@@ -137,6 +140,16 @@ class SetupParserBuildLoger(BuildStep):
 mastersrc=mastersrc_config,

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2023-06-05 Thread Magnus Granberg

commit: 599646e376fc489f091b4f53ec7574ca61cec63b
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun Jun  4 20:59:43 2023 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun Jun  4 20:59:43 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=599646e3

Disable distfiles on docker

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/workers.py | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/workers.py 
b/buildbot_gentoo_ci/config/workers.py
index 83eeda2..5f3a5a6 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -9,6 +9,8 @@ from buildbot.plugins import worker, util
 def log_docker_images(props):
 return 'bb-worker-log' + ':latest'
 
+#NOTE: Don't set distfiles if you have dist clean. Can get missing files
+#FIXME: Set in config
 @util.renderer
 def docker_volumes(props):
 volumes_list = []
@@ -16,7 +18,7 @@ def docker_volumes(props):
 src_dir = '/srv/gentoo/portage/' + props.getProperty('project_uuid')
 dest_dir = '/var/cache/portage'
 #add distdir
-volumes_list.append(src_dir + '/distfiles' + ':' + dest_dir + '/distfiles')
+#volumes_list.append(src_dir + '/distfiles' + ':' + dest_dir + 
'/distfiles')
 #add bindir
 volumes_list.append(src_dir + '/packages' + ':' + dest_dir + '/packages')
 return volumes_list
@@ -33,6 +35,7 @@ def docker_volumes_repositorys(props):
 volumes_list.append(':'.join([src_dir, dest_dir]))
 return volumes_list
 
+#FIXME: Move workers to DbConfig 
http://docs.buildbot.net/current/manual/configuration/dbconfig.html
 @util.renderer
 @defer.inlineCallbacks
 def getDockerHost(props, docker_worker):
@@ -45,6 +48,7 @@ def getDockerHost(props, docker_worker):
 print(node_data)
 return node_data['docker_host_url']
 
+#FIXME: Move workers to DbConfig 
http://docs.buildbot.net/current/manual/configuration/dbconfig.html
 @util.renderer
 @defer.inlineCallbacks
 def GetBuildDockerImage(props, docker_worker):

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2023-05-28 Thread Magnus Granberg

commit: 8f5231276de8cc3ad5a225a67f1f558de97d2410
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun May 28 09:46:22 2023 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun May 28 09:46:22 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8f523127

Set build_wait_timeout to 600 for log worker

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/workers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/workers.py 
b/buildbot_gentoo_ci/config/workers.py
index 6f27690..83eeda2 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -94,6 +94,6 @@ def gentoo_workers(worker_data):
 hostconfig=docker_hostconfig,
 followStartupLogs=True,
 masterFQDN='192.168.1.5',
-#build_wait_timeout=3600
+build_wait_timeout=600
 ))
 return w

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2023-05-28 Thread Magnus Granberg

commit: 2967519fc01d5041a31e47e98c630f222f5a9cf4
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun May 28 09:44:59 2023 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun May 28 09:44:59 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2967519f

Use full url for Bugs

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/reporters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/reporters.py 
b/buildbot_gentoo_ci/config/reporters.py
index bd55037..54472b0 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -12,7 +12,7 @@ irc_template = '''{% set resultsList = ["\x0303SUCCESS", 
"\x0308WARNINGS", "\x03
 {{ build['properties']['revision'][0]|truncate(10, True) }} {{ "\x0302" }}{{ 
build['properties']['owners'][0][0] }}{{ "\x03" }} \
 {{ "\x0306" }}{{ build['properties']['event'][0] }}{{ "\x03" }} {{ projects 
}}:{{ build['properties']['project_data'][0]['name'] }} \
 {{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{ 
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }} \
-{% if build['properties']['bgo'][0]['match'] is true %}{{ "\x0311" }}Bugid: 
{{build['properties']['bgo'][0]['id']}}{{ "\x03" }}{% endif %}\
+{% if build['properties']['bgo'][0]['match'] is true %}{{ "\x0311" }}Bug: {{ 
"\x03" }}https://bugs.gentoo.org/{{build['properties']['bgo'][0]['id']}}{% 
endif %}\
 '''
 
 def ircGenerators():

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2023-05-01 Thread Magnus Granberg

commit: b69003d51f5ed92cc38128cdc91c47e22eaa37b1
Author: Magnus Granberg  gentoo  org>
AuthorDate: Mon May  1 09:53:42 2023 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Mon May  1 09:53:42 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b69003d5

Don't use volumes for log worker

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/workers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/workers.py 
b/buildbot_gentoo_ci/config/workers.py
index 7d13f46..6f27690 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -90,7 +90,7 @@ def gentoo_workers(worker_data):
 None,
 docker_host='tcp://192.168.1.12:2375',
 image=log_docker_images,
-volumes=docker_volumes_repositorys,
+#volumes=docker_volumes_repositorys,
 hostconfig=docker_hostconfig,
 followStartupLogs=True,
 masterFQDN='192.168.1.5',

[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/, ...

2023-02-15 Thread Magnus Granberg

commit: b38a60cfc5b5dca25a674ebe5887737525aa353f
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Feb 15 21:30:32 2023 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Feb 15 21:30:32 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b38a60cf

Use command shell to build docker images

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py   |  24 +++---
 buildbot_gentoo_ci/config/buildfactorys.py  |   6 +-
 buildbot_gentoo_ci/config/schedulers.py |  22 ++---
 buildbot_gentoo_ci/config/workers.py|  12 ++-
 buildbot_gentoo_ci/db/model.py  |   1 +
 buildbot_gentoo_ci/db/workers.py|   3 +-
 buildbot_gentoo_ci/steps/nodes.py   | 121 
 docker/GentooBuildbotWorkerBuild.Dockerfile |  31 ---
 master.cfg  |  14 +++-
 sql/gentoo_ci_schema.sql|  14 +++-
 10 files changed, 186 insertions(+), 62 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index edcd0c1..b3ddb16 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -13,7 +13,7 @@ def getWorkersLocal(worker_type, workers):
 worker_list.append(worker['name'])
 return worker_list
 
-def getWorkersDocker(worker_type, workers):
+def getWorkers(worker_type, workers):
 worker_list = []
 for worker in workers:
 if worker['type'] == worker_type and worker['enable'] is True:
@@ -44,11 +44,11 @@ def CanWorkerUpdateV(builder, wfb, request):
 print('Find a worker: NO')
 return False
 
-def gentoo_builders(worker_data_local, worker_data_docker):
+def gentoo_builders(worker_data):
 b = []
 b.append(util.BuilderConfig(
 name='update_db_check',
-workername=getWorkersLocal('local', worker_data_local)[0],
+workername=getWorkersLocal('local', worker_data['local'])[0],
 workerbuilddir='builds',
 collapseRequests=False,
 factory=buildfactorys.update_db_check()
@@ -56,7 +56,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
 )
 b.append(util.BuilderConfig(
 name='update_repo_check',
-workername=getWorkersLocal('local', worker_data_local)[1],
+workername=getWorkersLocal('local', worker_data['local'])[1],
 workerbuilddir='builds',
 collapseRequests=True,
 factory=buildfactorys.update_repo_check()
@@ -67,7 +67,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
 # (builders.UpdateRepos step)
 b.append(util.BuilderConfig(
 name='update_cpv_data',
-workernames=getWorkersDocker('log', worker_data_docker)[0],
+workernames=getWorkers('log', worker_data['docker'])[0],
 workerbuilddir='builds',
 collapseRequests=False,
 factory=buildfactorys.update_db_cpv()
@@ -76,7 +76,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='update_v_data',
-workername=getWorkersDocker('log', worker_data_docker)[0],
+workername=getWorkers('log', worker_data['docker'])[0],
 workerbuilddir='builds',
 collapseRequests=False,
 canStartBuild=CanWorkerUpdateV,
@@ -86,7 +86,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='build_request_data',
-workernames=getWorkersLocal('local', worker_data_local),
+workernames=getWorkersLocal('local', worker_data['local']),
 collapseRequests=False,
 factory=buildfactorys.build_request_check()
 )
@@ -94,7 +94,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='run_build_request',
-workernames=getWorkersDocker('build', worker_data_docker),
+workernames=getWorkers('build', worker_data['docker']),
 canStartBuild=CanWorkerBuildProject,
 collapseRequests=False,
 factory=buildfactorys.run_build_request()
@@ -103,19 +103,19 @@ def gentoo_builders(worker_data_local, 
worker_data_docker):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='parse_build_log',
-workernames=getWorkersDocker('log', worker_data_docker)[1:],
+workernames=getWorkers('log', worker_data['docker'])[1:],
 collapseRequests=False,
 factory=buildfactorys.parse_build_log()
 )
 )
 # For node workers
 b.append(util.BuilderConfig(
-name='run_build_stage4_request',
-workernames=getWorkersLocal('local', worker_data_local),
+name='run_build_images_request',
+workernames=getWorkers('node', worker_data['node']),
 #FIXME: support more the one node

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, sql/

2023-02-03 Thread Magnus Granberg

commit: 102c51d88be07b23cd24d4cad6fad5e71edacb4c
Author: Magnus Granberg  gentoo  org>
AuthorDate: Fri Feb  3 22:21:42 2023 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Fri Feb  3 22:21:42 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=102c51d8

Add support for binhost on docker image build

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/workers.py |  2 +-
 buildbot_gentoo_ci/db/model.py   |  3 ++-
 buildbot_gentoo_ci/db/workers.py | 13 -
 sql/gentoo_ci_schema.sql |  3 ++-
 4 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/buildbot_gentoo_ci/config/workers.py 
b/buildbot_gentoo_ci/config/workers.py
index 3dcef1a..c29502e 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -43,7 +43,7 @@ def getDockerHost(props, docker_worker):
 node_data = yield gentooci.db.workers.getNodeByUuid(node['node_uuid'])
 print(worker_data)
 print(node_data)
-return node_data['host_url']
+return node_data['docker_host_url']
 
 @util.renderer
 @defer.inlineCallbacks

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 87d7dbc..a4b5935 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -372,7 +372,8 @@ class Model(base.DBConnectorComponent):
 sa.Column('uuid', sa.String(36), primary_key=True,
   default=lambda: str(uuid.uuid4())),
 sa.Column('name', sa.String(255), nullable=False),
-sa.Column('host_url', sa.String(255), nullable=False),
+sa.Column('docker_host_url', sa.String(255), nullable=False),
+sa.Column('bin_host_url', sa.String(255), nullable=False),
 sa.Column('enable', sa.Boolean, default=False),
 )
 

diff --git a/buildbot_gentoo_ci/db/workers.py b/buildbot_gentoo_ci/db/workers.py
index e82a6b2..d09d88c 100644
--- a/buildbot_gentoo_ci/db/workers.py
+++ b/buildbot_gentoo_ci/db/workers.py
@@ -130,6 +130,16 @@ class WorkersConnectorComponent(base.DBConnectorComponent):
 res = yield self.db.pool.do(thd)
 return res
 
+@defer.inlineCallbacks
+def getAllNodes(self):
+def thd(conn):
+tbl = self.db.model.nodes
+q = tbl.select()
+return [self._row2dict_node(conn, row)
+for row in conn.execute(q).fetchall()]
+res = yield self.db.pool.do(thd)
+return res
+
 def _row2dict(self, conn, row):
 return dict(
 uuid=row.uuid,
@@ -164,7 +174,8 @@ class WorkersConnectorComponent(base.DBConnectorComponent):
 return dict(
 uuid=row.uuid,
 name=row.name,
-host_url=row.host_url,
+docker_host_url=row.docker_host_url,
+bin_host_url=row.bin_host_url,
 enable=row.enable
 )
 def _row2dict_node_worker(self, conn, row):

diff --git a/sql/gentoo_ci_schema.sql b/sql/gentoo_ci_schema.sql
index ff81661..5956664 100644
--- a/sql/gentoo_ci_schema.sql
+++ b/sql/gentoo_ci_schema.sql
@@ -274,7 +274,8 @@ ALTER TABLE public.migrate_version OWNER TO buildbot;
 
 CREATE TABLE public.nodes (
 name character varying,
-host_url character varying,
+docker_host_url character varying,
+bin_host_url character varying,
 enable boolean,
 uuid character varying(36) NOT NULL
 );

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, sql/, /, buildbot_gentoo_ci/db/, patches/

2022-10-02 Thread Magnus Granberg

commit: 2202c53f0179002adde3095609d89fcee394b614
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun Oct  2 14:13:30 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun Oct  2 14:13:30 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2202c53f

Add support for workes, nodes, images and flavors in db

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py |  41 +++--
 buildbot_gentoo_ci/config/workers.py  |  81 +
 buildbot_gentoo_ci/db/connector.py|   2 +
 buildbot_gentoo_ci/db/model.py|  57 ++-
 buildbot_gentoo_ci/db/workers.py  | 175 +++
 master.cfg|  37 ++--
 patches/bb-props_master_fix.patch |  15 ++
 sql/gentoo_ci_schema.sql  | 308 +++---
 8 files changed, 467 insertions(+), 249 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index 82552fa..edcd0c1 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -1,11 +1,24 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2022 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from twisted.internet import defer
 
 from buildbot.plugins import util
 from buildbot_gentoo_ci.config import buildfactorys
-from buildbot_gentoo_ci.config.workers import gentoo_ci_workers
+
+def getWorkersLocal(worker_type, workers):
+worker_list = []
+for worker in workers:
+if worker['type'] == worker_type and worker['enable'] is True:
+worker_list.append(worker['name'])
+return worker_list
+
+def getWorkersDocker(worker_type, workers):
+worker_list = []
+for worker in workers:
+if worker['type'] == worker_type and worker['enable'] is True:
+worker_list.append(worker['uuid'])
+return worker_list
 
 @defer.inlineCallbacks
 def CanWorkerBuildProject(builder, wfb, request):
@@ -16,8 +29,9 @@ def CanWorkerBuildProject(builder, wfb, request):
 print(wfb)
 for worker in project_workers:
 if wfb.worker.workername == worker['worker_uuid']:
+print('Find a worker: YES')
 return True
-print('no worker')
+print('Find a worker: NO')
 return False
 
 # Use same worker as update_cpv_data was done by so we have same git commit
@@ -25,15 +39,16 @@ def CanWorkerUpdateV(builder, wfb, request):
 print(request.properties['cp_worker'])
 print(wfb)
 if wfb.worker.workername == request.properties['cp_worker']:
+print('Find a worker: YES')
 return True
+print('Find a worker: NO')
 return False
 
-def gentoo_builders(worker_data):
+def gentoo_builders(worker_data_local, worker_data_docker):
 b = []
-g_ci_w = gentoo_ci_workers(worker_data)
 b.append(util.BuilderConfig(
 name='update_db_check',
-workername=g_ci_w.getWorkersUuid('local')[0],
+workername=getWorkersLocal('local', worker_data_local)[0],
 workerbuilddir='builds',
 collapseRequests=False,
 factory=buildfactorys.update_db_check()
@@ -41,7 +56,7 @@ def gentoo_builders(worker_data):
 )
 b.append(util.BuilderConfig(
 name='update_repo_check',
-workername=g_ci_w.getWorkersUuid('local')[1],
+workername=getWorkersLocal('local', worker_data_local)[1],
 workerbuilddir='builds',
 collapseRequests=True,
 factory=buildfactorys.update_repo_check()
@@ -52,7 +67,7 @@ def gentoo_builders(worker_data):
 # (builders.UpdateRepos step)
 b.append(util.BuilderConfig(
 name='update_cpv_data',
-workernames=g_ci_w.getWorkersUuid('log')[0],
+workernames=getWorkersDocker('log', worker_data_docker)[0],
 workerbuilddir='builds',
 collapseRequests=False,
 factory=buildfactorys.update_db_cpv()
@@ -61,7 +76,7 @@ def gentoo_builders(worker_data):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='update_v_data',
-workername=g_ci_w.getWorkersUuid('log')[0],
+workername=getWorkersDocker('log', worker_data_docker)[0],
 workerbuilddir='builds',
 collapseRequests=False,
 canStartBuild=CanWorkerUpdateV,
@@ -71,7 +86,7 @@ def gentoo_builders(worker_data):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='build_request_data',
-workernames=g_ci_w.getWorkersUuid('local'),
+workernames=getWorkersLocal('local', worker_data_local),
 collapseRequests=False,
 factory=buildfactorys.build_request_check()
 )
@@ -79,7 +94,7 @@ def gentoo_builders(worker_data):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='run_build_request',
-workernames=g_ci_w.getWorkersUuid('docker'),
+workernames=getWorkersDocker('build',

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2022-08-10 Thread Magnus Granberg

commit: 26929ff836acfe559456bed9f400dd4879973d8b
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Aug 10 21:46:50 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Aug 10 21:46:50 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=26929ff8

Use one scheduler for push and mr

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/schedulers.py | 26 --
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/buildbot_gentoo_ci/config/schedulers.py 
b/buildbot_gentoo_ci/config/schedulers.py
index b769f00..60f1857 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -42,9 +42,17 @@ def getGitChanges(props):
 change_data['project'] = k['project']
 return change_data
 
-def mr_branch_fn(branch):
-# if branch end with -mr or -pr
-if branch[-3] == '-mr' or branch[-3] == '-pr':
+def bb_branch_fn(branch):
+# check branch
+print(f"Branch: {branch}")
+if branch.endswith('-mr') or branch.endswith('-pr') or branch == 'master':
+return True
+return False
+
+def bb_category_fn(category):
+# check event
+print(f"Category: {category}")
+if category == 'push' or category == 'merge_request':
 return True
 return False
 
@@ -56,16 +64,7 @@ def gentoo_schedulers():
 change_data = getGitChanges
 ),
 builderNames = builderUpdateDbNames,
-change_filter=util.ChangeFilter(branch='master', category='push'),
-)
-scheduler_update_db_mr= schedulers.SingleBranchScheduler(
-name='scheduler_update_db_mr',
-treeStableTimer=0,
-properties = dict(
-change_data = getGitChanges
-),
-builderNames = builderUpdateDbNames,
-change_filter=util.ChangeFilter(branch_fn=mr_branch_fn, 
category='merge_request'),
+change_filter=util.ChangeFilter(branch_fn=bb_branch_fn, 
category_fn=bb_category_fn),
 )
 create_stage4 = schedulers.ForceScheduler(
 name="create_stage4",
@@ -102,7 +101,6 @@ def gentoo_schedulers():
 s = []
 s.append(create_stage4)
 s.append(scheduler_update_db)
-s.append(scheduler_update_db_mr)
 s.append(update_repo_check)
 s.append(update_cpv_data)
 s.append(update_v_data)

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2022-07-30 Thread Magnus Granberg

commit: 457eadb511be7e44744cfa4a91a302614b5d754f
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Jul 30 22:43:32 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Jul 30 22:43:32 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=457eadb5

Add scheduler_update_db_mr for mr/pr support

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/schedulers.py | 19 ++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/schedulers.py 
b/buildbot_gentoo_ci/config/schedulers.py
index 2e5379c..b769f00 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -39,8 +39,15 @@ def getGitChanges(props):
 change_data['revision'] = k['revision']
 change_data['timestamp'] = k['when_timestamp']
 change_data['branch'] = k['branch']
+change_data['project'] = k['project']
 return change_data
 
+def mr_branch_fn(branch):
+# if branch end with -mr or -pr
+if branch[-3] == '-mr' or branch[-3] == '-pr':
+return True
+return False
+
 def gentoo_schedulers():
 scheduler_update_db = schedulers.SingleBranchScheduler(
 name='scheduler_update_db',
@@ -49,7 +56,16 @@ def gentoo_schedulers():
 change_data = getGitChanges
 ),
 builderNames = builderUpdateDbNames,
-change_filter=util.ChangeFilter(branch='master'),
+change_filter=util.ChangeFilter(branch='master', category='push'),
+)
+scheduler_update_db_mr= schedulers.SingleBranchScheduler(
+name='scheduler_update_db_mr',
+treeStableTimer=0,
+properties = dict(
+change_data = getGitChanges
+),
+builderNames = builderUpdateDbNames,
+change_filter=util.ChangeFilter(branch_fn=mr_branch_fn, 
category='merge_request'),
 )
 create_stage4 = schedulers.ForceScheduler(
 name="create_stage4",
@@ -86,6 +102,7 @@ def gentoo_schedulers():
 s = []
 s.append(create_stage4)
 s.append(scheduler_update_db)
+s.append(scheduler_update_db_mr)
 s.append(update_repo_check)
 s.append(update_cpv_data)
 s.append(update_v_data)

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2022-07-28 Thread Magnus Granberg

commit: 795fec9c1b421559ade5b82fc7232384d31a02d2
Author: Magnus Granberg  gentoo  org>
AuthorDate: Thu Jul 28 11:35:00 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Thu Jul 28 11:35:00 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=795fec9c

Add support branch,event and gitlab/github project name on IRC reporter

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/reporters.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/buildbot_gentoo_ci/config/reporters.py 
b/buildbot_gentoo_ci/config/reporters.py
index d37dd2a..9832305 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -8,9 +8,9 @@ from buildbot.reporters.message import MessageFormatter
 
 from buildbot_gentoo_ci.reporters import irc
 irc_template = '''{% set resultsList = ["\x0303SUCCESS", "\x0308WARNINGS", 
"\x0304FAILURE"] %}\
-{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303" 
}}repo/{{ projects }}{{ "\x03" }} \
+{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303" 
}}repo/{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
 {{ build['properties']['revision'][0]|truncate(10, True) }} {{ "\x0302" }}{{ 
build['properties']['owners'][0][0] }}{{ "\x03" }} \
-{{ build['properties']['project_data'][0]['name'] }} \
+{{ build['properties']['event'][0] }} {{ projects }}:{{ 
build['properties']['project_data'][0]['name'] }} \
 {{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{ 
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }}\
 '''

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, docker/

2022-07-27 Thread Magnus Granberg

commit: 1592e1d7979e55f71c676869c1d3cc1a21064860
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Jul 27 11:01:45 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Jul 27 11:01:45 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1592e1d7

Use worker for Version check (get aux db)

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py |  24 ++-
 buildbot_gentoo_ci/config/buildfactorys.py|  52 +++---
 buildbot_gentoo_ci/config/workers.py  |  18 ++-
 buildbot_gentoo_ci/steps/version.py   | 225 --
 docker/GentooBuildbotWorkerDefault.Dockerfile |  16 +-
 5 files changed, 241 insertions(+), 94 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index e168cf5..82552fa 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -20,6 +20,14 @@ def CanWorkerBuildProject(builder, wfb, request):
 print('no worker')
 return False
 
+# Use same worker as update_cpv_data was done by so we have same git commit
+def CanWorkerUpdateV(builder, wfb, request):
+print(request.properties['cp_worker'])
+print(wfb)
+if wfb.worker.workername == request.properties['cp_worker']:
+return True
+return False
+
 def gentoo_builders(worker_data):
 b = []
 g_ci_w = gentoo_ci_workers(worker_data)
@@ -39,24 +47,24 @@ def gentoo_builders(worker_data):
 factory=buildfactorys.update_repo_check()
 )
 )
-# Use multiplay workers depend on Property(cp)
-# if cp do not match next one, use diffrent worker then
-# or first cp have done its buildsteps.
-# first LocalWorker need to be done before we can use mulitplay workers 
(git pull)
+# update cpv in db and call build_request_data
+#FIXME: look so we don't run parallel with diffrent worker
+# (builders.UpdateRepos step)
 b.append(util.BuilderConfig(
 name='update_cpv_data',
-workernames=g_ci_w.getWorkersUuid('local'),
+workernames=g_ci_w.getWorkersUuid('log')[0],
 workerbuilddir='builds',
 collapseRequests=False,
-factory=buildfactorys.update_db_cp()
+factory=buildfactorys.update_db_cpv()
 )
 )
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='update_v_data',
-workernames=g_ci_w.getWorkersUuid('local'),
+workername=g_ci_w.getWorkersUuid('log')[0],
 workerbuilddir='builds',
 collapseRequests=False,
+canStartBuild=CanWorkerUpdateV,
 factory=buildfactorys.update_db_v()
 )
 )
@@ -80,7 +88,7 @@ def gentoo_builders(worker_data):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='parse_build_log',
-workernames=g_ci_w.getWorkersUuid('log'),
+workernames=g_ci_w.getWorkersUuid('log')[1:],
 collapseRequests=False,
 factory=buildfactorys.parse_build_log()
 )

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index d07e31b..67447aa 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -22,12 +22,13 @@ def update_db_check():
 #   return profile_repository, project
 f.addStep(update_db.GetDataGentooCiProject())
 # update the repos
-f.addStep(update_db.TriggerUpdateRepositorys())
+#f.addStep(update_db.TriggerUpdateRepositorys())
 # Make a for loop and trigger new builders with cpv from git_changes
 #   return cpv, repository, project_data
 f.addStep(update_db.TriggerCheckForCPV())
 return f
 
+# we run repo update in update_db_cpv instead
 def update_repo_check():
 f = util.BuildFactory()
 # FIXME: 6
@@ -40,9 +41,14 @@ def update_repo_check():
 f.addStep(repos.CheckRepository())
 return f
 
-def update_db_cp():
+def update_db_cpv():
 f = util.BuildFactory()
-# FIXME: 2
+# set needed Propertys
+f.addStep(package.SetupPropertys())
+# update the repositorys listed in project_repository
+f.addStep(builders.UpdateRepos())
+# add repo.conf
+#f.addStep(portage.SetReposConf())
 # if categorys in db
 #   return category_data
 #   add check category path step at end
@@ -57,33 +63,25 @@ def update_db_cp():
 #   add package to db step
 #   return package_data
 f.addStep(package.CheckP())
-# Trigger new builders with v from cpv
-#   return package_data, cpv, repository_data, project_data
+# Trigger update_db_v
 f.addStep(package.TriggerCheckForV())
 return f
 
 def update_db_v():
 f = util.BuildFactory()
-# FIXME: 3
-# if version in db
-#   return version_data
-f.addStep(version.GetVData())
-#   check path and hash
-f.addStep(version.CheckPathHash())
-#   if not path
-#   if not hash
-#   add

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/

2022-07-14 Thread Magnus Granberg

commit: dae4e75d90b65bc77a383c63cd0d6aa19d832531
Author: Magnus Granberg  gentoo  org>
AuthorDate: Thu Jul 14 23:43:18 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Thu Jul 14 23:43:18 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=dae4e75d

Reado the match stage

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  27 +
 buildbot_gentoo_ci/steps/builders.py   | 166 ++---
 2 files changed, 132 insertions(+), 61 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 426cee9..d07e31b 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -160,33 +160,10 @@ def run_build_request():
 f.addStep(portage.SetPackageDefault())
 # setup files in /etc if needed
 # run --regen if needed on repo
-# update packages before any tests
-# run pretend on packages update on worker
-f.addStep(builders.RunEmerge(step='pre-update'))
-#   look at the log to see if we need to do stuff
-#   run update package on worker and check log
-f.addStep(builders.RunEmerge(step='update'))
-# clean up the worker
-# look at the log to see if we need to do stuff
-# run pre-depclean and depclean if set
-f.addStep(builders.RunEmerge(step='pre-depclean'))
-# run preserved-libs and depclean
-f.addStep(builders.RunEmerge(step='preserved-libs'))
-f.addStep(builders.RunEmerge(step='depclean'))
-# setup make.conf if build id has changes make.conf as dict from 
SetMakeConf
-# setup package.* env if build id has changes
-# setup pkgcheck.conf if needed
-#f.addStep(builders.SetPkgCheckConf())
-# run pkgcheck if wanted
-#   check log
-f.addStep(builders.RunPkgCheck())
 # check cpv match
 f.addStep(builders.RunEmerge(step='match'))
-# Add the needed steps for build
-f.addStep(builders.RunBuild())
-# run eclean pkg and dist
-#f.addStep(builders.RunEclean(step='pkg')
-#f.addStep(builders.RunEclean(step='dist')
+# Setup the needed stages for update, pkgcheck and build
+f.addStep(builders.SetupStepts())
 return f
 
 def parse_build_log():

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index dcd54c2..ccb28e1 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -15,6 +15,7 @@ from buildbot.process.buildstep import BuildStep
 from buildbot.process.results import SUCCESS
 from buildbot.process.results import FAILURE
 from buildbot.process.results import SKIPPED
+from buildbot.process.results import WARNINGS
 from buildbot.plugins import steps
 
 #FIXME: should be set in config
@@ -26,9 +27,11 @@ def PersOutputOfEmerge(rc, stdout, stderr):
 emerge_output['preserved_libs'] = False
 emerge_output['change_use'] = False
 emerge_output['circular_deps'] = False
+emerge_output['masked'] = False
 emerge_output['failed'] = False
 package_dict = {}
 log_path_list = []
+print('stderr')
 print(stderr)
 # split the lines
 for line in stdout.split('\n'):
@@ -115,7 +118,6 @@ def PersOutputOfEmerge(rc, stdout, stderr):
 if change_use_list:
 change_use[cpv_split[0]] = change_use_list
 emerge_output['change_use'] = change_use
-err_line_list = []
 if line.startswith(' * '):
 if line.endswith('.log.gz'):
 log_path = line.split(' ')[3]
@@ -125,6 +127,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
 if line.endswith('circular dependencies:'):
 emerge_output['circular_deps'] = True
 stderr_line_list.append(line)
+if line.startswith('- '):
+if re.search('masked', line):
+stderr_line_list.append(line)
+emerge_output['masked'] = True
 emerge_output['stderr'] = stderr_line_list
 emerge_output['log_paths'] = log_path_list
 
@@ -397,6 +403,9 @@ class RunEmerge(BuildStep):
 'emerge',
 '-v'
 ]
+c = yield catpkgsplit(self.getProperty("cpv"))[0]
+p = yield catpkgsplit(self.getProperty("cpv"))[1]
+cp = c + '/' + p
 aftersteps_list = []
 #FIXME: Set build timeout in config
 self.build_timeout = 6600
@@ -510,25 +519,10 @@ class RunEmerge(BuildStep):
 aftersteps_list.append(CheckDepcleanLogs('depclean'))
 
 if self.step == 'match':
-packages_excludes = yield 
self.gentooci.db.projects.getProjectPortagePackageByUuidAndExclude(self.getProperty('project_data')['uuid'])
-cpv = self.getProperty("cpv")
-c = yield catpkgsplit(cpv)[0]
-p = yield catpkgsplit(cpv)[1]
-# Check if package is on the exclude list
-if

[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/config/

2022-07-13 Thread Magnus Granberg

commit: 45fe4420dd0bc2057e50ff2382c070c214cfd68a
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Jul 13 19:41:44 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Jul 13 19:41:44 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=45fe4420

Add support log docker worker

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py | 19 
 buildbot_gentoo_ci/config/workers.py  | 89 +++
 master.cfg|  1 +
 3 files changed, 48 insertions(+), 61 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index 6f3d606..e168cf5 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -23,12 +23,9 @@ def CanWorkerBuildProject(builder, wfb, request):
 def gentoo_builders(worker_data):
 b = []
 g_ci_w = gentoo_ci_workers(worker_data)
-LocalWorkers = g_ci_w.getLocalWorkersUuid()
-BuildWorkers = g_ci_w.getBuildWorkersUuid()
-NodeWorkers = g_ci_w.getNodeWorkersUuid()
 b.append(util.BuilderConfig(
 name='update_db_check',
-workername=LocalWorkers[0],
+workername=g_ci_w.getWorkersUuid('local')[0],
 workerbuilddir='builds',
 collapseRequests=False,
 factory=buildfactorys.update_db_check()
@@ -36,7 +33,7 @@ def gentoo_builders(worker_data):
 )
 b.append(util.BuilderConfig(
 name='update_repo_check',
-workername=LocalWorkers[1],
+workername=g_ci_w.getWorkersUuid('local')[1],
 workerbuilddir='builds',
 collapseRequests=True,
 factory=buildfactorys.update_repo_check()
@@ -48,7 +45,7 @@ def gentoo_builders(worker_data):
 # first LocalWorker need to be done before we can use mulitplay workers 
(git pull)
 b.append(util.BuilderConfig(
 name='update_cpv_data',
-workernames=LocalWorkers,
+workernames=g_ci_w.getWorkersUuid('local'),
 workerbuilddir='builds',
 collapseRequests=False,
 factory=buildfactorys.update_db_cp()
@@ -57,7 +54,7 @@ def gentoo_builders(worker_data):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='update_v_data',
-workernames=LocalWorkers,
+workernames=g_ci_w.getWorkersUuid('local'),
 workerbuilddir='builds',
 collapseRequests=False,
 factory=buildfactorys.update_db_v()
@@ -66,7 +63,7 @@ def gentoo_builders(worker_data):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='build_request_data',
-workernames=LocalWorkers,
+workernames=g_ci_w.getWorkersUuid('local'),
 collapseRequests=False,
 factory=buildfactorys.build_request_check()
 )
@@ -74,7 +71,7 @@ def gentoo_builders(worker_data):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='run_build_request',
-workernames=BuildWorkers,
+workernames=g_ci_w.getWorkersUuid('docker'),
 canStartBuild=CanWorkerBuildProject,
 collapseRequests=False,
 factory=buildfactorys.run_build_request()
@@ -83,7 +80,7 @@ def gentoo_builders(worker_data):
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='parse_build_log',
-workernames=LocalWorkers,
+workernames=g_ci_w.getWorkersUuid('log'),
 collapseRequests=False,
 factory=buildfactorys.parse_build_log()
 )
@@ -91,7 +88,7 @@ def gentoo_builders(worker_data):
 # For node workers
 b.append(util.BuilderConfig(
 name='run_build_stage4_request',
-workernames=NodeWorkers,
+workernames=g_ci_w.getWorkersUuid('node'),
 #FIXME: support more the one node
 #canStartBuild=CanWorkerBuildProject,
 collapseRequests=False,

diff --git a/buildbot_gentoo_ci/config/workers.py 
b/buildbot_gentoo_ci/config/workers.py
index d4c20f7..42c6631 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -7,50 +7,30 @@ class gentoo_ci_workers():
 def __init__(self, worker_data, **kwargs):
 self.worker_data = worker_data
 
-def getLocalWorkersUuid(self):
-local_worker = []
+def getWorkersUuid(self, worker_type):
+worker_list = []
 for worker in self.worker_data:
-if worker['type'] == 'local' and worker['enable'] is True:
-local_worker.append(worker['uuid'])
-print(local_worker)
-return local_worker
+if worker['type'] == worker_type and worker['enable'] is True:
+worker_list.append(worker['uuid'])
+print(worker_list)
+return worker_list
 
-def getBuildWorkersUuid(self):
-build_worker = []
+def getWorkersAllData(self, worker_type):
+worker_list = []
 for worker in self.worker_data:
-if (worker['type'] != 'local'

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2022-07-13 Thread Magnus Granberg

commit: e6adf0b1f7cad72329eed2d90021e024d43b1f83
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Jul 13 19:53:13 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Jul 13 19:53:13 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e6adf0b1

Use ReadEmergeInfoLog

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index ef68f2d..426cee9 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -207,7 +207,7 @@ def parse_build_log():
 # add sum log to buildbot log
 f.addStep(logs.setBuildbotLog())
 # pers the emerge info
-f.addStep(logs.SetupParserEmergeInfoLog())
+f.addStep(logs.ReadEmergeInfoLog())
 # add emerge info to log and db
 f.addStep(logs.setEmergeInfoLog())
 # add package info to log and db

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2022-06-11 Thread Magnus Granberg

commit: d50096480ef4f1737767d927ed76a9e53590761a
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Jun 11 09:52:10 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Jun 11 09:52:10 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d5009648

Disable log upload step to cloud

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index a714e47..ef68f2d 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -200,7 +200,7 @@ def parse_build_log():
 # pers the log from pkg check
 #f.addStep(logs.ParserPkgCheckLog())
 # Upload the log to the cloud and remove the log
-f.addStep(logs.Upload())
+#f.addStep(logs.Upload())
 # check the sum log if we need to make a issue/bug/pr report
 # set it SUCCESS/FAILURE/WARNINGS
 f.addStep(logs.MakeIssue())

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2022-05-31 Thread Magnus Granberg

commit: acb84d738be07a975aaeba84e97d1e28b0356269
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Jun  1 00:12:57 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Jun  1 00:12:57 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=acb84d73

Add support for buildbot 3.5.0

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/config.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/config.py 
b/buildbot_gentoo_ci/config/config.py
index 3fed5d2..faf2058 100644
--- a/buildbot_gentoo_ci/config/config.py
+++ b/buildbot_gentoo_ci/config/config.py
@@ -44,7 +44,8 @@ from buildbot.util import safeTranslate
 from buildbot.util import service as util_service
 from buildbot.warnings import ConfigWarning
 from buildbot.warnings import warn_deprecated
-from buildbot.config import ConfigErrors, error, loadConfigDict
+from buildbot.config import ConfigErrors, error
+from buildbot.config.master import loadConfigDict
 
 _errors = None

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2022-04-22 Thread Magnus Granberg

commit: 1c1e3c168bc09b5c43978b8880390262d9f6e9bd
Author: Magnus Granberg  gentoo  org>
AuthorDate: Fri Apr 22 12:31:43 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Fri Apr 22 12:31:43 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1c1e3c16

Add support for node workers

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py | 11 +++
 buildbot_gentoo_ci/config/workers.py  | 24 ++--
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index ca7fc73..6f3d606 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -25,6 +25,7 @@ def gentoo_builders(worker_data):
 g_ci_w = gentoo_ci_workers(worker_data)
 LocalWorkers = g_ci_w.getLocalWorkersUuid()
 BuildWorkers = g_ci_w.getBuildWorkersUuid()
+NodeWorkers = g_ci_w.getNodeWorkersUuid()
 b.append(util.BuilderConfig(
 name='update_db_check',
 workername=LocalWorkers[0],
@@ -87,4 +88,14 @@ def gentoo_builders(worker_data):
 factory=buildfactorys.parse_build_log()
 )
 )
+# For node workers
+b.append(util.BuilderConfig(
+name='run_build_stage4_request',
+workernames=NodeWorkers,
+#FIXME: support more the one node
+#canStartBuild=CanWorkerBuildProject,
+collapseRequests=False,
+factory=buildfactorys.run_build_stage4_request()
+)
+)
 return b

diff --git a/buildbot_gentoo_ci/config/workers.py 
b/buildbot_gentoo_ci/config/workers.py
index d9e7061..fbdc2a9 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -18,7 +18,7 @@ class gentoo_ci_workers():
 def getBuildWorkersUuid(self):
 build_worker = []
 for worker in self.worker_data:
-if worker['type'] != 'local' and worker['enable'] is True:
+if (worker['type'] != 'local' and worker['type'] != 'node') and 
worker['enable'] is True:
 build_worker.append(worker['uuid'])
 print(build_worker)
 return build_worker
@@ -26,19 +26,39 @@ class gentoo_ci_workers():
 def getBuildWorkersAllData(self):
 build_worker = []
 for worker in self.worker_data:
-if worker['type'] != 'local' and worker['enable'] is True:
+if (worker['type'] != 'local' and worker['type'] != 'node') and 
worker['enable'] is True:
 build_worker.append(worker)
 print(build_worker)
 return build_worker
 
+def getNodeWorkersUuid(self):
+node_worker = []
+for worker in self.worker_data:
+if worker['type'] == 'node' and worker['enable'] is True:
+node_worker.append(worker['uuid'])
+print(node_worker)
+return node_worker
+
+def getNodedWorkersAllData(self):
+node_worker = []
+for worker in self.worker_data:
+if worker['type'] == 'node' and worker['enable'] is True:
+node_worker.append(worker)
+print(node_worker)
+return node_worker
+
 def gentoo_workers(worker_data):
 w = []
 g_ci_w = gentoo_ci_workers(worker_data)
 LocalWorkers = g_ci_w.getLocalWorkersUuid()
 BuildWorkers = g_ci_w.getBuildWorkersAllData()
+NodeWorkers = g_ci_w.getNodedWorkersAllData()
 for local_worker in LocalWorkers:
 w.append(worker.LocalWorker(local_worker))
 for build_worker in BuildWorkers:
 if build_worker['type'] == 'default':
 w.append(worker.Worker(build_worker['uuid'], 
build_worker['password']))
+for node_worker in NodeWorkers:
+if node_worker['type'] == 'node':
+w.append(worker.Worker(node_worker['uuid'], 
node_worker['password']))
 return w

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2022-01-23 Thread Magnus Granberg

commit: ad1ecea15142bd7bf0165979e2bbda4d19a27207
Author: Magnus Granberg  gentoo  org>
AuthorDate: Mon Jan 24 00:45:11 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Mon Jan 24 00:45:11 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ad1ecea1

change how remove portage dir is done

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 37 +++---
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 53e7bb1..86fa162 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -3,6 +3,7 @@
 
 from buildbot.plugins import steps as buildbot_steps
 from buildbot.plugins import util
+from twisted.internet import defer
 
 from buildbot_gentoo_ci.steps import update_db
 from buildbot_gentoo_ci.steps import category
@@ -93,6 +94,7 @@ def build_request_check():
 f.addStep(builders.GetProjectRepositoryData())
 return f
 
+#@defer.inlineCallbacks
 def run_build_request():
 f = util.BuildFactory()
 # set needed Propertys
@@ -101,23 +103,42 @@ def run_build_request():
 f.addStep(builders.UpdateRepos())
 # Clean and add new /etc/portage
 #NOTE: remove the symlink befor the dir
+#f.addStep(buildbot_steps.ShellCommand(
+#flunkOnFailure=False,
+#name='Clean make.profile',
+#command=['rm', 'make.profile'],
+#workdir='/etc/portage/'
+#))
+if buildbot_steps.FileExists(file='portage/make.conf', workdir='/etc/', 
haltOnFailure = False):
+f.addStep(buildbot_steps.ShellCommand(
+flunkOnFailure=False,
+name='Remove portage dir',
+command=['rm', '-R', 'portage'],
+workdir='/etc/'
+))
 f.addStep(buildbot_steps.ShellCommand(
-command=['rm', 'make.profile'],
-workdir='/etc/portage/'
+flunkOnFailure=False,
+name='Create portage dir',
+command=['mkdir', 'portage'],
+workdir='/etc/'
 ))
-f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
-workdir='/etc/'))
-f.addStep(buildbot_steps.MakeDirectory(dir="portage",
-workdir='/etc/'))
+#f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
+#name='Remove portage dir',
+#workdir='/etc/'))
+#f.addStep(buildbot_steps.MakeDirectory(dir="portage",
+#name = 'Create the portage dir',
+#workdir='/etc/'))
 # Clean /var/cache/portage/logs and emerge.log
 f.addStep(buildbot_steps.ShellCommand(
+flunkOnFailure=False,
 name='Clean emerge.log',
 command=['rm', 'emerge.log'],
 workdir='/var/log/'
 ))
-f.addStep(buildbot_steps.ShellCommand(
+if buildbot_steps.FileExists(file='logs', 
workdir='/var/cache/portage/',haltOnFailure = False):
+f.addStep(buildbot_steps.ShellCommand(
 flunkOnFailure=False,
-name='Clean logs',
+name='Remove logs',
 command=['rm', '-R', 'logs'],
 workdir='/var/cache/portage/'
 ))

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2022-01-08 Thread Magnus Granberg

commit: 52f785d4268d9238e940e4c2b3297f7cdeeea0a8
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Jan  8 17:22:11 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Jan  8 17:22:11 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=52f785d4

clean portage logs

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 12 
 1 file changed, 12 insertions(+)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 6203a1f..53e7bb1 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -109,6 +109,18 @@ def run_build_request():
 workdir='/etc/'))
 f.addStep(buildbot_steps.MakeDirectory(dir="portage",
 workdir='/etc/'))
+# Clean /var/cache/portage/logs and emerge.log
+f.addStep(buildbot_steps.ShellCommand(
+name='Clean emerge.log',
+command=['rm', 'emerge.log'],
+workdir='/var/log/'
+))
+f.addStep(buildbot_steps.ShellCommand(
+flunkOnFailure=False,
+name='Clean logs',
+command=['rm', '-R', 'logs'],
+workdir='/var/cache/portage/'
+))
 # setup the profile
 #NOTE: pkgcheck do not support it as a dir
 #f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2021-10-27 Thread Magnus Granberg

commit: fa21ff6afa0f4ba9769a55e1d7bc59cf9dcca1c8
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Oct 27 20:08:54 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Oct 27 20:08:54 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=fa21ff6a

Move repo update step before portage steps for workers

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 45f8f48..d5f881b 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -95,10 +95,10 @@ def build_request_check():
 
 def run_build_request():
 f = util.BuildFactory()
-# FIXME: 5
-# Move the etc/portage stuff to is own file
 # set needed Propertys
 f.addStep(builders.SetupPropertys())
+# update the repositorys listed in project_repository
+f.addStep(builders.UpdateRepos())
 # Clean and add new /etc/portage
 #NOTE: remove the symlink befor the dir
 f.addStep(buildbot_steps.ShellCommand(
@@ -118,8 +118,6 @@ def run_build_request():
 f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
 workdir='/etc/portage/'))
 f.addStep(portage.SetReposConf())
-# update the repositorys listed in project_repository
-f.addStep(builders.UpdateRepos())
 # setup make.conf
 f.addStep(portage.SetMakeConf())
 # setup env

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2021-10-16 Thread Magnus Granberg

commit: df8b25b39d422fd8152e6113c1443835d5aee091
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Oct 16 14:57:34 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Oct 16 14:57:34 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=df8b25b3

Set the workers in worker_data in master.cfg

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py | 21 +++-
 buildbot_gentoo_ci/config/workers.py  | 45 +--
 2 files changed, 46 insertions(+), 20 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index c28d016..ca7fc73 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -5,16 +5,7 @@ from twisted.internet import defer
 
 from buildbot.plugins import util
 from buildbot_gentoo_ci.config import buildfactorys
-
-# FIXME: get LocalWorkers and BuildWorkers from db or file
-LocalWorkers = []
-LocalWorkers.append('updatedb_1')
-LocalWorkers.append('updatedb_2')
-LocalWorkers.append('updatedb_3')
-LocalWorkers.append('updatedb_4')
-
-BuildWorkers = []
-BuildWorkers.append('a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd')
+from buildbot_gentoo_ci.config.workers import gentoo_ci_workers
 
 @defer.inlineCallbacks
 def CanWorkerBuildProject(builder, wfb, request):
@@ -29,10 +20,14 @@ def CanWorkerBuildProject(builder, wfb, request):
 print('no worker')
 return False
 
-def gentoo_builders(b=[]):
+def gentoo_builders(worker_data):
+b = []
+g_ci_w = gentoo_ci_workers(worker_data)
+LocalWorkers = g_ci_w.getLocalWorkersUuid()
+BuildWorkers = g_ci_w.getBuildWorkersUuid()
 b.append(util.BuilderConfig(
 name='update_db_check',
-workername='updatedb_1',
+workername=LocalWorkers[0],
 workerbuilddir='builds',
 collapseRequests=False,
 factory=buildfactorys.update_db_check()
@@ -40,7 +35,7 @@ def gentoo_builders(b=[]):
 )
 b.append(util.BuilderConfig(
 name='update_repo_check',
-workername='updatedb_2',
+workername=LocalWorkers[1],
 workerbuilddir='builds',
 collapseRequests=True,
 factory=buildfactorys.update_repo_check()

diff --git a/buildbot_gentoo_ci/config/workers.py 
b/buildbot_gentoo_ci/config/workers.py
index 6141d89..d9e7061 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -3,11 +3,42 @@
 
 from buildbot.plugins import worker
 
-def gentoo_workers(w=[]):
-# FIXME: Get workers from db
-w.append(worker.LocalWorker('updatedb_1'))
-w.append(worker.LocalWorker('updatedb_2'))
-w.append(worker.LocalWorker('updatedb_3'))
-w.append(worker.LocalWorker('updatedb_4'))
-w.append(worker.Worker('bot-test', 'test1234'))
+class gentoo_ci_workers():
+def __init__(self, worker_data, **kwargs):
+self.worker_data = worker_data
+
+def getLocalWorkersUuid(self):
+local_worker = []
+for worker in self.worker_data:
+if worker['type'] == 'local' and worker['enable'] is True:
+local_worker.append(worker['uuid'])
+print(local_worker)
+return local_worker
+
+def getBuildWorkersUuid(self):
+build_worker = []
+for worker in self.worker_data:
+if worker['type'] != 'local' and worker['enable'] is True:
+build_worker.append(worker['uuid'])
+print(build_worker)
+return build_worker
+
+def getBuildWorkersAllData(self):
+build_worker = []
+for worker in self.worker_data:
+if worker['type'] != 'local' and worker['enable'] is True:
+build_worker.append(worker)
+print(build_worker)
+return build_worker
+
+def gentoo_workers(worker_data):
+w = []
+g_ci_w = gentoo_ci_workers(worker_data)
+LocalWorkers = g_ci_w.getLocalWorkersUuid()
+BuildWorkers = g_ci_w.getBuildWorkersAllData()
+for local_worker in LocalWorkers:
+w.append(worker.LocalWorker(local_worker))
+for build_worker in BuildWorkers:
+if build_worker['type'] == 'default':
+w.append(worker.Worker(build_worker['uuid'], 
build_worker['password']))
 return w

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/

2021-09-07 Thread Magnus Granberg

commit: 396cbf6ec3c530541c278155828677a341fda248
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Sep  8 00:21:03 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Sep  8 00:21:03 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=396cbf6e

Support dynamically projects and worker in run_build_request

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py | 24 +---
 buildbot_gentoo_ci/db/model.py| 20 
 buildbot_gentoo_ci/db/projects.py | 18 ++
 buildbot_gentoo_ci/steps/builders.py  | 28 +++-
 4 files changed, 78 insertions(+), 12 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index 56cdde1..c28d016 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -1,16 +1,34 @@
 # Copyright 2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+from twisted.internet import defer
+
 from buildbot.plugins import util
 from buildbot_gentoo_ci.config import buildfactorys
 
-# FIXME: get workers from db or file
+# FIXME: get LocalWorkers and BuildWorkers from db or file
 LocalWorkers = []
 LocalWorkers.append('updatedb_1')
 LocalWorkers.append('updatedb_2')
 LocalWorkers.append('updatedb_3')
 LocalWorkers.append('updatedb_4')
 
+BuildWorkers = []
+BuildWorkers.append('a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd')
+
+@defer.inlineCallbacks
+def CanWorkerBuildProject(builder, wfb, request):
+gentooci = 
builder.master.namedServices['services'].namedServices['gentooci']
+project_build_data = request.properties['project_build_data']
+project_workers = yield 
gentooci.db.projects.getWorkersByProjectUuid(project_build_data['project_uuid'])
+print(project_workers)
+print(wfb)
+for worker in project_workers:
+if wfb.worker.workername == worker['worker_uuid']:
+return True
+print('no worker')
+return False
+
 def gentoo_builders(b=[]):
 b.append(util.BuilderConfig(
 name='update_db_check',
@@ -57,11 +75,11 @@ def gentoo_builders(b=[]):
 factory=buildfactorys.build_request_check()
 )
 )
-# FIXME: get workers from db or file
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='run_build_request',
-workername='bot-test',
+workernames=BuildWorkers,
+canStartBuild=CanWorkerBuildProject,
 collapseRequests=False,
 factory=buildfactorys.run_build_request()
 )

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index ca9932a..d9a3972 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -232,6 +232,17 @@ class Model(base.DBConnectorComponent):
 sa.Column('search_type', sa.Enum('in', 'startswith', 'endswith', 
'search'), default='in'),
 )
 
+projects_workers = sautils.Table(
+"projects_workers", metadata,
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('project_uuid', sa.String(36),
+  sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+  nullable=False),
+sa.Column('worker_uuid', sa.String(36),
+  sa.ForeignKey('workers.uuid', ondelete='CASCADE'),
+  nullable=False),
+)
+
 keywords = sautils.Table(
 "keywords", metadata,
 # unique uuid per keyword
@@ -289,6 +300,15 @@ class Model(base.DBConnectorComponent):
 sa.Column('status', sa.Enum('stable','unstable','negative','all'), 
nullable=False),
 )
 
+workers = sautils.Table(
+"workers", metadata,
+# unique id per project
+sa.Column('uuid', sa.String(36), primary_key=True,
+  default=lambda: str(uuid.uuid4())),
+sa.Column('type', sa.Enum('local','default','latent'), nullable=False),
+sa.Column('enabled', sa.Boolean, default=False),
+)
+
 # Tables related to users
 # ---
 

diff --git a/buildbot_gentoo_ci/db/projects.py 
b/buildbot_gentoo_ci/db/projects.py
index 176be92..fbef435 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -189,6 +189,17 @@ class 
ProjectsConnectorComponent(base.DBConnectorComponent):
 res = yield self.db.pool.do(thd)
 return res
 
+@defer.inlineCallbacks
+def getWorkersByProjectUuid(self, uuid):
+def thd(conn):
+tbl = self.db.model.projects_workers
+q = tbl.select()
+q = q.where(tbl.c.project_uuid == uuid)
+return [self._row2dict_projects_workers(conn, row)
+for row in conn.execute(q).fetchall()]
+res = yield self.db.pool.do(thd)
+return res
+
 def _row2dict(self, conn, row):
 return dict(
 uuid=row.uuid,
@@ -217,6 +228,13

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/

2021-05-18 Thread Magnus Granberg

commit: cdbe9295f945bd4f40db2333e016c075e2c64784
Author: Magnus Granberg  gentoo  org>
AuthorDate: Tue May 18 09:13:59 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Tue May 18 09:13:59 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=cdbe9295

Add support to push log to Minio

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  2 +-
 buildbot_gentoo_ci/steps/logs.py   | 28 ++
 buildbot_gentoo_ci/steps/minio.py  | 60 ++
 3 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 417e385..d2e195c 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -171,7 +171,7 @@ def parse_build_log():
 # pers the log from pkg check
 #f.addStep(logs.ParserPkgCheckLog())
 # Upload the log to the cloud and remove the log
-#f.addStep(logs.Upload())
+f.addStep(logs.Upload())
 # check the sum log if we need to make a issue/bug/pr report
 # set it SUCCESS/FAILURE/WARNINGS
 f.addStep(logs.MakeIssue())

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 3c0ef8d..d69c447 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -18,6 +18,8 @@ from buildbot.process.results import FAILURE
 from buildbot.process.results import WARNINGS
 from buildbot.plugins import steps
 
+from buildbot_gentoo_ci.steps import minio
+
 class SetupPropertys(BuildStep):
 
 name = 'SetupPropertys'
@@ -277,6 +279,32 @@ class setEmergeInfoLog(BuildStep):
 yield log.addStdout(line + '\n')
 return SUCCESS
 
+class Upload(BuildStep):
+
+name = 'Upload'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = False
+flunkOnFailure = True
+warnOnWarnings = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+if self.getProperty('faild_cpv'):
+log_cpv = 
self.getProperty('log_build_data')[self.getProperty('faild_cpv')]
+else:
+log_cpv = 
self.getProperty('log_build_data')[self.getProperty('cpv')]
+bucket = self.getProperty('project_data')['uuid'] + '-' + 'logs'
+file_path = yield os.path.join(self.master.basedir, 'cpv_logs', 
log_cpv['full_logname'])
+aftersteps_list = []
+aftersteps_list.append(minio.putFileToMinio(file_path, 
log_cpv['full_logname'], bucket))
+yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+return SUCCESS
+
 class setBuildStatus(BuildStep):
 
 name = 'setBuildStatus'

diff --git a/buildbot_gentoo_ci/steps/minio.py 
b/buildbot_gentoo_ci/steps/minio.py
new file mode 100644
index 000..ac9ac3d
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/minio.py
@@ -0,0 +1,60 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from minio import Minio
+from minio.error import ResponseError, BucketAlreadyOwnedByYou, 
BucketAlreadyExists
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+
+#FIXME:
+# get url, user from config
+# get password from secret
+url = ''
+user = ''
+password = ''
+
+class putFileToMinio(BuildStep):
+
+name = 'putFileToMinio'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = False
+flunkOnFailure = True
+warnOnWarnings = True
+
+def __init__(self, filename, target, bucket, **kwargs):
+self.filename = filename
+self.bucket = bucket
+self.target = target
+super().__init__(**kwargs)
+
+def getMinioConnect(self, url, user, password):
+minioclient = Minio(
+url,
+access_key = user,
+secret_key = password,
+secure = False
+)
+return minioclient
+
+@defer.inlineCallbacks
+def pushFileToMinio(self):
+try:
+yield self.minio_connect.fput_object(self.bucket, self.target, 
self.filename)
+except ResponseError as err:
+print(err)
+return FAILURE
+return True
+
+@defer.inlineCallbacks
+def run(self):
+self.gentooci = 
self.master.namedServices['services'].namedServices['gentooci']
+self.minio_connect = yield self.getMinioConnect(url, user, password)
+success = yield self.pushFileToMinio()
+return SUCCESS

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/

2021-04-17 Thread Magnus Granberg

commit: bfa939daa5d6aad8f4a7b129bced7e9dba796828
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Apr 17 17:24:48 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Apr 17 17:24:48 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=bfa939da

Add support for more LocalWorkers

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py  | 28 +++---
 buildbot_gentoo_ci/config/buildfactorys.py |  8 
 buildbot_gentoo_ci/config/schedulers.py| 10 ++---
 buildbot_gentoo_ci/config/workers.py   |  3 ++
 buildbot_gentoo_ci/steps/category.py   |  2 +-
 buildbot_gentoo_ci/steps/package.py| 32 +++
 buildbot_gentoo_ci/steps/portage.py|  3 +-
 buildbot_gentoo_ci/steps/update_db.py  | 62 ++
 buildbot_gentoo_ci/steps/version.py| 46 +++---
 9 files changed, 85 insertions(+), 109 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index d022ac7..56cdde1 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -4,8 +4,14 @@
 from buildbot.plugins import util
 from buildbot_gentoo_ci.config import buildfactorys
 
+# FIXME: get workers from db or file
+LocalWorkers = []
+LocalWorkers.append('updatedb_1')
+LocalWorkers.append('updatedb_2')
+LocalWorkers.append('updatedb_3')
+LocalWorkers.append('updatedb_4')
+
 def gentoo_builders(b=[]):
-# FIXME: get workers from db
 b.append(util.BuilderConfig(
 name='update_db_check',
 workername='updatedb_1',
@@ -14,47 +20,44 @@ def gentoo_builders(b=[]):
 factory=buildfactorys.update_db_check()
 )
 )
-# FIXME: get workers from db
 b.append(util.BuilderConfig(
 name='update_repo_check',
-workername='updatedb_1',
+workername='updatedb_2',
 workerbuilddir='builds',
 collapseRequests=True,
 factory=buildfactorys.update_repo_check()
 )
 )
-# FIXME: get workers from db
-# Use multiplay workers depend on Property(cpv)
+# Use multiplay workers depend on Property(cp)
 # if cp do not match next one, use diffrent worker then
 # or first cp have done its buildsteps.
+# first LocalWorker need to be done before we can use mulitplay workers 
(git pull)
 b.append(util.BuilderConfig(
 name='update_cpv_data',
-workername='updatedb_1',
+workernames=LocalWorkers,
 workerbuilddir='builds',
 collapseRequests=False,
 factory=buildfactorys.update_db_cp()
 )
 )
-# FIXME: get workers from db
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='update_v_data',
-workername='updatedb_1',
+workernames=LocalWorkers,
 workerbuilddir='builds',
 collapseRequests=False,
 factory=buildfactorys.update_db_v()
 )
 )
-# FIXME: get workers from db
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='build_request_data',
-workername='updatedb_1',
+workernames=LocalWorkers,
 collapseRequests=False,
 factory=buildfactorys.build_request_check()
 )
 )
-# FIXME: get workers from db
+# FIXME: get workers from db or file
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='run_build_request',
@@ -63,11 +66,10 @@ def gentoo_builders(b=[]):
 factory=buildfactorys.run_build_request()
 )
 )
-# FIXME: get workers from db
 # Use multiplay workers
 b.append(util.BuilderConfig(
 name='parse_build_log',
-workername='updatedb_1',
+workernames=LocalWorkers,
 collapseRequests=False,
 factory=buildfactorys.parse_build_log()
 )

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 962eeb0..e08127b 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -21,14 +21,6 @@ def update_db_check():
 f.addStep(update_db.GetDataGentooCiProject())
 # update the repos
 f.addStep(update_db.TriggerUpdateRepositorys())
-# Check if needed path is there
-f.addStep(portage.CheckPathLocal())
-# setup the profile
-f.addStep(portage.SetMakeProfileLocal())
-# setup repos.conf dir
-f.addStep(portage.SetReposConfLocal())
-# setup make.conf
-f.addStep(portage.SetMakeConfLocal())
 # Make a for loop and trigger new builders with cpv from git_changes
 #   return cpv, repository, project_data
 f.addStep(update_db.TriggerCheckForCPV())

diff --git a/buildbot_gentoo_ci/config/schedulers.py 
b/buildbot_gentoo_ci/config/schedulers.py
index 107bb3c..e520076 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -12,7 +12,7 @@

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/

2021-04-05 Thread Magnus Granberg

commit: 34d24bb7295a9b3102464e2744b6b807fe2c6c22
Author: Magnus Granberg  gentoo  org>
AuthorDate: Mon Apr  5 08:07:51 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Mon Apr  5 08:07:51 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=34d24bb7

Add support for --depclean

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 22 --
 buildbot_gentoo_ci/steps/builders.py   | 70 +-
 2 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index fddfa2b..817b324 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -127,21 +127,15 @@ def run_build_request():
 # run pretend on packages update on worker
 f.addStep(builders.RunEmerge(step='pre-update'))
 #   look at the log to see if we need to do stuff
-#   run update package on worker
+#   run update package on worker and check log
 f.addStep(builders.RunEmerge(step='update'))
-#   check log
-# run depclean if set
-#   depclean pretend
+# clean up the worker
+# look at the log to see if we need to do stuff
+# run pre-depclean and depclean if set
 f.addStep(builders.RunEmerge(step='pre-depclean'))
-#   look at the log to see if we need to do stuff
-#   depclean
-f.addStep(builders.RunEmerge(step='depclean'))
-# run @preserved-rebuild if needed
+# run preserved-libs and depclean
 f.addStep(builders.RunEmerge(step='preserved-libs'))
-#   check log
-#   depclean
 f.addStep(builders.RunEmerge(step='depclean'))
-#   check log
 # setup make.conf if build id has changes make.conf as dict from 
SetMakeConf
 # setup package.* env if build id has changes
 # setup pkgcheck.conf if needed
@@ -154,10 +148,10 @@ def run_build_request():
 # Add the needed steps for build
 f.addStep(builders.RunBuild())
 # clean up the worker
+# look at the log to see if we need to do stuff
+# run pre-depclean and depclean if set
 f.addStep(builders.RunEmerge(step='pre-depclean'))
-#   look at the log to see if we need to do stuff
-#   run depclean and preserved-libs
-f.addStep(builders.RunEmerge(step='depclean'))
+# run preserved-libs and depclean
 f.addStep(builders.RunEmerge(step='preserved-libs'))
 f.addStep(builders.RunEmerge(step='depclean'))
 return f

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index 96572c0..af3be9d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -18,7 +18,6 @@ def PersOutputOfEmerge(rc, stdout, stderr):
 emerge_output = {}
 emerge_output['rc'] = rc
 emerge_output['preserved_libs'] = False
-emerge_output['depclean'] = False
 emerge_output['change_use'] = False
 package_dict = {}
 log_path_list = []
@@ -121,6 +120,21 @@ def PersOutputOfPkgCheck(rc, stdout, stderr):
 'pkgcheck_output' : pkgcheck_output
 }
 
+def PersOutputOfDepclean(rc, stdout, stderr):
+depclean_output = {}
+depclean_output['rc'] = rc
+print(stderr)
+depclean_output['stderr'] = stderr
+package_list = False
+for line in stdout.split('\n'):
+if line.startswith('All selected packages:'):
+line_tmp = line.replace('All selected packages: ', '')
+package_list = line_tmp.split(' ')
+depclean_output['packages'] = package_list
+return {
+'depclean_output' : depclean_output
+}
+
 class TriggerRunBuildRequest(BuildStep):
 
 name = 'TriggerRunBuildRequest'
@@ -364,22 +378,26 @@ class RunEmerge(BuildStep):
 steps.SetPropertyFromCommandNewStyle(
 command=shell_commad_list,
 strip=True,
-extract_fn=PersOutputOfEmerge,
+extract_fn=PersOutputOfDepclean,
 workdir='/'
 ))
-aftersteps_list.append(CheckEmergeLogs('depclean'))
+aftersteps_list.append(CheckDepcleanLogs('pre-depclean'))
+self.setProperty('depclean', False, 'depclean')
 
-if self.step == 'depclean' and self.getProperty('depclean'):
+if self.step == 'depclean' and projects_emerge_options['depclean']:
 shell_commad_list.append('-q')
 shell_commad_list.append('--depclean')
+# add exlude cpv if needed
+if self.getProperty('depclean'):
+pass
 aftersteps_list.append(
 steps.SetPropertyFromCommandNewStyle(
 command=shell_commad_list,
 strip=True,
-extract_fn=PersOutputOfEmerge,
+extract_fn=PersOutputOfDepclean,

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2021-04-04 Thread Magnus Granberg

commit: 67ae4c26aa06266d530fde15618bf6787486b4ef
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun Apr  4 20:22:24 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun Apr  4 20:22:24 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=67ae4c26

Don't collaps builds

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py | 6 ++
 1 file changed, 6 insertions(+)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index 0235753..25b180c 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -10,6 +10,7 @@ def gentoo_builders(b=[]):
 name='update_db_check',
 workername='updatedb_1',
 workerbuilddir='builds',
+collapseRequests=False,
 factory=buildfactorys.update_db_check()
 )
 )
@@ -21,6 +22,7 @@ def gentoo_builders(b=[]):
 name='update_cpv_data',
 workername='updatedb_1',
 workerbuilddir='builds',
+collapseRequests=False,
 factory=buildfactorys.update_db_cp()
 )
 )
@@ -30,6 +32,7 @@ def gentoo_builders(b=[]):
 name='update_v_data',
 workername='updatedb_1',
 workerbuilddir='builds',
+collapseRequests=False,
 factory=buildfactorys.update_db_v()
 )
 )
@@ -38,6 +41,7 @@ def gentoo_builders(b=[]):
 b.append(util.BuilderConfig(
 name='build_request_data',
 workername='updatedb_1',
+collapseRequests=False,
 factory=buildfactorys.build_request_check()
 )
 )
@@ -46,6 +50,7 @@ def gentoo_builders(b=[]):
 b.append(util.BuilderConfig(
 name='run_build_request',
 workername='bot-test',
+collapseRequests=False,
 factory=buildfactorys.run_build_request()
 )
 )
@@ -54,6 +59,7 @@ def gentoo_builders(b=[]):
 b.append(util.BuilderConfig(
 name='parse_build_log',
 workername='updatedb_1',
+collapseRequests=False,
 factory=buildfactorys.parse_build_log()
 )
 )

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2021-04-02 Thread Magnus Granberg

commit: bd8eed5bc56b32968b97d65e06ae1480fc39e556
Author: Magnus Granberg  gentoo  org>
AuthorDate: Fri Apr  2 21:59:48 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Fri Apr  2 21:59:48 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=bd8eed5b

Check all code changes in GitPoller pull

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/schedulers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/schedulers.py 
b/buildbot_gentoo_ci/config/schedulers.py
index 1941a82..5be8419 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -41,7 +41,7 @@ def gitUpdateDb(props):
 def gentoo_schedulers():
 scheduler_update_db = schedulers.SingleBranchScheduler(
 name='scheduler_update_db',
-treeStableTimer=60,
+treeStableTimer=0,
 properties = {
 'git_changes' : gitUpdateDb,
 },

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/

2021-03-18 Thread Magnus Granberg

commit: e940caa7779d5d0f0e3864670723e0ea3a0b3e1d
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Mar 13 18:39:23 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Mar 13 18:39:23 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e940caa7

Add support to use db for portage config insted of repo on local worker

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py  |   3 +
 buildbot_gentoo_ci/config/buildfactorys.py |  27 +++---
 buildbot_gentoo_ci/steps/portage.py| 106 ++
 buildbot_gentoo_ci/steps/update_db.py  | 140 +++--
 4 files changed, 218 insertions(+), 58 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index cba9fcc..c7b4469 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -9,6 +9,7 @@ def gentoo_builders(b=[]):
 b.append(util.BuilderConfig(
 name='update_db_check',
 workername='updatedb_1',
+workerbuilddir='builds',
 factory=buildfactorys.update_db_check()
 )
 )
@@ -19,6 +20,7 @@ def gentoo_builders(b=[]):
 b.append(util.BuilderConfig(
 name='update_cpv_data',
 workername='updatedb_1',
+workerbuilddir='builds',
 factory=buildfactorys.update_db_cp()
 )
 )
@@ -27,6 +29,7 @@ def gentoo_builders(b=[]):
 b.append(util.BuilderConfig(
 name='update_v_data',
 workername='updatedb_1',
+workerbuilddir='builds',
 factory=buildfactorys.update_db_v()
 )
 )

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 31543f9..f5a3b35 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -15,20 +15,21 @@ def update_db_check():
 f = util.BuildFactory()
 # FIXME: 1
 # Get base project data from db
-#   return project_repository, profile_repository,
-#   project
+#   return profile_repository, project
 f.addStep(update_db.GetDataGentooCiProject())
-# Check if base project repo is cloned
-# Check if profile repo is cloned
-# check if the profile link is there
-# check if the repository is cloned
-f.addStep(update_db.CheckPathGentooCiProject())
-# check if etc/portage has no error
-#   return config_root
-f.addStep(update_db.CheckProjectGentooCiProject())
-# Make a for loop and trigger new builders with cpv from cpv_changes
-#   return cpv, repository, project_data, config_root
-f.addStep(update_db.CheckCPVGentooCiProject())
+# Check if needed path is there
+f.addStep(update_db.CheckPath())
+# update the repos
+f.addStep(update_db.UpdateRepos())
+# setup the profile
+f.addStep(portage.SetMakeProfileLocal())
+# setup repos.conf dir
+f.addStep(portage.SetReposConfLocal())
+# setup make.conf
+f.addStep(portage.SetMakeConfLocal())
+# Make a for loop and trigger new builders with cpv from git_changes
+#   return cpv, repository, project_data
+f.addStep(update_db.TriggerCheckForCPV())
 return f
 
 def update_db_cp():

diff --git a/buildbot_gentoo_ci/steps/portage.py 
b/buildbot_gentoo_ci/steps/portage.py
index 30361e6..4fbe141 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -11,6 +11,15 @@ from buildbot.process.results import SUCCESS
 from buildbot.process.results import FAILURE
 from buildbot.plugins import steps
 
+@defer.inlineCallbacks
+def WriteTextToFile(path, text_list):
+separator = '\n'
+text_string = separator.join(text_list)
+with open(path, "a") as f:
+yield f.write(text_string)
+yield f.write(separator)
+yield f.close
+
 class SetMakeProfile(BuildStep):
 
 name = 'SetMakeProfile'
@@ -292,3 +301,100 @@ class SetEnvDefault(BuildStep):
 ])
 yield self.build.addStepsAfterCurrentStep(aftersteps_list)
 return SUCCESS
+
+class SetMakeProfileLocal(BuildStep):
+
+name = 'SetMakeProfileLocal'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+parent_path = yield os.path.join('portage', 'make.profile', 'parent')
+if os.path.isfile(parent_path):
+return SUCCESS
+self.gentooci = 
self.master.namedServices['services'].namedServices['gentooci']
+self.repository_basedir = 
self.gentooci.config.project['repository_basedir']
+makeprofiles_paths = []
+makeprofiles_data = yield 
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'],
 'make.profile')
+

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/

2021-02-27 Thread Magnus Granberg

commit: 0db09acc4ba75ea8343465083d8e16d897708b47
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Feb 27 22:59:29 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Feb 27 22:59:29 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0db09acc

Move the steps that have with portage to do to portage

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |   6 +-
 buildbot_gentoo_ci/steps/builders.py   | 197 -
 buildbot_gentoo_ci/steps/portage.py| 197 +
 3 files changed, 200 insertions(+), 200 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index acafed8..b592541 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -106,15 +106,15 @@ def run_build_request():
 #NOTE: pkgcheck do not support it as a dir
 #f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
 #workdir='/etc/portage/'))
-f.addStep(builders.SetMakeProfile())
+f.addStep(portage.SetMakeProfile())
 # setup repos.conf dir
 f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
 workdir='/etc/portage/'))
-f.addStep(builders.SetReposConf())
+f.addStep(portage.SetReposConf())
 # update the repositorys listed in project_repository
 f.addStep(builders.UpdateRepos())
 # setup make.conf
-f.addStep(builders.SetMakeConf())
+f.addStep(portage.SetMakeConf())
 # setup env
 f.addStep(portage.SetEnvDefault())
 # setup package.*

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index 3c19345..81d6e95 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -196,106 +196,6 @@ class SetupPropertys(BuildStep):
 self.setProperty('cpv_build', False, 'cpv_build')
 return SUCCESS
 
-class SetMakeProfile(BuildStep):
-
-name = 'SetMakeProfile'
-description = 'Running'
-descriptionDone = 'Ran'
-descriptionSuffix = None
-haltOnFailure = True
-flunkOnFailure = True
-
-def __init__(self, **kwargs):
-super().__init__(**kwargs)
-
-@defer.inlineCallbacks
-def run(self):
-self.gentooci = 
self.master.namedServices['services'].namedServices['gentooci']
-portage_repos_path = self.getProperty('portage_repos_path')
-project_data = self.getProperty('project_data')
-profile_repository_data = yield 
self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
-makeprofiles_paths = []
-#NOTE: pkgcheck don't support make.profile as a dir
-# we only support one line in db
-makeprofiles_data = yield 
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'],
 'make.profile')
-for makeprofile in makeprofiles_data:
-makeprofile_path = yield os.path.join(portage_repos_path, 
profile_repository_data['name'], 'profiles', makeprofile['value'], '')
-#makeprofiles_paths.append('../../..' + makeprofile_path)
-#separator = '\n'
-#makeprofile_path_string = separator.join(makeprofiles_paths)
-# yield self.build.addStepsAfterCurrentStep([
-#steps.StringDownload(makeprofile_path_string + separator,
-#workerdest="make.profile/parent",
-#workdir='/etc/portage/')
-#])
-#NOTE: pkgcheck profile link
-shell_commad_list = [
-'ln',
-'-s'
-]
-shell_commad_list.append(makeprofile_path)
-shell_commad_list.append('/etc/portage/make.profile')
-yield self.build.addStepsAfterCurrentStep([
-steps.ShellCommandNewStyle(
-command=shell_commad_list,
-workdir='/'
-)
-])
-return SUCCESS
-
-class SetReposConf(BuildStep):
-
-name = 'SetReposConf'
-description = 'Running'
-descriptionDone = 'Ran'
-descriptionSuffix = None
-haltOnFailure = True
-flunkOnFailure = True
-
-def __init__(self, **kwargs):
-super().__init__(**kwargs)
-
-@defer.inlineCallbacks
-def run(self):
-self.gentooci = 
self.master.namedServices['services'].namedServices['gentooci']
-portage_repos_path = self.getProperty('portage_repos_path')
-project_data = self.getProperty('project_data')
-# setup the default.conf
-repos_conf_data = yield 
self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'],
 'repos.conf')
-if repos_conf_data is None:
-print('Default repo is not set in repos.conf')
-return FAILURE
-#

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2021-02-27 Thread Magnus Granberg

commit: 8fb10ec717e99e1fd4e110112319514efeb724c8
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Feb 24 21:54:09 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Feb 24 21:54:09 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8fb10ec7

Remove the symlink for make.profile before we remove the portage dir

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index a6f2183..ac8bac1 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -92,7 +92,11 @@ def run_build_request():
 # set needed Propertys
 f.addStep(builders.SetupPropertys())
 # Clean and add new /etc/portage
-#FIXME: Is don't like symlinks
+#NOTE: remove the symlink befor the dir
+f.addStep(buildbot_steps.ShellCommandNewStyle(
+command=['rm', 'make.profile'],
+workdir='/etc/portage/'
+))
 f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
 workdir='/etc/'))
 f.addStep(buildbot_steps.MakeDirectory(dir="portage",

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2021-02-24 Thread Magnus Granberg

commit: 8972add1c81713e03444dee9c0847dd2a80b9cf0
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Feb 24 19:32:47 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Feb 24 19:32:47 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8972add1

move the depclean steps

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 10 ++
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index b22e1dc..a6f2183 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -122,15 +122,17 @@ def run_build_request():
 #   run update package on worker
 f.addStep(builders.RunEmerge(step='update'))
 #   check log
-# run pretend @preserved-rebuild if needed
-f.addStep(builders.RunEmerge(step='preserved-libs'))
-#   check log
 # run depclean if set
 #   depclean pretend
 f.addStep(builders.RunEmerge(step='pre-depclean'))
 #   look at the log to see if we need to do stuff
 #   depclean
 f.addStep(builders.RunEmerge(step='depclean'))
+# run @preserved-rebuild if needed
+f.addStep(builders.RunEmerge(step='preserved-libs'))
+#   check log
+#   depclean
+f.addStep(builders.RunEmerge(step='depclean'))
 #   check log
 # setup make.conf if build id has changes make.conf as dict from 
SetMakeConf
 # setup package.* env if build id has changes
@@ -146,7 +148,7 @@ def run_build_request():
 # clean up the worker
 f.addStep(builders.RunEmerge(step='pre-depclean'))
 #   look at the log to see if we need to do stuff
-#   depclean
+#   run depclean and preserved-libs
 f.addStep(builders.RunEmerge(step='depclean'))
 f.addStep(builders.RunEmerge(step='preserved-libs'))
 f.addStep(builders.RunEmerge(step='depclean'))

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/

2021-02-23 Thread Magnus Granberg

commit: c138ea64173bbc73930199f07f9eb6814ebc43f6
Author: Magnus Granberg  gentoo  org>
AuthorDate: Tue Feb 23 22:26:57 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Tue Feb 23 22:26:57 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c138ea64

Add RunBuild

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 11 
 buildbot_gentoo_ci/db/model.py |  1 +
 buildbot_gentoo_ci/db/projects.py  |  3 +-
 buildbot_gentoo_ci/steps/builders.py   | 84 +-
 4 files changed, 96 insertions(+), 3 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index cfe4131..b22e1dc 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -139,4 +139,15 @@ def run_build_request():
 # run pkgcheck if wanted
 #   check log
 f.addStep(builders.RunPkgCheck())
+# check cpv match
+f.addStep(builders.RunEmerge(step='match'))
+# Add the needed steps for build
+f.addStep(builders.RunBuild())
+# clean up the worker
+f.addStep(builders.RunEmerge(step='pre-depclean'))
+#   look at the log to see if we need to do stuff
+#   depclean
+f.addStep(builders.RunEmerge(step='depclean'))
+f.addStep(builders.RunEmerge(step='preserved-libs'))
+f.addStep(builders.RunEmerge(step='depclean'))
 return f

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 4d691ea..2153b79 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -129,6 +129,7 @@ class Model(base.DBConnectorComponent):
   nullable=False),
 sa.Column('auto', sa.Boolean, default=False),
 sa.Column('pkgcheck', sa.Enum('package','full','none'), 
default='none'),
+sa.Column('build', sa.Boolean, default=False),
 )
 
 # projects etc/portage settings

diff --git a/buildbot_gentoo_ci/db/projects.py 
b/buildbot_gentoo_ci/db/projects.py
index 9aeca50..37d401c 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -169,7 +169,8 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
 project_uuid=row.project_uuid,
 repository_uuid=row.repository_uuid,
 auto=row.auto,
-pkgcheck=pkgcheck
+pkgcheck=pkgcheck,
+build=row.build
 )
 
 def _row2dict_projects_portage(self, conn, row):

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index 6d46ae6..3ecb78b 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -21,6 +21,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
 emerge_output['depclean'] = False
 package_dict = {}
 print(stderr)
+emerge_output['stderr'] = stderr
 # split the lines
 for line in stdout.split('\n'):
 # package list
@@ -69,6 +70,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
 if line.startswith('!!! existing preserved libs'):
 pass
 #FIXME: Handling of depclean output dict of packages that get removed 
or saved
+emerge_output['package'] = package_dict
 # split the lines
 #FIXME: Handling of stderr output
 for line in stderr.split('\n'):
@@ -191,6 +193,7 @@ class SetupPropertys(BuildStep):
 self.setProperty('project_data', project_data, 'project_data')
 self.setProperty('preserved_libs', False, 'preserved-libs')
 self.setProperty('depclean', False, 'depclean')
+self.setProperty('cpv_build', False, 'cpv_build')
 return SUCCESS
 
 class SetMakeProfile(BuildStep):
@@ -511,7 +514,51 @@ class RunEmerge(BuildStep):
 workdir='/'
 ))
 aftersteps_list.append(CheckEmergeLogs('depclean'))
-if not self.step is None:
+
+if self.step == 'match':
+cpv = self.getProperty("cpv")
+c = yield catpkgsplit(cpv)[0]
+p = yield catpkgsplit(cpv)[1]
+shell_commad_list.append('-pO')
+shell_commad_list.append(c + '/' + p)
+aftersteps_list.append(
+steps.SetPropertyFromCommandNewStyle(
+command=shell_commad_list,
+strip=True,
+extract_fn=PersOutputOfEmerge,
+workdir='/',
+timeout=None
+))
+aftersteps_list.append(CheckEmergeLogs('match'))
+
+if self.step == 'pre-build':
+shell_commad_list.append('-p')
+shell_commad_list.append('=' + self.getProperty('cpv'))
+aftersteps_list.append(
+steps.SetPropertyFromCommandNewStyle(
+command=shell_commad_list,
+strip=True,

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/

2021-02-20 Thread Magnus Granberg

commit: 2dc7f260e99203e183d352601c6a479ae482697b
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Feb 20 18:39:19 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Feb 20 18:39:19 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2dc7f260

Add RunEmerge and CheckEmergeLogs

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  21 +---
 buildbot_gentoo_ci/db/model.py |  11 ++
 buildbot_gentoo_ci/db/projects.py  |  23 +
 buildbot_gentoo_ci/steps/builders.py   | 155 -
 buildbot_gentoo_ci/steps/update_db.py  |   2 +-
 5 files changed, 192 insertions(+), 20 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 2916847..3c77762 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -115,31 +115,20 @@ def run_build_request():
 # run --regen if needed on repo
 # update packages before any tests
 # run pretend on packages update on worker
-shell_commad_list = [
-'emerge',
-'-uDNv',
-'--changed-deps',
-'--changed-use',
-'--pretend',
-'@world'
-]
-f.addStep(buildbot_steps.SetPropertyFromCommandNewStyle(
-command=shell_commad_list,
-strip=True,
-extract_fn=builders.PersOutputOfEmerge,
-workdir='/'
-))
+f.addStep(builders.RunEmerge(step='pre-update'))
 #   look at the log to see if we need to do stuff
 #   run update package on worker
+f.addStep(builders.RunEmerge(step='update'))
 #   check log
 # run pretend @preserved-rebuild if needed
-#   look at the log to see if we need to do stuff
-#   run @preserved-rebuild
+f.addStep(builders.RunEmerge(step='preserved-libs'))
 #   check log
 # run depclean if set
 #   depclean pretend
+f.addStep(builders.RunEmerge(step='pre-depclean'))
 #   look at the log to see if we need to do stuff
 #   depclean
+f.addStep(builders.RunEmerge(step='depclean'))
 #   check log
 # setup make.conf if build id has changes make.conf as dict from 
SetMakeConf
 # setup package.* env if build id has changes

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 4c5e04c..0a2c145 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -161,6 +161,17 @@ class Model(base.DBConnectorComponent):
 sa.Column('value', sa.String(255), nullable=False),
 )
 
+projects_emerge_options = sautils.Table(
+"projects_emerge_options", metadata,
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('project_uuid', sa.String(36),
+  sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+  nullable=False),
+sa.Column('oneshot', sa.Boolean, default=True),
+sa.Column('depclean', sa.Boolean, default=True),
+sa.Column('preserved_libs', sa.Boolean, default=True),
+)
+
 keywords = sautils.Table(
 "keywords", metadata,
 # unique uuid per keyword

diff --git a/buildbot_gentoo_ci/db/projects.py 
b/buildbot_gentoo_ci/db/projects.py
index 1f19a00..eb8f727 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -130,6 +130,20 @@ class 
ProjectsConnectorComponent(base.DBConnectorComponent):
 res = yield self.db.pool.do(thd)
 return res
 
+@defer.inlineCallbacks
+def getProjectEmergeOptionsByUuid(self, uuid):
+def thd(conn):
+tbl = self.db.model.projects_emerge_options
+q = tbl.select()
+q = q.where(tbl.c.project_uuid == uuid)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict_projects_emerge_options(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
 def _row2dict(self, conn, row):
 return dict(
 uuid=row.uuid,
@@ -170,3 +184,12 @@ class 
ProjectsConnectorComponent(base.DBConnectorComponent):
 value=row.value,
 build_id=0
 )
+
+def _row2dict_projects_emerge_options(self, conn, row):
+return dict(
+id=row.id,
+project_uuid=row.project_uuid,
+oneshot=row.oneshot,
+depclean=row.depclean,
+preserved_libs=row.preserved_libs
+)

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index 862f0d4..c0567c1 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -15,13 +15,15 @@ from

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/

2021-01-31 Thread Magnus Granberg

commit: c667f7b39ec20d30678e758dc74d669f6634797a
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun Jan 31 17:43:03 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun Jan 31 17:43:03 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c667f7b3

Add SetMakeConf

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  3 +
 buildbot_gentoo_ci/db/connector.py |  2 +
 buildbot_gentoo_ci/db/model.py | 18 ++
 buildbot_gentoo_ci/db/portages.py  | 43 ++
 buildbot_gentoo_ci/db/projects.py  | 21 +++
 buildbot_gentoo_ci/steps/builders.py   | 95 ++
 6 files changed, 182 insertions(+)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 3fad219..861a4e1 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -88,6 +88,7 @@ def build_request_check():
 def run_build_request():
 f = util.BuildFactory()
 # FIXME: 5
+# Move the etc/portage stuff to is own file
 # set needed Propertys
 f.addStep(builders.SetupPropertys())
 # Clean and add new /etc/portage
@@ -105,4 +106,6 @@ def run_build_request():
 f.addStep(builders.SetReposConf())
 # update the repositorys listed in project_repository
 f.addStep(builders.UpdateRepos())
+# setup make.conf
+f.addStep(builders.SetMakeConf())
 return f

diff --git a/buildbot_gentoo_ci/db/connector.py 
b/buildbot_gentoo_ci/db/connector.py
index 35febe6..7d218ab 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -36,6 +36,7 @@ from buildbot_gentoo_ci.db import categorys
 from buildbot_gentoo_ci.db import packages
 from buildbot_gentoo_ci.db import versions
 from buildbot_gentoo_ci.db import keywords
+from buildbot_gentoo_ci.db import portages
 
 upgrade_message = textwrap.dedent("""\
 
@@ -81,6 +82,7 @@ class DBConnector(service.ReconfigurableServiceMixin,
 self.packages = packages.PackagesConnectorComponent(self)
 self.versions = versions.VersionsConnectorComponent(self)
 self.keywords = keywords.KeywordsConnectorComponent(self)
+self.portages = portages.PortagesConnectorComponent(self)
 
 @defer.inlineCallbacks
 def setup(self, config, check_version=True, verbose=True):

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 596d04e..bd111be 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -143,6 +143,24 @@ class Model(base.DBConnectorComponent):
 sa.Column('value', sa.String(255), nullable=False),
 )
 
+portages_makeconf = sautils.Table(
+"portages_makeconf", metadata,
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('variable', sa.String(255), nullable=False),
+)
+
+projects_portages_makeconf = sautils.Table(
+"projects_portages_makeconf", metadata,
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('project_uuid', sa.String(36),
+  sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+  nullable=False),
+sa.Column('makeconf_id', sa.String(255),
+  sa.ForeignKey('portages_makeconf.id', ondelete='CASCADE'),
+  nullable=False),
+sa.Column('value', sa.String(255), nullable=False),
+)
+
 keywords = sautils.Table(
 "keywords", metadata,
 # unique uuid per keyword

diff --git a/buildbot_gentoo_ci/db/portages.py 
b/buildbot_gentoo_ci/db/portages.py
new file mode 100644
index 000..428fb84
--- /dev/null
+++ b/buildbot_gentoo_ci/db/portages.py
@@ -0,0 +1,43 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors. 
+# Buildbot is free software: you can redistribute it and/or modify it 
+# under the terms of the GNU General Public License as published by the 
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class PortagesConnectorComponent(base.DBConnectorComponent):
+
+@defer.inlineCallbacks
+def getVariables(self):
+def thd(conn):
+tbl = self.db.model.portages_makeconf
+q = tbl.select()

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/

2021-01-27 Thread Magnus Granberg

commit: 7c12da2ec9470fcf82b37943e2a1cc523351aefe
Author: Magnus Granberg  gentoo  org>
AuthorDate: Thu Jan 28 01:09:38 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Thu Jan 28 01:09:38 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7c12da2e

Add SetReposConf and UpdateRepos

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 13 ++--
 buildbot_gentoo_ci/db/model.py |  3 +-
 buildbot_gentoo_ci/db/projects.py  | 29 -
 buildbot_gentoo_ci/steps/builders.py   | 98 ++
 4 files changed, 123 insertions(+), 20 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index a6bafb5..3fad219 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -88,8 +88,8 @@ def build_request_check():
 def run_build_request():
 f = util.BuildFactory()
 # FIXME: 5
-# update repo for the profile
-f.addStep(builders.UpdateProfileRepo())
+# set needed Propertys
+f.addStep(builders.SetupPropertys())
 # Clean and add new /etc/portage
 f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
 workdir='/etc/'))
@@ -99,9 +99,10 @@ def run_build_request():
 f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
 workdir='/etc/portage/'))
 f.addStep(builders.SetMakeProfile())
-# setup repo.conf dir
-#f.addStep(buildbot_steps.MakeDirectory(dir="repo.conf",
+# setup repos.conf dir
+f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
 workdir='/etc/portage/'))
-# check if we have all repository's in repos.conf listed in 
project_repository's
-# update all repos listed in project_repository's
+f.addStep(builders.SetReposConf())
+# update the repositorys listed in project_repository
+f.addStep(builders.UpdateRepos())
 return f

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 0defb0c..596d04e 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -138,7 +138,8 @@ class Model(base.DBConnectorComponent):
 sa.Column('project_uuid', sa.String(36),
   sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
   nullable=False),
-sa.Column('directorys', sa.Enum('make.profile'), nullable=False),
+# FIXME: directorys should be moved to own table
+sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'), 
nullable=False),
 sa.Column('value', sa.String(255), nullable=False),
 )
 

diff --git a/buildbot_gentoo_ci/db/projects.py 
b/buildbot_gentoo_ci/db/projects.py
index 5c3406a..00e1569 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -80,7 +80,19 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
 return res
 
 @defer.inlineCallbacks
-def getProjectPortageByUuidAndDirectory(self, uuid, directory):
+def getRepositorysByProjectUuid(self, uuid, auto=True):
+def thd(conn):
+tbl = self.db.model.projects_repositorys
+q = tbl.select()
+q = q.where(tbl.c.project_uuid == uuid)
+q = q.where(tbl.c.auto == auto)
+return [self._row2dict_projects_repositorys(conn, row)
+for row in conn.execute(q).fetchall()]
+res = yield self.db.pool.do(thd)
+return res
+
+@defer.inlineCallbacks
+def getAllProjectPortageByUuidAndDirectory(self, uuid, directory):
 def thd(conn):
 tbl = self.db.model.projects_portage
 q = tbl.select()
@@ -91,6 +103,21 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
 res = yield self.db.pool.do(thd)
 return res
 
+@defer.inlineCallbacks
+def getProjectPortageByUuidAndDirectory(self, uuid, directory):
+def thd(conn):
+tbl = self.db.model.projects_portage
+q = tbl.select()
+q = q.where(tbl.c.project_uuid == uuid)
+q = q.where(tbl.c.directorys == directory)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict_projects_portage(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
 def _row2dict(self, conn, row):
 return dict(
 uuid=row.uuid,

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index 1c8cbb0..d3b3607 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -80,9 +80,9 @@ class GetProjectRepositoryData(BuildStep):
 yield 
self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
 return

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/

2021-01-17 Thread Magnus Granberg

commit: 500c23afb393cc5330b82ba9d86a966e0abe6289
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun Jan 17 21:20:23 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun Jan 17 21:20:23 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=500c23af

Add BuildFactory build_request_check and start of run_build_request

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py  |  8 +++
 buildbot_gentoo_ci/config/buildfactorys.py | 12 
 buildbot_gentoo_ci/config/schedulers.py|  3 +
 buildbot_gentoo_ci/db/model.py |  6 +-
 buildbot_gentoo_ci/db/projects.py  | 37 ++-
 buildbot_gentoo_ci/steps/builders.py   | 98 ++
 buildbot_gentoo_ci/steps/version.py|  4 +-
 7 files changed, 163 insertions(+), 5 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index 9fef46a..69dd840 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -38,4 +38,12 @@ def gentoo_builders(b=[]):
 factory=buildfactorys.build_request_check()
 )
 )
+# FIXME: get workers from db
+# Use multiplay workers
+b.append(util.BuilderConfig(
+name='run_build_request',
+workername='updatedb_1',
+factory=buildfactorys.run_build_request()
+)
+)
 return b

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index ba96e5a..e0a6634 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -8,6 +8,7 @@ from buildbot_gentoo_ci.steps import update_db
 from buildbot_gentoo_ci.steps import category
 from buildbot_gentoo_ci.steps import package
 from buildbot_gentoo_ci.steps import version
+from buildbot_gentoo_ci.steps import builders
 
 def update_db_check():
 f = util.BuildFactory()
@@ -78,4 +79,15 @@ def update_db_v():
 def build_request_check():
 f = util.BuildFactory()
 # FIXME: 4
+# get project_data
+# check what tests to do
+# triggger build request
+f.addStep(builders.GetProjectRepositoryData())
+return f
+
+def run_build_request():
+f = util.BuildFactory()
+# FIXME: 5
+# Check if all is setup on the worker
+f.addStep(builders.SetupBuildWorker())
 return f

diff --git a/buildbot_gentoo_ci/config/schedulers.py 
b/buildbot_gentoo_ci/config/schedulers.py
index 687b52f..7cfb122 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -56,10 +56,13 @@ def gentoo_schedulers():
builderNames=["update_v_data"])
 build_request_data = schedulers.Triggerable(name="build_request_data",
builderNames=["build_request_data"])
+run_build_request = schedulers.Triggerable(name="run_build_request",
+   builderNames=["run_build_request"])
 s = []
 s.append(test_updatedb)
 #s.append(scheduler_update_db)
 s.append(update_cpv_data)
 s.append(update_v_data)
 s.append(build_request_data)
+s.append(run_build_request)
 return s

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 9ad1331..b051b23 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -109,7 +109,7 @@ class Model(base.DBConnectorComponent):
 sa.Column('keyword_id', sa.Integer,
   sa.ForeignKey('keywords.id', ondelete='CASCADE'),
   nullable=False),
-sa.Column('unstable', sa.Boolean, default=False),
+sa.Column('status', sa.Enum('stable','unstable','all'), 
nullable=False),
 sa.Column('auto', sa.Boolean, default=False),
 sa.Column('enabled', sa.Boolean, default=False),
 sa.Column('created_by', sa.Integer,
@@ -121,12 +121,14 @@ class Model(base.DBConnectorComponent):
 projects_repositorys = sautils.Table(
 "projects_repositorys", metadata,
 sa.Column('id', sa.Integer, primary_key=True),
-sa.Column('projects_uuid', sa.String(36),
+sa.Column('project_uuid', sa.String(36),
   sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
   nullable=False),
 sa.Column('repository_uuid', sa.String(36),
   sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
   nullable=False),
+sa.Column('auto', sa.Boolean, default=False),
+sa.Column('pkgcheck', sa.Boolean, default=False),
 )
 
 keywords = sautils.Table(

diff --git a/buildbot_gentoo_ci/db/projects.py 
b/buildbot_gentoo_ci/db/projects.py
index 4a14bac..065c9a0 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -53,6 +53,32 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
 res = yield self.db.pool.do(thd)

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, /

2021-01-02 Thread Magnus Granberg

commit: 78b01143652300f1eb98df5d17d6dd4edadd789a
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat Jan  2 16:03:07 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat Jan  2 16:03:07 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=78b01143

Add config option project['repository_basedir']

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/config.py | 17 +++--
 gentooci.cfg|  3 +++
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/buildbot_gentoo_ci/config/config.py 
b/buildbot_gentoo_ci/config/config.py
index 7ed59f1..e58e67c 100644
--- a/buildbot_gentoo_ci/config/config.py
+++ b/buildbot_gentoo_ci/config/config.py
@@ -15,7 +15,7 @@
 # Copyright Buildbot Team Members
 # Origins: buildbot.config.py
 # Modifyed by Gentoo Authors.
-# Copyright 2020 Gentoo Authors
+# Copyright 2021 Gentoo Authors
 
 import datetime
 import inspect
@@ -49,6 +49,7 @@ from buildbot.config import ConfigErrors, error, 
loadConfigDict
 _errors = None
 
 DEFAULT_DB_URL = 'sqlite:///gentoo.sqlite'
+DEFAULT_REPOSITORY_BASEDIR = 'repository'
 
 #Use GentooCiConfig.loadFromDict
 @implementer(interfaces.IConfigLoader)
@@ -89,7 +90,8 @@ class GentooCiConfig(util.ComparableMixin):
 
 _known_config_keys = set([
 "db_url",
-"project"
+"project",
+"repository_basedir"
 ])
 
 compare_attrs = list(_known_config_keys)
@@ -137,9 +139,12 @@ class GentooCiConfig(util.ComparableMixin):
 self.db = dict(db_url=self.getDbUrlFromConfig(config_dict))
 
 def load_project(self, config_dict):
-if  'project' in config_dict:
-self.project = dict(
-project=config_dict['project']
-)
+self.project = {}
+if 'project' in config_dict:
+self.project['project'] = config_dict['project']
 else:
 error("project are not configured")
+if 'repository_basedir' in config_dict:
+self.project['repository_basedir'] = 
config_dict['repository_basedir']
+else:
+self.project['repository_basedir'] = DEFAULT_REPOSITORY_BASEDIR

diff --git a/gentooci.cfg b/gentooci.cfg
index 198f6e4..ff12a75 100644
--- a/gentooci.cfg
+++ b/gentooci.cfg
@@ -17,3 +17,6 @@ c['db_url'] = 
"mysql://buildbot:@192.168.1.x/gentooci?max_idle=300"
 
 # This specifies what project buildbot uses for Gentoo Ci as default
 c['project'] = "gosbsbase"
+
+# This specifies what the repository base dir is
+c['repository_basedir'] = "repository"

[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/config/

2020-12-30 Thread Magnus Granberg

commit: 56ac2f08767150fd8659b8d9beac7d56d182b233
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Dec 30 18:35:17 2020 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Dec 30 18:35:17 2020 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=56ac2f08

Add config option project

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/config.py | 10 ++
 gentooci.cfg|  4 +++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/config.py 
b/buildbot_gentoo_ci/config/config.py
index 3ad8595..7ed59f1 100644
--- a/buildbot_gentoo_ci/config/config.py
+++ b/buildbot_gentoo_ci/config/config.py
@@ -89,6 +89,7 @@ class GentooCiConfig(util.ComparableMixin):
 
 _known_config_keys = set([
 "db_url",
+"project"
 ])
 
 compare_attrs = list(_known_config_keys)
@@ -113,6 +114,7 @@ class GentooCiConfig(util.ComparableMixin):
 # and defer the rest to sub-functions, for code clarity
 try:
 config.load_db(config_dict)
+config.load_project(config_dict)
 finally:
 _errors = None
 
@@ -133,3 +135,11 @@ class GentooCiConfig(util.ComparableMixin):
 
 def load_db(self, config_dict):
 self.db = dict(db_url=self.getDbUrlFromConfig(config_dict))
+
+def load_project(self, config_dict):
+if  'project' in config_dict:
+self.project = dict(
+project=config_dict['project']
+)
+else:
+error("project are not configured")

diff --git a/gentooci.cfg b/gentooci.cfg
index 5036ae9..198f6e4 100644
--- a/gentooci.cfg
+++ b/gentooci.cfg
@@ -8,10 +8,12 @@
 # a shorter alias to save typing.
 c = BuildmasterConfig = {}
 
-### DB URL
 ### DB URL
 # This specifies what database buildbot uses to store its state.
 # It's easy to start with sqlite, but it's recommended to switch to a dedicated
 # database, such as PostgreSQL or MySQL, for use in production environments.
 # 
http://docs.buildbot.net/current/manual/configuration/global.html#database-specification
 c['db_url'] = "mysql://buildbot:@192.168.1.x/gentooci?max_idle=300"
+
+# This specifies what project buildbot uses for Gentoo Ci as default
+c['project'] = "gosbsbase"

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/

2020-12-30 Thread Magnus Granberg

commit: ad0e3e17558f34e3c404a855109ec3dddbc38a01
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Dec 30 09:17:57 2020 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Dec 30 09:17:57 2020 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ad0e3e17

Fixme list for f_update_db_packages

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py  | 10 --
 buildbot_gentoo_ci/config/buildfactory.py  |  8 
 buildbot_gentoo_ci/config/buildfactorys.py | 28 
 buildbot_gentoo_ci/config/workers.py   |  1 +
 4 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index b663fba..114bb83 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -2,8 +2,14 @@
 # Distributed under the terms of the GNU General Public License v2
 
 from buildbot.plugins import util
-from buildbot_gentoo_ci.config import buildfactory
+from buildbot_gentoo_ci.config import buildfactorys
 
 def gentoo_builders(b=[]):
-b.append(util.BuilderConfig(name='update_db_packages', 
workername='updatedb_1', factory=buildfactory.f_update_db_packages()))
+# FIXME: get workers from db
+b.append(util.BuilderConfig(
+name='update_db_packages', 
+workername=[ 'updatedb_1', ], 
+factory=buildfactorys.f_update_db_packages()
+)
+)
 return b

diff --git a/buildbot_gentoo_ci/config/buildfactory.py 
b/buildbot_gentoo_ci/config/buildfactory.py
deleted file mode 100644
index 0943031..000
--- a/buildbot_gentoo_ci/config/buildfactory.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2020 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-from buildbot.plugins import util
-
-def f_update_db_packages():
-f = util.BuildFactory()
-return f

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
new file mode 100644
index 000..ef479a2
--- /dev/null
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -0,0 +1,28 @@
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from buildbot.plugins import util, steps
+
+def f_update_db_packages():
+f = util.BuildFactory()
+# FIXME:
+# Get base project from db
+# Check if base project repo for is cloned
+# Check if gentoo repo is cloned
+# check if the profile link is there
+# check if etc/portage has no error
+# Get the repository some way
+repository=util.Property('repository')
+# For loop
+# for cpv in util.Property('cpv_changes')
+#   check if not categorys in db
+#   run update categorys db step
+#   check if not package in db
+#   run update package db step
+#   and get foo{cpv] with metadata back
+#   if new or updated package
+#   else 
+#   run update package db step in a pool of workers
+#   and get foo{cpv] with metadata back
+#   if new or updated package
+return f

diff --git a/buildbot_gentoo_ci/config/workers.py 
b/buildbot_gentoo_ci/config/workers.py
index 50a4751..3005129 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -4,5 +4,6 @@
 from buildbot.plugins import worker
 
 def gentoo_workers(w=[]):
+# FIXME: Get workers from db
 w.append(worker.LocalWorker('updatedb_1'))
 return w

[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/config/

2020-12-30 Thread Magnus Granberg

commit: 584e5b5dd357706b1d1614ccc393ba367def5eaa
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Dec 30 08:20:41 2020 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Dec 30 08:21:24 2020 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=584e5b5d

Add GentooCiService

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/service.py | 43 
 master.cfg   |  6 +++--
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/buildbot_gentoo_ci/config/service.py 
b/buildbot_gentoo_ci/config/service.py
new file mode 100644
index 000..af283b5
--- /dev/null
+++ b/buildbot_gentoo_ci/config/service.py
@@ -0,0 +1,43 @@
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from twisted.python import log
+from twisted.internet import reactor, defer
+
+from buildbot import config as master_config
+from buildbot.db import exceptions
+from buildbot.scripts.base import isBuildmasterDir
+from buildbot.util.service import BuildbotService
+from buildbot import version
+
+from buildbot_gentoo_ci.db import connector as dbconnector
+from buildbot_gentoo_ci.config import config
+
+class GentooCiService(BuildbotService):
+
+name="gentooci"
+
+def checkConfig(self, basedir, **kwargs):
+self.basedir = basedir
+if not isBuildmasterDir(self.basedir):
+master_config.error("Can't find buildbot.tac in basedir")
+return
+
+@defer.inlineCallbacks
+def reconfigService(self, **kwargs):
+return defer.succeed(None)
+
+@defer.inlineCallbacks
+def startService(self):
+self.config_loader = config.FileLoader(self.basedir, 'gentooci.cfg')
+self.config = self.config_loader.loadConfig()
+self.db = dbconnector.DBConnector(self.basedir)
+yield self.db.setServiceParent(self)
+log.msg("Starting Gentoo-Ci Service -- buildbot.version: 
{}".format(version))
+# setup the db
+try:
+yield self.db.setup(self.config)
+except exceptions.DatabaseNotReadyError:
+# (message was already logged)
+reactor.stop()
+return

diff --git a/master.cfg b/master.cfg
index 996f917..464d54d 100644
--- a/master.cfg
+++ b/master.cfg
@@ -1,6 +1,6 @@
 # -*- python -*-
 # ex: set filetype=python:
-from buildbot_gentoo_ci.config import schedulers, workers, builders
+from buildbot_gentoo_ci.config import schedulers, workers, builders, service
 
 # This is a sample buildmaster config file. It must be installed as
 # 'master.cfg' in your buildmaster's base directory.
@@ -51,7 +51,9 @@ c['builders'] = builders.gentoo_builders()
 # status of each build will be pushed to these targets. buildbot/reporters/*.py
 # has a variety to choose from, like IRC bots.
 
-#c['services'] = []
+c['services'] = [
+service.GentooCiService(basedir)
+]
 
 ### PROJECT IDENTITY

45 matches

Mail list logo