[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/, ...
commit: b38a60cfc5b5dca25a674ebe5887737525aa353f
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Feb 15 21:30:32 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Feb 15 21:30:32 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b38a60cf
Use command shell to build docker images
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/builders.py | 24 +++---
buildbot_gentoo_ci/config/buildfactorys.py | 6 +-
buildbot_gentoo_ci/config/schedulers.py | 22 ++---
buildbot_gentoo_ci/config/workers.py| 12 ++-
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/db/workers.py| 3 +-
buildbot_gentoo_ci/steps/nodes.py | 121
docker/GentooBuildbotWorkerBuild.Dockerfile | 31 ---
master.cfg | 14 +++-
sql/gentoo_ci_schema.sql| 14 +++-
10 files changed, 186 insertions(+), 62 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index edcd0c1..b3ddb16 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -13,7 +13,7 @@ def getWorkersLocal(worker_type, workers):
worker_list.append(worker['name'])
return worker_list
-def getWorkersDocker(worker_type, workers):
+def getWorkers(worker_type, workers):
worker_list = []
for worker in workers:
if worker['type'] == worker_type and worker['enable'] is True:
@@ -44,11 +44,11 @@ def CanWorkerUpdateV(builder, wfb, request):
print('Find a worker: NO')
return False
-def gentoo_builders(worker_data_local, worker_data_docker):
+def gentoo_builders(worker_data):
b = []
b.append(util.BuilderConfig(
name='update_db_check',
-workername=getWorkersLocal('local', worker_data_local)[0],
+workername=getWorkersLocal('local', worker_data['local'])[0],
workerbuilddir='builds',
collapseRequests=False,
factory=buildfactorys.update_db_check()
@@ -56,7 +56,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
)
b.append(util.BuilderConfig(
name='update_repo_check',
-workername=getWorkersLocal('local', worker_data_local)[1],
+workername=getWorkersLocal('local', worker_data['local'])[1],
workerbuilddir='builds',
collapseRequests=True,
factory=buildfactorys.update_repo_check()
@@ -67,7 +67,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
# (builders.UpdateRepos step)
b.append(util.BuilderConfig(
name='update_cpv_data',
-workernames=getWorkersDocker('log', worker_data_docker)[0],
+workernames=getWorkers('log', worker_data['docker'])[0],
workerbuilddir='builds',
collapseRequests=False,
factory=buildfactorys.update_db_cpv()
@@ -76,7 +76,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
# Use multiplay workers
b.append(util.BuilderConfig(
name='update_v_data',
-workername=getWorkersDocker('log', worker_data_docker)[0],
+workername=getWorkers('log', worker_data['docker'])[0],
workerbuilddir='builds',
collapseRequests=False,
canStartBuild=CanWorkerUpdateV,
@@ -86,7 +86,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
# Use multiplay workers
b.append(util.BuilderConfig(
name='build_request_data',
-workernames=getWorkersLocal('local', worker_data_local),
+workernames=getWorkersLocal('local', worker_data['local']),
collapseRequests=False,
factory=buildfactorys.build_request_check()
)
@@ -94,7 +94,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
# Use multiplay workers
b.append(util.BuilderConfig(
name='run_build_request',
-workernames=getWorkersDocker('build', worker_data_docker),
+workernames=getWorkers('build', worker_data['docker']),
canStartBuild=CanWorkerBuildProject,
collapseRequests=False,
factory=buildfactorys.run_build_request()
@@ -103,19 +103,19 @@ def gentoo_builders(worker_data_local,
worker_data_docker):
# Use multiplay workers
b.append(util.BuilderConfig(
name='parse_build_log',
-workernames=getWorkersDocker('log', worker_data_docker)[1:],
+workernames=getWorkers('log', worker_data['docker'])[1:],
collapseRequests=False,
factory=buildfactorys.parse_build_log()
)
)
# For node workers
b.append(util.BuilderConfig(
-name='run_build_stage4_request',
-workernames=getWorkersLocal('local', worker_data_local),
+name='run_build_images_request',
+workernames=getWorkers('node', worker_data['node']),
#FIXME: support more the one node
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/
commit: 396cbf6ec3c530541c278155828677a341fda248
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Sep 8 00:21:03 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Sep 8 00:21:03 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=396cbf6e
Support dynamically projects and worker in run_build_request
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/builders.py | 24 +---
buildbot_gentoo_ci/db/model.py| 20
buildbot_gentoo_ci/db/projects.py | 18 ++
buildbot_gentoo_ci/steps/builders.py | 28 +++-
4 files changed, 78 insertions(+), 12 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index 56cdde1..c28d016 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -1,16 +1,34 @@
# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from twisted.internet import defer
+
from buildbot.plugins import util
from buildbot_gentoo_ci.config import buildfactorys
-# FIXME: get workers from db or file
+# FIXME: get LocalWorkers and BuildWorkers from db or file
LocalWorkers = []
LocalWorkers.append('updatedb_1')
LocalWorkers.append('updatedb_2')
LocalWorkers.append('updatedb_3')
LocalWorkers.append('updatedb_4')
+BuildWorkers = []
+BuildWorkers.append('a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd')
+
+@defer.inlineCallbacks
+def CanWorkerBuildProject(builder, wfb, request):
+gentooci =
builder.master.namedServices['services'].namedServices['gentooci']
+project_build_data = request.properties['project_build_data']
+project_workers = yield
gentooci.db.projects.getWorkersByProjectUuid(project_build_data['project_uuid'])
+print(project_workers)
+print(wfb)
+for worker in project_workers:
+if wfb.worker.workername == worker['worker_uuid']:
+return True
+print('no worker')
+return False
+
def gentoo_builders(b=[]):
b.append(util.BuilderConfig(
name='update_db_check',
@@ -57,11 +75,11 @@ def gentoo_builders(b=[]):
factory=buildfactorys.build_request_check()
)
)
-# FIXME: get workers from db or file
# Use multiplay workers
b.append(util.BuilderConfig(
name='run_build_request',
-workername='bot-test',
+workernames=BuildWorkers,
+canStartBuild=CanWorkerBuildProject,
collapseRequests=False,
factory=buildfactorys.run_build_request()
)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index ca9932a..d9a3972 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -232,6 +232,17 @@ class Model(base.DBConnectorComponent):
sa.Column('search_type', sa.Enum('in', 'startswith', 'endswith',
'search'), default='in'),
)
+projects_workers = sautils.Table(
+"projects_workers", metadata,
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+sa.Column('worker_uuid', sa.String(36),
+ sa.ForeignKey('workers.uuid', ondelete='CASCADE'),
+ nullable=False),
+)
+
keywords = sautils.Table(
"keywords", metadata,
# unique uuid per keyword
@@ -289,6 +300,15 @@ class Model(base.DBConnectorComponent):
sa.Column('status', sa.Enum('stable','unstable','negative','all'),
nullable=False),
)
+workers = sautils.Table(
+"workers", metadata,
+# unique id per project
+sa.Column('uuid', sa.String(36), primary_key=True,
+ default=lambda: str(uuid.uuid4())),
+sa.Column('type', sa.Enum('local','default','latent'), nullable=False),
+sa.Column('enabled', sa.Boolean, default=False),
+)
+
# Tables related to users
# ---
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 176be92..fbef435 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -189,6 +189,17 @@ class
ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+@defer.inlineCallbacks
+def getWorkersByProjectUuid(self, uuid):
+def thd(conn):
+tbl = self.db.model.projects_workers
+q = tbl.select()
+q = q.where(tbl.c.project_uuid == uuid)
+return [self._row2dict_projects_workers(conn, row)
+for row in conn.execute(q).fetchall()]
+res = yield self.db.pool.do(thd)
+return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -217,6 +228,13
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/
commit: 7c12da2ec9470fcf82b37943e2a1cc523351aefe
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Jan 28 01:09:38 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Jan 28 01:09:38 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7c12da2e
Add SetReposConf and UpdateRepos
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 13 ++--
buildbot_gentoo_ci/db/model.py | 3 +-
buildbot_gentoo_ci/db/projects.py | 29 -
buildbot_gentoo_ci/steps/builders.py | 98 ++
4 files changed, 123 insertions(+), 20 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index a6bafb5..3fad219 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -88,8 +88,8 @@ def build_request_check():
def run_build_request():
f = util.BuildFactory()
# FIXME: 5
-# update repo for the profile
-f.addStep(builders.UpdateProfileRepo())
+# set needed Propertys
+f.addStep(builders.SetupPropertys())
# Clean and add new /etc/portage
f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
workdir='/etc/'))
@@ -99,9 +99,10 @@ def run_build_request():
f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
workdir='/etc/portage/'))
f.addStep(builders.SetMakeProfile())
-# setup repo.conf dir
-#f.addStep(buildbot_steps.MakeDirectory(dir="repo.conf",
+# setup repos.conf dir
+f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
workdir='/etc/portage/'))
-# check if we have all repository's in repos.conf listed in
project_repository's
-# update all repos listed in project_repository's
+f.addStep(builders.SetReposConf())
+# update the repositorys listed in project_repository
+f.addStep(builders.UpdateRepos())
return f
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 0defb0c..596d04e 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -138,7 +138,8 @@ class Model(base.DBConnectorComponent):
sa.Column('project_uuid', sa.String(36),
sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
nullable=False),
-sa.Column('directorys', sa.Enum('make.profile'), nullable=False),
+# FIXME: directorys should be moved to own table
+sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'),
nullable=False),
sa.Column('value', sa.String(255), nullable=False),
)
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 5c3406a..00e1569 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -80,7 +80,19 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
return res
@defer.inlineCallbacks
-def getProjectPortageByUuidAndDirectory(self, uuid, directory):
+def getRepositorysByProjectUuid(self, uuid, auto=True):
+def thd(conn):
+tbl = self.db.model.projects_repositorys
+q = tbl.select()
+q = q.where(tbl.c.project_uuid == uuid)
+q = q.where(tbl.c.auto == auto)
+return [self._row2dict_projects_repositorys(conn, row)
+for row in conn.execute(q).fetchall()]
+res = yield self.db.pool.do(thd)
+return res
+
+@defer.inlineCallbacks
+def getAllProjectPortageByUuidAndDirectory(self, uuid, directory):
def thd(conn):
tbl = self.db.model.projects_portage
q = tbl.select()
@@ -91,6 +103,21 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+@defer.inlineCallbacks
+def getProjectPortageByUuidAndDirectory(self, uuid, directory):
+def thd(conn):
+tbl = self.db.model.projects_portage
+q = tbl.select()
+q = q.where(tbl.c.project_uuid == uuid)
+q = q.where(tbl.c.directorys == directory)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict_projects_portage(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 1c8cbb0..d3b3607 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -80,9 +80,9 @@ class GetProjectRepositoryData(BuildStep):
yield
self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
return
