commit: 0db09acc4ba75ea8343465083d8e16d897708b47
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 27 22:59:29 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Feb 27 22:59:29 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0db09acc
Move the steps that have with portage to do to portage
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 6 +-
buildbot_gentoo_ci/steps/builders.py | 197 -----------------------------
buildbot_gentoo_ci/steps/portage.py | 197 +++++++++++++++++++++++++++++
3 files changed, 200 insertions(+), 200 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index acafed8..b592541 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -106,15 +106,15 @@ def run_build_request():
#NOTE: pkgcheck do not support it as a dir
#f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
# workdir='/etc/portage/'))
- f.addStep(builders.SetMakeProfile())
+ f.addStep(portage.SetMakeProfile())
# setup repos.conf dir
f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
workdir='/etc/portage/'))
- f.addStep(builders.SetReposConf())
+ f.addStep(portage.SetReposConf())
# update the repositorys listed in project_repository
f.addStep(builders.UpdateRepos())
# setup make.conf
- f.addStep(builders.SetMakeConf())
+ f.addStep(portage.SetMakeConf())
# setup env
f.addStep(portage.SetEnvDefault())
# setup package.*
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 3c19345..81d6e95 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -196,106 +196,6 @@ class SetupPropertys(BuildStep):
self.setProperty('cpv_build', False, 'cpv_build')
return SUCCESS
-class SetMakeProfile(BuildStep):
-
- name = 'SetMakeProfile'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = True
- flunkOnFailure = True
-
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
- @defer.inlineCallbacks
- def run(self):
- self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
- portage_repos_path = self.getProperty('portage_repos_path')
- project_data = self.getProperty('project_data')
- profile_repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
- makeprofiles_paths = []
- #NOTE: pkgcheck don't support make.profile as a dir
- # we only support one line in db
- makeprofiles_data = yield
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'],
'make.profile')
- for makeprofile in makeprofiles_data:
- makeprofile_path = yield os.path.join(portage_repos_path,
profile_repository_data['name'], 'profiles', makeprofile['value'], '')
- # makeprofiles_paths.append('../../..' + makeprofile_path)
- #separator = '\n'
- #makeprofile_path_string = separator.join(makeprofiles_paths)
- # yield self.build.addStepsAfterCurrentStep([
- # steps.StringDownload(makeprofile_path_string + separator,
- # workerdest="make.profile/parent",
- # workdir='/etc/portage/')
- # ])
- #NOTE: pkgcheck profile link
- shell_commad_list = [
- 'ln',
- '-s'
- ]
- shell_commad_list.append(makeprofile_path)
- shell_commad_list.append('/etc/portage/make.profile')
- yield self.build.addStepsAfterCurrentStep([
- steps.ShellCommandNewStyle(
- command=shell_commad_list,
- workdir='/'
- )
- ])
- return SUCCESS
-
-class SetReposConf(BuildStep):
-
- name = 'SetReposConf'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = True
- flunkOnFailure = True
-
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
- @defer.inlineCallbacks
- def run(self):
- self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
- portage_repos_path = self.getProperty('portage_repos_path')
- project_data = self.getProperty('project_data')
- # setup the default.conf
- repos_conf_data = yield
self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'],
'repos.conf')
- if repos_conf_data is None:
- print('Default repo is not set in repos.conf')
- return FAILURE
- # check if repos_conf_data['value'] is vaild repo name
- separator = '\n'
- default_conf = []
- default_conf.append('[DEFAULT]')
- default_conf.append('main-repo = ' + repos_conf_data['value'])
- default_conf.append('auto-sync = no')
- default_conf_string = separator.join(default_conf)
- yield self.build.addStepsAfterCurrentStep([
- steps.StringDownload(default_conf_string + separator,
- workerdest="repos.conf/default.conf",
- workdir='/etc/portage/')
- ])
- # add all repos that project have in projects_repositorys to
repos.conf/reponame.conf
- projects_repositorys_data = yield
self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
- for project_repository_data in projects_repositorys_data:
- repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
- repository_path = yield os.path.join(portage_repos_path,
repository_data['name'])
- repository_conf = []
- repository_conf.append('[' + repository_data['name'] + ']')
- repository_conf.append('location = ' + repository_path)
- repository_conf.append('sync-uri = ' +
repository_data['mirror_url'])
- repository_conf.append('sync-type = git')
- repository_conf.append('auto-sync = no')
- repository_conf_string = separator.join(repository_conf)
- yield self.build.addStepsAfterCurrentStep([
- steps.StringDownload(repository_conf_string + separator,
- workerdest='repos.conf/' +
repository_data['name'] + '.conf',
- workdir='/etc/portage/')
- ])
- return SUCCESS
-
class UpdateRepos(BuildStep):
name = 'UpdateRepos'
@@ -326,103 +226,6 @@ class UpdateRepos(BuildStep):
])
return SUCCESS
-class SetMakeConf(BuildStep):
-
- name = 'SetMakeConf'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = True
- flunkOnFailure = True
-
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
- @defer.inlineCallbacks
- def run(self):
- #FIXME: Make a dict before we pass it to the make.conf
- self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
- project_data = self.getProperty('project_data')
- makeconf_variables_data = yield
self.gentooci.db.portages.getVariables()
- separator1 = '\n'
- separator2 = ' '
- makeconf_list = []
- for k in makeconf_variables_data:
- makeconf_variables_values_data = yield
self.gentooci.db.projects.getProjectMakeConfById(project_data['uuid'], k['id'])
- makeconf_variable_list = []
- # we add some default values
- #FIXME:
- # we could set them in a config variables
- # FEATURES
- if k['variable'] == 'FEATURES':
- makeconf_variable_list.append('xattr')
- makeconf_variable_list.append('cgroup')
- makeconf_variable_list.append('-news')
- makeconf_variable_list.append('-collision-protect')
- # EMERGE_DEFAULT_OPTS
- if k['variable'] == 'EMERGE_DEFAULT_OPTS':
- makeconf_variable_list.append('--buildpkg=y')
- makeconf_variable_list.append('--rebuild-if-new-rev=y')
- makeconf_variable_list.append('--rebuilt-binaries=y')
- makeconf_variable_list.append('--usepkg=y')
- makeconf_variable_list.append('--binpkg-respect-use=y')
- makeconf_variable_list.append('--binpkg-changed-deps=y')
- makeconf_variable_list.append('--nospinner')
- makeconf_variable_list.append('--color=n')
- makeconf_variable_list.append('--ask=n')
- # CFLAGS
- if k['variable'] == 'CFLAGS' or k['variable'] == 'FCFLAGS':
- makeconf_variable_list.append('-O2')
- makeconf_variable_list.append('-pipe')
- makeconf_variable_list.append('-march=native')
- makeconf_variable_list.append('-fno-diagnostics-color')
- #FIXME:
- # Depend on worker we may have to add a diffrent march
- if k['variable'] == 'CXXFLAGS':
- makeconf_variable_list.append('${CFLAGS}')
- if k['variable'] == 'FFLAGS':
- makeconf_variable_list.append('${FCFLAGS}')
- if k['variable'] == 'ACCEPT_PROPERTIES':
- makeconf_variable_list.append('-interactive')
- if k['variable'] == 'ACCEPT_RESTRICT':
- makeconf_variable_list.append('-fetch')
- for v in makeconf_variables_values_data:
- if v['build_id'] == 0:
- makeconf_variable_list.append(v['value'])
- if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list !=
[]:
- makeconf_variable_list.append('ACCEPT_LICENSE="*"')
- if makeconf_variable_list != []:
- makeconf_variable_string = k['variable'] + '="' +
separator2.join(makeconf_variable_list) + '"'
- makeconf_list.append(makeconf_variable_string)
- # add hardcoded variables and values
- #FIXME:
- # we could set them in a config variables
- makeconf_list.append('LC_MESSAGES=C')
- makeconf_list.append('NOCOLOR="true"')
- makeconf_list.append('GCC_COLORS=""')
- makeconf_list.append('PORTAGE_TMPFS="/dev/shm"')
- makeconf_list.append('CLEAN_DELAY=0')
- makeconf_list.append('NOCOLOR=true')
- makeconf_list.append('PORT_LOGDIR="/var/cache/portage/logs"')
- makeconf_list.append('PKGDIR="/var/cache/portage/packages"')
- makeconf_list.append('PORTAGE_ELOG_CLASSES="qa"')
- makeconf_list.append('PORTAGE_ELOG_SYSTEM="save"')
- # add ACCEPT_KEYWORDS from the project_data info
- keyword_data = yield
self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
- if project_data['status'] == 'unstable':
- makeconf_keyword = '~' + keyword_data['name']
- else:
- makeconf_keyword = keyword_data['name']
- makeconf_list.append('ACCEPT_KEYWORDS="' + makeconf_keyword + '"')
- makeconf_string = separator1.join(makeconf_list)
- print(makeconf_string)
- yield self.build.addStepsAfterCurrentStep([
- steps.StringDownload(makeconf_string + separator1,
- workerdest="make.conf",
- workdir='/etc/portage/')
- ])
- return SUCCESS
-
class RunEmerge(BuildStep):
name = 'RunEmerge'
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 52daf83..30361e6 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -11,6 +11,203 @@ from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.plugins import steps
+class SetMakeProfile(BuildStep):
+
+ name = 'SetMakeProfile'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ portage_repos_path = self.getProperty('portage_repos_path')
+ project_data = self.getProperty('project_data')
+ profile_repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
+ makeprofiles_paths = []
+ #NOTE: pkgcheck don't support make.profile as a dir
+ # we only support one line in db
+ makeprofiles_data = yield
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'],
'make.profile')
+ for makeprofile in makeprofiles_data:
+ makeprofile_path = yield os.path.join(portage_repos_path,
profile_repository_data['name'], 'profiles', makeprofile['value'], '')
+ # makeprofiles_paths.append('../../..' + makeprofile_path)
+ #separator = '\n'
+ #makeprofile_path_string = separator.join(makeprofiles_paths)
+ # yield self.build.addStepsAfterCurrentStep([
+ # steps.StringDownload(makeprofile_path_string + separator,
+ # workerdest="make.profile/parent",
+ # workdir='/etc/portage/')
+ # ])
+ #NOTE: pkgcheck profile link
+ shell_commad_list = [
+ 'ln',
+ '-s'
+ ]
+ shell_commad_list.append(makeprofile_path)
+ shell_commad_list.append('/etc/portage/make.profile')
+ yield self.build.addStepsAfterCurrentStep([
+ steps.ShellCommandNewStyle(
+ command=shell_commad_list,
+ workdir='/'
+ )
+ ])
+ return SUCCESS
+
+class SetReposConf(BuildStep):
+
+ name = 'SetReposConf'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ portage_repos_path = self.getProperty('portage_repos_path')
+ project_data = self.getProperty('project_data')
+ # setup the default.conf
+ repos_conf_data = yield
self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'],
'repos.conf')
+ if repos_conf_data is None:
+ print('Default repo is not set in repos.conf')
+ return FAILURE
+ # check if repos_conf_data['value'] is vaild repo name
+ separator = '\n'
+ default_conf = []
+ default_conf.append('[DEFAULT]')
+ default_conf.append('main-repo = ' + repos_conf_data['value'])
+ default_conf.append('auto-sync = no')
+ default_conf_string = separator.join(default_conf)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(default_conf_string + separator,
+ workerdest="repos.conf/default.conf",
+ workdir='/etc/portage/')
+ ])
+ # add all repos that project have in projects_repositorys to
repos.conf/reponame.conf
+ projects_repositorys_data = yield
self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
+ for project_repository_data in projects_repositorys_data:
+ repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
+ repository_path = yield os.path.join(portage_repos_path,
repository_data['name'])
+ repository_conf = []
+ repository_conf.append('[' + repository_data['name'] + ']')
+ repository_conf.append('location = ' + repository_path)
+ repository_conf.append('sync-uri = ' +
repository_data['mirror_url'])
+ repository_conf.append('sync-type = git')
+ repository_conf.append('auto-sync = no')
+ repository_conf_string = separator.join(repository_conf)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(repository_conf_string + separator,
+ workerdest='repos.conf/' +
repository_data['name'] + '.conf',
+ workdir='/etc/portage/')
+ ])
+ return SUCCESS
+
+class SetMakeConf(BuildStep):
+
+ name = 'SetMakeConf'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ #FIXME: Make a dict before we pass it to the make.conf
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ makeconf_variables_data = yield
self.gentooci.db.portages.getVariables()
+ separator1 = '\n'
+ separator2 = ' '
+ makeconf_list = []
+ for k in makeconf_variables_data:
+ makeconf_variables_values_data = yield
self.gentooci.db.projects.getProjectMakeConfById(project_data['uuid'], k['id'])
+ makeconf_variable_list = []
+ # we add some default values
+ #FIXME:
+ # we could set them in a config variables
+ # FEATURES
+ if k['variable'] == 'FEATURES':
+ makeconf_variable_list.append('xattr')
+ makeconf_variable_list.append('cgroup')
+ makeconf_variable_list.append('-news')
+ makeconf_variable_list.append('-collision-protect')
+ # EMERGE_DEFAULT_OPTS
+ if k['variable'] == 'EMERGE_DEFAULT_OPTS':
+ makeconf_variable_list.append('--buildpkg=y')
+ makeconf_variable_list.append('--rebuild-if-new-rev=y')
+ makeconf_variable_list.append('--rebuilt-binaries=y')
+ makeconf_variable_list.append('--usepkg=y')
+ makeconf_variable_list.append('--binpkg-respect-use=y')
+ makeconf_variable_list.append('--binpkg-changed-deps=y')
+ makeconf_variable_list.append('--nospinner')
+ makeconf_variable_list.append('--color=n')
+ makeconf_variable_list.append('--ask=n')
+ # CFLAGS
+ if k['variable'] == 'CFLAGS' or k['variable'] == 'FCFLAGS':
+ makeconf_variable_list.append('-O2')
+ makeconf_variable_list.append('-pipe')
+ makeconf_variable_list.append('-march=native')
+ makeconf_variable_list.append('-fno-diagnostics-color')
+ #FIXME:
+ # Depend on worker we may have to add a diffrent march
+ if k['variable'] == 'CXXFLAGS':
+ makeconf_variable_list.append('${CFLAGS}')
+ if k['variable'] == 'FFLAGS':
+ makeconf_variable_list.append('${FCFLAGS}')
+ if k['variable'] == 'ACCEPT_PROPERTIES':
+ makeconf_variable_list.append('-interactive')
+ if k['variable'] == 'ACCEPT_RESTRICT':
+ makeconf_variable_list.append('-fetch')
+ for v in makeconf_variables_values_data:
+ if v['build_id'] == 0:
+ makeconf_variable_list.append(v['value'])
+ if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list !=
[]:
+ makeconf_variable_list.append('ACCEPT_LICENSE="*"')
+ if makeconf_variable_list != []:
+ makeconf_variable_string = k['variable'] + '="' +
separator2.join(makeconf_variable_list) + '"'
+ makeconf_list.append(makeconf_variable_string)
+ # add hardcoded variables and values
+ #FIXME:
+ # we could set them in a config variables
+ makeconf_list.append('LC_MESSAGES=C')
+ makeconf_list.append('NOCOLOR="true"')
+ makeconf_list.append('GCC_COLORS=""')
+ makeconf_list.append('PORTAGE_TMPFS="/dev/shm"')
+ makeconf_list.append('CLEAN_DELAY=0')
+ makeconf_list.append('NOCOLOR=true')
+ makeconf_list.append('PORT_LOGDIR="/var/cache/portage/logs"')
+ makeconf_list.append('PKGDIR="/var/cache/portage/packages"')
+ makeconf_list.append('PORTAGE_ELOG_CLASSES="qa"')
+ makeconf_list.append('PORTAGE_ELOG_SYSTEM="save"')
+ # add ACCEPT_KEYWORDS from the project_data info
+ keyword_data = yield
self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
+ if project_data['status'] == 'unstable':
+ makeconf_keyword = '~' + keyword_data['name']
+ else:
+ makeconf_keyword = keyword_data['name']
+ makeconf_list.append('ACCEPT_KEYWORDS="' + makeconf_keyword + '"')
+ makeconf_string = separator1.join(makeconf_list)
+ print(makeconf_string)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(makeconf_string + separator1,
+ workerdest="make.conf",
+ workdir='/etc/portage/')
+ ])
+ return SUCCESS
+
class SetPackageDefault(BuildStep):
name = 'SetPackageDefault'
