commit: 63ea1fb9235a4585754d5b8acc41c281596ab4fc Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Tue Jul 26 12:57:35 2022 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Tue Jul 26 12:57:35 2022 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=63ea1fb9
Clean up and add use of rootworkdir Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org> buildbot_gentoo_ci/steps/portage.py | 204 ++++-------------------------------- 1 file changed, 21 insertions(+), 183 deletions(-) diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py index 431fe9a..ff06c58 100644 --- a/buildbot_gentoo_ci/steps/portage.py +++ b/buildbot_gentoo_ci/steps/portage.py @@ -20,6 +20,7 @@ from buildbot.process.buildstep import BuildStep from buildbot.process.results import SUCCESS from buildbot.process.results import FAILURE from buildbot.process.results import SKIPPED +from buildbot.process import remotecommand from buildbot.plugins import steps from buildbot_gentoo_ci.steps import master as master_steps @@ -34,39 +35,6 @@ def WriteTextToFile(path, text_list): yield f.write(separator) yield f.close -def PersOutputOfEbuildSH(rc, stdout, stderr): - metadata = None - NoSplit = [] - NoSplit.append('DESCRIPTION') - #make dict of the stout - index = 1 - metadata_line_dict = {} - for text_line in stdout.splitlines(): - metadata_line_dict[index] = text_line - index = index + 1 - # should have 22 lines - if len(auxdbkeys) != index -1: - # number of lines is incorrect. - return { - 'auxdb' : metadata - } - # split all keys to list instead of speces - metadata = {} - i = 1 - for key in auxdbkeys: - if metadata_line_dict[i] == '': - metadata[key] = None - else: - if ' ' in metadata_line_dict[i] and key not in NoSplit: - metadata[key] = metadata_line_dict[i].split(' ') - else: - metadata[key] = [] - metadata[key].append(metadata_line_dict[i]) - i = i + 1 - return { - 'auxdb' : metadata - } - class SetMakeProfile(BuildStep): name = 'SetMakeProfile' @@ -76,8 +44,7 @@ class SetMakeProfile(BuildStep): haltOnFailure = True flunkOnFailure = True - def __init__(self, workdir=False, **kwargs): - self.rootworkdir = workdir + def __init__(self, **kwargs): super().__init__(**kwargs) @defer.inlineCallbacks @@ -105,8 +72,8 @@ class SetMakeProfile(BuildStep): 'ln', '-s' ] - if self.rootworkdir: - symlink_makeprofile_path = yield os.path.join(self.rootworkdir, 'etc/portage/make.profile') + if self.getProperty('rootworkdir'): + symlink_makeprofile_path = yield os.path.join(self.getProperty('rootworkdir'), 'etc/portage/make.profile') else: symlink_makeprofile_path = '/etc/portage/make.profile' shell_commad_list.append(makeprofile_path) @@ -130,8 +97,7 @@ class SetReposConf(BuildStep): haltOnFailure = True flunkOnFailure = True - def __init__(self, workdir=False, **kwargs): - self.rootworkdir = workdir + def __init__(self, **kwargs): super().__init__(**kwargs) @defer.inlineCallbacks @@ -139,8 +105,8 @@ class SetReposConf(BuildStep): self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] portage_repos_path = self.getProperty('portage_repos_path') project_data = self.getProperty('project_data') - if self.rootworkdir: - portage_etc_path = yield os.path.join(self.rootworkdir, 'etc/portage/') + if self.getProperty('rootworkdir'): + portage_etc_path = yield os.path.join(self.getProperty('rootworkdir'), 'etc/portage/') else: portage_etc_path = '/etc/portage/' # setup the default.conf @@ -148,6 +114,13 @@ class SetReposConf(BuildStep): if repos_conf_data is None: print('Default repo is not set in repos.conf') return FAILURE + self.aftersteps_list = [] + # check if we have the dir + cmd = remotecommand.RemoteCommand('stat', {'file': portage_etc_path + 'repos.conf'}) + yield self.runCommand(cmd) + if cmd.didFail(): + self.aftersteps_list.append(steps.MakeDirectory(dir="repos.conf", + workdir=portage_etc_path)) log = yield self.addLog('repos.conf') # check if repos_conf_data['value'] is vaild repo name separator = '\n' @@ -156,11 +129,11 @@ class SetReposConf(BuildStep): default_conf.append('main-repo = ' + repos_conf_data['value']) default_conf.append('auto-sync = no') default_conf_string = separator.join(default_conf) - yield self.build.addStepsAfterCurrentStep([ + self.aftersteps_list.append( steps.StringDownload(default_conf_string + separator, workerdest="repos.conf/default.conf", workdir=portage_etc_path) - ]) + ) # display the default.conf yield log.addStdout('File: ' + 'default.conf' + '\n') for line in default_conf: @@ -178,14 +151,15 @@ class SetReposConf(BuildStep): repository_conf.append('auto-sync = no') repository_conf_string = separator.join(repository_conf) filename = repository_data['name'] + '.conf' - yield self.build.addStepsAfterCurrentStep([ + self.aftersteps_list.append( steps.StringDownload(repository_conf_string + separator, workerdest='repos.conf/' + filename, workdir=portage_etc_path) - ]) + ) yield log.addStdout('File: ' + filename + '\n') for line in repository_conf: yield log.addStdout(line + '\n') + yield self.build.addStepsAfterCurrentStep(self.aftersteps_list) return SUCCESS class SetMakeConf(BuildStep): @@ -207,8 +181,8 @@ class SetMakeConf(BuildStep): self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] project_data = self.getProperty('project_data') makeconf_variables_data = yield self.gentooci.db.portages.getVariables() - if self.rootworkdir: - portage_etc_path = yield os.path.join(self.rootworkdir, 'etc/portage/') + if self.getProperty('rootworkdir'): + portage_etc_path = yield os.path.join(self.getProperty('rootworkdir'), 'etc/portage/') else: portage_etc_path = '/etc/portage/' separator1 = '\n' @@ -541,139 +515,3 @@ class SetMakeConfLocal(BuildStep): makeconf_list.append('FEATURES="sandbox"') yield WriteTextToFile(make_conf_path, makeconf_list) return SUCCESS - -class SetEnvForEbuildSH(BuildStep): - - name = 'SetEnvForEbuildSH' - description = 'Running' - descriptionDone = 'Ran' - descriptionSuffix = None - haltOnFailure = True - flunkOnFailure = True - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def getEapiFromFile(self): - with io.open(_unicode_encode(self.getProperty("ebuild_file"), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') as f: - _eapi, _eapi_lineno = _parse_eapi_ebuild_head(f) - - return _eapi - - @defer.inlineCallbacks - def run(self): - addStepEbuildSH = [] - ebuild_commands = [] - ebuild_env = {} - config_root = yield os.path.join(self.getProperty("builddir"), '') - mysettings = yield portage_config(config_root = config_root) - - #Get EAPI from file and add it to env - eapi = yield self.getEapiFromFile() - print(eapi) - if eapi is None or not eapi_is_supported(eapi): - print('invalid eapi') - eapi = '0' - print(eapi_is_supported(eapi)) - ebuild_env['EAPI'] = eapi - - #FIXME: check manifest on ebuild_file - - #Setup ENV - category = yield catpkgsplit(self.getProperty("cpv"))[0] - package = yield catpkgsplit(self.getProperty("cpv"))[1] - version = yield catpkgsplit(self.getProperty("cpv"))[2] - revision = yield catpkgsplit(self.getProperty("cpv"))[3] - portage_bin_path = mysettings["PORTAGE_BIN_PATH"] - ebuild_sh_path = yield os.path.join(portage_bin_path, 'ebuild.sh') - #ebuild_env['PORTAGE_DEBUG'] = '1' - ebuild_env['EBUILD_PHASE'] = 'depend' - ebuild_env['CATEGORY'] = category - ebuild_env['P'] = package + '-' + version - ebuild_env['PN'] = package - ebuild_env['PR'] = revision - ebuild_env['PV'] = version - if revision == 'r0': - ebuild_env['PF'] = ebuild_env['P'] - ebuild_env['PVR'] = version - else: - ebuild_env['PF'] = ebuild_env['P'] + '-' + revision - ebuild_env['PVR'] = version + '-' + revision - ebuild_env['PORTAGE_BIN_PATH'] = portage_bin_path - ebuild_env['EBUILD'] = self.getProperty("ebuild_file") - ebuild_env['PORTAGE_PIPE_FD'] = '1' - ebuild_env['WORKDIR'] = yield os.path.join(mysettings["PORTAGE_TMPDIR"], 'portage', category, ebuild_env['PF'], 'work') - ebuild_env['PORTAGE_ECLASS_LOCATIONS'] = self.getProperty("repository_path") - - #FIXME: use sandbox if in FEATURES - ebuild_commands.append(ebuild_sh_path) - ebuild_commands.append('depend') - - addStepEbuildSH.append(master_steps.MasterSetPropertyFromCommand( - name = 'RunEbuildSH', - haltOnFailure = True, - flunkOnFailure = True, - command=ebuild_commands, - env=ebuild_env, - workdir=self.getProperty("builddir"), - strip=False, - extract_fn=PersOutputOfEbuildSH - )) - yield self.build.addStepsAfterCurrentStep(addStepEbuildSH) - return SUCCESS - -class GetAuxMetadata(BuildStep): - - name = 'GetAuxMetadata' - description = 'Running' - descriptionDone = 'Ran' - descriptionSuffix = None - haltOnFailure = True - flunkOnFailure = True - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - #@defer.inlineCallbacks - def thd_getAuxDbKeys(self): - auxdbs = self.myportdb.aux_get(self.getProperty("cpv"), auxdbkeys, myrepo=self.getProperty("repository_data")['name']) - return auxdbs - - @defer.inlineCallbacks - def run(self): - config_root = yield os.path.join(self.getProperty("builddir"), '') - # setup mysettings and myportdb - mysettings = yield portage_config(config_root = config_root) - self.myportdb = yield portdbapi(mysettings=mysettings) - auxdbs = yield self.thd_getAuxDbKeys() - metadata = None - NoSplit = [] - NoSplit.append('DESCRIPTION') - # should have 22 lines - if len(auxdbkeys) != len(auxdbs) or not isinstance(auxdbs, list): - # number of lines is incorrect or not a list. - print("Lines don't match or not a list") - yield self.myportdb.close_caches() - yield portdbapi.portdbapi_instances.remove(self.myportdb) - #self.setProperty('auxdb', metadata, 'auxdb') - return FAILURE - # split all keys to list instead of speces - metadata = {} - i = 0 - for key in auxdbkeys: - if auxdbs[i] == '': - metadata[key] = None - else: - if ' ' in auxdbs[i] and key not in NoSplit: - metadata[key] = auxdbs[i].split(' ') - else: - metadata[key] = [] - metadata[key].append(auxdbs[i]) - i = i + 1 - self.setProperty('auxdb', metadata, 'auxdb') - yield self.myportdb.close_caches() - yield portdbapi.portdbapi_instances.remove(self.myportdb) - return SUCCESS