attached is a patch that attempts to generalize checking out the
components used to build an SRPM. this patch supports CVS, GIT, and
SVN, but only CVS and SVN have been tested. the idea is to provide the
infrastructure for different SCM systems to be configured at run-time so
that users can choose their favorite system.
is there a better approach? did i miss something obvious? general
comments?
thanks.
rob.
Initial attempt to generalize checking out SRPM components, which currently
includes support for CVS, GIT, and SVN.
---
builder/kojid | 283 +++++++++++++++++++++++++++++++------
builder/kojid.conf | 7 +
builder/kojid.init | 2 +-
cli/koji | 20 ++--
koji/__init__.py | 2 +-
www/kojiweb/taskinfo.chtml | 10 +-
www/kojiweb/tasks.chtml | 2 +-
7 files changed, 267 insertions(+), 59 deletions(-)
diff --git a/builder/kojid b/builder/kojid
index 16f18dd..0e41e9d 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -1423,7 +1423,7 @@ class ChainBuildTask(BaseTaskHandler):
subtasks = []
build_tasks = []
for src in build_level:
- if src.startswith('cvs://'):
+ if scm.is_scm_url(src):
task_id = session.host.subtask(method='build',
arglist=[src, target, opts],
parent=self.id)
@@ -1464,7 +1464,7 @@ class BuildTask(BaseTaskHandler):
raise koji.BuildError, "arch_override is only allowed for scratch builds"
task_info = session.getTaskInfo(self.id)
# only allow admins to perform non-scratch builds from srpm
- if not src.startswith('cvs://') and not opts.get('scratch') \
+ if not scm.is_scm_url(src) and not opts.get('scratch') \
and not 'admin' in session.getUserPerms(task_info['owner']):
raise koji.BuildError, "only admins may peform non-scratch builds from srpm"
target_info = session.getBuildTarget(target)
@@ -1521,8 +1521,8 @@ class BuildTask(BaseTaskHandler):
def getSRPM(self, src):
"""Get srpm from src"""
if isinstance(src,str):
- if src.startswith('cvs://'):
- return self.getSRPMFromCVS(src)
+ if scm.is_scm_url(src):
+ return self.getSRPMFromSCM(src)
else:
#assume this is a path under uploads
return src
@@ -1530,9 +1530,9 @@ class BuildTask(BaseTaskHandler):
raise koji.BuildError, 'Invalid source specification: %s' % src
#XXX - other methods?
- def getSRPMFromCVS(self, url):
+ def getSRPMFromSCM(self, url):
#TODO - allow different ways to get the srpm
- task_id = session.host.subtask(method='buildSRPMFromCVS',
+ task_id = session.host.subtask(method='buildSRPMFromSCM',
arglist=[url],
label='srpm',
parent=self.id)
@@ -1795,9 +1795,9 @@ class TagBuildTask(BaseTaskHandler):
session.host.tagNotification(False, tag_id, fromtag, build_id, user_id, ignore_success, "%s: %s" % (exctype, value))
raise e
-class BuildSRPMFromCVSTask(BaseTaskHandler):
+class BuildSRPMFromSCMTask(BaseTaskHandler):
- Methods = ['buildSRPMFromCVS']
+ Methods = ['buildSRPMFromSCM']
_taskWeight = 0.75
def spec_sanity_checks(self, filename):
@@ -1810,42 +1810,21 @@ class BuildSRPMFromCVSTask(BaseTaskHandler):
raise koji.BuildError, "%s is not allowed to be defined in spec file" % tag
def handler(self,url):
- if not url.startswith('cvs://'):
- raise koji.BuildError("invalid cvs URL: %s" % url)
-
- # Hack it because it refuses to parse it properly otherwise
- scheme, netloc, path, params, query, fragment = urlparse.urlparse('http'+url[3:])
- if not (netloc and path and fragment and query):
- raise koji.BuildError("invalid cvs URL: %s" % url)
-
- # Steps:
- # 1. CVS checkout into tempdir
- # 2. create sources hardlinks
- # 3. Run 'make srpm'
-
- cvsdir = self.workdir + '/cvs'
- self.logger.debug(cvsdir)
- koji.ensuredir(cvsdir)
+ if not scm.is_scm_url(url):
+ raise koji.BuildError("invalid %s URL: %s" % (scm.get_scm_type(), url))
+
+ # Setup files and directories for SRPM creation
+ scmdir = self.workdir + '/scmroot'
+ self.logger.debug(scmdir)
+ koji.ensuredir(scmdir)
logfile = self.workdir + "/srpm.log"
uploadpath = self.getUploadDir()
- sourcedir = '%s/%s' % (cvsdir, query)
+ sourcedir = '%s/%s' % (scmdir, scm.get_pkg_name(url))
- #perform checkouts
- cmd = ['cvs', '-d', ':pserver:[EMAIL PROTECTED]:%s' % (netloc, path),
- 'checkout', '-r', fragment, query]
- if log_output(cmd[0], cmd, logfile, uploadpath, cwd=cvsdir, logerror=1):
- output = "(none)"
- try:
- output = open(logfile).read()
- except IOError:
- pass
- raise koji.BuildError, "Error with checkout ':pserver:[EMAIL PROTECTED]:%s': %s" % (netloc, path, output)
- cmd = ['cvs', '-d', ':pserver:[EMAIL PROTECTED]:%s' % (netloc, path),
- 'checkout', 'common']
- if log_output(cmd[0], cmd, logfile, uploadpath, cwd=cvsdir, logerror=1, append=1):
- raise koji.BuildError, "Error with checkout :pserver:[EMAIL PROTECTED]:%s" % (netloc, path)
- os.symlink('%s/common' % cvsdir, '%s/../common' % sourcedir)
+ # Check out spec file etc., from SCM
+ scm.do_checkout(url, scmdir, sourcedir, uploadpath, logfile)
+ # Find and verify that there is only one spec file.
spec_files = glob.glob("%s/*.spec" % sourcedir)
if len(spec_files) == 0:
raise koji.BuildError("No spec file found")
@@ -1858,7 +1837,7 @@ class BuildSRPMFromCVSTask(BaseTaskHandler):
#build srpm
cmd = ['make', '-C', '%s' % sourcedir, 'srpm']
- if log_output(cmd[0], cmd, logfile, uploadpath, cwd=cvsdir, logerror=1, append=1):
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=scmdir, logerror=1, append=1):
raise koji.BuildError, "Error building SRPM"
srpms = glob.glob('%s/*.src.rpm' % sourcedir)
@@ -2330,6 +2309,221 @@ class WaitrepoTask(BaseTaskHandler):
time.sleep(self.PAUSE)
return "Successfully waited %s seconds for a %s repo (%s)" % (int(time.time() - start), build_target['build_tag_name'], repo['id'])
+class SCM(object):
+ "SCM abstraction class"
+
+ scmdict = { "CVS":('cvs://'),
+ "GIT":('git://', 'http://', 'https://', 'rsync://'),
+ "GIT+SSH":('git+ssh://'),
+ "SVN":('svn://', 'http://', 'https://'),
+ "SVN+SSH":('svn+ssh://') }
+
+ def __init__(self,scmtype=None,scmusername=None):
+ global options
+ if scmtype is None:
+ self.set_scm_type(options.scmtype)
+ else:
+ self.set_scm_type(scmtype)
+ if scmusername is None:
+ self.set_scm_user(options.scmusername)
+ else:
+ self.set_scm_user(scmusername)
+
+ def get_scm_type(self):
+ "return the SCM type."
+ return self.scmtype
+
+ def set_scm_type(self, scmtype):
+ "set the SCM type."
+ if scmtype in self.scmdict.keys():
+ self.scmtype = scmtype
+ else:
+ raise GenericError, 'Unknown SCM type %s. Known types include: %s.' % (scmtype, self.scmdict.keys())
+
+ def get_scm_user(self):
+ "return the SCM user's name."
+ return self.scmusername
+
+ def set_scm_user(self, newscmuser):
+ "set the SCM user's name."
+ global options
+ if newscmuser is None:
+ self.scmusername = options.scmusername
+ else:
+ self.scmusername = newscmuser
+
+ def is_scm_url(self,url):
+ """
+ return True if this string is a valid scm location for the configured scmtype
+ and return False otherwise.
+ """
+
+ # acceptable schemes from this scmtype
+ scmtup = self.scmdict[self.scmtype]
+
+ # is the scheme for the given url acceptable for this scmtype?
+ if self.get_scheme(url) in scmtup:
+ return True
+ else:
+ return False
+
+ def get_scheme(self,url):
+ "parse the scheme from the SCM url"
+
+ # parse scheme from url
+ scheme = url.split('://')[0]
+ scheme += str('://')
+
+ return scheme
+
+ def parse_url(self,url):
+ "parse the SCM url into usable components"
+
+ # ensure the url is valid for the given scmtype
+ if not self.is_scm_url(url):
+ raise GenericError("invalid %s URL: %s" % (self.scmtype, url))
+
+ # get the url's scheme
+ scheme = self.get_scheme(url)
+
+ # replace the scheme with http:// so that the urlparse works in all cases
+ hackedurl = url.replace(scheme, 'http://', 1)
+ dummyscheme, netloc, path, params, query, fragment = urlparse.urlparse(hackedurl)
+
+ # check for failure
+ if not (scheme and netloc and path and fragment and query):
+ raise GenericError("Unable to parse %s URL: %s" % (self.scmtype, url))
+
+ # return parsed values
+ return [scheme, netloc, path, params, query, fragment]
+
+ def get_pkg_name(self,url):
+ "return the package name parsed from the SCM url"
+ return self.parse_url(url)[4]
+
+ def do_checkout(self, url, scmdir, sourcedir, uploadpath, logfile):
+ "checkout common and package directories from the given SCM URL."
+
+ # TODO: sanity check arguments
+
+ if self.scmtype == "CVS":
+
+ # parse the url
+ [scheme, netloc, path, params, query, fragment] = self.parse_url(url)
+
+ # perform checkouts
+ cmd = ['cvs', '-d', ':pserver:[EMAIL PROTECTED]:%s' % (netloc, path),
+ 'checkout', '-r', fragment, query]
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=scmdir, logerror=1):
+ output = "(none)"
+ try:
+ output = open(logfile).read()
+ except IOError:
+ pass
+ raise BuildError, "Error with checkout ':pserver:[EMAIL PROTECTED]:%s': %s" % (netloc, path, output)
+ cmd = ['cvs', '-d', ':pserver:[EMAIL PROTECTED]:%s' % (netloc, path),
+ 'checkout', 'common']
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=scmdir, logerror=1, append=1):
+ raise BuildError, "Error with checkout :pserver:[EMAIL PROTECTED]:%s" % (netloc, path)
+ os.symlink('%s/common' % scmdir, '%s/../common' % sourcedir)
+
+ elif self.scmtype == "GIT":
+
+ # parse the url
+ [scheme, netloc, path, params, query, fragment] = self.parse_url(url)
+
+ # perform checkouts
+ cmd = ['git', 'clone', 'git://%s%s/%s' % (netloc, path, query), query]
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=scmdir, logerror=1, append=1):
+ output = "(none)"
+ try:
+ output = open(logfile).read()
+ except IOError:
+ pass
+ raise BuildError, "Error with clone 'git://%s%s/%s: %s" % (netloc, path, query, output)
+
+ cmd = ['git', 'clone', 'git://%s%s/common' % (netloc, path)]
+ self.logger.debug("executing: %s" % cmd)
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=scmdir, logerror=1, append=1):
+ output = "(none)"
+ try:
+ output = open(logfile).read()
+ except IOError:
+ pass
+ raise BuildError, "Error with clone 'git://%s%s/common: %s" % (netloc, path, output)
+
+ elif self.scmtype == "GIT+SSH":
+
+ # parse the url
+ [scheme, netloc, path, params, query, fragment] = self.parse_url(url)
+
+ # Remove username@ from netloc if it is specified
+ netloc = re.sub('^.*@', '',netloc)
+
+ # perform checkouts
+ cmd = ['git', 'clone', '[EMAIL PROTECTED]/%s' % (scheme, self.get_scm_user(), netloc, path, query), query]
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=scmdir, logerror=1, append=1):
+ output = "(none)"
+ try:
+ output = open(logfile).read()
+ except IOError:
+ pass
+ raise BuildError, "Error with clone '[EMAIL PROTECTED]/%s: %s" % (scheme, self.get_scm_user(), netloc, path, query, output)
+
+ cmd = ['git', 'clone', '[EMAIL PROTECTED]/common' % (scheme, self.get_scm_user(), netloc, path)]
+ self.logger.debug("executing: %s" % cmd)
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=scmdir, logerror=1, append=1):
+ output = "(none)"
+ try:
+ output = open(logfile).read()
+ except IOError:
+ pass
+ raise BuildError, "Error with clone '[EMAIL PROTECTED]/common: %s" % (scheme, self.get_scm_user(), netloc, path, output)
+
+ elif self.scmtype == "SVN":
+
+ # parse the url
+ [scheme, netloc, path, params, query, fragment] = self.parse_url(url)
+
+ # perform checkouts
+ cmd = ['svn', 'checkout', '-r', fragment, '%s%s/%s/%s' % (scheme, netloc, path, query)]
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=scmdir, logerror=1, append=1):
+ output = "(none)"
+ try:
+ output = open(logfile).read()
+ except IOError:
+ pass
+ raise BuildError, "Error with checkout %s%s/%s/%s revsion %s: %s" % (scheme, netloc, path, query, fragment, output)
+
+ cmd = ['svn', 'checkout', '%s%s/%s/common' % (scheme, netloc, path)]
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=scmdir, logerror=1, append=1):
+ raise BuildError, "Error with checkout %s://%s/%s/common: %s" % (scheme, netloc, path, output)
+
+ elif self.scmtype == "SVN+SSH":
+
+ # parse the url
+ [scheme, netloc, path, params, query, fragment] = self.parse_url(url)
+
+ # Remove username@ from netloc if it is specified
+ netloc = re.sub('^.*@', '',netloc)
+
+ # perform checkouts
+ cmd = ['svn', 'checkout', '-r', fragment, '[EMAIL PROTECTED]/%s/%s' % (scheme, self.get_scm_user(), netloc, path, query)]
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=scmdir, logerror=1, append=1):
+ output = "(none)"
+ try:
+ output = open(logfile).read()
+ except IOError:
+ pass
+ raise BuildError, "Error with checkout [EMAIL PROTECTED]/%s/%s revsion %s: %s" % (scheme, self.get_scm_user(), netloc, path, query, fragment, output)
+
+ cmd = ['svn', 'checkout', '[EMAIL PROTECTED]/%s/common' % (scheme, self.get_scm_user(), netloc, path)]
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=scmdir, logerror=1, append=1):
+ raise BuildError, "Error with checkout [EMAIL PROTECTED]/%s/common: %s" % (scheme, self.get_scm_user(), netloc, path, output)
+
+ else:
+ raise koji.GenericError, "Unknown SCM type %s" % (scmtype)
+
def get_options():
"""process options from command line and config file"""
global options
@@ -2402,7 +2596,9 @@ def get_options():
'password': None,
'cert': '/etc/kojid/client.crt',
'ca': '/etc/kojid/clientca.crt',
- 'serverca': '/etc/kojid/serverca.crt'}
+ 'serverca': '/etc/kojid/serverca.crt',
+ 'scmtype': 'CVS',
+ 'scmusername': 'anscmuser'}
if config.has_section('kojid'):
for name, value in config.items('kojid'):
if name in ['sleeptime', 'maxjobs', 'minspace']:
@@ -2449,6 +2645,9 @@ if __name__ == "__main__":
if options.admin_emails:
koji.add_mail_logger("koji", options.admin_emails)
+ # get an SCM
+ scm = SCM()
+
#build session options
session_opts = {}
for k in ('user','password','debug_xmlrpc', 'debug'):
diff --git a/builder/kojid.conf b/builder/kojid.conf
index c6d5ee4..bb0b0b3 100644
--- a/builder/kojid.conf
+++ b/builder/kojid.conf
@@ -48,3 +48,10 @@ from_addr=Koji Build System <[EMAIL PROTECTED]>
;certificate of the CA that issued the HTTP server certificate
;serverca = /etc/kojid/serverca.crt
+
+; scm types include: CVS, GIT, GIT+SSH, SVN, SVN+SSH
+scmtype = CVS
+
+; username for svn+ssh:// or git+ssh://
+scmusername = kojiscmusername
+
diff --git a/builder/kojid.init b/builder/kojid.init
index 0de1203..5487271 100755
--- a/builder/kojid.init
+++ b/builder/kojid.init
@@ -36,7 +36,7 @@ start() {
[ "$KOJID_DEBUG" == "Y" ] && ARGS="$ARGS --debug"
[ "$KOJID_VERBOSE" == "Y" ] && ARGS="$ARGS --verbose"
# XXX Fix for make download-checks in kernel builds
- # Remove once we're running the buildSRPMFromCVS task
+ # Remove once we're running the buildSRPMFromSCM task
# as an unpriviledged user with their own environment
export HOME="/root"
daemon /usr/sbin/kojid $ARGS
diff --git a/cli/koji b/cli/koji
index c2e3b4b..8bda8d2 100755
--- a/cli/koji
+++ b/cli/koji
@@ -645,7 +645,7 @@ def handle_build(options, session, args):
help=_("Run the build at a lower priority"))
(build_opts, args) = parser.parse_args(args)
if len(args) != 2:
- parser.error(_("Exactly two arguments (a build target and a CVS URL or srpm file) are required"))
+ parser.error(_("Exactly two arguments (a build target and a SCM URL or srpm file) are required"))
assert False
if build_opts.arch_override and not build_opts.scratch:
parser.error(_("--arch_override is only allowed for --scratch builds"))
@@ -669,7 +669,8 @@ def handle_build(options, session, args):
if build_opts.background:
#relative to koji.PRIO_DEFAULT
priority = 5
- if not source.startswith('cvs://'):
+ # try to check that source is an SRPM
+ if '/' not in source and source.endswith('.src.rpm') and len(source.split('-')) >= 3:
# only allow admins to perform non-scratch builds from srpm
if not opts['scratch'] and not session.hasPerm('admin'):
parser.error(_("builds from srpm must use --scratch"))
@@ -707,7 +708,7 @@ def handle_chain_build(options, session, args):
help=_("Run the build at a lower priority"))
(build_opts, args) = parser.parse_args(args)
if len(args) < 2:
- parser.error(_("At least two arguments (a build target and a CVS URL) are required"))
+ parser.error(_("At least two arguments (a build target and a SCM URL) are required"))
assert False
activate_session(session)
target = args[0]
@@ -740,13 +741,14 @@ def handle_chain_build(options, session, args):
if build_level:
src_list.append(build_level)
build_level = []
- elif src.startswith('cvs://'):
+ elif '://' in src:
+ # quick check that src might be a url
build_level.append(src)
elif '/' not in src and len(src.split('-')) >= 3:
# quick check that it looks like a N-V-R
build_level.append(src)
else:
- print _('"%s" is not a CVS URL or package N-V-R')
+ print _('"%s" is not a SCM URL or package N-V-R')
return 1
if build_level:
src_list.append(build_level)
@@ -2428,11 +2430,11 @@ def _parseTaskParams(session, method, task_id):
lines = []
- if method == 'buildFromCVS':
- lines.append("CVS URL: %s" % params[0])
+ if method == 'buildFromSCM':
+ lines.append("SCM URL: %s" % params[0])
lines.append("Build Target: %s" % params[1])
- elif method == 'buildSRPMFromCVS':
- lines.append("CVS URL: %s" % params[0])
+ elif method == 'buildSRPMFromSCM':
+ lines.append("SCM URL: %s" % params[0])
elif method == 'multiArchBuild':
lines.append("SRPM: %s/work/%s" % (options.topdir, params[0]))
lines.append("Build Target: %s" % params[1])
diff --git a/docs/HOWTO.html b/docs/HOWTO.html
diff --git a/hub/kojihub.py b/hub/kojihub.py
diff --git a/koji.spec b/koji.spec
diff --git a/koji/__init__.py b/koji/__init__.py
index d808126..89289df 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -1520,7 +1520,7 @@ def taskLabel(taskInfo):
else:
source = os.path.basename(source)
extra = '%s, %s' % (target, source)
- elif method == 'buildSRPMFromCVS':
+ elif method == 'buildSRPMFromSCM':
if taskInfo.has_key('request'):
url = taskInfo['request'][0]
url = url[url.rfind('/') + 1:]
diff --git a/www/kojiweb/includes/header.chtml b/www/kojiweb/includes/header.chtml
diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml
index 76df4bc..f99f7e6 100644
--- a/www/kojiweb/taskinfo.chtml
+++ b/www/kojiweb/taskinfo.chtml
@@ -62,11 +62,11 @@
<tr>
<th>Parameters</th>
<td>
- #if $task.method == 'buildFromCVS'
- <strong>CVS URL:</strong> $params[0]<br/>
+ #if $task.method == 'buildFromSCM'
+ <strong>SCM URL:</strong> $params[0]<br/>
<strong>Build Target:</strong> <a href="buildtargetinfo?name=$params[1]">$params[1]</a>
- #elif $task.method == 'buildSRPMFromCVS'
- <strong>CVS URL:</strong> $params[0]
+ #elif $task.method == 'buildSRPMFromSCM'
+ <strong>SCM URL:</strong> $params[0]
#elif $task.method == 'multiArchBuild'
<strong>SRPM:</strong> $params[0]<br/>
<strong>Build Target:</strong> <a href="buildtargetinfo?name=$params[1]">$params[1]</a><br/>
@@ -276,7 +276,7 @@ $cgi.escape($result.faultString.strip())
<a href="getfile?taskID=$task.id&name=$urllib.quote($filename)">$filename</a><br/>
#end for
#if $task.state not in ($koji.TASK_STATES.CLOSED, $koji.TASK_STATES.CANCELED, $koji.TASK_STATES.FAILED) and \
- $task.method in ('buildSRPMFromCVS', 'buildArch', 'createrepo')
+ $task.method in ('buildSRPMFromSCM', 'buildArch', 'createrepo')
<br/>
<a href="watchlogs?taskID=$task.id">Watch logs</a>
#end if
diff --git a/www/kojiweb/tasks.chtml b/www/kojiweb/tasks.chtml
index 70de06c..c34e504 100644
--- a/www/kojiweb/tasks.chtml
+++ b/www/kojiweb/tasks.chtml
@@ -103,7 +103,7 @@ All
<select name="method" class="filterlist" onchange="javascript: window.location = 'tasks?method=' + this.value + '$util.passthrough($self, 'owner', 'hostID', 'state', 'order')';">
<option value="all" #if $method == 'all' then 'selected="selected"' else ''#>all</option>
<option value="build" #if $method == 'build' then 'selected="selected"' else ''#>build</option>
- <option value="buildSRPMFromCVS" #if $method == 'buildSRPMFromCVS' then 'selected="selected"' else ''#>buildSRPMFromCVS</option>
+ <option value="buildSRPMFromSCM" #if $method == 'buildSRPMFromSCM' then 'selected="selected"' else ''#>buildSRPMFromSCM</option>
<option value="buildArch" #if $method == 'buildArch' then 'selected="selected"' else ''#>buildArch</option>
<option value="buildNotification" #if $method == 'buildNotification' then 'selected="selected"' else ''#>buildNotification</option>
<option value="tagBuild" #if $method == 'tagBuild' then 'selected="selected"' else ''#>tagBuild</option>
--
Fedora-buildsys-list mailing list
Fedora-buildsys-list@redhat.com
https://www.redhat.com/mailman/listinfo/fedora-buildsys-list
