Hello,
I've written a patch to make reposync more modular. The essential thing
is moving the argument parsing to the spacewalk-repo-sync script and out
of python/site-packages, but there are also a lot of other style
improvements.
We've already applied those changes and others in our branch, I just
merged some of them into the spacewalk branch now.
I hope you like it,
-Ionuț
>From e8a14d14817f57432c14c1d469bc07559359bf40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ionu=C8=9B=20Ar=C8=9B=C4=83ri=C8=99i?= <iartar...@suse.cz>
Date: Fri, 2 Dec 2011 17:09:13 +0100
Subject: [PATCH] make reposync more modular
- move argument parsing to the spacewalk-repo-sync script
- move repo setup to a special method in yum_src
- fixed other minor issues (indentation, class constructor, other pep-8 issues)
---
backend/satellite_tools/repo_plugins/yum_src.py | 60 +++++----
backend/satellite_tools/reposync.py | 155 +++++++++--------------
backend/satellite_tools/spacewalk-repo-sync | 64 +++++++---
3 files changed, 142 insertions(+), 137 deletions(-)
diff --git a/backend/satellite_tools/repo_plugins/yum_src.py b/backend/satellite_tools/repo_plugins/yum_src.py
index f0949da..8d0a557 100644
--- a/backend/satellite_tools/repo_plugins/yum_src.py
+++ b/backend/satellite_tools/repo_plugins/yum_src.py
@@ -1,6 +1,6 @@
#
# Copyright (c) 2008--2011 Red Hat, Inc.
-# Copyright (c) 2010--2011 SUSE Linux Products GmbH
+# Copyright (c) 2010--2011 SUSE LINUX Products GmbH, Nuernberg, Germany.
#
# This software is licensed to you under the GNU General Public License,
# version 2 (GPLv2). There is NO WARRANTY for this software, express or
@@ -8,15 +8,17 @@
# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
# along with this software; if not, see
# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
-#
+#
# Red Hat trademarks are not licensed under GPLv2. No permission is
# granted to use or replicate Red Hat trademarks that are incorporated
-# in this software or its documentation.
+# in this software or its documentation.
#
-import yum
+
import shutil
import sys
import gzip
+
+import yum
from yum.update_md import UpdateMetadata, UpdateNoticeException, UpdateNotice
from yum.yumRepo import YumRepository
try:
@@ -30,6 +32,8 @@ except ImportError:
from spacewalk.satellite_tools.reposync import ContentPackage
from spacewalk.common.rhnConfig import CFG, initCFG
+CACHE_DIR = '/var/cache/rhn/reposync/'
+
class YumWarnings:
def write(self, s):
pass
@@ -75,10 +79,10 @@ class YumUpdateMetadata(UpdateMetadata):
no.add(un)
class ContentSource:
- repo = None
- cache_dir = '/var/cache/rhn/reposync/'
def __init__(self, url, name):
- self._clean_cache(self.cache_dir + name)
+ self.url = url
+ self.name = name
+ self._clean_cache(CACHE_DIR + name)
# read the proxy configuration in /etc/rhn/rhn.conf
initCFG('server.satellite')
@@ -92,15 +96,25 @@ class ContentSource:
self.proxy_url = "http://%s"; %(self.proxy_addr)
else:
self.proxy_url = None
+
repo = yum.yumRepo.YumRepository(name)
self.repo = repo
+ self.sack = None
+
+ self.setup_repo(repo)
+ self.num_packages = 0
+ self.num_excluded = 0
+
+ def setup_repo(self, repo):
+ """Fetch repository metadata"""
repo.cache = 0
repo.metadata_expire = 0
- repo.mirrorlist = url
- repo.baseurl = [url]
- repo.basecachedir = self.cache_dir
+ repo.mirrorlist = self.url
+ repo.baseurl = [self.url]
+ repo.basecachedir = CACHE_DIR
if hasattr(repo, 'base_persistdir'):
- repo.base_persistdir = self.cache_dir
+ repo.base_persistdir = CACHE_DIR
+
if self.proxy_url is not None:
repo.proxy = self.proxy_url
@@ -108,28 +122,24 @@ class ContentSource:
warnings.disable()
repo.baseurlSetup()
warnings.restore()
-
repo.setup(False)
- self.num_packages = 0
- self.num_excluded = 0
-
+ self.sack = self.repo.getPackageSack()
def list_packages(self, filters):
""" list packages"""
- sack = self.repo.getPackageSack()
- sack.populate(self.repo, 'metadata', None, 0)
- list = sack.returnPackages()
+ self.sack.populate(self.repo, 'metadata', None, 0)
+ list = self.sack.returnPackages()
self.num_packages = len(list)
if filters:
list = self._filter_packages(list, filters)
- list = self._get_package_dependencies(sack, list)
+ list = self._get_package_dependencies(self.sack, list)
self.num_excluded = self.num_packages - len(list)
to_return = []
for pack in list:
if pack.arch == 'src':
continue
new_pack = ContentPackage()
- new_pack.setNVREA(pack.name, pack.version, pack.release,
+ new_pack.setNVREA(pack.name, pack.version, pack.release,
pack.epoch, pack.arch)
new_pack.unique_id = pack
new_pack.checksum_type = pack.checksums[0][0]
@@ -201,8 +211,8 @@ class ContentSource:
shutil.rmtree(directory, True)
def get_updates(self):
- if not self.repo.repoXML.repoData.has_key('updateinfo'):
- return []
- um = YumUpdateMetadata()
- um.add(self.repo, all=True)
- return um.notices
+ if not self.repo.repoXML.repoData.has_key('updateinfo'):
+ return []
+ um = YumUpdateMetadata()
+ um.add(self.repo, all=True)
+ return um.notices
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py
index 2ba1d7d..1893545 100644
--- a/backend/satellite_tools/reposync.py
+++ b/backend/satellite_tools/reposync.py
@@ -13,12 +13,13 @@
# granted to use or replicate Red Hat trademarks that are incorporated
# in this software or its documentation.
#
-import sys, os, time
import hashlib
+import os
import re
+import sys
+import time
from datetime import datetime
-import traceback
-from optparse import OptionParser
+
from spacewalk.server import rhnPackage, rhnSQL, rhnChannel, rhnPackageUpload
from spacewalk.common import rhnLog
from spacewalk.common.rhnLog import log_debug
@@ -40,84 +41,66 @@ def set_filter_opt(option, opt_str, value, parser):
else: f_type = '-'
parser.values.filters.append((f_type, re.split('[,\s]+', value)))
-class RepoSync:
- parser = None
- type = None
- urls = None
- channel_label = None
- channel = None
- fail = False
- quiet = False
- regen = False
- filters = []
+class RepoSync(object):
+ def __init__(self, channel_label, repo_type, url=None, fail=False,
+ quiet=False, filters=[]):
+ self.regen = False
+ self.fail = fail
+ self.quiet = quiet
+ self.filters = filters
- def main(self):
initCFG('server')
db_string = CFG.DEFAULT_DB #"rhnsat/rhnsat@rhnsat"
rhnSQL.initDB(db_string)
- (options, args) = self.process_args()
+ # setup logging
log_filename = 'reposync.log'
- if options.channel_label:
- date = time.localtime()
- datestr = '%d.%02d.%02d-%02d:%02d:%02d' % (date.tm_year, date.tm_mon, date.tm_mday, date.tm_hour, date.tm_min, date.tm_sec)
- log_filename = options.channel_label + '-' + datestr + '.log'
-
+ date = time.localtime()
+ datestr = '%d.%02d.%02d-%02d:%02d:%02d' % (
+ date.tm_year, date.tm_mon, date.tm_mday, date.tm_hour,
+ date.tm_min, date.tm_sec)
+ log_filename = channel_label + '-' + datestr + '.log'
rhnLog.initLOG(default_log_location + log_filename)
#os.fchown isn't in 2.4 :/
os.system("chgrp apache " + default_log_location + log_filename)
- if options.type not in ["yum"]:
- print "Error: Unknown type %s" % options.type
- sys.exit(2)
-
- quit = False
- if not options.url:
- if options.channel_label:
- # TODO:need to look at user security across orgs
- h = rhnSQL.prepare("""select s.id, s.source_url
- from rhnContentSource s,
- rhnChannelContentSource cs,
- rhnChannel c
- where s.id = cs.source_id
- and cs.channel_id = c.id
- and c.label = :label""")
- h.execute(label=options.channel_label)
- source_data = h.fetchall_dict() or []
- if source_data:
- self.urls = [(row['id'], row['source_url']) for row in source_data]
- else:
- quit = True
- self.error_msg("Channel has no URL associated")
- else:
- self.urls = [(None, options.url)]
- if not options.channel_label:
- quit = True
- self.error_msg("--channel must be specified")
-
- self.filters = options.filters
self.log_msg("\nSync started: %s" % (time.asctime(time.localtime())))
self.log_msg(str(sys.argv))
+ if not url:
+ # TODO:need to look at user security across orgs
+ h = rhnSQL.prepare("""select s.id, s.source_url
+ from rhnContentSource s,
+ rhnChannelContentSource cs,
+ rhnChannel c
+ where s.id = cs.source_id
+ and cs.channel_id = c.id
+ and c.label = :label""")
+ h.execute(label=channel_label)
+ source_data = h.fetchall_dict()
+ if source_data:
+ self.urls = [(row['id'], row['source_url']) for row in source_data]
+ else:
+ self.error_msg("Channel has no URL associated")
+ sys.exit(1)
+ else:
+ self.urls = [(None, url)]
- if quit:
- sys.exit(1)
+ self.repo_plugin = self.load_plugin(repo_type)
+ self.channel_label = channel_label
- self.type = options.type
- self.channel_label = options.channel_label
- self.fail = options.fail
- self.quiet = options.quiet
self.channel = self.load_channel()
-
if not self.channel or not rhnChannel.isCustomChannel(self.channel['id']):
- print "Channel does not exist or is not custom"
+ self.print_msg("Channel does not exist or is not custom.")
sys.exit(1)
+ def sync(self):
+ """Trigger a reposync"""
start_time = datetime.now()
- for (id, url) in self.urls:
- plugin = self.load_plugin()(url, self.channel_label)
- self.import_packages(plugin, id, url)
+ for (repo_id, url) in self.urls:
+ plugin = self.repo_plugin(url, self.channel_label)
+ self.import_packages(plugin, repo_id, url)
self.import_updates(plugin, url)
if self.regen:
taskomatic.add_to_repodata_queue_for_channel_package_subscription(
@@ -136,29 +119,17 @@ class RepoSync:
where label = :channel""")
h.execute(channel=self.channel['label'])
- def process_args(self):
- self.parser = OptionParser()
- self.parser.add_option('-u', '--url', action='store', dest='url', help='The url to sync')
- self.parser.add_option('-c', '--channel', action='store', dest='channel_label', help='The label of the channel to sync packages to')
- self.parser.add_option('-t', '--type', action='store', dest='type', help='The type of repo, currently only "yum" is supported', default='yum')
- self.parser.add_option('-f', '--fail', action='store_true', dest='fail', default=False , help="If a package import fails, fail the entire operation")
- self.parser.add_option('-q', '--quiet', action='store_true', dest='quiet', default=False, help="Print no output, still logs output")
- self.parser.add_option('-i', '--include', action='callback', callback=set_filter_opt, type='str', nargs=1, dest='filters', default=[], help="List of included packages")
- self.parser.add_option('-e', '--exclude', action='callback', callback=set_filter_opt, type='str', nargs=1, dest='filters', default=[], help="List of excluded packages")
-
- return self.parser.parse_args()
-
- def load_plugin(self):
- name = self.type + "_src"
+ def load_plugin(self, repo_type):
+ name = repo_type + "_src"
mod = __import__('spacewalk.satellite_tools.repo_plugins', globals(), locals(), [name])
submod = getattr(mod, name)
return getattr(submod, "ContentSource")
def import_updates(self, plug, url):
- notices = plug.get_updates()
- self.print_msg("Repo " + url + " has " + str(len(notices)) + " errata.")
- if len(notices) > 0:
- self.upload_updates(notices)
+ notices = plug.get_updates()
+ self.print_msg("Repo %s has %s errata." % (url, len(notices)))
+ if notices:
+ self.upload_updates(notices)
def upload_updates(self, notices):
batch = []
@@ -240,7 +211,7 @@ class RepoSync:
and at.label = 'rpm'
and ch.label = :channel_label
""" % (orgStatement, epochStatement))
- apply(h.execute, (), param_dict)
+ h.execute(**param_dict)
cs = h.fetchone_dict() or None
if not cs:
@@ -358,9 +329,9 @@ class RepoSync:
if db_pack['channel_label'] == self.channel_label:
# package is already in the channel
to_link = False
- elif db_pack['channel_label'] == self.channel_label:
+ elif db_pack['channel_label'] == self.channel_label:
# different package with SAME NVREA
- self.disassociate_package(db_pack)
+ self.disassociate_package(db_pack)
if to_download or to_link:
to_process.append((pack, to_download, to_link))
@@ -376,11 +347,12 @@ class RepoSync:
self.regen=True
is_non_local_repo = (url.find("file://") < 0)
- # try/except/finally doesn't work in python 2.4 (RHEL5), so here's a hack
+
def finally_remove(path):
if is_non_local_repo and path and os.path.exists(path):
os.remove(path)
+ # try/except/finally doesn't work in python 2.4 (RHEL5), so here's a hack
for (index, what) in enumerate(to_process):
pack, to_download, to_link = what
localpath = None
@@ -469,22 +441,19 @@ class RepoSync:
checksum_type=pack['checksum_type'], checksum=pack['checksum'])
def _importer_run(self, package, caller, backend):
- importer = ChannelPackageSubscription(
- [IncompletePackage().populate(package)],
- backend, caller=caller, repogen=False)
- importer.run()
-
+ importer = ChannelPackageSubscription(
+ [IncompletePackage().populate(package)],
+ backend, caller=caller, repogen=False)
+ importer.run()
def load_channel(self):
return rhnChannel.channel_info(self.channel_label)
-
def print_msg(self, message):
rhnLog.log_clean(0, message)
if not self.quiet:
print message
-
def error_msg(self, message):
rhnLog.log_clean(0, message)
if not self.quiet:
@@ -499,10 +468,10 @@ class RepoSync:
def _to_db_date(self, date):
ret = ""
if date.isdigit():
- ret = datetime.fromtimestamp(float(date)).isoformat(' ')
+ ret = datetime.fromtimestamp(float(date)).isoformat(' ')
else:
- # we expect to get ISO formated date
- ret = date
+ # we expect to get ISO formated date
+ ret = date
return ret
def fix_notice(self, notice):
@@ -614,7 +583,7 @@ class ContentPackage:
def load_checksum_from_header(self):
if self.path is None:
- raise rhnFault(50, "Unable to load package", explain=0)
+ raise rhnFault(50, "Unable to load package", explain=0)
self.file = open(self.path, 'rb')
self.header, self.payload_stream, self.header_start, self.header_end = \
rhnPackageUpload.load_package(self.file)
diff --git a/backend/satellite_tools/spacewalk-repo-sync b/backend/satellite_tools/spacewalk-repo-sync
index ae06a96..69c9568 100755
--- a/backend/satellite_tools/spacewalk-repo-sync
+++ b/backend/satellite_tools/spacewalk-repo-sync
@@ -1,6 +1,7 @@
#!/usr/bin/python -u
#
# Copyright (c) 2008--2011 Red Hat, Inc.
+# Copyright (c) 2011 SUSE LINUX Products GmbH, Nuernberg, Germany.
#
# This software is licensed to you under the GNU General Public License,
# version 2 (GPLv2). There is NO WARRANTY for this software, express or
@@ -8,30 +9,25 @@
# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
# along with this software; if not, see
# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
-#
+#
# Red Hat trademarks are not licensed under GPLv2. No permission is
# granted to use or replicate Red Hat trademarks that are incorporated
-# in this software or its documentation.
+# in this software or its documentation.
#
-LOCK = None
+import sys
+import os
+import socket
+from optparse import OptionParser
-if __name__ != '__main__':
- raise ImportError, "module cannot be imported"
+LOCK = None
-import sys
-def systemExit(code, msgs=None):
+def systemExit(code, msg=None):
"Exit with a code and optional message(s). Saved a few lines of code."
- if msgs:
- if type(msgs) not in [type([]), type(())]:
- msgs = (msgs, )
- for msg in msgs:
- sys.stderr.write(str(msg)+'\n')
+ sys.stderr.write(str(msg)+'\n')
sys.exit(code)
try:
- import os
- import socket
from rhn import rhnLockfile
from spacewalk.common.rhnConfig import CFG
from spacewalk.common.rhnTB import fetchTraceback
@@ -39,8 +35,8 @@ try:
except KeyboardInterrupt:
systemExit(0, "\nUser interrupted process.")
except ImportError:
- sys.stderr.write("Unable to find RHN code tree.\n"
- "Path not correct? '%s'\n" % _LIBPATH)
+ systemExit(1, "Unable to find code tree.\n"
+ "Path not correct? '%s'" % sys.path)
def releaseLOCK():
global LOCK
@@ -49,22 +45,52 @@ def releaseLOCK():
LOCK = None
def main():
-
# quick check to see if you are a super-user.
if os.getuid() != 0:
sys.stderr.write('ERROR: must be root to execute\n')
sys.exit(8)
+ global LOCK
LOCK = None
try:
LOCK = rhnLockfile.Lockfile('/var/run/spacewalk-repo-sync.pid')
except rhnLockfile.LockfileLockedException:
systemExit(1, "ERROR: attempting to run more than one instance of spacewalk-repo-sync Exiting.")
- sync = reposync.RepoSync()
- sync.main()
+
+ parser = OptionParser()
+ parser.add_option('-u', '--url', action='store', dest='url',
+ help='The url to sync')
+ parser.add_option('-c', '--channel', action='store',
+ dest='channel_label',
+ help='The label of the channel to sync packages to')
+ parser.add_option('-t', '--type', action='store', dest='repo_type',
+ help='The type of repo, currently only "yum" is supported',
+ default='yum')
+ parser.add_option('-f', '--fail', action='store_true', dest='fail',
+ default=False ,
+ help="If a package import fails, fail the entire operation")
+ parser.add_option('-q', '--quiet', action='store_true', dest='quiet',
+ default=False, help="Print no output, still logs output")
+ parser.add_option('-i', '--include', action='callback', callback=reposync.set_filter_opt,
+ type='str', nargs=1, dest='filters', default=[], help="List of included packages")
+ parser.add_option('-e', '--exclude', action='callback', callback=reposync.set_filter_opt,
+ type='str', nargs=1, dest='filters', default=[], help="List of excluded packages")
+ (options, args) = parser.parse_args()
+
+ if not options.channel_label:
+ systemExit(1, "--channel must be specified")
+
+ sync = reposync.RepoSync(channel_label=options.channel_label,
+ repo_type=options.repo_type,
+ url=options.url,
+ fail=options.fail,
+ quiet=options.quiet,
+ filters=options.filters)
+ sync.sync()
releaseLOCK()
return 0
+
if __name__ == '__main__':
try:
sys.exit(abs(main() or 0))
--
1.7.3.4
_______________________________________________
Spacewalk-devel mailing list
Spacewalk-devel@redhat.com
https://www.redhat.com/mailman/listinfo/spacewalk-devel
