RPM Package Manager, CVS Repository http://rpm5.org/cvs/ ____________________________________________________________________________
Server: rpm5.org Name: Jeff Johnson Root: /v/rpm/cvs Email: [email protected] Module: createrepo Date: 07-Aug-2010 01:40:50 Branch: VENDOR Handle: 2010080623404900 Modified files: (Branch: VENDOR) createrepo ChangeLog Makefile createrepo.spec genpkgmetadata.py createrepo/docs createrepo.8 Touched files: (Branch: VENDOR) createrepo README createrepo/bin Makefile createrepo createrepo/docs Makefile Log: Import createrepo-0.4.4 [Release Tag: createrepo-0_4_4] Summary: Revision Changes Path 1.1.1.8 +248 -132 createrepo/ChangeLog 1.1.1.17 +1 -1 createrepo/Makefile 1.1.1.17 +4 -1 createrepo/createrepo.spec 1.1.1.3 +0 -0 createrepo/docs/createrepo.8 1.1.1.17 +386 -266 createrepo/genpkgmetadata.py ____________________________________________________________________________ patch -p0 <<'@@ .' Index: createrepo/ChangeLog ============================================================================ $ cvs diff -u -r1.1.1.7 -r1.1.1.8 ChangeLog --- createrepo/ChangeLog 6 Aug 2010 23:40:02 -0000 1.1.1.7 +++ createrepo/ChangeLog 6 Aug 2010 23:40:49 -0000 1.1.1.8 @@ -1,151 +1,266 @@ +2006-03-04 01:28 pnasrat + + * createrepo.spec: release + +2006-02-21 18:30 pnasrat + + * Makefile, createrepo.spec, genpkgmetadata.py, docs/createrepo.8: + Documentation and version updates + +2006-02-21 15:10 pnasrat + + * dumpMetadata.py, genpkgmetadata.py: Enable seperate outputdir + (dgregor) + +2006-02-17 21:22 lmacken + + * genpkgmetadata.py, docs/createrepo.8: Add support for -U + (--update-info-location) flag to query a specified server for + package update metadata. The metadata will currently be stored + in 'repodata/update-info' and each package in the primary.xml + will have an <update id="FEDORA-XXXX-XX" + location="update-info/pkg-ver-rel.xml"/> tag which points to it's + corresponding update information. + +2006-01-13 01:24 skvidal + + * dumpMetadata.py, genpkgmetadata.py: + apply two patches from [email protected] - verifies that the + checksum cache file is more recent than the corresponding rpm - + move around cmds dict initialization to make it more consistent. + +2005-12-08 14:25 pnasrat + + * genpkgmetadata.py: Fix cachedir/groupfile handling with --basedir + and using paths not relative to cwd when run without --basedir. + +2005-12-08 14:24 pnasrat + + * genpkgmetadata.py: Support --split option to label media with + urls across directories. + +2005-12-08 14:22 pnasrat + + * genpkgmetadata.py: Split out processing into smaller methods. + Make ts internal. Files and base/file/other data attributes. + +2005-12-08 14:21 pnasrat + + * genpkgmetadata.py: Cleanup of generator class to use cmds + internally as an attribute. + +2005-12-08 14:19 pnasrat + + * genpkgmetadata.py: Initial work to form metadata generator class. + +2005-11-27 18:01 skvidal + + * dumpMetadata.py: speed up by caching file mode lookup by Dennis + Gregorovic + +2005-11-11 16:03 pnasrat + + * dumpMetadata.py, genpkgmetadata.py: Enable basedir to be used - + dgregor + +2005-11-02 15:23 skvidal + + * dumpMetadata.py, genpkgmetadata.py: + patch from Christoph Thiel to make it work on suse 9.3 and to + allow for non absolute-path cache dirs. + +2005-08-11 17:09 skvidal + + * dumpMetadata.py: + turn off all signature checking when reading in headers + +2005-07-24 19:54 skvidal + + * genpkgmetadata.py: + document that the -g option is for a file relative to the + directory you are creating the repository for. + +2005-07-14 02:34 skvidal + + * docs/createrepo.8: + man page for cachedir + +2005-07-14 02:31 skvidal + + * Makefile, createrepo.spec, genpkgmetadata.py: + 0.4.3 + +2005-07-11 01:45 skvidal + + * dumpMetadata.py, genpkgmetadata.py: + - disable the checksum flag - default and only use sha1sum's - + add in -c,--cachedir option to setup a cachedir for the cache + files of the checksums of the packages. Uses name-hdrid from the + package hdr as filenames. Contents of the file is a single line + of the package's checksum. This dramatically speeds up + rebuilding a repository's metadata b/c the checksum of the + package file was the item taking the most time. + +2005-05-28 23:14 skvidal + + * dumpMetadata.py, genpkgmetadata.py: + Apply Arun Bhanu's patch to add in --quiet and --verbose options + instead of just -q and -v + +2005-03-30 00:32 skvidal + + * docs/Makefile: + fix mandir path for docs + 2005-01-18 02:14 skvidal - * Makefile, docs/Makefile: - fix the Makefiles, f'real + * Makefile, docs/Makefile: + fix the Makefiles, f'real 2005-01-18 01:59 skvidal - * docs/: Makefile, createrepo.8: - real commit + * docs/: Makefile, createrepo.8: + real commit 2005-01-18 01:58 skvidal - * Makefile, createrepo.spec: - adding man page and upating the Makefiles and specfile accordingly. - Thanks Bob Kashani for the man page. + * Makefile, createrepo.spec: + adding man page and upating the Makefiles and specfile + accordingly. Thanks Bob Kashani for the man page. 2005-01-18 01:24 skvidal - * dumpMetadata.py: - need to seek to the beginning before doing a new read operation. + * dumpMetadata.py: + need to seek to the beginning before doing a new read operation. 2005-01-17 02:52 skvidal - * Makefile, createrepo.spec: - spec and Makefile to 0.4.2 + * Makefile, createrepo.spec: + spec and Makefile to 0.4.2 2005-01-17 02:52 skvidal - * dumpMetadata.py, genpkgmetadata.py: - move around gzipOpen for use in another program relabel 0.4.2 + * dumpMetadata.py, genpkgmetadata.py: + move around gzipOpen for use in another program relabel 0.4.2 2005-01-07 16:39 skvidal - * dumpMetadata.py, genpkgmetadata.py: - single open for all file operations. about a 30% time savings. + * dumpMetadata.py, genpkgmetadata.py: + single open for all file operations. about a 30% time savings. 2004-11-02 00:55 skvidal - * genpkgmetadata.py: - small fix for --exclude to work. -x works, but --exclude didn't, - now it is fixed + * genpkgmetadata.py: + small fix for --exclude to work. -x works, but --exclude didn't, + now it is fixed 2004-10-21 03:42 skvidal - * Makefile, createrepo.spec, genpkgmetadata.py: - update version numbers + * Makefile, createrepo.spec, genpkgmetadata.py: + update version numbers 2004-10-21 03:40 skvidal - * dumpMetadata.py: - problem with ghost entries not showing up in primary.xml even if - they matched the regex strings. + * dumpMetadata.py: + problem with ghost entries not showing up in primary.xml even if + they matched the regex strings. 2004-10-18 01:41 skvidal - * bin/createrepo: - whoops! need to quote that var string + * bin/createrepo: + whoops! need to quote that var string 2004-10-11 07:43 skvidal - * Makefile, createrepo.spec, genpkgmetadata.py: - correct problem with handling dirs with a space in the filename - update version number + * Makefile, createrepo.spec, genpkgmetadata.py: + correct problem with handling dirs with a space in the filename + update version number 2004-10-03 23:43 skvidal - * genpkgmetadata.py: - clean up argument parsing to handle --version and --help more - correctly. Not quite the patch Ville Skyttä submitted. + * genpkgmetadata.py: + clean up argument parsing to handle --version and --help more + correctly. Not quite the patch Ville Skyttä submitted. 2004-09-30 01:54 skvidal - * genpkgmetadata.py: - one more place to tag + * genpkgmetadata.py: + one more place to tag 2004-09-30 01:52 skvidal - * Makefile, createrepo.spec: - update to 0.3.9 + * Makefile, createrepo.spec: + update to 0.3.9 2004-09-30 01:49 skvidal - * dumpMetadata.py: - checksum of group file will be wrong if specified - didn't seek(0) - after copying it. + * dumpMetadata.py: + checksum of group file will be wrong if specified - didn't + seek(0) after copying it. 2004-09-20 01:13 skvidal - * genpkgmetadata.py: - made 'cannot remove old metadata dir' a non-fatal error. It just - warns. + * genpkgmetadata.py: + made 'cannot remove old metadata dir' a non-fatal error. It just + warns. 2004-09-20 01:06 skvidal - * genpkgmetadata.py: - updated to default to sha-1 checksums + * genpkgmetadata.py: + updated to default to sha-1 checksums 2004-09-11 16:19 skvidal - * createrepo.spec, genpkgmetadata.py: - update spec file as 0.3.8 fix for bug in command handling of groups - location + * createrepo.spec, genpkgmetadata.py: + update spec file as 0.3.8 fix for bug in command handling of + groups location 2004-09-11 16:07 skvidal - * Makefile: - fix for group file path being wrong - Bill Nottingham mark as 0.3.8 + * Makefile: + fix for group file path being wrong - Bill Nottingham mark as + 0.3.8 2004-09-11 16:07 skvidal - * dumpMetadata.py, genpkgmetadata.py: - fix for error when string is None for utf8 conversion + * dumpMetadata.py, genpkgmetadata.py: + fix for error when string is None for utf8 conversion 2004-09-03 02:29 skvidal - * Makefile: - Makefile update to fix a bug reported by Anvil + * Makefile: + Makefile update to fix a bug reported by Anvil 2004-09-01 02:56 skvidal - * Makefile, createrepo.spec, genpkgmetadata.py: - 0.3.7 + * Makefile, createrepo.spec, genpkgmetadata.py: + 0.3.7 2004-08-27 03:03 skvidal - * dumpMetadata.py, genpkgmetadata.py: - patch from Ville Skytta (this a will be wrong, sorry) to correct - decoding/encoding problems. + * dumpMetadata.py, genpkgmetadata.py: + patch from Ville Skytta (this a will be wrong, sorry) to correct + decoding/encoding problems. 2004-07-23 16:22 skvidal - * README: - updated readme with anoncvs location + * README: + updated readme with anoncvs location 2004-07-23 15:41 skvidal - * Makefile, createrepo.spec, genpkgmetadata.py: - ver to 0.3.6 + * Makefile, createrepo.spec, genpkgmetadata.py: + ver to 0.3.6 2004-07-23 14:45 skvidal - * dumpMetadata.py: - fix filelists to be complete + * dumpMetadata.py: + fix filelists to be complete 2004-07-23 11:56 skvidal - * dumpMetadata.py: - remove a debug print call + * dumpMetadata.py: + remove a debug print call 2004-07-23 11:51 skvidal @@ -153,142 +268,143 @@ 2004-07-23 11:50 skvidal - * dumpMetadata.py: - fix up for broken filelists in packages + * dumpMetadata.py: + fix up for broken filelists in packages 2004-07-23 01:23 skvidal - * genpkgmetadata.py: - silly string fix + * genpkgmetadata.py: + silly string fix 2004-07-19 21:24 skvidal - * Makefile, createrepo.spec, genpkgmetadata.py: - bump number to 0.3.4 + * Makefile, createrepo.spec, genpkgmetadata.py: + bump number to 0.3.4 2004-07-19 21:23 skvidal - * dumpMetadata.py, genpkgmetadata.py: - re-enabled group files documented it + * dumpMetadata.py, genpkgmetadata.py: + re-enabled group files documented it 2004-06-30 04:23 skvidal - * dumpMetadata.py: - add pre=1 to requires entries for prereq marking + * dumpMetadata.py: + add pre=1 to requires entries for prereq marking 2004-06-30 03:08 skvidal - * dumpMetadata.py, genpkgmetadata.py: - - include xmlns:rpm in main metadata tag rather than per-format - node - fix output for sorta-list, sorta-string rpm header tags + * dumpMetadata.py, genpkgmetadata.py: + - include xmlns:rpm in main metadata tag rather than per-format + node - fix output for sorta-list, sorta-string rpm header tags 2004-06-28 11:24 skvidal - * dumpMetadata.py: - fix for namespace for license, vendor, group, buildhost and - sourcerpm was None, should have been formatns (rpm namespace) + * dumpMetadata.py: + fix for namespace for license, vendor, group, buildhost and + sourcerpm was None, should have been formatns (rpm namespace) 2004-06-08 23:09 skvidal - * Makefile, createrepo.spec, genpkgmetadata.py: - mark as 0.3.3 + * Makefile, createrepo.spec, genpkgmetadata.py: + mark as 0.3.3 2004-06-06 18:38 skvidal - * genpkgmetadata.py: - included a not-that-terribly accurate package count + * genpkgmetadata.py: + included a not-that-terribly accurate package count 2004-06-05 14:38 skvidal - * dumpMetadata.py, genpkgmetadata.py: - generate uncompressed checksums a much easier way. + * dumpMetadata.py, genpkgmetadata.py: + generate uncompressed checksums a much easier way. 2004-06-05 14:31 skvidal - * dumpMetadata.py, genpkgmetadata.py: - revert some changes + * dumpMetadata.py, genpkgmetadata.py: + revert some changes 2004-06-03 02:14 skvidal - * genpkgmetadata.py: - fix stupid version thing + * genpkgmetadata.py: + fix stupid version thing 2004-06-03 02:14 skvidal - * dumpMetadata.py, genpkgmetadata.py: - keep checksum of uncompressed metadata files in repomd.xml + * dumpMetadata.py, genpkgmetadata.py: + keep checksum of uncompressed metadata files in repomd.xml 2004-06-03 01:26 skvidal - * dumpMetadata.py, genpkgmetadata.py: - move versioned prco from separate string to properties of the entry + * dumpMetadata.py, genpkgmetadata.py: + move versioned prco from separate string to properties of the + entry 2004-04-16 01:15 skvidal - * Makefile: - fix makefile + * Makefile: + fix makefile 2004-04-16 01:10 skvidal - * Makefile, createrepo.spec, genpkgmetadata.py: - update to 0.3.2 added -p or --pretty flag to make it pretty-print - the xml output not pretty printing the output makes the import a - lot faster and the data smaller + * Makefile, createrepo.spec, genpkgmetadata.py: + update to 0.3.2 added -p or --pretty flag to make it pretty-print + the xml output not pretty printing the output makes the import a + lot faster and the data smaller 2004-01-18 16:43 skvidal - * Makefile, createrepo.spec, dumpMetadata.py, genpkgmetadata.py: - 1. make it actually work :) 2. bump to 0.3.1 + * Makefile, createrepo.spec, dumpMetadata.py, genpkgmetadata.py: + 1. make it actually work :) 2. bump to 0.3.1 2004-01-18 12:57 skvidal - * Makefile, createrepo.spec: - add README for real *boggle* + * Makefile, createrepo.spec: + add README for real *boggle* 2004-01-18 12:56 skvidal - * Makefile, README, createrepo.spec: - tagged Makefile and createrepo as 0.3 Add README to both of the - above + * Makefile, README, createrepo.spec: + tagged Makefile and createrepo as 0.3 Add README to both of the + above 2004-01-18 11:45 skvidal - * genpkgmetadata.py: - make metadata files be written to repodata/ + * genpkgmetadata.py: + make metadata files be written to repodata/ 2004-01-17 01:04 skvidal - * dumpMetadata.py, genpkgmetadata.py: - fix bug where not all files were getting included make the - directory detection more reliable + * dumpMetadata.py, genpkgmetadata.py: + fix bug where not all files were getting included make the + directory detection more reliable 2004-01-13 22:56 skvidal * Makefile, createrepo.spec, dumpMetadata.py, genpkgmetadata.py, - bin/Makefile: - fixed memory use problem updated spec for 0.2 fixed makefile - dumbness fixed problems with broken symlinks + bin/Makefile: + fixed memory use problem updated spec for 0.2 fixed makefile + dumbness fixed problems with broken symlinks 2004-01-13 16:16 skvidal - * dumpMetadata.py: - catch some errors on broken symlinks + * dumpMetadata.py: + catch some errors on broken symlinks 2004-01-11 17:07 skvidal * Makefile, createrepo.spec, dumpMetadata.py, genpkgmetadata.py, - bin/Makefile, bin/createrepo: - - translation stubs - makefiles - spec file - bin wrapper + bin/Makefile, bin/createrepo: + - translation stubs - makefiles - spec file - bin wrapper 2004-01-10 03:39 skvidal - * dumpMetadata.py, genpkgmetadata.py: - silly updates in comments + * dumpMetadata.py, genpkgmetadata.py: + silly updates in comments 2004-01-10 03:34 skvidal - * dumpMetadata.py, genpkgmetadata.py: - CVS Id Tags + * dumpMetadata.py, genpkgmetadata.py: + CVS Id Tags 2004-01-10 03:34 skvidal @@ -296,21 +412,21 @@ 2004-01-10 03:33 skvidal - * genpkgmetadata.py: - added --version and __version__ string + * genpkgmetadata.py: + added --version and __version__ string 2004-01-10 03:04 skvidal - * dumpMetadata.py, genpkgmetadata.py: - move two functions around to more logically arrange the repomd.xml - generating function + * dumpMetadata.py, genpkgmetadata.py: + move two functions around to more logically arrange the + repomd.xml generating function 2004-01-09 02:48 skvidal - * dumpMetadata.py, genpkgmetadata.py: metadata generate first - import + * dumpMetadata.py, genpkgmetadata.py: Initial revision 2004-01-09 02:48 skvidal - * dumpMetadata.py, genpkgmetadata.py: Initial revision + * dumpMetadata.py, genpkgmetadata.py: metadata generate first + import @@ . patch -p0 <<'@@ .' Index: createrepo/Makefile ============================================================================ $ cvs diff -u -r1.1.1.16 -r1.1.1.17 Makefile --- createrepo/Makefile 6 Aug 2010 23:40:30 -0000 1.1.1.16 +++ createrepo/Makefile 6 Aug 2010 23:40:49 -0000 1.1.1.17 @@ -1,5 +1,5 @@ PACKAGE = createrepo -VERSION = 0.4.3 +VERSION = 0.4.4 SHELL = /bin/sh top_srcdir = . srcdir = . @@ . patch -p0 <<'@@ .' Index: createrepo/createrepo.spec ============================================================================ $ cvs diff -u -r1.1.1.16 -r1.1.1.17 createrepo.spec --- createrepo/createrepo.spec 6 Aug 2010 23:40:30 -0000 1.1.1.16 +++ createrepo/createrepo.spec 6 Aug 2010 23:40:49 -0000 1.1.1.17 @@ -1,6 +1,6 @@ Summary: Creates a common metadata repository Name: createrepo -Version: 0.4.3 +Version: 0.4.4 Release: 1 License: GPL Group: System Environment/Base @@ -34,6 +34,9 @@ %{_mandir}/man8/createrepo.8* %changelog +* Sat Mar 04 2006 Paul Nasrat <[email protected]> +- 0.4.4 + * Thu Jul 14 2005 Seth Vidal <[email protected]> - enable caching option - 0.4.3 @@ . patch -p0 <<'@@ .' Index: createrepo/docs/createrepo.8 ============================================================================ $ cvs diff -u -r1.1.1.2 -r1.1.1.3 docs/createrepo.8 @@ . patch -p0 <<'@@ .' Index: createrepo/genpkgmetadata.py ============================================================================ $ cvs diff -u -r1.1.1.16 -r1.1.1.17 genpkgmetadata.py --- createrepo/genpkgmetadata.py 6 Aug 2010 23:40:30 -0000 1.1.1.16 +++ createrepo/genpkgmetadata.py 6 Aug 2010 23:40:49 -0000 1.1.1.17 @@ -17,7 +17,7 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2004 Duke University -# $Id: genpkgmetadata.py,v 1.1.1.16 2010/08/06 23:40:30 jbj Exp $ +# $Id: genpkgmetadata.py,v 1.1.1.17 2010/08/06 23:40:49 jbj Exp $ import os @@ -27,6 +27,7 @@ import libxml2 import string import fnmatch +import urlgrabber import dumpMetadata from dumpMetadata import _gzipOpen @@ -38,62 +39,294 @@ def _(args): """Stub function for translation""" return args - + def usage(retval=1): print _(""" createrepo [options] directory-of-packages - + Options: - -u, --baseurl = optional base url location for all files + -u, --baseurl <url> = optional base url location for all files + -o, --outputdir <dir> = optional directory to output to -x, --exclude = files globs to exclude, can be specified multiple times -q, --quiet = run quietly -g, --groupfile <filename> to point to for group information (precreated) + (<filename> relative to directory-of-packages) -v, --verbose = run verbosely -c, --cachedir <dir> = specify which dir to use for the checksum cache + -U, --update-info-location <url> = acquire package update metadata -h, --help = show this help -V, --version = output version -p, --pretty = output xml files in pretty format. - """) + """) sys.exit(retval) +class MetaDataGenerator: + def __init__(self, cmds): + self.cmds = cmds + self.ts = rpm.TransactionSet() + self.pkgcount = 0 + self.files = [] + + def getFileList(self, basepath, path, ext, filelist): + """Return all files in path matching ext, store them in filelist, + recurse dirs. Returns a list object""" -def getFileList(path, ext, filelist): - """Return all files in path matching ext, store them in filelist, recurse dirs - return list object""" - - extlen = len(ext) - try: - dir_list = os.listdir(path) - except OSError, e: - errorprint(_('Error accessing directory %s, %s') % (path, e)) - sys.exit(1) - - for d in dir_list: - if os.path.isdir(path + '/' + d): - filelist = getFileList(path + '/' + d, ext, filelist) - else: - if string.lower(d[-extlen:]) == '%s' % (ext): - newpath = os.path.normpath(path + '/' + d) - filelist.append(newpath) - - return filelist + extlen = len(ext) + totalpath = os.path.normpath(os.path.join(basepath, path)) + try: + dir_list = os.listdir(totalpath) + except OSError, e: + errorprint(_('Error accessing directory %s, %s') % (totalpath, e)) + sys.exit(1) + for d in dir_list: + if os.path.isdir(totalpath + '/' + d): + filelist = self.getFileList(basepath, os.path.join(path, d), ext, filelist) + else: + if string.lower(d[-extlen:]) == '%s' % (ext): + if totalpath.find(basepath) == 0: + relativepath = totalpath.replace(basepath, "", 1) + relativepath = relativepath.lstrip("/") + filelist.append(os.path.join(relativepath, d)) + else: + raise "basepath '%s' not found in path '%s'" % (basepath, totalpath) + + return filelist + + + def trimRpms(self, files): + badrpms = [] + for file in files: + for glob in self.cmds['excludes']: + if fnmatch.fnmatch(file, glob): + # print 'excluded: %s' % file + if file not in badrpms: + badrpms.append(file) + for file in badrpms: + if file in files: + files.remove(file) + return files + + def doPkgMetadata(self, directory): + """all the heavy lifting for the package metadata""" + + # rpms we're going to be dealing with + files = self.getFileList(self.cmds['basedir'], directory, '.rpm', []) + files = self.trimRpms(files) + self.pkgcount = len(files) + self.openMetadataDocs() + self.writeMetadataDocs(files) + self.closeMetadataDocs() + + + def openMetadataDocs(self): + self._setupBase() + self._setupFilelists() + self._setupOther() + + def _setupBase(self): + # setup the base metadata doc + self.basedoc = libxml2.newDoc("1.0") + self.baseroot = self.basedoc.newChild(None, "metadata", None) + basens = self.baseroot.newNs('http://linux.duke.edu/metadata/common', None) + self.formatns = self.baseroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm') + self.baseroot.setNs(basens) + basefilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['primaryfile']) + self.basefile = _gzipOpen(basefilepath, 'w') + self.basefile.write('<?xml version="1.0" encoding="UTF-8"?>\n') + self.basefile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">\n' % + self.pkgcount) + + def _setupFilelists(self): + # setup the file list doc + self.filesdoc = libxml2.newDoc("1.0") + self.filesroot = self.filesdoc.newChild(None, "filelists", None) + filesns = self.filesroot.newNs('http://linux.duke.edu/metadata/filelists', None) + self.filesroot.setNs(filesns) + filelistpath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['filelistsfile']) + self.flfile = _gzipOpen(filelistpath, 'w') + self.flfile.write('<?xml version="1.0" encoding="UTF-8"?>\n') + self.flfile.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists" packages="%s">\n' % + self.pkgcount) + + def _setupOther(self): + # setup the other doc + self.otherdoc = libxml2.newDoc("1.0") + self.otherroot = self.otherdoc.newChild(None, "otherdata", None) + otherns = self.otherroot.newNs('http://linux.duke.edu/metadata/other', None) + self.otherroot.setNs(otherns) + otherfilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['otherfile']) + self.otherfile = _gzipOpen(otherfilepath, 'w') + self.otherfile.write('<?xml version="1.0" encoding="UTF-8"?>\n') + self.otherfile.write('<otherdata xmlns="http://linux.duke.edu/metadata/other" packages="%s">\n' % + self.pkgcount) + + def writeMetadataDocs(self, files, current=0): + for file in files: + current+=1 + try: + mdobj = dumpMetadata.RpmMetaData(self.ts, self.cmds['basedir'], file, self.cmds) + if not self.cmds['quiet']: + if self.cmds['verbose']: + print '%d/%d - %s' % (current, len(files), file) + else: + sys.stdout.write('\r' + ' ' * 80) + sys.stdout.write("\r%d/%d - %s" % (current, self.pkgcount, file)) + sys.stdout.flush() + except dumpMetadata.MDError, e: + errorprint('\n%s - %s' % (e, file)) + continue + else: + try: + node = dumpMetadata.generateXML(self.basedoc, self.baseroot, self.formatns, mdobj, self.cmds['sumtype']) + except dumpMetadata.MDError, e: + errorprint(_('\nAn error occurred creating primary metadata: %s') % e) + continue + else: + try: + # Fetch the update metadata for this package + if self.cmds['update-info-location']: + metadata = urlgrabber.urlopen( + self.cmds['update-info-location'] + + '?pkg=%s' % file) + filename = file.replace('.rpm', '.xml') + metadata.filename = os.path.join( + self.cmds['basedir'], self.cmds['tempdir'], + self.cmds['update-info-dir'], filename) + metadata._do_grab() + metadata.close() + + # Get the update ID from the metadata + md = libxml2.parseFile(metadata.filename) + update_root = md.children + update = node.newChild(None, 'update', None) + update.newProp('id', update_root.prop('id')) + update.newProp('location', os.path.join( + self.cmds['update-info-dir'], filename)) + del md, metadata + except Exception, e: + pass + output = node.serialize('UTF-8', self.cmds['pretty']) + self.basefile.write(output) + self.basefile.write('\n') + node.unlinkNode() + node.freeNode() + del node + + try: + node = dumpMetadata.fileListXML(self.filesdoc, self.filesroot, mdobj) + except dumpMetadata.MDError, e: + errorprint(_('\nAn error occurred creating filelists: %s') % e) + continue + else: + output = node.serialize('UTF-8', self.cmds['pretty']) + self.flfile.write(output) + self.flfile.write('\n') + node.unlinkNode() + node.freeNode() + del node + + try: + node = dumpMetadata.otherXML(self.otherdoc, self.otherroot, mdobj) + except dumpMetadata.MDError, e: + errorprint(_('\nAn error occurred: %s') % e) + continue + else: + output = node.serialize('UTF-8', self.cmds['pretty']) + self.otherfile.write(output) + self.otherfile.write('\n') + node.unlinkNode() + node.freeNode() + del node + return current + + + def closeMetadataDocs(self): + if not self.cmds['quiet']: + print '' + + # save them up to the tmp locations: + if not self.cmds['quiet']: + print _('Saving Primary metadata') + self.basefile.write('\n</metadata>') + self.basefile.close() + self.basedoc.freeDoc() + + if not self.cmds['quiet']: + print _('Saving file lists metadata') + self.flfile.write('\n</filelists>') + self.flfile.close() + self.filesdoc.freeDoc() + + if not self.cmds['quiet']: + print _('Saving other metadata') + self.otherfile.write('\n</otherdata>') + self.otherfile.close() + self.otherdoc.freeDoc() + + def doRepoMetadata(self): + """wrapper to generate the repomd.xml file that stores the info on the other files""" + repodoc = libxml2.newDoc("1.0") + reporoot = repodoc.newChild(None, "repomd", None) + repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None) + reporoot.setNs(repons) + repofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['repomdfile']) + + try: + dumpMetadata.repoXML(reporoot, self.cmds) + except dumpMetadata.MDError, e: + errorprint(_('Error generating repo xml file: %s') % e) + sys.exit(1) + + try: + repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1) + except: + errorprint(_('Error saving temp file for rep xml: %s') % repofilepath) + sys.exit(1) + + del repodoc + +class SplitMetaDataGenerator(MetaDataGenerator): + + def __init__(self, cmds): + MetaDataGenerator.__init__(self, cmds) + self.initialdir = self.cmds['basedir'] + + def _getFragmentUrl(self, url, fragment): + import urlparse + urlparse.uses_fragment.append('media') + if not url: + return url + (scheme, netloc, path, query, fragid) = urlparse.urlsplit(url) + return urlparse.urlunsplit((scheme, netloc, path, query, str(fragment))) + + def doPkgMetadata(self, directories): + """all the heavy lifting for the package metadata""" + import types + if type(directories) == types.StringType: + MetaDataGenerator.doPkgMetadata(self, directories) + return + filematrix = {} + for mydir in directories: + filematrix[mydir] = self.getFileList(os.path.join(self.initialdir, mydir), '.', '.rpm', []) + self.trimRpms(filematrix[mydir]) + self.pkgcount += len(filematrix[mydir]) + + mediano = 1 + current = 0 + self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], mediano) + self.cmds['basedir'] = os.path.join(self.initialdir, directories[0]) + self.openMetadataDocs() + for mydir in directories: + self.cmds['basedir'] = os.path.join(self.initialdir, mydir) + self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], mediano) + current = self.writeMetadataDocs(filematrix[mydir], current) + mediano += 1 + self.cmds['basedir'] = os.path.join(self.initialdir, directories[0]) + self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], 1) + self.closeMetadataDocs() -def trimRpms(rpms, excludeGlobs): - # print 'Pre-Trim Len: %d' % len(rpms) - badrpms = [] - for file in rpms: - for glob in excludeGlobs: - if fnmatch.fnmatch(file, glob): - # print 'excluded: %s' % file - if file not in badrpms: - badrpms.append(file) - for file in badrpms: - if file in rpms: - rpms.remove(file) - # print 'Post-Trim Len: %d' % len(rpms) - return rpms def checkAndMakeDir(dir): """ @@ -119,7 +352,6 @@ result = True return result - def parseArgs(args): """ Parse the command line args return a commands dict and directory. @@ -135,42 +367,48 @@ cmds['pretty'] = 0 # cmds['updategroupsonly'] = 0 cmds['cachedir'] = None + cmds['basedir'] = os.getcwd() cmds['cache'] = False + cmds['split'] = False + cmds['outputdir'] = "" cmds['file-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$'] cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*'] try: - gopts, argsleft = getopt.getopt(args, 'phqVvg:s:x:u:c:', ['help', 'exclude=', - 'quiet', 'verbose', 'cachedir=', - 'baseurl=', 'groupfile=', 'checksum=', - 'version', 'pretty']) + gopts, argsleft = getopt.getopt(args, 'phqVvg:s:x:u:c:U:o:', ['help', 'exclude=', + 'quiet', 'verbose', 'cachedir=', 'basedir=', + 'baseurl=', 'groupfile=', 'checksum=', + 'version', 'pretty', 'split', 'outputdir=', + 'update-info-location=']) except getopt.error, e: errorprint(_('Options Error: %s.') % e) usage() - try: + try: for arg,a in gopts: if arg in ['-h','--help']: usage(retval=0) elif arg in ['-V', '--version']: print '%s' % __version__ sys.exit(0) + elif arg == '--split': + cmds['split'] = True except ValueError, e: errorprint(_('Options Error: %s') % e) usage() - + # make sure our dir makes sense before we continue - if len(argsleft) > 1: + if len(argsleft) > 1 and not cmds['split']: errorprint(_('Error: Only one directory allowed per run.')) usage() elif len(argsleft) == 0: errorprint(_('Error: Must specify a directory to index.')) usage() else: - directory = argsleft[0] - - try: + directories = argsleft + + try: for arg,a in gopts: if arg in ['-v', '--verbose']: cmds['verbose'] = 1 @@ -187,11 +425,7 @@ errorprint(_('Error: Only one groupfile allowed.')) usage() else: - if os.path.exists(directory + '/' + a): - cmds['groupfile'] = a - else: - errorprint(_('Error: groupfile %s cannot be found.' % a)) - usage() + cmds['groupfile'] = a elif arg in ['-x', '--exclude']: cmds['excludes'].append(a) elif arg in ['-p', '--pretty']: @@ -203,167 +437,49 @@ elif arg in ['-c', '--cachedir']: cmds['cache'] = True cmds['cachedir'] = a - if not checkAndMakeDir(a): - errorprint(_('Error: cannot open/write to cache dir %s' % a)) - usage() + elif arg in ['-U', '--update-info-location']: + cmds['update-info-location'] = a + elif arg == '--basedir': + cmds['basedir'] = a + elif arg in ['-o','--outputdir']: + cmds['outputdir'] = a except ValueError, e: errorprint(_('Options Error: %s') % e) usage() - - return cmds, directory - -def doPkgMetadata(cmds, ts): - """all the heavy lifting for the package metadata""" - - # rpms we're going to be dealing with - files = [] - files = getFileList('./', '.rpm', files) - files = trimRpms(files, cmds['excludes']) - pkgcount = len(files) - - # setup the base metadata doc - basedoc = libxml2.newDoc("1.0") - baseroot = basedoc.newChild(None, "metadata", None) - basens = baseroot.newNs('http://linux.duke.edu/metadata/common', None) - formatns = baseroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm') - baseroot.setNs(basens) - basefilepath = os.path.join(cmds['tempdir'], cmds['primaryfile']) - basefile = _gzipOpen(basefilepath, 'w') - basefile.write('<?xml version="1.0" encoding="UTF-8"?>\n') - basefile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">\n' % - pkgcount) - - # setup the file list doc - filesdoc = libxml2.newDoc("1.0") - filesroot = filesdoc.newChild(None, "filelists", None) - filesns = filesroot.newNs('http://linux.duke.edu/metadata/filelists', None) - filesroot.setNs(filesns) - filelistpath = os.path.join(cmds['tempdir'], cmds['filelistsfile']) - flfile = _gzipOpen(filelistpath, 'w') - flfile.write('<?xml version="1.0" encoding="UTF-8"?>\n') - flfile.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists" packages="%s">\n' % - pkgcount) - - - # setup the other doc - otherdoc = libxml2.newDoc("1.0") - otherroot = otherdoc.newChild(None, "otherdata", None) - otherns = otherroot.newNs('http://linux.duke.edu/metadata/other', None) - otherroot.setNs(otherns) - otherfilepath = os.path.join(cmds['tempdir'], cmds['otherfile']) - otherfile = _gzipOpen(otherfilepath, 'w') - otherfile.write('<?xml version="1.0" encoding="UTF-8"?>\n') - otherfile.write('<otherdata xmlns="http://linux.duke.edu/metadata/other" packages="%s">\n' % - pkgcount) - - - current = 0 - for file in files: - current+=1 - try: - mdobj = dumpMetadata.RpmMetaData(ts, file, cmds) - if not cmds['quiet']: - if cmds['verbose']: - print '%d/%d - %s' % (current, len(files), file) - else: - sys.stdout.write('\r' + ' ' * 80) - sys.stdout.write("\r%d/%d - %s" % (current, len(files), file)) - sys.stdout.flush() - except dumpMetadata.MDError, e: - errorprint('\n%s - %s' % (e, file)) - continue - else: - try: - node = dumpMetadata.generateXML(basedoc, baseroot, formatns, mdobj, cmds['sumtype']) - except dumpMetadata.MDError, e: - errorprint(_('\nAn error occurred creating primary metadata: %s') % e) - continue - else: - output = node.serialize('UTF-8', cmds['pretty']) - basefile.write(output) - basefile.write('\n') - node.unlinkNode() - node.freeNode() - del node - - try: - node = dumpMetadata.fileListXML(filesdoc, filesroot, mdobj) - except dumpMetadata.MDError, e: - errorprint(_('\nAn error occurred creating filelists: %s') % e) - continue - else: - output = node.serialize('UTF-8', cmds['pretty']) - flfile.write(output) - flfile.write('\n') - node.unlinkNode() - node.freeNode() - del node - - try: - node = dumpMetadata.otherXML(otherdoc, otherroot, mdobj) - except dumpMetadata.MDError, e: - errorprint(_('\nAn error occurred: %s') % e) - continue - else: - output = node.serialize('UTF-8', cmds['pretty']) - otherfile.write(output) - otherfile.write('\n') - node.unlinkNode() - node.freeNode() - del node - - - if not cmds['quiet']: - print '' - - # save them up to the tmp locations: - if not cmds['quiet']: - print _('Saving Primary metadata') - basefile.write('\n</metadata>') - basefile.close() - basedoc.freeDoc() - - if not cmds['quiet']: - print _('Saving file lists metadata') - flfile.write('\n</filelists>') - flfile.close() - filesdoc.freeDoc() - - if not cmds['quiet']: - print _('Saving other metadata') - otherfile.write('\n</otherdata>') - otherfile.close() - otherdoc.freeDoc() - -def doRepoMetadata(cmds): - """wrapper to generate the repomd.xml file that stores the info on the other files""" - repodoc = libxml2.newDoc("1.0") - reporoot = repodoc.newChild(None, "repomd", None) - repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None) - reporoot.setNs(repons) - repofilepath = os.path.join(cmds['tempdir'], cmds['repomdfile']) - - try: - dumpMetadata.repoXML(reporoot, cmds) - except dumpMetadata.MDError, e: - errorprint(_('Error generating repo xml file: %s') % e) - sys.exit(1) - - try: - repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1) - except: - errorprint(_('Error saving temp file for rep xml: %s') % repofilepath) - sys.exit(1) - - del repodoc - - + directory = directories[0] +# Fix paths + directory = os.path.normpath(directory) + if cmds['split']: + pass + elif os.path.isabs(directory): + cmds['basedir'] = directory + directory = '.' + else: + cmds['basedir'] = os.path.realpath(os.path.join(cmds['basedir'], directory)) + directory = '.' + if not cmds['outputdir']: + cmds['outputdir'] = cmds['basedir'] + if cmds['groupfile']: + a = cmds['groupfile'] + if cmds['split']: + a = os.path.join(cmds['basedir'], directory, cmds['groupfile']) + elif not os.path.isabs(a): + a = os.path.join(cmds['basedir'], cmds['groupfile']) + if not os.path.exists(a): + errorprint(_('Error: groupfile %s cannot be found.' % a)) + usage() + cmds['groupfile'] = a + if cmds['cachedir']: + a = cmds ['cachedir'] + if not os.path.isabs(a): + a = os.path.join(cmds['basedir'] ,a) + if not checkAndMakeDir(a): + errorprint(_('Error: cannot open/write to cache dir %s' % a)) + usage() + cmds['cachedir'] = a -def main(args): - cmds, directory = parseArgs(args) - #setup some defaults cmds['primaryfile'] = 'primary.xml.gz' cmds['filelistsfile'] = 'filelists.xml.gz' @@ -372,108 +488,112 @@ cmds['tempdir'] = '.repodata' cmds['finaldir'] = 'repodata' cmds['olddir'] = '.olddata' - - # save where we are right now - curdir = os.getcwd() + cmds['update-info-dir'] = 'update-info' + + return cmds, directories + +def main(args): + cmds, directories = parseArgs(args) + directory = directories[0] # start the sanity/stupidity checks - if not os.path.exists(directory): + if not os.path.exists(os.path.join(cmds['basedir'], directory)): errorprint(_('Directory must exist')) sys.exit(1) - - if not os.path.isdir(directory): + + if not os.path.isdir(os.path.join(cmds['basedir'], directory)): errorprint(_('Directory of packages must be a directory.')) sys.exit(1) - - if not os.access(directory, os.W_OK): + + if not os.access(cmds['outputdir'], os.W_OK): errorprint(_('Directory must be writable.')) sys.exit(1) - - if not checkAndMakeDir(os.path.join(directory, cmds['tempdir'])): + if cmds['split']: + oldbase = cmds['basedir'] + cmds['basedir'] = os.path.join(cmds['basedir'], directory) + if not checkAndMakeDir(os.path.join(cmds['outputdir'], cmds['tempdir'])): sys.exit(1) - - if not checkAndMakeDir(os.path.join(directory, cmds['finaldir'])): + + if not checkAndMakeDir(os.path.join(cmds['outputdir'], cmds['finaldir'])): sys.exit(1) - - if os.path.exists(os.path.join(directory, cmds['olddir'])): + + if os.path.exists(os.path.join(cmds['outputdir'], cmds['olddir'])): errorprint(_('Old data directory exists, please remove: %s') % cmds['olddir']) sys.exit(1) - - # change to the basedir to work from w/i the path - for relative url paths - os.chdir(directory) + + if cmds.has_key('update-info-location'): + if not checkAndMakeDir(os.path.join(cmds['basedir'], + cmds['tempdir'], cmds['update-info-dir'])): + errorprint(_('Error: cannot open/write to update info dir %s' % a)) + usage() # make sure we can write to where we want to write to: for direc in ['tempdir', 'finaldir']: for file in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile']: - filepath = os.path.join(cmds[direc], cmds[file]) + filepath = os.path.join(cmds['outputdir'], cmds[direc], cmds[file]) if os.path.exists(filepath): if not os.access(filepath, os.W_OK): errorprint(_('error in must be able to write to metadata files:\n -> %s') % filepath) - os.chdir(curdir) usage() - - ts = rpm.TransactionSet() - try: - doPkgMetadata(cmds, ts) - except: - # always clean up your messes - os.chdir(curdir) - raise - - try: - doRepoMetadata(cmds) - except: - os.chdir(curdir) - raise - - if os.path.exists(cmds['finaldir']): + + if cmds['split']: + cmds['basedir'] = oldbase + mdgen = SplitMetaDataGenerator(cmds) + mdgen.doPkgMetadata(directories) + else: + mdgen = MetaDataGenerator(cmds) + mdgen.doPkgMetadata(directory) + mdgen.doRepoMetadata() + + if os.path.exists(os.path.join(cmds['outputdir'], cmds['finaldir'])): try: - os.rename(cmds['finaldir'], cmds['olddir']) + os.rename(os.path.join(cmds['outputdir'], cmds['finaldir']), + os.path.join(cmds['outputdir'], cmds['olddir'])) except: - errorprint(_('Error moving final to old dir')) - os.chdir(curdir) + errorprint(_('Error moving final %s to old dir %s' % (os.path.join(cmds['outputdir'], cmds['finaldir']), + os.path.join(cmds['outputdir'], cmds['olddir'])))) sys.exit(1) - + try: - os.rename(cmds['tempdir'], cmds['finaldir']) + os.rename(os.path.join(cmds['outputdir'], cmds['tempdir']), + os.path.join(cmds['outputdir'], cmds['finaldir'])) except: errorprint(_('Error moving final metadata into place')) # put the old stuff back - os.rename(cmds['olddir'], cmds['finaldir']) - os.chdir(curdir) + os.rename(os.path.join(cmds['outputdir'], cmds['olddir']), + os.path.join(cmds['outputdir'], cmds['finaldir'])) sys.exit(1) - + for file in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', 'groupfile']: if cmds[file]: fn = os.path.basename(cmds[file]) else: continue - oldfile = os.path.join(cmds['olddir'], fn) + oldfile = os.path.join(cmds['outputdir'], cmds['olddir'], fn) if os.path.exists(oldfile): try: os.remove(oldfile) except OSError, e: errorprint(_('Could not remove old metadata file: %s') % oldfile) errorprint(_('Error was %s') % e) - os.chdir(curdir) sys.exit(1) - + + # Clean up any update metadata + mdpath = os.path.join(cmds['basedir'], cmds['olddir'], cmds['update-info-dir']) + if os.path.isdir(mdpath): + for file in os.listdir(mdpath): + os.remove(os.path.join(mdpath, file)) + os.rmdir(mdpath) + + +#XXX: fix to remove tree as we mung basedir try: - os.rmdir(cmds['olddir']) + os.rmdir(os.path.join(cmds['outputdir'], cmds['olddir'])) except OSError, e: errorprint(_('Could not remove old metadata dir: %s') % cmds['olddir']) errorprint(_('Error was %s') % e) errorprint(_('Please clean up this directory manually.')) - os.chdir(curdir) - - - - - # take us home mr. data - os.chdir(curdir) - - if __name__ == "__main__": if len(sys.argv) > 1: if sys.argv[1] == 'profile': @@ . ______________________________________________________________________ RPM Package Manager http://rpm5.org CVS Sources Repository [email protected]
