RPM Package Manager, CVS Repository http://rpm5.org/cvs/ ____________________________________________________________________________
Server: rpm5.org Name: Jeff Johnson Root: /v/rpm/cvs Email: [email protected] Module: createrepo Date: 07-Aug-2010 01:41:44 Branch: VENDOR Handle: 2010080623414300 Added files: (Branch: VENDOR) createrepo modifyrepo.py createrepo/bin modifyrepo Modified files: (Branch: VENDOR) createrepo ChangeLog Makefile createrepo.spec dumpMetadata.py genpkgmetadata.py createrepo/bin Makefile createrepo/docs createrepo.8 Touched files: (Branch: VENDOR) createrepo/bin createrepo createrepo/docs Makefile Log: Import createrepo-0.4.7 [Release Tag: createrepo-0_4_7] Summary: Revision Changes Path 1.1.1.10 +14 -0 createrepo/ChangeLog 1.1.1.19 +3 -2 createrepo/Makefile 1.1.1.3 +0 -0 createrepo/bin/Makefile 1.1.1.1 +0 -0 createrepo/bin/modifyrepo 1.1.1.19 +9 -1 createrepo/createrepo.spec 1.1.1.5 +0 -0 createrepo/docs/createrepo.8 1.1.1.17 +82 -4 createrepo/dumpMetadata.py 1.1.1.19 +10 -6 createrepo/genpkgmetadata.py 1.1.1.1 +124 -0 createrepo/modifyrepo.py ____________________________________________________________________________ patch -p0 <<'@@ .' Index: createrepo/ChangeLog ============================================================================ $ cvs diff -u -r1.1.1.9 -r1.1.1.10 ChangeLog --- createrepo/ChangeLog 6 Aug 2010 23:41:19 -0000 1.1.1.9 +++ createrepo/ChangeLog 6 Aug 2010 23:41:43 -0000 1.1.1.10 @@ -1,3 +1,17 @@ +2006-10-21 10:54 lmacken + + * modifyrepo.py: use the mdname for the 'href' element, so it doesn't + explode when dealing with xml.dom.minidom.Document objects. + +2006-10-14 01:30 lmacken + + * Makefile, bin/Makefile, createrepo.spec: Makefile changes for + modifyrepo, and added it to the spec as well. + +2006-08-23 15:40 lmacken + + * modifyrepo.py, bin/modifyrepo: Initial import + 2006-08-11 16:03 pnasrat * createrepo.spec: update date @@ . patch -p0 <<'@@ .' Index: createrepo/Makefile ============================================================================ $ cvs diff -u -r1.1.1.18 -r1.1.1.19 Makefile --- createrepo/Makefile 6 Aug 2010 23:41:19 -0000 1.1.1.18 +++ createrepo/Makefile 6 Aug 2010 23:41:43 -0000 1.1.1.19 @@ -1,5 +1,5 @@ PACKAGE = createrepo -VERSION = 0.4.6 +VERSION = 0.4.7 SHELL = /bin/sh top_srcdir = . srcdir = . @@ -40,7 +40,8 @@ SUBDIRS = bin docs MODULES = $(srcdir)/genpkgmetadata.py \ - $(srcdir)/dumpMetadata.py + $(srcdir)/dumpMetadata.py \ + $(srcdir)/modifyrepo.py .SUFFIXES: .py .pyc .py.pyc: @@ . patch -p0 <<'@@ .' Index: createrepo/bin/Makefile ============================================================================ $ cvs diff -u -r1.1.1.2 -r1.1.1.3 bin/Makefile @@ . patch -p0 <<'@@ .' Index: createrepo/bin/modifyrepo ============================================================================ $ cvs diff -u -r0 -r1.1.1.1 bin/modifyrepo @@ . patch -p0 <<'@@ .' Index: createrepo/createrepo.spec ============================================================================ $ cvs diff -u -r1.1.1.18 -r1.1.1.19 createrepo.spec --- createrepo/createrepo.spec 6 Aug 2010 23:41:19 -0000 1.1.1.18 +++ createrepo/createrepo.spec 6 Aug 2010 23:41:43 -0000 1.1.1.19 @@ -1,6 +1,6 @@ Summary: Creates a common metadata repository Name: createrepo -Version: 0.4.6 +Version: 0.4.7 Release: 1 License: GPL Group: System Environment/Base @@ -9,6 +9,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}root BuildArchitectures: noarch Requires: python >= 2.1, rpm-python, rpm >= 0:4.1.1, libxml2-python +Requires: yum-metadata-parser %description This utility will generate a common metadata repository from a directory of @@ -31,9 +32,16 @@ %doc ChangeLog README %{_datadir}/%{name}/* %{_bindir}/%{name} +%{_bindir}/modifyrepo %{_mandir}/man8/createrepo.8* %changelog +* Tue Feb 6 2007 Seth Vidal <skvidal at linux.duke.edu> +- 0.4.7 and yum-metadata-parser dependency + +* Sat Oct 14 2006 Luke Macken <[email protected]> +- Add modifyrepo + * Fri Aug 11 2006 Paul Nasrat <[email protected]> - 0.4.6 @@ . patch -p0 <<'@@ .' Index: createrepo/docs/createrepo.8 ============================================================================ $ cvs diff -u -r1.1.1.4 -r1.1.1.5 docs/createrepo.8 @@ . patch -p0 <<'@@ .' Index: createrepo/dumpMetadata.py ============================================================================ $ cvs diff -u -r1.1.1.16 -r1.1.1.17 dumpMetadata.py --- createrepo/dumpMetadata.py 6 Aug 2010 23:41:19 -0000 1.1.1.16 +++ createrepo/dumpMetadata.py 6 Aug 2010 23:41:43 -0000 1.1.1.17 @@ -16,7 +16,7 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2004 Duke University -# $Id: dumpMetadata.py,v 1.1.1.16 2010/08/06 23:41:19 jbj Exp $ +# $Id: dumpMetadata.py,v 1.1.1.17 2010/08/06 23:41:43 jbj Exp $ import os import rpm @@ -27,6 +27,8 @@ import struct import re import stat +import bz2 +import sqlitecachec # done to fix gzip randomly changing the checksum import gzip @@ -54,7 +56,20 @@ def _gzipOpen(filename, mode="rb", compresslevel=9): return GzipFile(filename, mode, compresslevel) +def bzipFile(source, dest): + + s_fn = open(source, 'rb') + destination = bz2.BZ2File(dest, 'w', compresslevel=9) + while True: + data = s_fn.read(1024000) + + if not data: break + destination.write(data) + + destination.close() + s_fn.close() + def returnFD(filename): try: @@ -748,14 +763,75 @@ workfiles = [(cmds['otherfile'], 'other',), (cmds['filelistsfile'], 'filelists'), (cmds['primaryfile'], 'primary')] + repoid='garbageid' + repopath = os.path.join(cmds['outputdir'], cmds['tempdir']) + if cmds['database']: + dbversion = '9' + rp = sqlitecachec.RepodataParserSqlite(repopath, repoid, None) + for (file, ftype) in workfiles: - zfo = _gzipOpen(os.path.join(cmds['outputdir'], cmds['tempdir'], file)) + complete_path = os.path.join(repopath, file) + + zfo = _gzipOpen(complete_path) uncsum = getChecksum(sumtype, zfo) zfo.close() - csum = getChecksum(sumtype, os.path.join(cmds['outputdir'], cmds['tempdir'], file)) - timestamp = os.stat(os.path.join(cmds['outputdir'], cmds['tempdir'], file))[8] + csum = getChecksum(sumtype, complete_path) + timestamp = os.stat(complete_path)[8] + + db_csums = {} + db_compressed_sums = {} + + if cmds['database']: + if ftype == 'primary': + rp.getPrimary(complete_path, csum) + + elif ftype == 'filelists': + rp.getFilelists(complete_path, csum) + + elif ftype == 'other': + rp.getOtherdata(complete_path, csum) + + + tmp_result_name = '%s.xml.gz.sqlite' % ftype + tmp_result_path = os.path.join(repopath, tmp_result_name) + good_name = '%s.sqlite' % ftype + resultpath = os.path.join(repopath, good_name) + + # rename from silly name to not silly name + os.rename(tmp_result_path, resultpath) + compressed_name = '%s.bz2' % good_name + result_compressed = os.path.join(repopath, compressed_name) + db_csums[ftype] = getChecksum(sumtype, resultpath) + + # compress the files + bzipFile(resultpath, result_compressed) + # csum the compressed file + db_compressed_sums[ftype] = getChecksum(sumtype, result_compressed) + # remove the uncompressed file + os.unlink(resultpath) + + # timestamp the compressed file + db_timestamp = os.stat(result_compressed)[8] + + # add this data as a section to the repomdxml + db_data_type = '%s_db' % ftype + data = node.newChild(None, 'data', None) + data.newProp('type', db_data_type) + location = data.newChild(None, 'location', None) + if cmds['baseurl'] is not None: + location.newProp('xml:base', cmds['baseurl']) + + location.newProp('href', os.path.join(cmds['finaldir'], compressed_name)) + checksum = data.newChild(None, 'checksum', db_compressed_sums[ftype]) + checksum.newProp('type', sumtype) + db_tstamp = data.newChild(None, 'timestamp', str(db_timestamp)) + unchecksum = data.newChild(None, 'open-checksum', db_csums[ftype]) + unchecksum.newProp('type', sumtype) + database_version = data.newChild(None, 'database_version', dbversion) + + data = node.newChild(None, 'data', None) data.newProp('type', ftype) location = data.newChild(None, 'location', None) @@ -790,3 +866,5 @@ checksum = data.newChild(None, 'checksum', csum) checksum.newProp('type', sumtype) timestamp = data.newChild(None, 'timestamp', str(timestamp)) + + @@ . patch -p0 <<'@@ .' Index: createrepo/genpkgmetadata.py ============================================================================ $ cvs diff -u -r1.1.1.18 -r1.1.1.19 genpkgmetadata.py --- createrepo/genpkgmetadata.py 6 Aug 2010 23:41:19 -0000 1.1.1.18 +++ createrepo/genpkgmetadata.py 6 Aug 2010 23:41:43 -0000 1.1.1.19 @@ -17,7 +17,7 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2004 Duke University -# $Id: genpkgmetadata.py,v 1.1.1.18 2010/08/06 23:41:19 jbj Exp $ +# $Id: genpkgmetadata.py,v 1.1.1.19 2010/08/06 23:41:43 jbj Exp $ import os @@ -31,7 +31,7 @@ import dumpMetadata from dumpMetadata import _gzipOpen -__version__ = '0.4.6' +__version__ = '0.4.7' def errorprint(stuff): print >> sys.stderr, stuff @@ -61,6 +61,7 @@ -h, --help = show this help -V, --version = output version -p, --pretty = output xml files in pretty format. + -d, --database = generate the sqlite databases. """) sys.exit(retval) @@ -377,15 +378,16 @@ cmds['mdtimestamp'] = 0 cmds['split'] = False cmds['outputdir'] = "" + cmds['database'] = False cmds['file-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$'] cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*'] try: - gopts, argsleft = getopt.getopt(args, 'phqVvng:s:x:u:c:o:C', ['help', 'exclude=', + gopts, argsleft = getopt.getopt(args, 'phqVvndg:s:x:u:c:o:C', ['help', 'exclude=', 'quiet', 'verbose', 'cachedir=', 'basedir=', 'baseurl=', 'groupfile=', 'checksum=', 'version', 'pretty', 'split', 'outputdir=', - 'noepoch', 'checkts']) + 'noepoch', 'checkts', 'database']) except getopt.error, e: errorprint(_('Options Error: %s.') % e) usage() @@ -451,7 +453,9 @@ cmds['outputdir'] = a elif arg in ['-n', '--noepoch']: cmds['noepoch'] = True - + elif arg in ['-d', '--database']: + cmds['database'] = True + except ValueError, e: errorprint(_('Options Error: %s') % e) usage() @@ -461,7 +465,7 @@ sys.exit(1) directory = directories[0] -# + directory = os.path.normpath(directory) if cmds['split']: pass @@ . patch -p0 <<'@@ .' Index: createrepo/modifyrepo.py ============================================================================ $ cvs diff -u -r0 -r1.1.1.1 modifyrepo.py --- /dev/null 2010-08-07 01:38:43.000000000 +0200 +++ modifyrepo.py 2010-08-07 01:41:44.664052410 +0200 @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# $Id: modifyrepo.py,v 1.1.1.1 2010/08/06 23:41:43 jbj Exp $ +# +# This tools is used to insert arbitrary metadata into an RPM repository. +# Example: +# ./modifyrepo.py updateinfo.xml myrepo/repodata +# or in Python: +# >>> from modifyrepo import RepoMetadata +# >>> repomd = RepoMetadata('myrepo/repodata') +# >>> repomd.add('updateinfo.xml') +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# (C) Copyright 2006 Red Hat, Inc. +# Luke Macken <[email protected]> + +import os +import sys +import sha +import gzip + +from xml.dom import minidom + +class RepoMetadata: + + def __init__(self, repo): + """ Parses the repomd.xml file existing in the given repo directory. """ + self.repodir = os.path.abspath(repo) + self.repomdxml = os.path.join(self.repodir, 'repomd.xml') + if not os.path.exists(self.repomdxml): + raise Exception('%s not found' % self.repomdxml) + self.doc = minidom.parse(self.repomdxml) + + def _insert_element(self, parent, name, attrs={}, text=None): + child = self.doc.createElement(name) + for item in attrs.items(): + child.setAttribute(item[0], item[1]) + if text: + txtnode = self.doc.createTextNode(text) + child.appendChild(txtnode) + parent.appendChild(child) + return child + + def add(self, metadata): + """ Insert arbitrary metadata into this repository. + metadata can be either an xml.dom.minidom.Document object, or + a filename. + """ + md = None + if not metadata: + raise Exception('metadata cannot be None') + if isinstance(metadata, minidom.Document): + md = metadata.toxml() + mdname = 'updateinfo.xml' + elif isinstance(metadata, str): + if os.path.exists(metadata): + oldmd = file(metadata, 'r') + md = oldmd.read() + oldmd.close() + mdname = os.path.basename(metadata) + else: + raise Exception('%s not found' % metadata) + else: + raise Exception('invalid metadata type') + + ## Compress the metadata and move it into the repodata + mdname += '.gz' + mdtype = mdname.split('.')[0] + destmd = os.path.join(self.repodir, mdname) + newmd = gzip.GzipFile(destmd, 'wb') + newmd.write(md) + newmd.close() + print "Wrote:", destmd + + ## Read the gzipped metadata + f = file(destmd, 'r') + newmd = f.read() + f.close() + + ## Remove any stale metadata + for elem in self.doc.getElementsByTagName('data'): + if elem.attributes['type'].value == mdtype: + self.doc.firstChild.removeChild(elem) + + ## Build the metadata + root = self.doc.firstChild + data = self._insert_element(root, 'data', attrs={ 'type' : mdtype }) + self._insert_element(data, 'location', + attrs={ 'href' : 'repodata/' + mdname }) + self._insert_element(data, 'checksum', attrs={ 'type' : 'sha' }, + text=sha.new(newmd).hexdigest()) + self._insert_element(data, 'timestamp', + text=str(os.stat(destmd).st_mtime)) + self._insert_element(data, 'open-checksum', attrs={ 'type' : 'sha' }, + text=sha.new(md).hexdigest()) + + print " type =", mdtype + print " location =", 'repodata/' + mdname + print " checksum =", sha.new(newmd).hexdigest() + print " timestamp =", str(os.stat(destmd).st_mtime) + print " open-checksum =", sha.new(md).hexdigest() + + ## Write the updated repomd.xml + outmd = file(self.repomdxml, 'w') + self.doc.writexml(outmd) + outmd.close() + print "Wrote:", self.repomdxml + + +if __name__ == '__main__': + if len(sys.argv) != 3 or '-h' in sys.argv: + print "Usage: %s <input metadata> <output repodata>" % sys.argv[0] + sys.exit() + + repomd = RepoMetadata(sys.argv[2]) + repomd.add(sys.argv[1]) @@ . ______________________________________________________________________ RPM Package Manager http://rpm5.org CVS Sources Repository [email protected]
