---
yum/deltamd.py | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
yum/yumRepo.py | 26 ++++++++++++-
2 files changed, 141 insertions(+), 1 deletion(-)
create mode 100644 yum/deltamd.py
diff --git a/yum/deltamd.py b/yum/deltamd.py
new file mode 100644
index 0000000..686eacd
--- /dev/null
+++ b/yum/deltamd.py
@@ -0,0 +1,116 @@
+# Delta metadata support
+# Copyright 2013 Zdenek Pavlas
+
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330,
+# Boston, MA 02111-1307 USA
+
+from yum.Errors import MiscError
+from yum.misc import _decompress_chunked
+
+def splitter(read, pattern):
+ ''' Read the stream, splitting data at each pattern instance.
+ Split at the last '</', too, so XML with N elements below
+ the root yields exactly N+2 items.
+ '''
+ buf = ''
+ while True:
+ more = read(0x4000)
+ if not more: break
+ buf += more
+ i = 0
+ while True:
+ j = buf.find(pattern, i + len(pattern))
+ if j == -1: break
+ yield buf[i:j]
+ i = j
+ buf = buf[i:]
+ i = buf.rfind('</')
+ if i != -1:
+ yield buf[:i]
+ buf = buf[i:]
+ yield buf
+
+from hashlib import sha1 as hash_func
+
+def apply_delta(old, delta, new, opt='', pattern=None):
+ ''' Use the compressed delta file to update from old to new.
+ Delta is line-oriented stream of literals (encoded as +<N>\n
+ followed by N bytes) and old data references (guaranteed not
+ to start with the '+' sign). Supported options are:
+
+ [h]ashrefs: reference old data with hash values, not ordinals.
+ This is more flexible but less efficient.
+
+ [s]equential: delta uses each piece of old data at most once
+ and in the original order. This implies that we don't need
+ to store them for later use, and may use prefix matching.
+ '''
+
+ hash_refs = 'h' in opt
+ sequential = 's' in opt
+
+ # copying uncompressed old to new..
+ next = splitter(open(old, 'rb').read, pattern or '<package ').next
+ write = open(new, 'wb').write
+
+ n = -1 # last index used
+ lookup = {} if hash_refs else []
+ f = _decompress_chunked(delta, None, None)
+ while True:
+ l = f.readline()
+ if not l: break
+ if l[0] == '+':
+
+ # literal chunk
+ write(f.read(int(l[1:])))
+ continue
+
+ if hash_refs:
+ if sequential:
+ hint = int(l[0], 16)
+ needle = l[1:-1]
+ while True:
+ data = next()
+ if len(data) & 0xf != hint:
+ continue
+ if hash_func(data).hexdigest().startswith(needle):
+ break
+ else:
+ needle = l[:-1]
+ while needle not in lookup:
+ data = next()
+ hv = hash_func(data).hexdigest()
+ lookup[hv] = data
+ data = lookup[needle]
+ else:
+ if sequential:
+ if l != '\n':
+ skip = int(l)
+ while skip:
+ skip -= 1
+ next()
+ data = next()
+ else:
+ n += 1
+ if l != '\n':
+ n = int(l)
+ while len(lookup) <= n:
+ data = next()
+ lookup.append(data)
+ data = lookup[n]
+
+ # copy from old
+ write(data)
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index f409485..3b49f56 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -51,6 +51,7 @@ import shutil
import stat
import errno
import tempfile
+from yum.deltamd import apply_delta
# This is unused now, probably nothing uses it but it was global/public.
skip_old_DBMD_check = False
@@ -1627,11 +1628,34 @@ Insufficient space in download directory %s
newmdfiles.append(self._get_mdtype_fname(ndata, False))
return downloading
+ def _applyDelta(self, deltamd):
+ """ Apply delta metadata file and checksum the destination """
+
+ full, opt = deltamd.delta
+ local = self._get_mdtype_fname(deltamd)
+ new = self.cachedir + '/gen/%s.xml' % full
+ try:
+ apply_delta(new + '.old.tmp', local, new, opt)
+ self._checkMD(new, full, openchecksum=True)
+ except (Errors.MiscError, URLGrabError), e:
+ logger.warning(_('deltamd %s/%s failure: %s'), self, deltamd.type,
e)
+ return False
+ ts = int(self.repoXML.getData(full).timestamp)
+ os.utime(new, (ts, ts))
+ os.unlink(local)
+ return True
+
def _commonRetrieveDataMD_done(self, downloading):
""" Uncompress the downloaded metadata """
for (ndata, nmdtype) in downloading:
- local = self._get_mdtype_fname(ndata, False)
+ if ndata.delta and not self._applyDelta(ndata):
+ # fallback to full metadata or revert
+ full, opt = deltamd.delta
+ if not self._retrieveMD(full, retrieve_can_fail=True):
+ self._revertOldRepoXML()
+ return False
+
self._doneOldRepoXML()
return True
--
1.7.11.7
_______________________________________________
Yum-devel mailing list
[email protected]
http://lists.baseurl.org/mailman/listinfo/yum-devel
