Hi!
I was curious if all these ideas really translate into a performance
improvement. Attached patch reenables delta resolving and resolves
FileRequires globally.
Delta Resolving:
I moved the delta list into the TransactionData object to get really all
changes (got caught by the installonlyn plugin). Now packages are only
removed from the delta if there are no unresolved dependencies. This reliefs
the depsolver from having to perfectly solve everything (see rev 1.138 of
depsolver.py)
I am still a bit unsure about TransactionData.remove(). To make delta
resolving work removed installs have to be treated as removes. Can this
currently happen at all?
Global File Require Resolving:
Most stuff about it is already said. Current implementation always checks
file requirements. This could be avoided if there are no erases/updates
since the last successful (everything ok) run.
On my PC "echo n |yum update" is now appr. 100% faster when trying
to update 60 pkgs. Larger and more complicated cases or packages with lots
of files may gain even more.
When I create PkgPrco indices in the sqlite I get another 7% speed up which
is not much but not surprising as we only load a few pkg from the sqlite db.
have fun
Florian
PS: I have the feeling that the yum sqlite db interface is slow. It might be
a problem of either generating a cursor on every search or/and pkgId=pkgKey
merge. But didn't do any measurements yet.
Index: yum/depsolve.py
===================================================================
RCS file: /cvsroot/yum/cvs/yum/yum/depsolve.py,v
retrieving revision 1.142
diff -u -r1.142 depsolve.py
--- yum/depsolve.py 19 Mar 2007 20:29:03 -0000 1.142
+++ yum/depsolve.py 20 Mar 2007 14:35:41 -0000
@@ -43,7 +43,6 @@
self.dsCallback = None
self.logger = logging.getLogger("yum.Depsolve")
self.verbose_logger = logging.getLogger("yum.verbose.Depsolve")
- self.tsInfoDelta = []
def doTsSetup(self):
warnings.warn('doTsSetup() will go away in a future version of Yum.\n',
@@ -510,7 +509,6 @@
requiringPo, needname)
txmbr = self.tsInfo.addErase(requiringPo)
txmbr.setAsDep(po=needpo)
- self.tsInfoDelta.append(txmbr)
checkdeps = 1
if needmode in ['i', 'u']:
@@ -532,7 +530,6 @@
txmbr = self.tsInfo.addObsoleting(po, requiringPo)
self.tsInfo.addObsoleted(requiringPo, po)
txmbr.setAsDep(po=needpo)
- self.tsInfoDelta.append(txmbr)
self.verbose_logger.log(logginglevels.DEBUG_2, 'TSINFO: Obsoleting %s with %s to resolve dep.',
requiringPo, po)
checkdeps = 1
@@ -564,7 +561,6 @@
if po.pkgtup == new:
txmbr = self.tsInfo.addUpdate(po, requiringPo)
txmbr.setAsDep(po=needpo)
- self.tsInfoDelta.append(txmbr)
self.verbose_logger.log(logginglevels.DEBUG_2, 'TSINFO: Updating %s to resolve dep.', po)
checkdeps = 1
@@ -706,13 +702,11 @@
# FIXME: we should probably handle updating multiple packages...
txmbr = self.tsInfo.addUpdate(best, inst[0])
txmbr.setAsDep()
- self.tsInfoDelta.append(txmbr)
else:
self.verbose_logger.debug('TSINFO: Marking %s as install for %s', best,
name)
txmbr = self.tsInfo.addInstall(best)
txmbr.setAsDep()
- self.tsInfoDelta.append(txmbr)
checkdeps = 1
@@ -773,7 +767,6 @@
'TSINFO: Updating %s to resolve conflict.', po)
txmbr = self.tsInfo.addUpdate(po, confpkg)
txmbr.setAsDep()
- self.tsInfoDelta.append(txmbr)
CheckDeps = 1
else:
@@ -806,6 +799,8 @@
self.deps = {}
self.path = []
self.loops = []
+ self.installedFileRequires = None
+ self.installedUnresolvedFileRequires = None
def _provideToPkg(self, req):
best = None
@@ -833,7 +828,7 @@
if self.tsInfo.getMembers(po.pkgtup):
self.deps[req] = po
return po
-
+
for txmbr in self.tsInfo.getMembers(None, TS_INSTALL_STATES):
if txmbr.po.checkPrco('provides', (r, f, v)):
self.deps[req] = txmbr.po
@@ -841,12 +836,6 @@
return None # for new ts check attempt
- def _undoDepInstalls(self):
- # clean up after ourselves in the case of failures
- for txmbr in self.tsInfo:
- if txmbr.isDep:
- self.tsInfo.remove(txmbr.pkgtup)
-
def prof_resolveDeps(self):
fn = "anaconda.prof.0"
import hotshot, hotshot.stats
@@ -857,7 +846,9 @@
stats = hotshot.stats.load(fn)
stats.strip_dirs()
stats.sort_stats('time', 'calls')
- stats.print_stats(20)
+ stats.print_stats(40)
+ stats.sort_stats("cumulative")
+ stats.print_stats(40)
return rc
def cprof_resolveDeps(self):
@@ -877,7 +868,8 @@
# returns a list of tuples
# ((name, version, release), (needname, needversion), flags, suggest, sense)
ret = []
- for txmbr in self.tsInfo.getMembers():
+ for txmbr in self.tsInfo.delta.copy():
+ l = len(ret)
self.verbose_logger.log(logginglevels.INFO_2,
"Checking deps for %s" %(txmbr,))
if txmbr.output_state in (TS_INSTALL, TS_TRUEINSTALL, TS_OBSOLETING):
@@ -887,7 +879,14 @@
elif txmbr.output_state in TS_REMOVE_STATES:
ret.extend(self._checkRemove(txmbr))
- self.tsInfoDelta = []
+ if len(ret) == l:
+ # no unresolved
+ self.tsInfo.delta.remove(txmbr)
+
+ self.verbose_logger.log(logginglevels.INFO_2,
+ "Checking file requirements")
+ ret.extend(self._checkFileRequires())
+
return ret
def _resolveDeps(self):
@@ -900,7 +899,6 @@
errors = []
if self.dsCallback: self.dsCallback.start()
- self.tsInfoDelta = self.tsInfo.getMembers()
while CheckDeps:
self.cheaterlookup = {} # short cache for some information we'd resolve
# (needname, needversion) = pkgtup
@@ -1015,9 +1013,9 @@
# if this is an update, we should check what the old
# requires were to make things faster
- oldreqs = []
+ oldreqs = set()
for oldpo in txmbr.updates:
- oldreqs.extend(oldpo.returnPrco('requires'))
+ oldreqs.update(oldpo.returnPrco('requires'))
ret = []
for req in reqs:
@@ -1073,11 +1071,6 @@
po = txmbr.po
provs = po.returnPrco('provides')
- # get the files in the package and express them as "provides"
- files = po.filelist
- filesasprovs = map(lambda f: (f, None, (None,None,None)), files)
- provs.extend(filesasprovs)
-
# if this is an update, we should check what the new package
# provides to make things faster
newpoprovs = []
@@ -1090,8 +1083,6 @@
for prov in provs:
if prov[0].startswith('rpmlib('): # ignore rpmlib() provides
continue
- if prov[0].startswith("/usr/share/doc"): # XXX: ignore doc files
- continue
if prov in newpoprovs:
continue
@@ -1207,3 +1198,53 @@
flags[f], None, rpm.RPMDEP_SENSE_REQUIRES) )
return ret
+
+ def _checkFileRequires(self):
+ # XXX move to TransactionData
+ newPkgs = self._tsInfo.getMembers(output_states=TS_INSTALL_STATES)
+ removedPkgs = set((txmbr.po for txmbr in self._tsInfo.getMembers(
+ output_states=TS_REMOVE_STATES)))
+
+ fileRequires = set()
+ reverselookup = {}
+ ret = []
+
+ # get file requirements from new packages
+ for txmbr in newPkgs:
+ for name, flag, evr in txmbr.po.requires:
+ if name.startswith('/'):
+ fileRequires.add(name)
+ reverselookup.setdefault(name, []).append(txmbr.po)
+
+ # generate list of file requirement in rpmdb
+ if self.installedFileRequires is None:
+ self.installedFileRequires = {}
+ self.installedUnresolvedFileRequires = set()
+ resolved = set()
+ for pkg in self.rpmdb.returnPackages():
+ for name, flag, evr in pkg.requires:
+ if not name.startswith('/'):
+ continue
+ self.installedFileRequires.setdefault(pkg, []).append(name)
+ if name not in resolved:
+ dep = self.rpmdb.whatProvides(name, flag, evr)
+ resolved.add(name)
+ if not dep:
+ self.installedUnresolvedFileRequires.add(name)
+
+ for po, files in self.installedFileRequires.iteritems():
+ if po not in removedPkgs:
+ fileRequires.update(files)
+ for filename in files:
+ reverselookup.setdefault(filename, []).append(po)
+
+ fileRequires -= self.installedUnresolvedFileRequires
+
+ for filename in fileRequires:
+ dep = self._provideToPkg((filename, None, (None, None, None)))
+ if not dep:
+ for po in reverselookup[filename]:
+ ret.append( ((po.name, po.version, po.release),
+ (filename, ''), 0, None,
+ rpm.RPMDEP_SENSE_REQUIRES) )
+ return ret
Index: yum/transactioninfo.py
===================================================================
RCS file: /cvsroot/yum/cvs/yum/yum/transactioninfo.py,v
retrieving revision 1.37
diff -u -r1.37 transactioninfo.py
--- yum/transactioninfo.py 22 Feb 2007 03:47:23 -0000 1.37
+++ yum/transactioninfo.py 20 Mar 2007 14:35:41 -0000
@@ -32,6 +32,7 @@
self.probFilterFlags = []
self.root = '/'
self.pkgdict = {} # key = pkgtup, val = list of TransactionMember obj
+ self.delta = set()
self.debug = 0
self.changed = False
@@ -146,7 +147,7 @@
return
self.pkgdict[txmember.pkgtup].append(txmember)
self.changed = True
-
+ self.delta.add(txmember)
if self.conditionals.has_key(txmember.name):
for po in self.conditionals[txmember.name]:
condtxmbr = self.addInstall(po)
@@ -160,6 +161,13 @@
return
for txmbr in self.pkgdict[pkgtup]:
txmbr.po.state = None
+ self.delta.discard(txmbr)
+ if txmbr.output_state in TS_INSTALL_STATES:
+ # insert dummy into delta to check if this breaks deps
+ # XXX really needed?
+ check_txmbr = TransactionMember(txmbr.po)
+ check_txmbr.output_state = TS_ERASE
+ self.delta.add(check_txmbr)
del self.pkgdict[pkgtup]
self.changed = True
_______________________________________________
Yum-devel mailing list
[email protected]
https://lists.dulug.duke.edu/mailman/listinfo/yum-devel
