On Fri, 2009-03-20 at 11:35 +0100, Markus Duft wrote: > Hey guys :) > > Just wanted to stop by and get some opinions on a patch i wrote for the > prefix branch.
argh... managed to fail to attach the patch _again_ :) at least i managed to add the output stuff i mentioned before to the patch in the meantime. so this patch should be pretty much complete. Thanks and Cheers, Markus > > i'll try and explain what i want in the first place: i'm porting things > to native windows. since windows isn't too cooperative, i'm unable to > merge most things (and with other things, i simply don't want to), and > thus i need to take those things from somewhere else (more or less the > complete @system). I _am_ able to build all those things for interix > (which is the host system in the windows case). So what i want is a > setup of two prefix instances with a certain relation to each other: the > native windows prefix should be able to recognize installed packages > from the other instance, and resolve dependencies by eventually using > the other .../var/db/... > > This could be (and is) quite usefull for all other platforms too. For > exmaple i could use prefix chaining on a linux box. I could create a > prefix containing a base system, and then for testing of > i-don't-know-whatever, i could create another small prefix inheriting > all installed packages from the other one. this way new prefixes can > stay very slim, but still the "parent" prefix is not altered on merges. > > one issue not handled by the current patch is, that prefixes can have > different CHOST/ARCH/... (which is the case with x86-interix and > x86-winnt for example). > > another thing is, that i plan to add some output in the merge list, > telling the user, which packages have been readonly-resolved from > another portage instance. right now the dependency is treated as if it > didn't exist. > > all together, i'm pretty sure i did the one or the other forbidden thing > in my patch, but that's why i'm asking the guys-who-know :) it was hard > enough to read the portage source and get where i am, so i'm happy with > the result ;) > > Waiting for comments, suggestions, etc. > > Thanks in advance, > Cheers, Markus > >
diff -ru portage.orig/pym/_emerge/__init__.py portage/pym/_emerge/__init__.py --- portage.orig/pym/_emerge/__init__.py 2009-03-18 10:13:34.000000000 +0100 +++ portage/pym/_emerge/__init__.py 2009-03-20 12:08:49.000000000 +0100 @@ -49,7 +49,7 @@ import portage.xpak, commands, errno, re, socket, time from portage.output import blue, bold, colorize, darkblue, darkgreen, darkred, green, \ nc_len, red, teal, turquoise, xtermTitle, \ - xtermTitleReset, yellow + xtermTitleReset, yellow, purple from portage.output import create_color_func good = create_color_func("GOOD") bad = create_color_func("BAD") @@ -69,6 +69,7 @@ from portage.util import cmp_sort_key, writemsg, writemsg_level from portage.sets import load_default_config, SETPREFIX from portage.sets.base import InternalPackageSet +from portage.dbapi.vartree import vartree from itertools import chain, izip @@ -4692,6 +4693,7 @@ self._unsatisfied_deps_for_display = [] self._unsatisfied_blockers_for_display = None self._circular_deps_for_display = None + self._injected_readonly_deps_for_display = [] self._dep_stack = [] self._unsatisfied_deps = [] self._initially_unsatisfied_deps = [] @@ -5270,18 +5272,35 @@ if removal_action and self.myopts.get("--with-bdeps", "y") == "n": edepend["DEPEND"] = "" + # MDUFT: create additional vartrees for every readonly root here + # (maybe FakeVartree's?). Then below search those trees and set + # mypriority.satisfied to True. + # the ro_vartrees instances are created below as they are needed to + # avoid reading vartrees of portage instances which aren't required + # while resolving this dependencies. + ro_trees = {} + ro_vartrees = {} + + for type in ("DEPEND","RDEPEND", "PDEPEND"): + ro_trees[type] = [] + + for ro_root, ro_dep_types in self.settings.readonly_roots.items(): + if type in ro_dep_types: + ro_trees[type].append(ro_root) + deps = ( - ("/", edepend["DEPEND"], + ("/", "DEPEND", self._priority(buildtime=(not bdeps_optional), optional=bdeps_optional)), - (myroot, edepend["RDEPEND"], self._priority(runtime=True)), - (myroot, edepend["PDEPEND"], self._priority(runtime_post=True)) + (myroot, "RDEPEND", self._priority(runtime=True)), + (myroot, "PDEPEND", self._priority(runtime_post=True)) ) debug = "--debug" in self.myopts strict = mytype != "installed" try: - for dep_root, dep_string, dep_priority in deps: + for dep_root, dep_type, dep_priority in deps: + dep_string = edepend[dep_type] if not dep_string: continue if debug: @@ -5309,6 +5328,32 @@ if not atom.blocker and vardb.match(atom): mypriority.satisfied = True + # MDUFT: How erver we do it - if we find "atom" beeing installed + # in a valid readonly root for the current dependency type, then + # _DONT_ call the below, but rather return 1 immediately. + for ro_root in ro_trees[dep_type]: + if not ro_vartrees.has_key(ro_root): + # target_root=ro_root ok? or should it be the real target_root? + _tmp_settings = portage.config(config_root=ro_root, target_root=ro_root, + config_incrementals=portage.const.INCREMENTALS) + #_tmp_trees = portage.util.LazyItemsDict() + #_tmp_trees.addLazySingleton("vartree", vartree, ro_root, + # categories=_tmp_settings.categories, settings=_tmp_settings) + + #setconfig = load_default_config(_tmp_trees["vartree"].settings, _tmp_trees) + #ro_vartrees[ro_root] = FakeVartree(RootConfig(_tmp_trees["vartree"].settings, + # _tmp_trees, setconfig), acquire_lock=0) + + ro_vartrees[ro_root] = vartree(root=ro_root, categories=_tmp_settings.categories, settings=_tmp_settings, kill_eprefix=True) + + ro_matches = ro_vartrees[ro_root].dbapi.match(atom) + + if ro_matches: + #TODO: print notice, just the same as the rest of the + # self.*_for_display stuff. + self._injected_readonly_deps_for_display.append({"ro_root" : ro_root, "atom" : atom, "matches": ro_matches, "type": dep_type}) + return 1 + if not self._add_dep(Dependency(atom=atom, blocker=atom.blocker, depth=depth, parent=pkg, priority=mypriority, root=dep_root), @@ -8457,6 +8502,19 @@ for x in blockers: print x + # print _injected_readonly_deps_for_display. + if len(self._injected_readonly_deps_for_display) > 0: + out.write("\n%s\n" % (darkgreen("Packages resolved from readonly installations:"))) + + for x in self._injected_readonly_deps_for_display: + tmp_type = x["type"] + while len(tmp_type) < 9: + tmp_type += " " + out.write(" [ %s ] %s %s %s (matched: %s)" % (teal(tmp_type), green(str(x["atom"])), yellow("from"), blue(x["ro_root"]), turquoise(str(x["matches"])))) + + if len(self._injected_readonly_deps_for_display) > 0: + out.write("\n") + if verbosity == 3: print print counters @@ -11825,6 +11883,7 @@ for x in candidate_catpkgs: # cycle through all our candidate deps and determine # what will and will not get unmerged + try: mymatch = vartree.dbapi.match(x) except portage.exception.AmbiguousPackageName, errpkgs: diff -ru portage.orig/pym/portage/__init__.py portage/pym/portage/__init__.py --- portage.orig/pym/portage/__init__.py 2009-03-18 10:13:35.000000000 +0100 +++ portage/pym/portage/__init__.py 2009-03-20 09:05:56.000000000 +0100 @@ -1236,6 +1236,7 @@ self._accept_license = copy.deepcopy(clone._accept_license) self._plicensedict = copy.deepcopy(clone._plicensedict) + self.readonly_roots = copy.deepcopy(clone.readonly_roots) else: def check_var_directory(varname, var): @@ -1802,6 +1803,50 @@ # but needs to be available using portageq self["EPREFIX"] = EPREFIX + # expand READONLY_EROOT to a list of all readonly portage instances + # all the way down to the last one. beware that ATM a deeper instance + # in the chain can provide more than the toplevel! this means that + # if you only inherit DEPENDS from one instance, that instance may + # inherit RDEPENDs from another one, making the top-level instance + # inherit RDEPENDs from there too - even if the intermediate prefix + # does not do this. + self.readonly_roots = {} + my_ro_current_instance = config_root + + while True: + my_ro_current_make_conf_file = os.path.join(my_ro_current_instance,MAKE_CONF_FILE.lstrip(os.path.sep)) + + if os.path.exists(my_ro_current_make_conf_file): + my_ro_cfg = getconfig(my_ro_current_make_conf_file) + + if my_ro_cfg.has_key("READONLY_EROOT"): + + if not my_ro_cfg["READONLY_EROOT"].find(":"): + raise portage.exception.InvalidReadonlyRoot("ERROR: malformed READONLY_EROOT in %s" % (my_ro_current_make_conf_file)) + + (my_ro_cfg_root,my_ro_cfg_root_deps) = my_ro_cfg["READONLY_EROOT"].rsplit(":",1) + + if not os.path.exists(my_ro_cfg_root): + raise portage.exception.InvalidReadonlyRoot("ERROR: malformed READONLY_EROOT in %s: path does not exist!" % (my_ro_current_instance)) + + if self.readonly_roots.has_key(my_ro_cfg_root): + raise portage.exception.InvalidReadonlyRoot("ERROR: circular READONLY_EROOT's in %s. %s already checked for %s" % (my_ro_current_make_conf_file, my_ro_cfg_root, self.readonly_roots[my_ro_cfg_root])) + + if my_ro_cfg_root == config_root: + raise portage.exception.InvalidReadonlyRoot("ERROR: cannot add this instance as READONLY_EROOT in %s." % (my_ro_current_make_conf_file)) + + self.readonly_roots[my_ro_cfg_root] = my_ro_cfg_root_deps.split(",") + my_ro_current_instance = my_ro_cfg_root + + continue + + break + + if len(self.readonly_roots) > 0: + writemsg("Found readonly portage instance to resolve dependencies:\n",noiselevel=0) + for k,v in self.readonly_roots.items(): + writemsg("%20s %-s\n" % (v,k), noiselevel=0) + self._init_dirs() if mycpv: diff -ru portage.orig/pym/portage/dbapi/vartree.py portage/pym/portage/dbapi/vartree.py --- portage.orig/pym/portage/dbapi/vartree.py 2009-03-18 10:13:35.000000000 +0100 +++ portage/pym/portage/dbapi/vartree.py 2009-03-20 11:09:49.000000000 +0100 @@ -1270,9 +1270,20 @@ self._counter_path = os.path.join(root, CACHE_PATH.lstrip(os.path.sep), "counter") + plibreg_path = os.path.join(self.root, EPREFIX_LSTRIP, PRIVATE_PATH, "preserved_libs_registry") + + if vartree: + self._kill_eprefix = vartree._kill_eprefix + else: + self._kill_eprefix = False + + if self._kill_eprefix: + self._aux_cache_filename = os.path.join(self.root, self._aux_cache_filename.lstrip(EPREFIX.rstrip('/'))) + self._counter_path = os.path.join(self.root, self._counter_path.lstrip(EPREFIX.rstrip('/'))) + plibreg_path = os.path.join(self.root, plibreg_path.lstrip(EPREFIX.rstrip('/'))) + try: - self.plib_registry = PreservedLibsRegistry( - os.path.join(self.root, EPREFIX_LSTRIP, PRIVATE_PATH, "preserved_libs_registry")) + self.plib_registry = PreservedLibsRegistry(plibreg_path) except PermissionDenied: # apparently this user isn't allowed to access PRIVATE_PATH self.plib_registry = None @@ -1285,6 +1296,10 @@ def getpath(self, mykey, filename=None): rValue = os.path.join(self.root, VDB_PATH, mykey) + + if self._kill_eprefix: + rValue = os.path.join(self.root, rValue.lstrip(EPREFIX.rstrip('/'))) + if filename != None: rValue = os.path.join(rValue, filename) return rValue @@ -1433,6 +1448,9 @@ returnme = [] basepath = os.path.join(self.root, VDB_PATH) + os.path.sep + if self._kill_eprefix: + basepath = os.path.join(self.root, basepath.lstrip(EPREFIX.rstrip('/'))) + if use_cache: from portage import listdir else: @@ -1522,7 +1540,12 @@ return list(self._iter_match(mydep, self.cp_list(mydep.cp, use_cache=use_cache))) try: - curmtime = os.stat(self.root+VDB_PATH+"/"+mycat).st_mtime + _tmp_path = self.root+VDB_PATH+"/"+mycat + + if self._kill_eprefix: + _tmp_path = os.path.join(self.root, _tmp_path.lstrip(EPREFIX.rstrip('/'))) + + curmtime = os.stat(_tmp_path).st_mtime except (IOError, OSError): curmtime=0 @@ -2063,7 +2086,7 @@ class vartree(object): "this tree will scan a var/db/pkg database located at root (passed to init)" def __init__(self, root="/", virtual=None, clone=None, categories=None, - settings=None): + settings=None, kill_eprefix=False): if clone: writemsg("vartree.__init__(): deprecated " + \ "use of clone parameter\n", noiselevel=-1) @@ -2072,6 +2095,7 @@ self.populated = 1 from portage import config self.settings = config(clone=clone.settings) + self._kill_eprefix = clone._kill_eprefix else: self.root = root[:] if settings is None: @@ -2079,6 +2103,7 @@ self.settings = settings # for key_expand calls if categories is None: categories = settings.categories + self._kill_eprefix=kill_eprefix self.dbapi = vardbapi(self.root, categories=categories, settings=settings, vartree=self) self.populated = 1 @@ -2110,6 +2135,10 @@ raise except Exception, e: mydir = os.path.join(self.root, VDB_PATH, mycpv) + + if self._kill_eprefix: + mydir = os.path.join(self.root, mydir.lstrip(EPREFIX.rstrip('/'))) + writemsg("\nParse Error reading PROVIDE and USE in '%s'\n" % mydir, noiselevel=-1) if mylines: