--- action_import.py_FINAL_PHASE2 2008-09-19 19:35:40.000000000 +0200 +++ action_import.py_FINAL_PHASE2_evolved 2008-09-21 22:55:11.000000000 +0200 @@ -37,6 +37,7 @@ TRY_DICT = { 'debian': [ "pool" ], + 'ubuntu': [ "pool" ], 'redhat': [ "Fedora", "Packages", "RedHat", "Client", "Server", "Centos", "CentOS", "Fedora/RPMS", "RedHat/RPMS", "Client/RPMS", "Server/RPMS", "Centos/RPMS", "CentOS/RPMS", "RPMS" ] @@ -64,7 +65,6 @@ self.profiles = config.profiles() self.systems = config.systems() self.settings = config.settings() - self.distros_added = [] self.kickstart_file = kickstart_file self.rsync_flags = rsync_flags self.arch = arch @@ -91,6 +91,9 @@ # FIXME : Raise exception even when network_root is given ? raise CX(_("Something already exists at this import location (%s). You must specify --arch to avoid potentially overwriting existing files.") % mpath) + if self.kickstart_file and not self.breed: + raise CX(_("Kickstart file can only be specified when a specific breed is selected")) + if self.arch: # append the arch path to the name if the arch is not already # found in the name. @@ -107,7 +110,7 @@ # accessible support location already exists if self.network_root is None: - self.path = "%s/ks_mirror/%s" % (self.settings.webdir, self.mirror_name) + self.path = os.path.normpath( "%s/ks_mirror/%s" % (self.settings.webdir, self.mirror_name) ) self.mkdir(self.path) # prevent rsync from creating the directory name twice @@ -138,7 +141,7 @@ if not self.network_root.endswith("/"): self.network_root = self.network_root + "/" - self.path = self.mirror + self.path = os.path.normpath( self.mirror ) valid_roots = [ "nfs://", "ftp://", "http://" ] for valid_root in valid_roots: if self.network_root.startswith(valid_root): @@ -154,16 +157,17 @@ self.processed_repos = {} print _("---------------- (adding distros)") + distros_added = [] # FIXME : search below self.path for isolinux configurations or known directories from TRY_LIST - os.path.walk(self.path, self.distro_adder, {}) + os.path.walk(self.path, self.distro_adder, distros_added) if self.network_root is None: print _("---------------- (associating repos)") # FIXME: this automagic is not possible (yet) without mirroring - self.repo_finder() + self.repo_finder(distros_added) print _("---------------- (associating kickstarts)") - self.kickstart_finder() + self.kickstart_finder(distros_added) print _("---------------- (syncing)") self.api.sync() @@ -192,7 +196,7 @@ # ---------------------------------------------------------------------- - def kickstart_finder(self): + def kickstart_finder(self,distros_added): """ For all of the profiles in the config w/o a kickstart, use the given kickstart file, or look at the kernel path, from that, @@ -202,7 +206,7 @@ for profile in self.profiles: distro = self.distros.find(name=profile.distro) - if distro is None or not (distro in self.distros_added): + if distro is None or not (distro in distros_added): # print _("- skipping distro %s since it wasn't imported this time") % profile.distro continue @@ -211,37 +215,32 @@ importer = import_factory(distro.breed,self.network_root,self.path) base_dir = importer.get_new_tree_root(kdir) - for try_key in TRY_DICT.keys(): - for try_entry in TRY_DICT[try_key]: + for try_entry in TRY_DICT[distro.breed]: try_dir = os.path.join(base_dir, try_entry) if os.path.exists(try_dir): - if try_key == "redhat": + if distro.breed == "redhat": rpms = glob.glob(os.path.join(try_dir, "*release-*")) for rpm in rpms: if rpm.find("notes") != -1: continue - results = self.scan_rpm_filename(rpm) + results = importer.scan_rpm_filename(rpm) if results is None: continue (flavor, major, minor) = results # print _("- finding default kickstart template for %(flavor)s %(major)s") % { "flavor" : flavor, "major" : major } - kickstart = self.set_variance(profile, flavor, major, minor, distro) - elif try_key == "debian": + kickstart = importer.set_variance(profile, flavor, major, minor, distro) + elif distro.breed == "debian" or distro.breed == "ubuntu": # search for base-files or base-installer ? rpms = glob.glob(os.path.join(try_dir, "main/b/base-files" , "base-files_*")) for rpm in rpms: - results = self.scan_deb_filename(rpm) + results = importer.scan_deb_filename(rpm) if results is None: continue (flavor, major, minor) = results if self.breed and self.breed != flavor: raise CX(_("Error: given breed does not match imported tree")) # print _("- finding default kickstart template for %(flavor)s %(major)s") % { "flavor" : flavor, "major" : major } - kickstart = self.set_variance(profile, flavor, major, minor, distro) - else: - # FIXME : Why not fix this while initially creating the profile ? - print _("- using kickstart file %s") % self.kickstart_file - profile.set_kickstart(self.kickstart_file) + kickstart = importer.set_variance(profile, flavor, major, minor, distro) self.configure_tree_location(distro) self.distros.add(distro,save=True) # re-save @@ -302,149 +301,11 @@ result = "/" + result return result - # --------------------------------------------------------------------- - - def set_variance(self, profile, flavor, major, minor, distro): - - # find the profile kickstart and set the distro breed/os-version based on what - # we can find out from the rpm filenames and then return the kickstart - # path to use. - - if flavor == "fedora": - - # this may actually fail because the libvirt/virtinst database - # is not always up to date. We keep a simplified copy of this - # in codes.py. If it fails we set it to something generic - # and don't worry about it. - distro.set_breed("redhat") - try: - distro.set_os_version("fedora%s" % int(major)) - except: - print "- warning: could not store os-version fedora%s" % int(major) - distro.set_os_version("other") - - if major >= 8: - return profile.set_kickstart("/etc/cobbler/sample_end.ks") - if major >= 6: - return profile.set_kickstart("/etc/cobbler/sample.ks") - - if flavor == "redhat" or flavor == "centos": - distro.set_breed("redhat") - if major <= 2: - # rhel2.1 is the only rhel2 - distro.set_os_version("rhel2.1") - else: - try: - distro.set_os_version("rhel%s" % int(major)) - except: - print "- warning: could not store os-version %s" % int(major) - distro.set_os_version("other") - - if major >= 5: - return profile.set_kickstart("/etc/cobbler/sample.ks") - - if flavor == "debian": - - distro.set_breed("debian") - dist_names = { '4.0' : "Etch" , '5.0' : "Lenny" } - dist_vers = "%s.%s" % ( major , minor ) - distro.set_os_version("debian%s" % dist_names[dist_vers]) - return profile.set_kickstart("/etc/cobbler/sample.seed") - - if flavor == "ubuntu": - - distro.set_breed("ubuntu") - # Release names taken from wikipedia - dist_names = { '4.10':"WartyWarthog", '5.4':"HoaryHedgehog", '5.10':"BreezyBadger", '6.4':"DapperDrake", '6.10':"EdgyEft", '7.4':"FeistyFawn", '7.10':"GutsyGibbon", '8.4':"HardyHeron", '8.10':"IntrepidIbex", '9.4':"JauntyJackalope" } - dist_vers = "%s.%s" % ( major , minor ) - distro.set_os_version("ubuntu%s" % dist_names[dist_vers]) - return profile.set_kickstart("/etc/cobbler/sample.seed") - - print _("- using default kickstart file choice") - return profile.set_kickstart("/etc/cobbler/legacy.ks") - - # --------------------------------------------------------------------- - - def scan_rpm_filename(self, rpm): - """ - Determine what the distro is based on the release RPM filename. - """ - - rpm = os.path.basename(rpm) - print "- processing rpm : %s" % rpm - - # if it looks like a RHEL RPM we'll cheat. - # it may be slightly wrong, but it will be close enough - # for RHEL5 we can get it exactly. - - for x in [ "4AS", "4ES", "4WS" ]: - if rpm.find(x) != -1: - return ("redhat", 4, 0) - for x in [ "3AS", "3ES", "3WS" ]: - if rpm.find(x) != -1: - return ("redhat", 3, 0) - for x in [ "2AS", "2ES", "2WS" ]: - if rpm.find(x) != -1: - return ("redhat", 2, 0) - - # now get the flavor: - flavor = "redhat" - if rpm.lower().find("fedora") != -1: - flavor = "fedora" - if rpm.lower().find("centos") != -1: - flavor = "centos" - - # get all the tokens and try to guess a version - accum = [] - tokens = rpm.split(".") - for t in tokens: - tokens2 = t.split("-") - for t2 in tokens2: - try: - float(t2) - accum.append(t2) - except: - pass - - major = float(accum[0]) - minor = float(accum[1]) - return (flavor, major, minor) - - def scan_deb_filename(self, deb): - """ - Determine what the distro is based on the base-files dpkg filename. - """ - - deb = os.path.basename(deb) - print "- processing deb : %s" % deb - - # now get the flavor: - flavor = "debian" - if deb.lower().find("ubuntu") != -1: - flavor = "ubuntu" - - # get all the tokens and try to guess a version - accum = [] - tokens = deb.split("_") - tokens2 = tokens[1].split(".") - for t2 in tokens2: - try: - val = int(t2) - accum.append(val) - except: - pass - if flavor == "ubuntu": - accum.pop(0) - accum.pop(0) - accum.append(0) - - return (flavor, accum[0], accum[1]) - # ---------------------------------------------------------------------- - def repo_finder(self): + def repo_finder(self,distros_added): - for distro in self.distros_added: + for distro in distros_added: print _("- traversing distro %s") % distro.name if distro.kernel.find("ks_mirror") != -1: basepath = os.path.dirname(distro.kernel) @@ -580,7 +441,7 @@ print "- WARNING - symlink name (%s) does not fit the breed (%s)" % ( x , self.breed ) continue print "- following symlink: %s" % fullname - os.path.walk(fullname, self.distro_adder, {}) + os.path.walk(fullname, self.distro_adder, foo) if x.startswith("initrd"): initrd = os.path.join(dirname,x) @@ -590,17 +451,28 @@ continue kernel = os.path.join(dirname,x) if initrd is not None and kernel is not None and dirname.find("isolinux") == -1: - self.add_entry(dirname,kernel,initrd) + foo.extend( self.add_entry(dirname,kernel,initrd) ) # The values are reset because debian media has extra initrd images initrd = None kernel = None def add_entry(self,dirname,kernel,initrd): - # FIXME : Raise exception if kernel and initrd come from different directories ? + if os.path.dirname(kernel) != os.path.dirname(initrd): + raise CX(_("- scanning %s for distro signature") % dirname2) + # + # As a single media can generate multiple distros, we need to change the given name + # FIXME : Might be better tu use the real imported root, but that will lost xen and other distros/profiles + proposed_name = self.get_proposed_name(dirname) + # importer = import_factory(self.breed,self.network_root,self.path) - for pxe_arch in importer.get_pxe_arch(dirname): + # + archs = importer.get_pxe_arch(dirname) + if importer.breed == "redhat" and len(archs)>1 : + raise CX(_("Directory %s holds multiple arches : %s") % ( dirname , archs ) ) + distros_added = [] + for pxe_arch in archs: # FIXME : Compare the many places where arch can be guessed (including CLI) - name = self.get_proposed_name(dirname, pxe_arch) + name = proposed_name + "-" + pxe_arch existing_distro = self.distros.find(name=name) @@ -620,7 +492,7 @@ distro.set_breed(self.breed) distro.source_repos = [] self.distros.add(distro,save=True) - self.distros_added.append(distro) + distros_added.append(distro) existing_profile = self.profiles.find(name=name) @@ -634,8 +506,8 @@ profile.set_name(name) profile.set_distro(name) - #if self.kickstart_file: - # profile.set_kickstart(self.kickstart_file) + if self.kickstart_file: + profile.set_kickstart(self.kickstart_file) if name.find("-xen") != -1: profile.set_virt_type("xenpv") else: @@ -664,36 +536,31 @@ self.profiles.add(profile,save=True) - self.api.serialize() + self.api.serialize() + return distros_added - def get_proposed_name(self,dirname,pxe_arch): - archname = pxe_arch - if archname == "x86": - # be consistent - archname = "i386" + def get_proposed_name(self,dirname): # FIXME: this is new, needs testing ... if self.network_root is not None: - name = "-".join(self.path_tail(self.path,dirname).split("/")) + name = self.mirror_name + "-".join(self.path_tail(os.path.dirname(self.path),dirname).split("/")) else: # remove the part that says /var/www/cobbler/ks_mirror/name - name = "-".join(dirname.split("/")[6:]) - if name.startswith("-"): - name = name[1:] - name = self.mirror_name + "-" + name - # FIXME : Why do we clean all these suffixes ? - name = name.replace("-os","") + name = "-".join(dirname.split("/")[5:]) + # We clean some strings from the proposed name + # some kernel paths within the media name = name.replace("-images","") + name = name.replace("-pxeboot","") + name = name.replace("-install","") + # some paths above the media root present with multiple media imports + name = name.replace("-os","") name = name.replace("-tree","") name = name.replace("var-www-cobbler-", "") name = name.replace("ks_mirror-","") - name = name.replace("-pxeboot","") - name = name.replace("-install","") name = name.replace("--","-") + # any architecture name related string, as real arch will be appended later for separator in [ '-' , '_' , '.' ] : for arch in [ "i386" , "x86_64" , "ia64" , "x86" , "s390x" , "386" , "amd" ]: name = name.replace("%s%s" % ( separator , arch ),"") - # ensure arch is on the end, regardless of path used. - name = name + "-" + archname return name @@ -802,8 +669,7 @@ result["i386"] = 1 return result.keys() - def get_pxe_arch(self,dirname): - t = dirname.lower() + def get_pxe_arch(self,t): if t.find("x86_64") != -1 or t.find("amd") != -1: return [ "x86_64" ] if t.find("ia64") != -1: @@ -812,5 +678,141 @@ return [ "i386" ] if t.find("s390") != -1: return [ "s390x" ] - return self.learn_arch_from_tree(dirname) + return self.learn_arch_from_tree(t) + + # --------------------------------------------------------------------- + + def set_variance(self, profile, flavor, major, minor, distro): + + # find the profile kickstart and set the distro breed/os-version based on what + # we can find out from the rpm filenames and then return the kickstart + # path to use. + + if flavor == "fedora": + + # this may actually fail because the libvirt/virtinst database + # is not always up to date. We keep a simplified copy of this + # in codes.py. If it fails we set it to something generic + # and don't worry about it. + distro.set_breed("redhat") + try: + distro.set_os_version("fedora%s" % int(major)) + except: + print "- warning: could not store os-version fedora%s" % int(major) + distro.set_os_version("other") + + if major >= 8: + return profile.set_kickstart("/etc/cobbler/sample_end.ks") + if major >= 6: + return profile.set_kickstart("/etc/cobbler/sample.ks") + + if flavor == "redhat" or flavor == "centos": + distro.set_breed("redhat") + if major <= 2: + # rhel2.1 is the only rhel2 + distro.set_os_version("rhel2.1") + else: + try: + distro.set_os_version("rhel%s" % int(major)) + except: + print "- warning: could not store os-version %s" % int(major) + distro.set_os_version("other") + + if major >= 5: + return profile.set_kickstart("/etc/cobbler/sample.ks") + + if flavor == "debian": + + distro.set_breed("debian") + dist_names = { '4.0' : "Etch" , '5.0' : "Lenny" } + dist_vers = "%s.%s" % ( major , minor ) + distro.set_os_version("debian%s" % dist_names[dist_vers]) + return profile.set_kickstart("/etc/cobbler/sample.seed") + + if flavor == "ubuntu": + + distro.set_breed("ubuntu") + # Release names taken from wikipedia + dist_names = { '4.10':"WartyWarthog", '5.4':"HoaryHedgehog", '5.10':"BreezyBadger", '6.4':"DapperDrake", '6.10':"EdgyEft", '7.4':"FeistyFawn", '7.10':"GutsyGibbon", '8.4':"HardyHeron", '8.10':"IntrepidIbex", '9.4':"JauntyJackalope" } + dist_vers = "%s.%s" % ( major , minor ) + distro.set_os_version("ubuntu%s" % dist_names[dist_vers]) + return profile.set_kickstart("/etc/cobbler/sample.seed") + + print _("- using default kickstart file choice") + return profile.set_kickstart("/etc/cobbler/legacy.ks") + + def scan_rpm_filename(self, rpm): + """ + Determine what the distro is based on the release RPM filename. + """ + + rpm = os.path.basename(rpm) + print "- processing rpm : %s" % rpm + + # if it looks like a RHEL RPM we'll cheat. + # it may be slightly wrong, but it will be close enough + # for RHEL5 we can get it exactly. + + for x in [ "4AS", "4ES", "4WS" ]: + if rpm.find(x) != -1: + return ("redhat", 4, 0) + for x in [ "3AS", "3ES", "3WS" ]: + if rpm.find(x) != -1: + return ("redhat", 3, 0) + for x in [ "2AS", "2ES", "2WS" ]: + if rpm.find(x) != -1: + return ("redhat", 2, 0) + + # now get the flavor: + flavor = "redhat" + if rpm.lower().find("fedora") != -1: + flavor = "fedora" + if rpm.lower().find("centos") != -1: + flavor = "centos" + + # get all the tokens and try to guess a version + accum = [] + tokens = rpm.split(".") + for t in tokens: + tokens2 = t.split("-") + for t2 in tokens2: + try: + float(t2) + accum.append(t2) + except: + pass + + major = float(accum[0]) + minor = float(accum[1]) + return (flavor, major, minor) + + def scan_deb_filename(self, deb): + """ + Determine what the distro is based on the base-files dpkg filename. + """ + + deb = os.path.basename(deb) + print "- processing deb : %s" % deb + + # now get the flavor: + flavor = "debian" + if deb.lower().find("ubuntu") != -1: + flavor = "ubuntu" + + # get all the tokens and try to guess a version + accum = [] + tokens = deb.split("_") + tokens2 = tokens[1].split(".") + for t2 in tokens2: + try: + val = int(t2) + accum.append(val) + except: + pass + if flavor == "ubuntu": + accum.pop(0) + accum.pop(0) + accum.append(0) + + return (flavor, accum[0], accum[1])