From: Jan Provaznik <[email protected]> to install pulp: 1) install pulp packages (described on https://fedorahosted.org/pulp/wiki/Install)
2) create testing repository $ pulp-admin -u admin -p admin repo create --id jboss --name jboss \ --feed='yum:http://file.rdu.redhat.com/~imcleod/repos/oddthesis/F11-alternate/noarch/' \ --arch=x86_64 $ pulp-admin -u admin -p admin repo sync --id=jboss 3) you should see this repository in template UI on package selection page --- src/app/util/image_descriptor_xml.rb | 24 +-- src/app/util/repository_manager.rb | 165 ++++---------------- .../util/repository_manager/abstract_repository.rb | 10 ++ .../util/repository_manager/comps_repository.rb | 116 ++++++++++++++ src/app/util/repository_manager/pulp_repository.rb | 73 +++++++++ src/config/environment.rb | 1 + .../image_descriptor_package_repositories.yml | 23 ++- 7 files changed, 257 insertions(+), 155 deletions(-) create mode 100644 src/app/util/repository_manager/abstract_repository.rb create mode 100644 src/app/util/repository_manager/comps_repository.rb create mode 100644 src/app/util/repository_manager/pulp_repository.rb diff --git a/src/app/util/image_descriptor_xml.rb b/src/app/util/image_descriptor_xml.rb index d9b8348..2509cbf 100644 --- a/src/app/util/image_descriptor_xml.rb +++ b/src/app/util/image_descriptor_xml.rb @@ -51,9 +51,11 @@ class ImageDescriptorXML end def platform=(str) - # FIXME: we remove all repos beacouse we don't know which one is for + # FIXME: we remove all repos because we don't know which one is for # platform - recreate_repo_nodes(str) + # update: we don't add platform repo, image builder chooses right one from OS + # name, but we add all other repos + recreate_repo_nodes node = get_or_create_node('os') node.content = str end @@ -206,19 +208,12 @@ class ImageDescriptorXML parent << n end - def recreate_repo_nodes(platform) - unless repconf = platforms[platform] - raise "unknown platform #{platform}" - end - + def recreate_repo_nodes repo_node = get_or_create_node('repos') repo_node.xpath('.//repo').remove - rnode = get_or_create_node('repo', repo_node) - rnode.content = repconf['baseurl'] - - repository_manager.repositories.each do |rname, repo| + repository_manager.repositories.each do |repo| rnode = get_or_create_node('repo', repo_node) - rnode.content = repo['baseurl'] + rnode.content = repo.yumurl end end @@ -246,9 +241,6 @@ class ImageDescriptorXML end def repository_manager - unless @repository_manager - @repository_manager = RepositoryManager.new - end - return @repository_manager + @repository_manager ||= RepositoryManager.new end end diff --git a/src/app/util/repository_manager.rb b/src/app/util/repository_manager.rb index 05cc5da..1075bf4 100644 --- a/src/app/util/repository_manager.rb +++ b/src/app/util/repository_manager.rb @@ -17,145 +17,24 @@ # MA 02110-1301, USA. A copy of the GNU General Public License is # also available at http://www.gnu.org/copyleft/gpl.html. -require 'open-uri' require 'yaml' +require 'util/repository_manager/comps_repository' +require 'util/repository_manager/pulp_repository' class RepositoryManager - class CompsRepository - def initialize(baseurl, id) - @id = id - @baseurl = baseurl - end - - def get_packages - packages = [] - get_packages_nodes.each do |node| - name = node.at_xpath('./xmlns:name/child::text()') - group = node.at_xpath('./xmlns:format/rpm:group/child::text()') - description = node.at_xpath('./xmlns:description/child::text()') - next unless name and group - packages << { - :repository_id => @id, - :name => name.text, - :group => group.text, - :description => description ? description.text : '', - } - end - return packages - end - - def get_groups - groups = {} - get_groups_nodes.each do |g| - pkgs = get_group_packages(g) - next if pkgs.empty? - name = g.at_xpath('name').text - groups[name] = { - :name => name, - :description => (t = g.at_xpath('description')) ? t.text : '', - :packages => pkgs, - } - end - return groups - end - - def download_xml(type) - begin - url = get_url(type) - rescue - return '' - end - - xml_data = open(url) - if url =~ /\.gz$/ - return Zlib::GzipReader.new(xml_data).read - else - return xml_data.read - end - end - - private - - def get_xml(type) - # FIXME: I'm not sure config is right dir for automatic storing of - # xml files, but this should be temporary solution until Image Repo is - # done - xml_dir = "#{RAILS_ROOT}/config/image_descriptor_xmls" - xml_file = "#{xml_dir}/#...@id}.#{type}.xml" - begin - return File.open(xml_file) { |f| f.read } - rescue - xml = download_xml(type) - Dir.mkdir(xml_dir) unless File.directory?(xml_dir) - File.open(xml_file, 'w') { |f| f.write xml } - return xml - end - end - - def get_group_packages(group_node) - pkgs = {} - group_node.xpath('packagelist/packagereq').each do |p| - pkgs[p.text] = p.attr('type') - end - return pkgs - end - - def get_packages_nodes - unless @packages_nodes - data = get_xml('primary') - xml = Nokogiri::XML(data) - @packages_nodes = xml.xpath('/xmlns:metadata/xmlns:package') - end - return @packages_nodes - end + attr_reader :repositories - def get_groups_nodes - unless @groups_nodes - data = get_xml('group') - xml = Nokogiri::XML(data) - @groups_nodes = xml.xpath('/comps/group') - end - return @groups_nodes - end - - def get_url(type) - if type == 'repomd' - return File.join(@baseurl, 'repodata', 'repomd.xml') - else - location = repomd.xpath("/xmlns:repomd/xmlns:da...@type=\"#{type}\"]/xmlns:location").first - raise "location for #{type} data not found" unless location - return File.join(@baseurl, location['href']) - end - end - - def repomd - unless @repomd - @repomd = Nokogiri::XML(get_xml('repomd')) - end - return @repomd - end - end - - def initialize - @config = YAML.load_file("#{RAILS_ROOT}/config/image_descriptor_package_repositories.yml") - end - - def get_repository(repository_id) - repo = @config[repository_id] - raise "Repository '#{repository_id}' doesn't exist" unless repo - return CompsRepository.new(repo['baseurl'], repository_id) - end - - def repositories - return @config + def initialize(opts = {}) + @config = opts[:config] + @repositories = get_repositories end def all_groups(repository = nil) unless @all_groups @all_groups = {} - repositories.keys.each do |r| - next if repository and repository != 'all' and repository != r - get_repository(r).get_groups.each do |group, data| + repositories.each do |r| + next if repository and repository != 'all' and repository != r.id + r.groups.each do |group, data| if @all_groups[group] @all_groups[group][:packages].merge!(data[:packages]) else @@ -170,11 +49,31 @@ class RepositoryManager def all_packages(repository = nil) unless @all_packages @all_packages = [] - repositories.keys.each do |r| - next if repository and repository != 'all' and repository != r - @all_packages += get_repository(r).get_packages + repositories.each do |r| + next if repository and repository != 'all' and repository != r.id + @all_packages += r.packages end end return @all_packages end + + private + + def get_repositories + repositories = [] + if config + config.each do |rep| + if rep['type'] == 'xml' + repositories << CompsRepository.new(rep) + elsif rep['type'] == 'pulp' + repositories += PulpRepository.repositories(rep) + end + end + end + return repositories + end + + def config + @config ||= YAML.load_file("#{RAILS_ROOT}/config/image_descriptor_package_repositories.yml") + end end diff --git a/src/app/util/repository_manager/abstract_repository.rb b/src/app/util/repository_manager/abstract_repository.rb new file mode 100644 index 0000000..baa72e7 --- /dev/null +++ b/src/app/util/repository_manager/abstract_repository.rb @@ -0,0 +1,10 @@ +class AbstractRepository + attr_reader :id, :name, :baseurl, :yumurl + + def initialize(conf) + @id = conf['id'] + @name = conf['name'] + @baseurl = conf['baseurl'] + @yumurl = conf['yumurl'] || conf['baseurl'] + end +end diff --git a/src/app/util/repository_manager/comps_repository.rb b/src/app/util/repository_manager/comps_repository.rb new file mode 100644 index 0000000..d7e192a --- /dev/null +++ b/src/app/util/repository_manager/comps_repository.rb @@ -0,0 +1,116 @@ +require 'util/repository_manager/abstract_repository' +require 'open-uri' + +class CompsRepository < AbstractRepository + def initialize(conf) + super + @baseurl = conf['baseurl'] + end + + def packages + packages = [] + get_packages_nodes.each do |node| + name = node.at_xpath('./xmlns:name/child::text()') + group = node.at_xpath('./xmlns:format/rpm:group/child::text()') + description = node.at_xpath('./xmlns:description/child::text()') + next unless name and group + packages << { + :repository_id => @id, + :name => name.text, + :description => description ? description.text : '', + } + end + return packages + end + + def groups + groups = {} + get_groups_nodes.each do |g| + pkgs = get_group_packages(g) + next if pkgs.empty? + name = g.at_xpath('name').text + groups[name] = { + :name => name, + :description => (t = g.at_xpath('description')) ? t.text : '', + :packages => pkgs, + } + end + return groups + end + + def download_xml(type) + begin + url = get_url(type) + rescue + return '' + end + + xml_data = open(url) + if url =~ /\.gz$/ + return Zlib::GzipReader.new(xml_data).read + else + return xml_data.read + end + end + + private + + def get_xml(type) + # FIXME: I'm not sure config is right dir for automatic storing of + # xml files, but this should be temporary solution until Image Repo is + # done + xml_dir = "#{RAILS_ROOT}/config/image_descriptor_xmls" + xml_file = "#{xml_dir}/#...@id}.#{type}.xml" + begin + return File.open(xml_file) { |f| f.read } + rescue + xml = download_xml(type) + Dir.mkdir(xml_dir) unless File.directory?(xml_dir) + File.open(xml_file, 'w') { |f| f.write xml } + return xml + end + end + + def get_group_packages(group_node) + pkgs = {} + group_node.xpath('packagelist/packagereq').each do |p| + pkgs[p.text] = p.attr('type') + end + return pkgs + end + + def get_packages_nodes + unless @packages_nodes + data = get_xml('primary') + xml = Nokogiri::XML(data) + @packages_nodes = xml.xpath('/xmlns:metadata/xmlns:package') + end + return @packages_nodes + end + + def get_groups_nodes + unless @groups_nodes + data = get_xml('group') + xml = Nokogiri::XML(data) + @groups_nodes = xml.xpath('/comps/group') + end + return @groups_nodes + end + + def get_url(type) + if type == 'repomd' + return File.join(@baseurl, 'repodata', 'repomd.xml') + else + location = repomd.xpath("/xmlns:repomd/xmlns:da...@type=\"#{type}\"]/xmlns:location").first + raise "location for #{type} data not found" unless location + return File.join(@baseurl, location['href']) + end + end + + def repomd + unless @repomd + @repomd = Nokogiri::XML(get_xml('repomd')) + end + return @repomd + end +end diff --git a/src/app/util/repository_manager/pulp_repository.rb b/src/app/util/repository_manager/pulp_repository.rb new file mode 100644 index 0000000..5641f0c --- /dev/null +++ b/src/app/util/repository_manager/pulp_repository.rb @@ -0,0 +1,73 @@ +require 'json' +require 'typhoeus' +require 'util/repository_manager/abstract_repository' + +class PulpRepository < AbstractRepository + class WrappedRestClient + def self.method_missing(method, *args) + resp = Typhoeus::Request.send(method, *args) + unless resp.code == 200 + raise "pulp repository: failed to fetch #{args.first} (code #{resp.code}): #{resp.body}" + end + begin + JSON.parse(resp.body) + rescue JSON::ParserError + raise "pulp repository: failed to parse response (#{args.first})" + end + end + end + + HTTP_OPTS = { :disable_ssl_peer_verification => true, :timeout => 30000 } + + def self.repositories(conf) + WrappedRestClient.get(File.join(conf['baseurl'], "/repositories/"), HTTP_OPTS).map do |r| + next if conf['except'] and conf['except'].include?(r['id']) + PulpRepository.new(r.merge('baseurl' => conf['baseurl'], + 'yumurl' => File.join(conf['yumurl'], r['id']))) + end.compact + end + + def initialize(conf) + super + @packages_url = File.join(strip_path(@baseurl), conf['packages']) + @groups_url = File.join(strip_path(@baseurl), conf['packagegroups']) + end + + def packages + packages = [] + WrappedRestClient.get(@packages_url, HTTP_OPTS).each_value do |info| + packages << { + :repository_id => @id, + :name => info['name'], + :description => info['description'], + } + end + return packages + end + + def groups + groups = {} + WrappedRestClient.get(@groups_url, HTTP_OPTS).each do |id, info| + pkgs = {} + info['default_package_names'].each {|p| pkgs[p] = 'default'} + info['optional_package_names'].each {|p| pkgs[p] = 'optional'} + info['mandatory_package_names'].each {|p| pkgs[p] = 'mandatory'} + next if pkgs.empty? + name = info['name'] + groups[name] = { + :name => name, + :description => info['description'].to_s, + :packages => pkgs, + } + end + return groups + end + + private + + def strip_path(url) + uri = URI.parse(url) + uri.path = '' + uri.to_s + end +end diff --git a/src/config/environment.rb b/src/config/environment.rb index ba14e06..c445f91 100644 --- a/src/config/environment.rb +++ b/src/config/environment.rb @@ -51,6 +51,7 @@ Rails::Initializer.run do |config| config.gem "compass", :version => ">= 0.10.2" config.gem "compass-960-plugin", :lib => "ninesixty" config.gem "simple-navigation" + config.gem "typhoeus" config.active_record.observers = :instance_observer, :task_observer # Only load the plugins named here, in the order given. By default, all plugins diff --git a/src/config/image_descriptor_package_repositories.yml b/src/config/image_descriptor_package_repositories.yml index 40335ea..a014bda 100644 --- a/src/config/image_descriptor_package_repositories.yml +++ b/src/config/image_descriptor_package_repositories.yml @@ -1,6 +1,17 @@ -fedora: - name: Fedora - baseurl: http://download.fedoraproject.org/pub/fedora/linux/releases/11/Everything/x86_64/os/ -jboss: - name: JBoss - baseurl: http://repo.oddthesis.org/cirras/packages/fedora/11/RPMS/noarch/ +#- +# id: fedora +# name: Fedora +# #baseurl: http://download.fedoraproject.org/pub/fedora/linux/releases/11/Everything/x86_64/os/ +# baseurl: http://localhost/fedora/ +# type: xml +# +#- +# id: jboss +# name: JBoss +# baseurl: http://file.rdu.redhat.com/~imcleod/repos/oddthesis/F11-alternate/noarch/ +# type: xml +# +- + baseurl: https://admin:ad...@localhost/pulp/api/ + yumurl: https://localhost/pulp/repos/ + type: pulp -- 1.7.2.2 _______________________________________________ deltacloud-devel mailing list [email protected] https://fedorahosted.org/mailman/listinfo/deltacloud-devel
