Avishai Ish-Shalom has proposed merging lp:~avishai-ish-shalom/cloud-init/chef-refactor into lp:cloud-init.
Requested reviews: cloud init development team (cloud-init-dev) Related bugs: Bug #847358 in cloud-init: "chef integration should support chef-solo" https://bugs.launchpad.net/cloud-init/+bug/847358 Bug #1027188 in cloud-init: "Cloud-init chef plugin should support more distros and installation methods" https://bugs.launchpad.net/cloud-init/+bug/1027188 For more details, see: https://code.launchpad.net/~avishai-ish-shalom/cloud-init/chef-refactor/+merge/164009 Refactored cc_chef, added omnibus and chef-solo support -- https://code.launchpad.net/~avishai-ish-shalom/cloud-init/chef-refactor/+merge/164009 Your team cloud init development team is requested to review the proposed merge of lp:~avishai-ish-shalom/cloud-init/chef-refactor into lp:cloud-init.
=== modified file 'cloudinit/config/cc_chef.py' --- cloudinit/config/cc_chef.py 2012-12-12 15:39:43 +0000 +++ cloudinit/config/cc_chef.py 2013-05-15 17:20:33 +0000 @@ -20,10 +20,15 @@ import json import os +import urllib from cloudinit import templater from cloudinit import url_helper from cloudinit import util +from cloudinit.settings import PER_INSTANCE +from urlparse import urlparse + +frequency = PER_INSTANCE RUBY_VERSION_DEFAULT = "1.8" @@ -38,20 +43,51 @@ OMNIBUS_URL = "https://www.opscode.com/chef/install.sh"; +BIN_PATHS = ( + "/usr/bin", "/usr/local/bin", + "/var/lib/gems/2.0.0/bin", + "/var/lib/gems/1.9.1/bin", + "/var/lib/gems/1.9/bin", + "/var/lib/gems/1.8/bin" +) +DEFAULT_ARGS = { + 'interval': ('-i', 1800), + 'splay': ('-s', 300), + 'daemonize': ('-d', True), + 'fork': ('-f', True) +} +DEFAULT_CFG = { + 'repo_path': '/var/lib/cloud/chef_repo', + 'repo_source_type': 'tarball' +} +JSON_ATTRIB_FILE = '/etc/chef/firstboot.json' + def handle(name, cfg, cloud, log, _args): - + log.info("Starting cc_chef") # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: log.debug(("Skipping module named %s," " no 'chef' key in configuration"), name) return - chef_cfg = cfg['chef'] + chef_cfg = dict(DEFAULT_CFG.items() + cfg['chef'].items()) + log.debug("Chef config: %r", chef_cfg) # Ensure the chef directories we use exist for d in CHEF_DIRS: util.ensure_dir(d) + if 'mode' in chef_cfg: + chef_mode = chef_cfg['mode'] + else: + if 'server_url' in chef_cfg and \ + ('validation_key' in chef_cfg or 'validation_cert' in chef_cfg): + chef_mode = "client" + else: + chef_mode = "solo" + + log.debug("Chef mode is %s", chef_mode) + # Set the validation key based on the presence of either 'validation_key' # or 'validation_cert'. In the case where both exist, 'validation_key' # takes precedence @@ -61,80 +97,191 @@ break # Create the chef config from template - template_fn = cloud.get_template_filename('chef_client.rb') - if template_fn: - iid = str(cloud.datasource.get_instance_id()) - params = { - 'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name'] - } - templater.render_to_file(template_fn, '/etc/chef/client.rb', params) - else: - log.warn("No template found, not rendering to /etc/chef/client.rb") + if not write_chef_config(chef_cfg, cloud, chef_mode, log): + return False # set the firstboot json - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in list(initial_attributes.keys()): - initial_json[k] = initial_attributes[k] - util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json)) + write_json_attrib_file(chef_cfg) # If chef is not installed, we install chef based on 'install_type' + install_chef(chef_cfg, cloud) + + if chef_mode == 'solo': + get_cookbooks(chef_cfg, cloud) + + chef_args = chef_args_from_cfg(chef_cfg, ['-j', JSON_ATTRIB_FILE]) + if util.get_cfg_option_bool(chef_cfg, 'autostart', default=True): + run_chef(log, chef_mode, chef_args) + + +def install_chef(chef_cfg, cloud): if (not os.path.isfile('/usr/bin/chef-client') or - util.get_cfg_option_bool(chef_cfg, 'force_install', default=False)): + util.get_cfg_option_bool(chef_cfg, 'force_install', default=False)): install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') + chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) + if install_type == "gems": # this will install and run the chef-client from gems - chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', RUBY_VERSION_DEFAULT) - install_chef_from_gems(cloud.distro, ruby_version, chef_version) - # and finally, run chef-client - log.debug('Running chef-client') - util.subp(['/usr/bin/chef-client', - '-d', '-i', '1800', '-s', '20'], capture=False) + ohai_version = util.get_cfg_option_str(chef_cfg, 'ohai_version', None) + install_chef_from_gems(cloud.distro, ruby_version, chef_version, ohai_version) elif install_type == 'packages': # this will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) elif install_type == 'omnibus': url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - content = url_helper.readurl(url=url, retries=5) + content = url_helper.readurl(url=url, retries=5).contents with util.tempdir() as tmpd: # use tmpd over tmpfile to avoid 'Text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd util.write_file(tmpf, content, mode=0700) - util.subp([tmpf], capture=False) + args = [] + if chef_version: + args.append("-v") + args.append(chef_version) + util.subp([tmpf] + args, capture=False) else: - log.warn("Unknown chef install type %s", install_type) + raise RuntimeError("Unknown chef install type %s" % install_type) + + +def write_chef_config(chef_cfg, cloud, chef_mode, log): + "Write chef config file from template" + template_fn = cloud.get_template_filename('chef_client.rb') + cfg_filename = "/etc/chef/" + ("client.rb" if chef_mode == "client" else "solo.rb") + if template_fn: + iid = str(cloud.datasource.get_instance_id()) + params = { + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'mode': chef_mode + } + + if chef_mode == "client": + # server_url is required + params['server_url'] = chef_cfg['server_url'] + params['validation_name'] = chef_cfg.get('validation_name', None) + elif chef_mode == "solo": + params['repo_path'] = chef_cfg['repo_path'] + + templater.render_to_file(template_fn, cfg_filename, params) + return True + else: + log.warn("No template found, not rendering to %s", cfg_filename) + return False + + +def write_json_attrib_file(chef_cfg): + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in list(initial_attributes.keys()): + initial_json[k] = initial_attributes[k] + util.write_file(JSON_ATTRIB_FILE, json.dumps(initial_json)) + + +def run_chef(log, chef_type, chef_args): + chef_bin = "chef-%s" % chef_type + chef_exec = None + for path in BIN_PATHS: + f = os.path.join(path, chef_bin) + if os.path.isfile(f) and os.access(f, os.X_OK): + chef_exec = f + break + if chef_exec is None: + raise RuntimeError("Couldn't find chef executable for %s" % chef_bin) + log.debug("Running %s", chef_exec) + util.subp([chef_exec] + chef_args, capture=False) def get_ruby_packages(version): # return a list of packages needed to install ruby at version - pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] + pkgs = ['ruby%s' % version, 'ruby%s-dev' % version, 'build-essential'] if version == "1.8": pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) return pkgs -def install_chef_from_gems(ruby_version, chef_version, distro): +def install_chef_from_gems(distro, ruby_version, chef_version, ohai_version=None): distro.install_packages(get_ruby_packages(ruby_version)) + + def gem_install(gem, version=None): + cmd_args = ['/usr/bin/gem', 'install', gem] + if version is not None: + cmd_args.append('-v') + cmd_args.append(version) + + cmd_args += ['--no-rdoc', '--bindir', '/usr/bin', '-q'] + + return util.subp(cmd_args, capture=False) + if not os.path.exists('/usr/bin/gem'): util.sym_link('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') if not os.path.exists('/usr/bin/ruby'): util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') - if chef_version: - util.subp(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) + if ohai_version: + gem_install('ohai', ohai_version) + gem_install('chef', chef_version) + + +def chef_args_from_cfg(cfg, extra_args=[]): + merged_args = {} + for (k, v) in DEFAULT_ARGS.iteritems(): + merged_args[k] = (v[0], cfg.get(k, v[1])) + + if merged_args['daemonize'][1] is False: + del merged_args['interval'] + del merged_args['splay'] + + args = [] + for (k, (arg, v)) in merged_args.iteritems(): + if type(v) == bool: + if util.get_cfg_option_bool(cfg, k, v): + args.append(arg) + else: + args.append(arg) + args.append(str(cfg.get(k, v))) + + return args + extra_args + + +def get_cookbooks(cfg, cloud): + repo_source_type = util.get_cfg_option_str( + cfg, 'repo_source_type', default='tarball') + if repo_source_type == 'git': + get_cookbooks_from_git(cfg, cloud) + elif repo_source_type == 'tarball': + get_cookbooks_from_tarball(cfg) else: - util.subp(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q'], capture=False) + raise RuntimeError("Unknown cookbooks source type %s" % repo_source_type) + + +def get_cookbooks_from_git(cfg, cloud): + cloud.distro.install_packages('git') + enclosing_dir = os.path.dirname(cfg['repo_path']) + if not os.path.isdir(enclosing_dir): + os.makedirs(enclosing_dir) + util.subp(['git', 'clone', '--recurse-submodules', cfg['repo_source'], cfg['repo_path']]) + + +def get_cookbooks_from_tarball(cfg): + with util.tempdir() as tmpd: + filename = os.path.basename(urlparse(cfg['repo_source']).path) + tmpfile = os.path.join(tmpd, filename) + urllib.urlretrieve(cfg['repo_source'], tmpfile) + if not os.path.isdir(cfg['repo_path']): + os.makedirs(cfg['repo_path']) + util.subp(['tar', '-xf', tmpfile, '-C', cfg['repo_path']]) + + +def get_cookbooks_from_berkshelf(cfg): + raise NotImplementedError() + + +def get_cookbooks_from_librarian(cfg): + raise NotImplementedError() === modified file 'doc/examples/cloud-config-chef.txt' --- doc/examples/cloud-config-chef.txt 2012-12-12 15:39:43 +0000 +++ doc/examples/cloud-config-chef.txt 2013-05-15 17:20:33 +0000 @@ -84,8 +84,23 @@ maxclients: 100 keepalive: "off" - # if install_type is 'omnibus', change the url to download - omnibus_url: "https://www.opscode.com/chef/install.sh"; + # For chef-solo, we want to download cookbooks. Currently git and tarball can be used + # tarball/git should contain cookbooks, roles and data_bags folders + #repo_source_type: git + #repo_source: https://github.com/some-org/cookbooks-repo.git + #repo_path: /var/lib/cloud/chef_repo + + # if install_type is 'omnibus', change the url to the download script + #omnibus_url: "https://www.opscode.com/chef/install.sh"; + + # Daemonize and run every 'interval' + #daemonize: true + #interval: 1800 + # Random wait before starting the run + #splay: 300 + + # fork off a worker for every chef run, effective against memory leaks + fork: true # Capture all subprocess output into a logfile === modified file 'templates/chef_client.rb.tmpl' --- templates/chef_client.rb.tmpl 2012-07-09 20:45:26 +0000 +++ templates/chef_client.rb.tmpl 2013-05-15 17:20:33 +0000 @@ -11,13 +11,18 @@ log_level :info log_location "/var/log/chef/client.log" ssl_verify_mode :verify_none +#if $mode == 'client' validation_client_name "$validation_name" validation_key "/etc/chef/validation.pem" client_key "/etc/chef/client.pem" chef_server_url "$server_url" environment "$environment" +#else if $mode == 'solo' +cookbook_path "$repo_path/cookbooks" +data_bag_path "$repo_path/data_bags" +role_path "$repo_path/roles" +#end if node_name "$node_name" -json_attribs "/etc/chef/firstboot.json" file_cache_path "/var/cache/chef" file_backup_path "/var/backups/chef" pid_file "/var/run/chef/client.pid"
_______________________________________________ Mailing list: https://launchpad.net/~cloud-init-dev Post to : [email protected] Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
