Gehel has uploaded a new change for review. https://gerrit.wikimedia.org/r/276427
Change subject: Make r8s module use base::expose_puppet_certs ...................................................................... Make r8s module use base::expose_puppet_certs WiP - do not merge This enable reuse of code exposing Puppet SSL certificates Tests have been written before changing any code, but there is enough that I do not understand that I probably got it completely wrong. Puppet is not deployed in the same way on Prod and Labs, which leads to a few tricky parts. Bug: T124444 Change-Id: I5e2b4d01023fad27dddcb0c7c34a20f14f943ad3 --- A modules/k8s/Rakefile M modules/k8s/manifests/apiserver.pp M modules/k8s/manifests/controller.pp M modules/k8s/manifests/ssl.pp A modules/k8s/spec/defines/apiserver_rspec.rb A modules/k8s/spec/defines/controller_rspec.rb A modules/k8s/spec/defines/infrastructure_config_rspec.rb A modules/k8s/spec/defines/kubelet_rspec.rb A modules/k8s/spec/defines/role_toollabs_k8s_master_rspec.rb A modules/k8s/spec/defines/ssl_rspec.rb A modules/k8s/spec/fixtures/hiera/hiera.yaml A modules/k8s/spec/fixtures/hiera/test.yaml A modules/k8s/spec/fixtures/manifests/site.pp A modules/k8s/spec/spec_helper.rb M modules/k8s/templates/initscripts/controller-manager.systemd.erb M modules/k8s/templates/initscripts/kube-apiserver.systemd.erb M modules/k8s/templates/initscripts/kubelet.systemd.erb M modules/k8s/templates/kubeconfig-client.yaml.erb M modules/toollabs/manifests/kube2proxy.pp 19 files changed, 186 insertions(+), 57 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/27/276427/1 diff --git a/modules/k8s/Rakefile b/modules/k8s/Rakefile new file mode 100644 index 0000000..6471966 --- /dev/null +++ b/modules/k8s/Rakefile @@ -0,0 +1,7 @@ +require 'rake' + +require 'rspec/core/rake_task' + +RSpec::Core::RakeTask.new(:spec) do |t| + t.pattern = 'spec/*/*_rspec.rb' +end diff --git a/modules/k8s/manifests/apiserver.pp b/modules/k8s/manifests/apiserver.pp index 889afd4..3746bc3 100644 --- a/modules/k8s/manifests/apiserver.pp +++ b/modules/k8s/manifests/apiserver.pp @@ -1,6 +1,7 @@ class k8s::apiserver( $etcd_servers, $master_host, + $users = hiera('k8s_users'), ) { require_package('kube-apiserver') @@ -13,7 +14,6 @@ mode => '0700', } - $users = hiera('k8s_users') file { '/etc/kubernetes/tokenauth': content => template('k8s/tokenauth.csv.erb'), owner => 'kubernetes', diff --git a/modules/k8s/manifests/controller.pp b/modules/k8s/manifests/controller.pp index 6a4d52d..5253abb 100644 --- a/modules/k8s/manifests/controller.pp +++ b/modules/k8s/manifests/controller.pp @@ -1,5 +1,6 @@ class k8s::controller { require_package('kube-controller-manager') + require ::k8s::ssl include k8s::users diff --git a/modules/k8s/manifests/ssl.pp b/modules/k8s/manifests/ssl.pp index 6ffa111..e5a5784 100644 --- a/modules/k8s/manifests/ssl.pp +++ b/modules/k8s/manifests/ssl.pp @@ -9,54 +9,14 @@ $ssldir = '/var/lib/puppet/ssl', $target_basedir = '/var/lib/kubernetes' ) { - $puppet_cert_name = $::fqdn + $ca_cert_path = '/etc/ssl/certs/Puppet_Internal_CA.pem' #TODO: check if this is the correct cert (also for labs) + $server_cert_path = "${target_basedir}/ssl/cert.pem" + $server_private_key_path = "${target_basedir}/ssl/server.key" - file { $target_basedir: - ensure => directory, - owner => $user, - group => $group, - mode => '0755', # more permissive! - } - - file { [ - "${target_basedir}/ssl", - "${target_basedir}/ssl/certs", - "${target_basedir}/ssl/private_keys", - ]: - ensure => directory, - owner => $user, - group => $group, - mode => '0555', - require => File[$target_basedir], # less permissive - } - - - file { "${target_basedir}/ssl/certs/ca.pem": - ensure => present, - owner => $user, - group => $group, - mode => '0444', - source => "${ssldir}/certs/ca.pem", - require => File["${target_basedir}/ssl/certs"], - } - - file { "${target_basedir}/ssl/certs/cert.pem": - ensure => present, - owner => $user, - group => $group, - mode => '0400', - source => "${ssldir}/certs/${puppet_cert_name}.pem", - require => File["${target_basedir}/ssl/certs/ca.pem"], - } - - if $provide_private { - file { "${target_basedir}/ssl/private_keys/server.key": - ensure => present, - owner => $user, - group => $group, - mode => '0400', - source => "${ssldir}/private_keys/${puppet_cert_name}.pem", - require => File["${target_basedir}/ssl/private_keys"], - } + ::base::expose_puppet_certs { $target_basedir: + provide_private => $provide_private, + user => $user, + group => $group, + ssldir => $ssldir, } } diff --git a/modules/k8s/spec/defines/apiserver_rspec.rb b/modules/k8s/spec/defines/apiserver_rspec.rb new file mode 100644 index 0000000..9523677 --- /dev/null +++ b/modules/k8s/spec/defines/apiserver_rspec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' + +describe 'k8s::apiserver', :type => :class do + let(:facts) { {:fqdn => 'host.example.net'} } + let(:params) { { + :etcd_servers => 'https://etcd.example.net:2379', + :master_host => 'example.net', + :users => [], + } } + + context 'with systemd as init' do + let(:facts) { {:initsystem => 'systemd'} } + + it 'should containt a systemd unit file with correct certificate path' do + should contain_file('/lib/systemd/system/kube-apiserver.service') + .with({ 'ensure' => 'present' }) + .with_content(%r{--tls-private-key-file=/var/lib/kubernetes/ssl/server.key}) + .with_content(%r{--tls-cert-file=/var/lib/kubernetes/ssl/cert.pem}) + .with_content(%r{--service-account-key-file=/var/lib/kubernetes/ssl/server.key}) + + end + + end + +end diff --git a/modules/k8s/spec/defines/controller_rspec.rb b/modules/k8s/spec/defines/controller_rspec.rb new file mode 100644 index 0000000..6f06106 --- /dev/null +++ b/modules/k8s/spec/defines/controller_rspec.rb @@ -0,0 +1,17 @@ +require 'spec_helper' + +describe 'k8s::controller', :type => :class do + let(:facts) { {:fqdn => 'host.example.net'} } + + context 'with systemd as init' do + let(:facts) { {:initsystem => 'systemd'} } + + it 'should containt a systemd unit file with correct certificate path' do + should contain_file('/lib/systemd/system/controller-manager.service') + .with({ 'ensure' => 'present' }) + .with_content(%r{--service-account-private-key-file=/var/lib/kubernetes/ssl/server.key}) + end + + end + +end diff --git a/modules/k8s/spec/defines/infrastructure_config_rspec.rb b/modules/k8s/spec/defines/infrastructure_config_rspec.rb new file mode 100644 index 0000000..f49cca5 --- /dev/null +++ b/modules/k8s/spec/defines/infrastructure_config_rspec.rb @@ -0,0 +1,12 @@ +require 'spec_helper' + +describe 'k8s::infrastructure_config', :type => :class do + + it 'should containt kubeconfig file with correct certificate path' do + should contain_file('/etc/kubernetes/kubeconfig') + .with({ 'ensure' => 'present' }) + .with_content(%r{certificate-authority: /etc/ssl/certs/Puppet_Internal_CA.pem}) + + end + +end diff --git a/modules/k8s/spec/defines/kubelet_rspec.rb b/modules/k8s/spec/defines/kubelet_rspec.rb new file mode 100644 index 0000000..61c3724 --- /dev/null +++ b/modules/k8s/spec/defines/kubelet_rspec.rb @@ -0,0 +1,22 @@ +require 'spec_helper' + +describe 'k8s::kubelet', :type => :class do + let(:facts) { {:fqdn => 'host.example.net'} } + let(:params) { { + :master_host => 'example.net', + } } + + context 'with systemd as init' do + let(:facts) { {:initsystem => 'systemd'} } + + it 'should containt a systemd unit file with correct certificate path' do + should contain_file('/lib/systemd/system/kubelet.service') + .with({ 'ensure' => 'present' }) + .with_content(%r{--tls-private-key-file=/var/lib/kubernetes/ssl/server.key}) + .with_content(%r{--tls-cert-file=/var/lib/kubernetes/ssl/cert.pem}) + + end + + end + +end diff --git a/modules/k8s/spec/defines/role_toollabs_k8s_master_rspec.rb b/modules/k8s/spec/defines/role_toollabs_k8s_master_rspec.rb new file mode 100644 index 0000000..bcf15df --- /dev/null +++ b/modules/k8s/spec/defines/role_toollabs_k8s_master_rspec.rb @@ -0,0 +1,31 @@ +require 'spec_helper' + +# This test could be moved to the `role` module, but all I am interested here +# is the parts that are related to k8s, so I'm keeping it here for the moment. +# +# `role::toollabs::k8s::master` has too many dependencies to be testable under +# reasonable efforts. This test can still be run by commenting out everything +# not related to k8s in `role::toollabs::k8s::master` + +describe 'role::toollabs::k8s::master', :type => :class do + let(:facts) { {:fqdn => 'host.example.net'} } + + context 'with systemd as init' do + let(:facts) { {:initsystem => 'systemd'} } + + it 'should contain Kubernetes apiserver' do + pending 'role::toollabs::k8s::master should be refactored to be actually testable' + should contain_class('k8s::apiserver') + end + it 'should contain Kubernetes scheduler' do + pending 'role::toollabs::k8s::master should be refactored to be actually testable' + should contain_class('k8s::scheduler') + end + it 'should contain Kubernetes controller' do + pending 'role::toollabs::k8s::master should be refactored to be actually testable' + should contain_class('k8s::controller') + end + + end + +end diff --git a/modules/k8s/spec/defines/ssl_rspec.rb b/modules/k8s/spec/defines/ssl_rspec.rb new file mode 100644 index 0000000..7a36794 --- /dev/null +++ b/modules/k8s/spec/defines/ssl_rspec.rb @@ -0,0 +1,27 @@ +require 'spec_helper' + +describe 'k8s::ssl', :type => :class do + let(:facts) { { :fqdn => 'host.example.net'} } + + describe 'certificates are exposed' do + it { should contain_file('/var/lib/kubernetes/ssl').with({ 'ensure' => 'directory' }) } + it { should contain_file('/var/lib/kubernetes/ssl/cert.pem').with({ 'ensure' => 'present' }) } + end + + describe 'private key is not exposed by default' do + it { should_not contain_file('/var/lib/kubernetes/ssl/private_keys/server.key') } + end + + describe 'private key is exposed if required' do + let(:params) { { :provide_private => true } } + + it { should contain_file('/var/lib/kubernetes/ssl/server.key') + .with({ + 'ensure' => 'present', + 'mode' => '0400', + 'source' => '/var/lib/puppet/ssl/private_keys/host.example.net.pem', + }) + } + end + +end diff --git a/modules/k8s/spec/fixtures/hiera/hiera.yaml b/modules/k8s/spec/fixtures/hiera/hiera.yaml new file mode 100644 index 0000000..f99d200 --- /dev/null +++ b/modules/k8s/spec/fixtures/hiera/hiera.yaml @@ -0,0 +1,7 @@ +--- +:backends: + - yaml +:yaml: + :datadir: spec/fixtures/hiera +:hierarchy: + - test diff --git a/modules/k8s/spec/fixtures/hiera/test.yaml b/modules/k8s/spec/fixtures/hiera/test.yaml new file mode 100644 index 0000000..111fdb0 --- /dev/null +++ b/modules/k8s/spec/fixtures/hiera/test.yaml @@ -0,0 +1,6 @@ +--- +k8s_users: + - name: client-infrastructure + token: test-token +k8s::etcd_hosts: + - etcd.example.net \ No newline at end of file diff --git a/modules/k8s/spec/fixtures/manifests/site.pp b/modules/k8s/spec/fixtures/manifests/site.pp new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/modules/k8s/spec/fixtures/manifests/site.pp diff --git a/modules/k8s/spec/spec_helper.rb b/modules/k8s/spec/spec_helper.rb new file mode 100644 index 0000000..3200f59 --- /dev/null +++ b/modules/k8s/spec/spec_helper.rb @@ -0,0 +1,14 @@ +require 'puppet' +require 'rspec' +require 'rspec-puppet' + +def param_value(subject, type, title, param) + subject.resource(type, title).send(:parameters)[param.to_sym] +end + +RSpec.configure do |c| + c.module_path = File.expand_path(File.join(File.dirname(__FILE__), '../..')) + # Using an empty site.pp file to avoid: https://github.com/rodjek/rspec-puppet/issues/15 + c.manifest_dir = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures/manifests')) + c.hiera_config = 'spec/fixtures/hiera/hiera.yaml' +end diff --git a/modules/k8s/templates/initscripts/controller-manager.systemd.erb b/modules/k8s/templates/initscripts/controller-manager.systemd.erb index 9cc9f25..107e4a2 100644 --- a/modules/k8s/templates/initscripts/controller-manager.systemd.erb +++ b/modules/k8s/templates/initscripts/controller-manager.systemd.erb @@ -7,7 +7,7 @@ ExecStart=/usr/bin/kube-controller-manager \ --cluster-cidr=192.168.0.0/24 \ --master=http://127.0.0.1:8080 \ - --service-account-private-key-file=/var/lib/kubernetes/ssl/private_keys/server.key + --service-account-private-key-file=<%= scope['::k8s::ssl::server_private_key_path'] %> [Install] WantedBy=multi-user.target diff --git a/modules/k8s/templates/initscripts/kube-apiserver.systemd.erb b/modules/k8s/templates/initscripts/kube-apiserver.systemd.erb index 1f31a3f..1fa7b14 100644 --- a/modules/k8s/templates/initscripts/kube-apiserver.systemd.erb +++ b/modules/k8s/templates/initscripts/kube-apiserver.systemd.erb @@ -11,9 +11,9 @@ --admission-control=NamespaceLifecycle,ServiceAccount,ResourceQuota,LimitRanger,UidEnforcer \ --authorization-mode=ABAC \ --authorization-policy-file=/etc/kubernetes/abac \ - --tls-private-key-file=/var/lib/kubernetes/ssl/private_keys/server.key \ - --tls-cert-file=/var/lib/kubernetes/ssl/certs/cert.pem \ - --service-account-key-file=/var/lib/kubernetes/ssl/private_keys/server.key + --tls-private-key-file=<%= scope['::k8s::ssl::server_private_key_path'] %> \ + --tls-cert-file=<%= scope['::k8s::ssl::server_cert_path'] %> \ + --service-account-key-file=<%= scope['::k8s::ssl::server_private_key_path'] %> [Install] WantedBy=multi-user.target diff --git a/modules/k8s/templates/initscripts/kubelet.systemd.erb b/modules/k8s/templates/initscripts/kubelet.systemd.erb index 03f5d31..2747010 100644 --- a/modules/k8s/templates/initscripts/kubelet.systemd.erb +++ b/modules/k8s/templates/initscripts/kubelet.systemd.erb @@ -7,8 +7,8 @@ --kubeconfig=/etc/kubernetes/kubeconfig \ --api-servers=https://<%= @master_host %>:6443 \ --configure-cbr0=false \ - --tls-private-key-file=/var/lib/kubernetes/ssl/private_keys/server.key \ - --tls-cert-file=/var/lib/kubernetes/ssl/certs/cert.pem \ + --tls-private-key-file=<%= scope['::k8s::ssl::server_private_key_path'] %> \ + --tls-cert-file=<%= scope['::k8s::ssl::server_cert_path'] %> \ --cluster-domain=kube \ --hostname-override=<%= @fqdn %> diff --git a/modules/k8s/templates/kubeconfig-client.yaml.erb b/modules/k8s/templates/kubeconfig-client.yaml.erb index 2f1701a..4c366a6 100644 --- a/modules/k8s/templates/kubeconfig-client.yaml.erb +++ b/modules/k8s/templates/kubeconfig-client.yaml.erb @@ -4,7 +4,7 @@ clusters: - cluster: server: https://<%= @master_host %>:6443 - certificate-authority: /var/lib/kubernetes/ssl/certs/ca.pem + certificate-authority: /etc/ssl/certs/Puppet_Internal_CA.pem name: default contexts: - context: diff --git a/modules/toollabs/manifests/kube2proxy.pp b/modules/toollabs/manifests/kube2proxy.pp index 0dedda5..0fd364c 100644 --- a/modules/toollabs/manifests/kube2proxy.pp +++ b/modules/toollabs/manifests/kube2proxy.pp @@ -43,7 +43,7 @@ 'redis' => 'localhost:6379', 'kubernetes' => { 'master' => "https://${master_host}:6443", - 'ca_cert' => '/var/lib/kubernetes/ssl/certs/ca.pem', + 'ca_cert' => $::k8s::ssl::ca_cert_path, 'token' => $client_token, } } -- To view, visit https://gerrit.wikimedia.org/r/276427 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I5e2b4d01023fad27dddcb0c7c34a20f14f943ad3 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Gehel <gleder...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits