Gehel has uploaded a new change for review.
https://gerrit.wikimedia.org/r/276427
Change subject: Make r8s module use base::expose_puppet_certs
......................................................................
Make r8s module use base::expose_puppet_certs
WiP - do not merge
This enable reuse of code exposing Puppet SSL certificates
Tests have been written before changing any code, but there is enough that
I do not understand that I probably got it completely wrong. Puppet is not
deployed in the same way on Prod and Labs, which leads to a few tricky parts.
Bug: T124444
Change-Id: I5e2b4d01023fad27dddcb0c7c34a20f14f943ad3
---
A modules/k8s/Rakefile
M modules/k8s/manifests/apiserver.pp
M modules/k8s/manifests/controller.pp
M modules/k8s/manifests/ssl.pp
A modules/k8s/spec/defines/apiserver_rspec.rb
A modules/k8s/spec/defines/controller_rspec.rb
A modules/k8s/spec/defines/infrastructure_config_rspec.rb
A modules/k8s/spec/defines/kubelet_rspec.rb
A modules/k8s/spec/defines/role_toollabs_k8s_master_rspec.rb
A modules/k8s/spec/defines/ssl_rspec.rb
A modules/k8s/spec/fixtures/hiera/hiera.yaml
A modules/k8s/spec/fixtures/hiera/test.yaml
A modules/k8s/spec/fixtures/manifests/site.pp
A modules/k8s/spec/spec_helper.rb
M modules/k8s/templates/initscripts/controller-manager.systemd.erb
M modules/k8s/templates/initscripts/kube-apiserver.systemd.erb
M modules/k8s/templates/initscripts/kubelet.systemd.erb
M modules/k8s/templates/kubeconfig-client.yaml.erb
M modules/toollabs/manifests/kube2proxy.pp
19 files changed, 186 insertions(+), 57 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/27/276427/1
diff --git a/modules/k8s/Rakefile b/modules/k8s/Rakefile
new file mode 100644
index 0000000..6471966
--- /dev/null
+++ b/modules/k8s/Rakefile
@@ -0,0 +1,7 @@
+require 'rake'
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+ t.pattern = 'spec/*/*_rspec.rb'
+end
diff --git a/modules/k8s/manifests/apiserver.pp
b/modules/k8s/manifests/apiserver.pp
index 889afd4..3746bc3 100644
--- a/modules/k8s/manifests/apiserver.pp
+++ b/modules/k8s/manifests/apiserver.pp
@@ -1,6 +1,7 @@
class k8s::apiserver(
$etcd_servers,
$master_host,
+ $users = hiera('k8s_users'),
) {
require_package('kube-apiserver')
@@ -13,7 +14,6 @@
mode => '0700',
}
- $users = hiera('k8s_users')
file { '/etc/kubernetes/tokenauth':
content => template('k8s/tokenauth.csv.erb'),
owner => 'kubernetes',
diff --git a/modules/k8s/manifests/controller.pp
b/modules/k8s/manifests/controller.pp
index 6a4d52d..5253abb 100644
--- a/modules/k8s/manifests/controller.pp
+++ b/modules/k8s/manifests/controller.pp
@@ -1,5 +1,6 @@
class k8s::controller {
require_package('kube-controller-manager')
+ require ::k8s::ssl
include k8s::users
diff --git a/modules/k8s/manifests/ssl.pp b/modules/k8s/manifests/ssl.pp
index 6ffa111..e5a5784 100644
--- a/modules/k8s/manifests/ssl.pp
+++ b/modules/k8s/manifests/ssl.pp
@@ -9,54 +9,14 @@
$ssldir = '/var/lib/puppet/ssl',
$target_basedir = '/var/lib/kubernetes'
) {
- $puppet_cert_name = $::fqdn
+ $ca_cert_path = '/etc/ssl/certs/Puppet_Internal_CA.pem' #TODO: check if
this is the correct cert (also for labs)
+ $server_cert_path = "${target_basedir}/ssl/cert.pem"
+ $server_private_key_path = "${target_basedir}/ssl/server.key"
- file { $target_basedir:
- ensure => directory,
- owner => $user,
- group => $group,
- mode => '0755', # more permissive!
- }
-
- file { [
- "${target_basedir}/ssl",
- "${target_basedir}/ssl/certs",
- "${target_basedir}/ssl/private_keys",
- ]:
- ensure => directory,
- owner => $user,
- group => $group,
- mode => '0555',
- require => File[$target_basedir], # less permissive
- }
-
-
- file { "${target_basedir}/ssl/certs/ca.pem":
- ensure => present,
- owner => $user,
- group => $group,
- mode => '0444',
- source => "${ssldir}/certs/ca.pem",
- require => File["${target_basedir}/ssl/certs"],
- }
-
- file { "${target_basedir}/ssl/certs/cert.pem":
- ensure => present,
- owner => $user,
- group => $group,
- mode => '0400',
- source => "${ssldir}/certs/${puppet_cert_name}.pem",
- require => File["${target_basedir}/ssl/certs/ca.pem"],
- }
-
- if $provide_private {
- file { "${target_basedir}/ssl/private_keys/server.key":
- ensure => present,
- owner => $user,
- group => $group,
- mode => '0400',
- source => "${ssldir}/private_keys/${puppet_cert_name}.pem",
- require => File["${target_basedir}/ssl/private_keys"],
- }
+ ::base::expose_puppet_certs { $target_basedir:
+ provide_private => $provide_private,
+ user => $user,
+ group => $group,
+ ssldir => $ssldir,
}
}
diff --git a/modules/k8s/spec/defines/apiserver_rspec.rb
b/modules/k8s/spec/defines/apiserver_rspec.rb
new file mode 100644
index 0000000..9523677
--- /dev/null
+++ b/modules/k8s/spec/defines/apiserver_rspec.rb
@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe 'k8s::apiserver', :type => :class do
+ let(:facts) { {:fqdn => 'host.example.net'} }
+ let(:params) { {
+ :etcd_servers => 'https://etcd.example.net:2379',
+ :master_host => 'example.net',
+ :users => [],
+ } }
+
+ context 'with systemd as init' do
+ let(:facts) { {:initsystem => 'systemd'} }
+
+ it 'should containt a systemd unit file with correct certificate path' do
+ should contain_file('/lib/systemd/system/kube-apiserver.service')
+ .with({ 'ensure' => 'present' })
+
.with_content(%r{--tls-private-key-file=/var/lib/kubernetes/ssl/server.key})
+
.with_content(%r{--tls-cert-file=/var/lib/kubernetes/ssl/cert.pem})
+
.with_content(%r{--service-account-key-file=/var/lib/kubernetes/ssl/server.key})
+
+ end
+
+ end
+
+end
diff --git a/modules/k8s/spec/defines/controller_rspec.rb
b/modules/k8s/spec/defines/controller_rspec.rb
new file mode 100644
index 0000000..6f06106
--- /dev/null
+++ b/modules/k8s/spec/defines/controller_rspec.rb
@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe 'k8s::controller', :type => :class do
+ let(:facts) { {:fqdn => 'host.example.net'} }
+
+ context 'with systemd as init' do
+ let(:facts) { {:initsystem => 'systemd'} }
+
+ it 'should containt a systemd unit file with correct certificate path' do
+ should contain_file('/lib/systemd/system/controller-manager.service')
+ .with({ 'ensure' => 'present' })
+
.with_content(%r{--service-account-private-key-file=/var/lib/kubernetes/ssl/server.key})
+ end
+
+ end
+
+end
diff --git a/modules/k8s/spec/defines/infrastructure_config_rspec.rb
b/modules/k8s/spec/defines/infrastructure_config_rspec.rb
new file mode 100644
index 0000000..f49cca5
--- /dev/null
+++ b/modules/k8s/spec/defines/infrastructure_config_rspec.rb
@@ -0,0 +1,12 @@
+require 'spec_helper'
+
+describe 'k8s::infrastructure_config', :type => :class do
+
+ it 'should containt kubeconfig file with correct certificate path' do
+ should contain_file('/etc/kubernetes/kubeconfig')
+ .with({ 'ensure' => 'present' })
+ .with_content(%r{certificate-authority:
/etc/ssl/certs/Puppet_Internal_CA.pem})
+
+ end
+
+end
diff --git a/modules/k8s/spec/defines/kubelet_rspec.rb
b/modules/k8s/spec/defines/kubelet_rspec.rb
new file mode 100644
index 0000000..61c3724
--- /dev/null
+++ b/modules/k8s/spec/defines/kubelet_rspec.rb
@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe 'k8s::kubelet', :type => :class do
+ let(:facts) { {:fqdn => 'host.example.net'} }
+ let(:params) { {
+ :master_host => 'example.net',
+ } }
+
+ context 'with systemd as init' do
+ let(:facts) { {:initsystem => 'systemd'} }
+
+ it 'should containt a systemd unit file with correct certificate path' do
+ should contain_file('/lib/systemd/system/kubelet.service')
+ .with({ 'ensure' => 'present' })
+
.with_content(%r{--tls-private-key-file=/var/lib/kubernetes/ssl/server.key})
+
.with_content(%r{--tls-cert-file=/var/lib/kubernetes/ssl/cert.pem})
+
+ end
+
+ end
+
+end
diff --git a/modules/k8s/spec/defines/role_toollabs_k8s_master_rspec.rb
b/modules/k8s/spec/defines/role_toollabs_k8s_master_rspec.rb
new file mode 100644
index 0000000..bcf15df
--- /dev/null
+++ b/modules/k8s/spec/defines/role_toollabs_k8s_master_rspec.rb
@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+# This test could be moved to the `role` module, but all I am interested here
+# is the parts that are related to k8s, so I'm keeping it here for the moment.
+#
+# `role::toollabs::k8s::master` has too many dependencies to be testable under
+# reasonable efforts. This test can still be run by commenting out everything
+# not related to k8s in `role::toollabs::k8s::master`
+
+describe 'role::toollabs::k8s::master', :type => :class do
+ let(:facts) { {:fqdn => 'host.example.net'} }
+
+ context 'with systemd as init' do
+ let(:facts) { {:initsystem => 'systemd'} }
+
+ it 'should contain Kubernetes apiserver' do
+ pending 'role::toollabs::k8s::master should be refactored to be actually
testable'
+ should contain_class('k8s::apiserver')
+ end
+ it 'should contain Kubernetes scheduler' do
+ pending 'role::toollabs::k8s::master should be refactored to be actually
testable'
+ should contain_class('k8s::scheduler')
+ end
+ it 'should contain Kubernetes controller' do
+ pending 'role::toollabs::k8s::master should be refactored to be actually
testable'
+ should contain_class('k8s::controller')
+ end
+
+ end
+
+end
diff --git a/modules/k8s/spec/defines/ssl_rspec.rb
b/modules/k8s/spec/defines/ssl_rspec.rb
new file mode 100644
index 0000000..7a36794
--- /dev/null
+++ b/modules/k8s/spec/defines/ssl_rspec.rb
@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe 'k8s::ssl', :type => :class do
+ let(:facts) { { :fqdn => 'host.example.net'} }
+
+ describe 'certificates are exposed' do
+ it { should contain_file('/var/lib/kubernetes/ssl').with({ 'ensure' =>
'directory' }) }
+ it { should contain_file('/var/lib/kubernetes/ssl/cert.pem').with({
'ensure' => 'present' }) }
+ end
+
+ describe 'private key is not exposed by default' do
+ it { should_not
contain_file('/var/lib/kubernetes/ssl/private_keys/server.key') }
+ end
+
+ describe 'private key is exposed if required' do
+ let(:params) { { :provide_private => true } }
+
+ it { should contain_file('/var/lib/kubernetes/ssl/server.key')
+ .with({
+ 'ensure' => 'present',
+ 'mode' => '0400',
+ 'source' =>
'/var/lib/puppet/ssl/private_keys/host.example.net.pem',
+ })
+ }
+ end
+
+end
diff --git a/modules/k8s/spec/fixtures/hiera/hiera.yaml
b/modules/k8s/spec/fixtures/hiera/hiera.yaml
new file mode 100644
index 0000000..f99d200
--- /dev/null
+++ b/modules/k8s/spec/fixtures/hiera/hiera.yaml
@@ -0,0 +1,7 @@
+---
+:backends:
+ - yaml
+:yaml:
+ :datadir: spec/fixtures/hiera
+:hierarchy:
+ - test
diff --git a/modules/k8s/spec/fixtures/hiera/test.yaml
b/modules/k8s/spec/fixtures/hiera/test.yaml
new file mode 100644
index 0000000..111fdb0
--- /dev/null
+++ b/modules/k8s/spec/fixtures/hiera/test.yaml
@@ -0,0 +1,6 @@
+---
+k8s_users:
+ - name: client-infrastructure
+ token: test-token
+k8s::etcd_hosts:
+ - etcd.example.net
\ No newline at end of file
diff --git a/modules/k8s/spec/fixtures/manifests/site.pp
b/modules/k8s/spec/fixtures/manifests/site.pp
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/modules/k8s/spec/fixtures/manifests/site.pp
diff --git a/modules/k8s/spec/spec_helper.rb b/modules/k8s/spec/spec_helper.rb
new file mode 100644
index 0000000..3200f59
--- /dev/null
+++ b/modules/k8s/spec/spec_helper.rb
@@ -0,0 +1,14 @@
+require 'puppet'
+require 'rspec'
+require 'rspec-puppet'
+
+def param_value(subject, type, title, param)
+ subject.resource(type, title).send(:parameters)[param.to_sym]
+end
+
+RSpec.configure do |c|
+ c.module_path = File.expand_path(File.join(File.dirname(__FILE__), '../..'))
+ # Using an empty site.pp file to avoid:
https://github.com/rodjek/rspec-puppet/issues/15
+ c.manifest_dir = File.expand_path(File.join(File.dirname(__FILE__),
'fixtures/manifests'))
+ c.hiera_config = 'spec/fixtures/hiera/hiera.yaml'
+end
diff --git a/modules/k8s/templates/initscripts/controller-manager.systemd.erb
b/modules/k8s/templates/initscripts/controller-manager.systemd.erb
index 9cc9f25..107e4a2 100644
--- a/modules/k8s/templates/initscripts/controller-manager.systemd.erb
+++ b/modules/k8s/templates/initscripts/controller-manager.systemd.erb
@@ -7,7 +7,7 @@
ExecStart=/usr/bin/kube-controller-manager \
--cluster-cidr=192.168.0.0/24 \
--master=http://127.0.0.1:8080 \
-
--service-account-private-key-file=/var/lib/kubernetes/ssl/private_keys/server.key
+ --service-account-private-key-file=<%=
scope['::k8s::ssl::server_private_key_path'] %>
[Install]
WantedBy=multi-user.target
diff --git a/modules/k8s/templates/initscripts/kube-apiserver.systemd.erb
b/modules/k8s/templates/initscripts/kube-apiserver.systemd.erb
index 1f31a3f..1fa7b14 100644
--- a/modules/k8s/templates/initscripts/kube-apiserver.systemd.erb
+++ b/modules/k8s/templates/initscripts/kube-apiserver.systemd.erb
@@ -11,9 +11,9 @@
--admission-control=NamespaceLifecycle,ServiceAccount,ResourceQuota,LimitRanger,UidEnforcer
\
--authorization-mode=ABAC \
--authorization-policy-file=/etc/kubernetes/abac \
- --tls-private-key-file=/var/lib/kubernetes/ssl/private_keys/server.key \
- --tls-cert-file=/var/lib/kubernetes/ssl/certs/cert.pem \
- --service-account-key-file=/var/lib/kubernetes/ssl/private_keys/server.key
+ --tls-private-key-file=<%= scope['::k8s::ssl::server_private_key_path'] %>
\
+ --tls-cert-file=<%= scope['::k8s::ssl::server_cert_path'] %> \
+ --service-account-key-file=<%=
scope['::k8s::ssl::server_private_key_path'] %>
[Install]
WantedBy=multi-user.target
diff --git a/modules/k8s/templates/initscripts/kubelet.systemd.erb
b/modules/k8s/templates/initscripts/kubelet.systemd.erb
index 03f5d31..2747010 100644
--- a/modules/k8s/templates/initscripts/kubelet.systemd.erb
+++ b/modules/k8s/templates/initscripts/kubelet.systemd.erb
@@ -7,8 +7,8 @@
--kubeconfig=/etc/kubernetes/kubeconfig \
--api-servers=https://<%= @master_host %>:6443 \
--configure-cbr0=false \
- --tls-private-key-file=/var/lib/kubernetes/ssl/private_keys/server.key \
- --tls-cert-file=/var/lib/kubernetes/ssl/certs/cert.pem \
+ --tls-private-key-file=<%= scope['::k8s::ssl::server_private_key_path'] %>
\
+ --tls-cert-file=<%= scope['::k8s::ssl::server_cert_path'] %> \
--cluster-domain=kube \
--hostname-override=<%= @fqdn %>
diff --git a/modules/k8s/templates/kubeconfig-client.yaml.erb
b/modules/k8s/templates/kubeconfig-client.yaml.erb
index 2f1701a..4c366a6 100644
--- a/modules/k8s/templates/kubeconfig-client.yaml.erb
+++ b/modules/k8s/templates/kubeconfig-client.yaml.erb
@@ -4,7 +4,7 @@
clusters:
- cluster:
server: https://<%= @master_host %>:6443
- certificate-authority: /var/lib/kubernetes/ssl/certs/ca.pem
+ certificate-authority: /etc/ssl/certs/Puppet_Internal_CA.pem
name: default
contexts:
- context:
diff --git a/modules/toollabs/manifests/kube2proxy.pp
b/modules/toollabs/manifests/kube2proxy.pp
index 0dedda5..0fd364c 100644
--- a/modules/toollabs/manifests/kube2proxy.pp
+++ b/modules/toollabs/manifests/kube2proxy.pp
@@ -43,7 +43,7 @@
'redis' => 'localhost:6379',
'kubernetes' => {
'master' => "https://${master_host}:6443";,
- 'ca_cert' => '/var/lib/kubernetes/ssl/certs/ca.pem',
+ 'ca_cert' => $::k8s::ssl::ca_cert_path,
'token' => $client_token,
}
}
--
To view, visit https://gerrit.wikimedia.org/r/276427
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5e2b4d01023fad27dddcb0c7c34a20f14f943ad3
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Gehel <gleder...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
