BBlack has submitted this change and it was merged.
Change subject: protoproxy/sslcert/cache: nginx ssl_stapling_file support
......................................................................
protoproxy/sslcert/cache: nginx ssl_stapling_file support
Bug: T86666
Change-Id: If19dc78a8743cdcfff18b702a6c4502eeedcf393
---
M manifests/role/cache.pp
M manifests/role/protoproxy.pp
A modules/protoproxy/files/update-ocsp-all
M modules/protoproxy/manifests/localssl.pp
A modules/protoproxy/manifests/ocsp_updater.pp
M modules/protoproxy/templates/localssl.erb
A modules/sslcert/files/update-ocsp
M modules/sslcert/manifests/init.pp
8 files changed, 316 insertions(+), 13 deletions(-)
Approvals:
BBlack: Verified; Looks good to me, approved
diff --git a/manifests/role/cache.pp b/manifests/role/cache.pp
index 25b8418..d71d100 100644
--- a/manifests/role/cache.pp
+++ b/manifests/role/cache.pp
@@ -631,7 +631,7 @@
}
}
- define localssl($certname, $server_name=$::fqdn, $server_aliases=[],
$default_server=false) {
+ define localssl($certname, $do_ocsp=false, $server_name=$::fqdn,
$server_aliases=[], $default_server=false) {
# Assumes that LVS service IPs are setup elsewhere
install_certificate { $certname:
@@ -644,6 +644,7 @@
default_server => $default_server,
server_name => $server_name,
server_aliases => $server_aliases,
+ do_ocsp => $do_ocsp,
}
}
diff --git a/manifests/role/protoproxy.pp b/manifests/role/protoproxy.pp
index ddce309..f92f916 100644
--- a/manifests/role/protoproxy.pp
+++ b/manifests/role/protoproxy.pp
@@ -35,17 +35,6 @@
content => template('nginx/logrotate'),
tag => 'nginx', # workaround PUP-2689, can remove w/ puppetmaster
3.6.2+
}
-
- # reload protoproxies once a day for ticket keys
- # on legacy cache boxes (to be removed when no matching
- # hosts in the if clause here).
- if ! os_version('debian >= jessie') {
- cron { 'nginx_reload_daily':
- command => '/usr/sbin/service nginx reload >/dev/null 2>/dev/null',
- hour => fqdn_rand(24),
- minute => fqdn_rand(60),
- }
- }
}
class role::protoproxy::ssl::beta::common {
diff --git a/modules/protoproxy/files/update-ocsp-all
b/modules/protoproxy/files/update-ocsp-all
new file mode 100644
index 0000000..ea3a7ea
--- /dev/null
+++ b/modules/protoproxy/files/update-ocsp-all
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Executes update-ocsp for all existing OCSP files,
+# continuing through the list even if some fail, and then
+# reloads nginx configuration to apply updates and exits
+# with a status that reflects whether any updates failed
+
+if [ $# != 1 ]; then
+ echo One argument required: the proxy hostname:port
+ exit 1
+fi
+
+PROXY=$1
+OCSP_DIR=/var/cache/ocsp
+LCERT_DIR=/etc/ssl/localcerts
+
+some_failed=0
+for existing in ${OCSP_DIR}/*.ocsp; do
+ bn=$(basename $existing)
+ certname=${bn%.ocsp}
+ /usr/local/sbin/update-ocsp -c ${LCERT_DIR}/${certname}.crt -o $existing
-p $proxy
+ if [ $? -ne 0 ]; then
+ echo OCSP update failed for $certname
+ some_failed=1
+ fi
+done
+
+service nginx reload
+
+exit $some_failed
diff --git a/modules/protoproxy/manifests/localssl.pp
b/modules/protoproxy/manifests/localssl.pp
index 87f8279..51dfc6a 100644
--- a/modules/protoproxy/manifests/localssl.pp
+++ b/modules/protoproxy/manifests/localssl.pp
@@ -17,14 +17,22 @@
# [*default_server*]
# Boolean. Adds the 'default_server' option to the listen statement.
# Exactly one instance should have this set to true.
+#
+# [*do_ocsp*]
+# Boolean. Sets up OCSP Stapling for this server. This both enables the
+# correct configuration directives in the site's nginx config file as well
+# as creates the OCSP data file itself and ensures a cron is running to
+# keep it up to date.
define protoproxy::localssl(
$proxy_server_cert_name,
$server_name = $::fqdn,
$server_aliases = [],
$default_server = false,
- $upstream_port = '80'
+ $upstream_port = '80',
+ $do_ocsp = false
) {
+ require ::sslcert
# Ensure that exactly one definition exists with default_server = true
# if multiple defines have default_server set to true, this
@@ -38,6 +46,21 @@
# for localssl.erb below
$ssl_protos = 'ssl spdy'
+ if $do_ocsp {
+ include ::protoproxy::ocsp_updater
+
+ $certpath = "/etc/ssl/localcerts/${proxy_server_cert_name}.crt"
+ $output = "/var/cache/ocsp/${proxy_server_cert_name}.ocsp"
+ $proxy = "webproxy.${::site}.wmnet:8080"
+
+ exec { "${title}-create-ocsp":
+ command => "/usr/local/sbin/update-ocsp -c $cert -o $output -p
$proxy",
+ creates => $output,
+ require => Sslcert::Certificate[$proxy_server_cert_name],
+ before => Service['nginx']
+ }
+ }
+
nginx::site { $name:
require => Notify['protoproxy localssl default_server'], # Ensure a
default_server has been defined
content => template('protoproxy/localssl.erb')
diff --git a/modules/protoproxy/manifests/ocsp_updater.pp
b/modules/protoproxy/manifests/ocsp_updater.pp
new file mode 100644
index 0000000..4668885
--- /dev/null
+++ b/modules/protoproxy/manifests/ocsp_updater.pp
@@ -0,0 +1,59 @@
+# == Class: protoproxy::ocsp_updater
+#
+# This class defines a machine-global cronjob which updates
+# any existing OCSP files in /var/cache/ocsp once every two
+# hours, randomly splayed per-machine.
+#
+# It is intended to be used as "include protoproxy::ocsp_updater"
+# any time an ocsp file is defined for creation on a given machine.
+# See protoproxy::localssl for example.
+#
+# Note that everything about how we time/check this stuff today makes
+# assumptions based on GlobalSign's OCSP validity time windows. In the
+# future, it would be better to find a way to make the cron/check -timing
+# a bit more adaptive...
+#
+
+class protoproxy::ocsp_updater {
+ require ::sslcert
+
+ file { '/usr/local/sbin/update-ocsp-all':
+ mode => '0555',
+ owner => 'root',
+ group => 'root',
+ source => 'puppet:///modules/protoproxy/update-ocsp-all',
+ }
+
+ # This is "0" or "1" randomly by-host, used below with Linux crontab
+ # syntax to get every-two-hours timing with hosts splayed into even/odd
hours
+ $fqr01 = fqdn_rand(2, '97e54956f8c8e861')
+
+ cron { 'update-ocsp-all':
+ command => "/usr/local/sbin/update-ocsp-all
webproxy.${::site}.wmnet:8080",
+ minute => fqdn_rand(60, '1adf3dd699e51805'),
+ hour => "${fqr01}-23/2",
+ require => [
+ File['/usr/local/sbin/update-ocsp-all'],
+ Service['nginx'],
+ ],
+ }
+
+ # Generate icinga alert if OCSP files falling out of date due to errors
+ #
+ # The cron above attempts to get fresh data every 2 hours, and a good
+ # fresh fetch of data has a 12H lifetime with the windows we're seeing
+ # from GlobalSign today.
+ #
+ # The crit/warn values of 29100 and 14700 correspond are "8h5m" and
+ # "4h5m", so those are basically warning if two updates in a row failed
+ # for a given cert, and critical if 4 updates in a row fail (at which
+ # point we have 4h left to fix the situation before the validity window
+ # expires).
+
+ $check_args = '-c 29100 -w 14700 -d /var/cache/ocsp -g "*.ocsp"'
+ nrpe::monitor_service { 'ocsp-freshness':
+ description => 'Freshness of OCSP Stapling files',
+ nrpe_command => "/usr/lib/nagios/plugins/check-fresh-files-in-dir.py
$check_args",
+ require =>
File['/usr/lib/nagios/plugins/check-fresh-files-in-dir.py'],
+ }
+}
diff --git a/modules/protoproxy/templates/localssl.erb
b/modules/protoproxy/templates/localssl.erb
index 08e123e..2265f10 100644
--- a/modules/protoproxy/templates/localssl.erb
+++ b/modules/protoproxy/templates/localssl.erb
@@ -13,6 +13,11 @@
ssl_certificate /etc/ssl/localcerts/<%= @proxy_server_cert_name
%>.chained.crt;
ssl_certificate_key /etc/ssl/private/<%= @proxy_server_cert_name %>.key;
+ <% if @do_ocsp -%>
+ ssl_stapling on;
+ ssl_stapling_file /var/cache/ocsp/<%= @proxy_server_cert_name %>.ocsp;
+ <% end -%>
+
keepalive_timeout 60;
location / {
diff --git a/modules/sslcert/files/update-ocsp
b/modules/sslcert/files/update-ocsp
new file mode 100644
index 0000000..5f2ab69
--- /dev/null
+++ b/modules/sslcert/files/update-ocsp
@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# update-ocsp - creates or updates an OCSP stapling file for an SSL cert
+#
+# Copyright 2015 Brandon Black
+# Copyright 2015 Wikimedia Foundation, Inc.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import errno
+import argparse
+from subprocess import check_output
+import glob
+import tempfile
+import datetime
+
+
+def file_exists(fname):
+ """Helper for argparse to do check if a filename argument exists"""
+ if not os.path.exists(fname):
+ raise argparse.ArgumentTypeError("{0} does not exist".format(fname))
+ return fname
+
+
+def parse_options():
+ """Parse command-line options, return args hash"""
+ parser = argparse.ArgumentParser(description="OCSP Fetcher")
+ parser.add_argument('--certificate', '-c', dest="cert",
+ type=file_exists,
+ help="certificate filename",
+ required=True)
+ parser.add_argument('--output', '-o', dest="output",
+ help="output filename",
+ required=True)
+ parser.add_argument('--proxy', '-p', dest="proxy",
+ help="HTTP proxy server URL to use for OCSP request",
+ default=None)
+ parser.add_argument('--ca-certs', '-d', dest="cadir",
+ help="SSL CA certificates directory",
+ default='/etc/ssl/certs')
+ parser.add_argument('--time-offset-start', '-s', dest="offset_start",
+ help="validate thisUpdate <= X secs in the future",
+ type=int, default=60)
+ parser.add_argument('--time-offset-end', '-e', dest="offset_end",
+ help="validate nextUpdate >= X secs in the future",
+ type=int, default=3600)
+
+ return parser.parse_args()
+
+
+def cert_x509_option(filename, attrib):
+ """Returns output of an openssl x509 cert option w/ noout"""
+ return check_output([
+ "openssl", "x509", "-noout",
+ "-in", filename,
+ "-" + attrib,
+ ]).rstrip()
+
+
+def cert_x509_option_kv(filename, attrib):
+ """As above, but returns the value when output is k=v"""
+ k, v = cert_x509_option(filename, attrib).split("=", 1)
+ assert k == attrib
+ return v
+
+
+def cert_get_issuer_filename(cert, cadir):
+ """Get the filename of the immediate issuer of the given cert"""
+
+ # Note, this uses the pre-0.9.6 algorithm - it can be confused if
+ # there are 2+ distinct possible issuers in cadir with identical subjects!
+ # (is there a way to resolve that ambiguity that isn't unreasonable?)
+
+ issuer_subject = cert_x509_option_kv(cert, "issuer")
+ issuer_hash = cert_x509_option(cert, "issuer_hash")
+ for issuer in glob.glob(os.path.join(cadir, issuer_hash + '.[0-9]')):
+ if cert_x509_option_kv(issuer, "subject") == issuer_subject:
+ return issuer
+ raise Exception("No matching issuer file found at %s for %s",
+ (issuer_glob, cert))
+
+
+def cert_fetch_ocsp(cert, cadir, outfile, proxy):
+ """Fetch validated OCSP response for cert"""
+
+ issuer_path = cert_get_issuer_filename(cert, cadir)
+ ocsp_uri = cert_x509_option(cert, "ocsp_uri")
+
+ cmd = [
+ "openssl", "ocsp", "-nonce",
+ "-respout", outfile,
+ "-issuer", issuer_path,
+ "-cert", cert,
+ ]
+
+ if proxy:
+ cmd.extend([
+ "-path", ocsp_uri,
+ "-host", proxy,
+ ])
+ else:
+ cmd.extend([
+ "-url", ocsp_uri,
+ ])
+
+ return check_output(cmd)
+
+
+def ocsp_text_datetime_field(ocsp_text, field):
+ """Create datetime object from field:datetsring line in OCSP text"""
+ for line in ocsp_text.split("\n"):
+ if ":" in line:
+ k, v = line.strip().split(":", 1)
+ if k == field:
+ return datetime.datetime.strptime(v.lstrip(),
+ "%b %d %H:%M:%S %Y %Z")
+ raise Exception("Did not find OCSP datetime field for '%s'" % field)
+
+
+def ocsp_validate_window(ocspfile, offset_start, offset_end):
+ """Validate the validity range of the OCSP response"""
+
+ ocsp_text = check_output([
+ "openssl", "ocsp", "-noverify",
+ "-respin", ocspfile,
+ "-resp_text",
+ ])
+
+ thisup_dt = ocsp_text_datetime_field(ocsp_text, "This Update")
+ nextup_dt = ocsp_text_datetime_field(ocsp_text, "Next Update")
+ now_dt = datetime.datetime.utcnow()
+
+ thisup_notafter = now_dt + datetime.timedelta(0, offset_start)
+ if thisup_dt > thisup_notafter:
+ raise Exception("OCSP thisUpdate more than %i secs in the future"
+ % offset_start)
+
+ nextup_notbefore = now_dt + datetime.timedelta(0, offset_end)
+ if nextup_dt < nextup_notbefore:
+ raise Exception("OCSP nextUpdate less than %i secs in the future"
+ % offset_end)
+
+ return 0
+
+
+def mkdir_p(path):
+ try:
+ os.makedirs(path)
+ except OSError as exc:
+ if exc.errno == errno.EEXIST and os.path.isdir(path):
+ pass
+ else:
+ raise
+
+
+def main():
+ args = parse_options()
+
+ os.umask(022)
+ out_fn = os.path.basename(args.output)
+ out_basedir = os.path.dirname(args.output)
+ mkdir_p(out_basedir)
+ out_tempdir = tempfile.mkdtemp(".tmp", "update-ocsp-", out_basedir)
+ out_tempfile = os.path.join(out_tempdir, out_fn)
+
+ cert_fetch_ocsp(args.cert, args.cadir, out_tempfile, args.proxy)
+ ocsp_validate_window(out_tempfile, args.offset_start, args.offset_end)
+
+ os.rename(out_tempfile, args.output)
+ os.rmdir(out_tempdir)
+
+
+if __name__ == '__main__':
+ main()
+
+# vim: set ts=4 sw=4 et:
diff --git a/modules/sslcert/manifests/init.pp
b/modules/sslcert/manifests/init.pp
index d671fe9..619cf26 100644
--- a/modules/sslcert/manifests/init.pp
+++ b/modules/sslcert/manifests/init.pp
@@ -30,6 +30,14 @@
require => Package['ssl-cert'],
}
+ # generic script for fetching the OCSP file for a given cert
+ file { '/usr/local/sbin/update-ocsp':
+ mode => '0555',
+ owner => 'root',
+ group => 'root',
+ source => 'puppet:///modules/sslcert/update-ocsp',
+ }
+
# Limit AppArmor support to just Ubuntu, for now
if $::operatingsystem == 'Ubuntu' {
include apparmor
--
To view, visit https://gerrit.wikimedia.org/r/198110
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: If19dc78a8743cdcfff18b702a6c4502eeedcf393
Gerrit-PatchSet: 21
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack <[email protected]>
Gerrit-Reviewer: BBlack <[email protected]>
Gerrit-Reviewer: Faidon Liambotis <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
