The branch master has been updated via 61f488185e0736cf5196efc9d5f4f4b3370b3f8e (commit) from 539bea014de78db5ff5b0785a46bfd7647b0b589 (commit)
- Log ----------------------------------------------------------------- commit 61f488185e0736cf5196efc9d5f4f4b3370b3f8e Author: Richard Levitte <levi...@openssl.org> Date: Tue Jul 27 09:55:07 2021 +0200 Simplify the CDN purge Our CDN (Akamai) purge script was run as a standalone automation, in parallell with the automatic 'make' run. The consequence was that the CDN could catch a copy of our original web pages in a semi built state, as demonstrated by openssl/openssl#16152. The solution is the ensure that the purge is run in sequence after everything is built. We simplify this further by moving the actual script into the web source. Reviewed-by: Paul Dale <pa...@openssl.org> (Merged from https://github.com/openssl/web/pull/248) ----------------------------------------------------------------------- Summary of changes: Makefile | 7 +++-- bin/purge-one-hour | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 32b9244..fb15dad 100644 --- a/Makefile +++ b/Makefile @@ -56,15 +56,18 @@ SRCLISTS = $(foreach S,$(FUTURESERIES) $(SERIES) $(OLDSERIES2) fips,source/old/$ @rm -f $@ ./bin/md-to-html5 $< -all: suball manmaster mancross +all: suball manmaster mancross akamai-purge suball: $(SIMPLE) $(SRCLISTS) -relupd: suball manpages mancross +relupd: suball manpages mancross akamai-purge clean: rm -f $(SIMPLE) $(SRCLISTS) +akamai-purge: + ./bin/purge-one-hour + # Legacy targets hack-source_htaccess: all simple: all diff --git a/bin/purge-one-hour b/bin/purge-one-hour index 5e10e49..895967f 100755 --- a/bin/purge-one-hour +++ b/bin/purge-one-hour @@ -1,3 +1,89 @@ -#! /bin/sh +#! /usr/bin/perl +# +# script to purge the Akamai cache. +# +# Notes: +# +# - we limit the purging to files newer than an hour +# - there must be a file ~openssl/.edgerc with our Akamai credentials +# - the Akamai supplied program 'akamai-purge' must be installed in +# /usr/local/bin -/opt/openssl/maker/triggered-makers/akamai-purge +use strict; +use warnings; + +# Find all .html files that include a .inc file, and create a map +my %inc2html = (); + +my $debug = $ENV{DEBUG}; +my $dryrun = $ENV{DRYRUN}; + +my $base = '/var/www/openssl'; # MUST NOT end with a slash + +foreach ( `find $base -type f -name '*.html'` ) { + chomp; + my $file = $_; + my ($dn, $fn) = $_ =~ m/^(?:(.*)\/)?([^\/]*)$/; + my @incs = (); + + open HTML, $_; + foreach ( <HTML> ) { + if (/<!--\s*#include\s+virtual="([^"]*)"\s*-->/) { + my $vf = $1; + $vf = ($vf =~ m|^/|) ? "$base$vf" : "$dn/$vf"; + push @incs, "$vf"; + } + } + close HTML; + + foreach ( @incs ) { + push @{$inc2html{$_}}, $file; + } +} + +if ($debug) { + for ( sort keys %inc2html ) { + print STDERR "DEBUG: $_ => ", join(", ", @{$inc2html{$_}}), "\n"; + } +} + +# Find all files younger than an hour +# Discard those in .git/ and bin/ +# Discard any .ht* +# For any virtually included file, use the corresponding .html file instead +# For all remaining files, turn it into a valid URL +# For any valid index file, duplicate into two URLs without the file, +# one with an ending slash and one without. +my %files = (); + +foreach ( `find $base -type f -mtime -2` ) { + chomp; + next if /^\Q$base\E\/(\.git|bin)/; + next if /\/\.ht\w+$/; + my $x = $_; + my @files = defined $inc2html{$x} ? @{$inc2html{$x}} : ( $x ); + foreach ( @files ) { + s/^\Q$base\E\//https:\/\/www.openssl.org\//; + $files{$_} = 1; + if ( /^(.*)\/index.(html|cgi|pl|php|xhtml|htm)$/ ) { + $files{"$1/"} = $files{"$1"} = 1; + } + } +} + +# Finally, output the result to the akamai-purge program +my @files = sort keys %files; +while ( @files ) { + my $count = 500; # Try not to overwhelm Akamai + if ( $dryrun || open PURGE, '| /usr/local/bin/akamai-purge invalidate' ) { + printf STDERR + "DEBUG: Invoking '/usr/local/bin/akamai-purge invalidate' with:\n" + if $debug; + while ( @files && $count-- > 0 ) { + my $file = pop @files; + print STDERR " ",$file,"\n" if $debug; + print PURGE $file,"\n" unless $dryrun; + } + close PURGE unless $dryrun; + } +}