Dzahn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/405802 )
Change subject: httpd: copy apache-fast-test from apache module
......................................................................
httpd: copy apache-fast-test from apache module
This script is still in use to test rewrite rule changes.
Copy it from the apache module to the httpd module before
we remove the apache module later and forget about it.
While doing this also re-tabbed it and replaced literal
tabs with 4 spaces.
Change-Id: I0142b229bfccd77cb9457b974e5194686f5f1837
---
A modules/httpd/files/apache-fast-test
1 file changed, 253 insertions(+), 0 deletions(-)
Approvals:
jenkins-bot: Verified
Dzahn: Looks good to me, approved
diff --git a/modules/httpd/files/apache-fast-test
b/modules/httpd/files/apache-fast-test
new file mode 100755
index 0000000..64e82e4
--- /dev/null
+++ b/modules/httpd/files/apache-fast-test
@@ -0,0 +1,253 @@
+#!/usr/bin/perl
+#
+# Copyright (c) 2014,2015 Jeff Green <jgr...@wikimedia.org>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to
do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+use strict;
+no warnings 'threads';
+use threads;
+use threads::shared;
+use LWP::UserAgent;
+use Net::DNS::Resolver;
+use Time::HiRes;
+
+my $threads = 50; # how many concurrent threads?
+my $timeout_limit = 3; # how many timeouts to tolerate before dropping a server
+my $timeout = 10; # seconds before LWP gives up on a web request
+my $pad_length = 25; # server hostname column width in report
+my $default_staging_server = 'mwdebug1001.eqiad.wmnet';
+
+# get a list of servers to test
+my $servers;
+if ($ARGV[1]) {
+ # explicit list of servers
+ for (@ARGV[1..$#ARGV]) {
+ chomp;
+ if (/^pybal$/) {
+ # list of production servers from pybal config (URL or local file)
+ $servers = get_server_list_from_pybal_config(qw(
+ http://config-master.eqiad.wmnet/pybal/eqiad/apaches
+ http://config-master.eqiad.wmnet/pybal/eqiad/api
+ http://config-master.eqiad.wmnet/pybal/codfw/apaches
+ http://config-master.eqiad.wmnet/pybal/codfw/api
+ ));
+ } elsif (/^(eqiad|codfw)$/i) {
+ my $dc = $1;
+ $servers = get_server_list_from_pybal_config((
+ "http://config-master.eqiad.wmnet/pybal/$dc/apaches";,
+ "http://config-master.eqiad.wmnet/pybal/$dc/api";
+ ));
+ } elsif (/^([\w\.]+)$/) {
+ $servers->{$1} = 1;
+ }
+ }
+} else {
+ $servers->{$default_staging_server} = 1;
+}
+
+# thread-shared variables used for test/result
+my @queue : shared; # job_ids grouped in per-thread batches
+my %result :shared; # threads return stuff here
+my %timeouts :shared; # server timeout counter
+
+# read in the list of urls
+my $urls;
+if (defined $ARGV[0]) {
+ $urls = get_url_list($ARGV[0]);
+}
+
+# do the server DNS lookups in advance, threading DNS lookups is fraught with
peril
+my $resolver = Net::DNS::Resolver->new;
+for my $host (sort keys %{$servers}) {
+ my $answer = $resolver->search($host);
+ if ($answer) {
+ for my $r ($answer->answer) {
+ if ($r->type eq 'A') {
+ my $batch;
+ $servers->{$host} = $r->address; # grab the first A record
+ for my $url (sort keys %{$urls}) {
+ $batch .= "$servers->{$host}\t$url\n";
+ }
+ push @queue, $batch if defined $batch;
+ last;
+ }
+ }
+ }
+ unless ($servers->{$host} > 1) {
+ print "[31mno IP found for $host[0m\n";
+ delete $servers->{$host};
+ }
+}
+
+# informative output
+if (scalar @queue) {
+ print "testing " . (keys %{$urls}) . ' urls on ' . (keys %{$servers}) . '
servers, totalling ' .
+ ((keys %{$urls}) * (keys %{$servers})) . " requests\n";
+} else {
+ print "\n usage: $0 url_file [server spec]\n\n" .
+ " url_file: (web or local file containing one http/https URL per
line\n" .
+ " ./urls.txt\n".
+ " http://somehost.eqiad.wmnet/test_urls.txt\n\n"; .
+ " server spec:\n" .
+ " (none) use staging server $default_staging_server\n" .
+ " pybal fetch full webserver list from live pybal
configuration\n" .
+ " eqiad|codfw fetch full webserver list for a specific
datacenter\n" .
+ " host1 host2 host3\n\n";
+ exit;
+}
+
+# spawn worker threads to do api web requests, then reap them
+print "spawning threads";
+for (1..$threads) {
+ last unless scalar @queue;
+ threads->new(sub { process_queue_task() });
+}
+while (threads->list) {
+ sleep 0.1;
+ for my $thr (threads->list(threads::joinable)) {
+ $thr->join;
+ }
+}
+print "\n";
+
+# can't nest hashes with threads::shared, so we retrieve from shared %result
hash
+print "\n";
+for my $url (sort keys %{$urls}) {
+ my ($highlight, $previous_result, $report);
+ for my $host (sort keys %{$servers}) {
+ my $ip = $servers->{$host};
+ next unless defined $result{"$ip\t$url"};
+ $report .= " " . sprintf("%-${pad_length}s",$host) .
$result{"$ip\t$url"} . "\n";
+ $highlight++ if defined($previous_result) and ($previous_result ne
$result{"$ip\t$url"});
+ $previous_result = $result{"$ip\t$url"};
+ }
+ if (defined $highlight) {
+ print "[31m$url\n$report[0m";
+ } else {
+ print "$url\n * $previous_result\n"; # short form output
+ }
+}
+
+exit;
+
+
+
+
+
+
+# SUBROUTINES
+sub process_queue_task {
+ my $name = 'thread' . threads->self->tid();
+ print '.';
+ while (defined $queue[0]) {
+ for my $job (split("\n", shift @queue)) {
+ my ($ip,$url) = split("\t", $job);
+ if (($timeouts{$ip} < $timeout_limit) and ($url =~
/^(https?):\/\/([^\/]+)(.*)$/)) {
+ my ($protocol,$host,$path) = ($1,$2,$3);
+ $path = (defined $path) ? $path : '';
+ my $ua = LWP::UserAgent->new(timeout => $timeout);
+ my $request = HTTP::Request->new(GET => "http://$ip$path";);
+ $request->header(
+ 'Host' => $host,
+ 'User_Agent' => $0,
+ 'X-Forwarded-Proto' => $protocol, # proxies add this
+ );
+ my $r = $ua->simple_request($request);
+ if ($r->is_success) {
+ $result{"$ip\t$url"} = $r->status_line . ' ' .
get_mw_response_size($r->content);
+ } elsif (($r->is_error) and ($r->status_line =~ /^500 can't
connect/i)) { # simple timeout, drop the data
+ $timeouts{$ip}++;
+ print "$name dropped $ip, too many timeouts\n" if
$timeouts{$ip} >= $timeout_limit;
+ } elsif ($r->is_error) { # report other errors
+ $result{"$ip\t$url"} = $r->status_line;
+ } else {
+ $result{"$ip\t$url"} = $r->status_line . ' ' .
$r->header('Location');
+ }
+ }
+ }
+ }
+}
+
+sub get_mw_response_size {
+ my $body = shift;
+ $body =~ s/mw.config.set\([^()]+\);//mg; # remove transaction metadata
+ return length($body);
+}
+
+sub get_server_list_from_pybal_config {
+ my $servers;
+ for my $source (@_) {
+ my $content;
+ if ($source =~ /^http/) {
+ $content = slurp_http_doc($source);
+ } else {
+ $content = slurp_local_file($source);
+ }
+ for my $line (split /^/, $content) {
+ next if $line =~ /^\s*#/;
+ if ( ($line =~ /'enabled':\s+True/i) and ($line =~
/'host':\s+'([\.\w]+)'/) ) {
+ $servers->{$1} = 1;
+ }
+ }
+ }
+ return $servers;
+}
+
+sub get_url_list {
+ my ($content,$urls);
+ my $source = shift;
+ if ($source =~ /^http/) {
+ $content = slurp_http_doc($source);
+ } else {
+ $content = slurp_local_file($source);
+ }
+ for my $line (split /^/, $content) {
+ chomp $line;
+ if ($line =~ /^\s*(http\S+)/) {
+ $urls->{$1}++;
+ }
+ }
+ unless (keys %{$urls}) {
+ die "[31mno urls found in $source[0m\n";
+ }
+ return $urls;
+}
+
+sub slurp_http_doc {
+ my $url = shift;
+ my $ua = LWP::UserAgent->new(timeout => $timeout);
+ my $request = HTTP::Request->new(GET => $url);
+ my $r = $ua->simple_request($request);
+ if ($r->is_success) {
+ return $r->content;
+ } else {
+ print "[31merror fetching '$url'\n " . $r->status_line . "[0m\n";
+ }
+}
+
+sub slurp_local_file {
+ my $file = shift;
+ local $/ = undef;
+ open FILE, $file or die "can't open file '$file': $!";
+ binmode FILE;
+ my $content = <FILE>;
+ close FILE;
+ return $content;
+}
--
To view, visit https://gerrit.wikimedia.org/r/405802
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0142b229bfccd77cb9457b974e5194686f5f1837
Gerrit-PatchSet: 3
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Dzahn <dz...@wikimedia.org>
Gerrit-Reviewer: Dzahn <dz...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Jgreen <jgr...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
