From: Dave Dykstra <d...@fnal.gov>
Date: 7 July 2009 6:35:19 AM
To: Mark Nottingham <m...@yahoo-inc.com>
Subject: Re: squid -I experiences
On Fri, Jul 03, 2009 at 01:52:43PM +1000, Mark Nottingham wrote:
I think you need to provide a license for it, so that it can be
included
there...
If you do that I'll be happy to push for its inclusion.
Oh, ok. I put in a BSD License. The changed versions are below. I
didn't make any other changes.
Thanks,
- Dave
------------------------------ multisquid ---------------------------
#!/usr/bin/perl -w
#
# run multiple squids.
# If the command line options are for starting up and listening on a
# socket, first open a socket for them to share with squid -I.
# If either one results in an error exit code, return the first
error code.
# Written by Dave Dykstra, July 2007
#
# Copyright (c) 2007, Fermi National Accelerator Laboratory
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the Fermi National Accelerator
Laboratory nor
# the names of its contributors may be used to endorse or promote
products
# derived from this software without specific prior written
permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED
# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
use strict;
use Socket;
use IO::Handle;
use Fcntl;
if ($#ARGV < 2) {
print STDERR "Usage: multisquid squidprefix numsquids http_port
[squid_args ...]\n";
exit 2;
}
my $prefix = shift(@ARGV);
my $numsquids = shift(@ARGV);
my $port = shift(@ARGV);
my $proto = getprotobyname('tcp');
if (!(($#ARGV >= 0) && (($ARGV[0] eq "-k") || ($ARGV[0] eq "-z")))) {
#open the socket for both squids to listen on if not doing an
# operation that doesn't use the socket (that is, -k or -z)
close STDIN;
my $fd;
socket($fd, PF_INET, SOCK_STREAM, $proto) || die "socket: $!";
setsockopt($fd, SOL_SOCKET, SO_REUSEADDR, 1) || die "setsockopt:
$!";
bind($fd, sockaddr_in($port, INADDR_ANY)) || die "bind of port
$port: $!";
}
my $childn;
for ($childn = 0; $childn < $numsquids; $childn++) {
if (fork() == 0) {
exec "$prefix/sbin/squid -f $prefix/etc/.squid-$childn.conf -I
@ARGV" || die "exec: $!";
}
# get them to start at different times so they're identifiable by
squidclient
sleep 2;
}
my $exitcode = 0;
while(wait() > 0) {
if (($? != 0) && ($exitcode == 0)) {
# Take the first non-zero exit code and ignore the other one.
# exit expects a byte, but the exit code from wait() has signal
# numbers in low byte and exit code in high byte. Combine them.
$exitcode = ($? >> 8) | ($? & 255);
}
}
exit $exitcode;
------------------------------ init-squid ---------------------------
#!/bin/bash
# This script will work with one squid or up to 4 squids on the same
http port.
# The number of squids is determined by the existence of cache
directories
# as follows. The main path to the cache directories is determined
by the
# cache_dir option in squid.conf. To run multiple squids, create
directories
# of the form
# `dirname $cache_dir`/$N/`basename $cache_dir`
# where N goes from 0 to the number of squids minus 1. Also create
# cache_log directories of the same form. Note that the cache_log
option
# in squid.conf is a file, not a directory, so the $N is up one level:
# cache_log_file=`basename $cache_log`
# cache_log_dir=`dirname $cache_log`
# cache_log_dir=`dirname $cache_log_dir`/$N/`basename
$cache_log_dir`
# The access_log should be in the same directory as the cache_log, and
# the pid_filename also needs to be in similarly named directories
(the
# same directories as the cache_log is a good choice).
#
# Written by Dave Dykstra, July 2007
#
# Copyright (c) 2007, Fermi National Accelerator Laboratory
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the Fermi National Accelerator
Laboratory nor
# the names of its contributors may be used to endorse or promote
products
# derived from this software without specific prior written
permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED
# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
. /etc/init.d/functions
RETVAL=0
INSTALL_DIR
=_your_base_install_dir_with_squid_and_utils_subdirectories_
#size at which rotateiflarge will rotate access.log
LARGE_ACCESS_LOG=1000000000
CONF_FILE=$INSTALL_DIR/squid/etc/squid.conf
CACHE_DIR=`awk '$1 == "cache_dir" {x=$3} END{print x}' $CONF_FILE`
CACHE_LOG=`awk '$1 == "cache_log" {x=$2} END{print x}' $CONF_FILE`
ACCESS_LOG=`awk '$1 == "access_log" {x=$2} END{print x}' $CONF_FILE`
squid_dirs()
{
# if $NUMSQUIDS is 1, echo the parameter, otherwise echo the parameter
# N times with $N before the basename of the parameter, where N is
# from 0 to $NUMSQUIDS-1
if [ $NUMSQUIDS = 1 ]; then
echo $1
return
fi
typeset N
N=0
while [ $N -lt $NUMSQUIDS ]; do
echo `dirname $1`/$N/`basename $1`
let N=$N+1
done
}
#see how many squid cache directories exist (up to 4)
NUMSQUIDS=0
for D in `NUMSQUIDS=4 squid_dirs $CACHE_DIR`; do
if [ ! -d $D ]; then
break
fi
let NUMSQUIDS=$NUMSQUIDS+1
done
if [ $NUMSQUIDS = 0 ]; then
NUMSQUIDS=1
SQUID=$INSTALL_DIR/squid/sbin/squid
PLURAL=""
else
PLURAL="s"
# create the squid.conf files for each squid
PID_FILENAME=`awk '$1 == "pid_filename" {x=$2} END{print x}'
$CONF_FILE`
HTTP_PORT=`awk '$1 == "http_port" {print $2;exit}' $CONF_FILE`
SNMP_PORT=`awk '$1 == "snmp_port" {x=$2} END{print x}' $CONF_FILE`
ICP_PORT=`awk '$1 == "icp_port" {x=$2} END{print x}' $CONF_FILE`
ICP_PORT=`awk '$1 == "icp_port" {x=$2} END{print x}' $CONF_FILE`
#visible_hostname is the name that shows up in X-Cache and Via headers
VISIBLE_HOSTNAME=`awk '$1 == "visible_hostname" {x=$2} END{print x}'
$CONF_FILE`
HOSTNAME=`hostname`
SQUID="$INSTALL_DIR/utils/sbin/multisquid $INSTALL_DIR/squid
$NUMSQUIDS ${HTTP_PORT:-3128}"
CACHEBASE="$(dirname $CACHE_DIR)"
LOGBASE="$(dirname $(dirname $CACHE_LOG))"
PIDBASE="$(dirname $(dirname $PID_FILENAME))"
N=0
while [ $N -lt $NUMSQUIDS ]; do
SEDCMDS="-e \"s,$CACHEBASE/,$CACHEBASE/$N/,\""
if [ "$LOGBASE" != "$CACHEBASE" ]; then
SEDCMDS="$SEDCMDS -e \"s,$LOGBASE/,$LOGBASE/$N/,\""
fi
if [ "$PIDBASE" != "$CACHEBASE" ] && [ "$PIDBASE" != "$LOGBASE" ];
then
SEDCMDS="$SEDCMDS -e \"s,$PIDBASE/,$PIDBASE/$N/,\""
fi
NEWCONF=$INSTALL_DIR/squid/etc/.squid-$N.conf
rm -f $NEWCONF
eval sed $SEDCMDS $INSTALL_DIR/squid/etc/squid.conf | awk '
BEGIN{print "# DO NOT EDIT -- Automatically generated by
'$0'";print}
/^[ \t#]*http_port[ \t]/ {
if (!got_http_port) {
#on first squid, add additional http_port after first one
specified,
# or after first commented http_port if none specified
portnum=0
if (substr($0,1,1) != "#") portnum=$2
else if ("'$HTTP_PORT'" == "") {portnum=3128; print}
if (portnum != 0) {
print "#first port overridden by command line -I"
print "http_port", portnum
# have each squid also listen on their own port, the first one to
# be parent of others and the others for forced cache reloads
print "http_port", portnum - ('$N' + 1)
got_http_port=1
next
}
}
else if ((substr($0,1,1) != "#") && ("'$N'" != 0)) {
# only first squid listens on other http ports, skip this one
next
}
}
/^[ \t#]*snmp_port[ \t]/ {
#replace snmp_port with separate number for each squid
portnum=0
if (substr($0,1,1) != "#") portnum=$2
else if ("'$SNMP_PORT'" == "") {portnum=3401; print}
if (portnum != 0) {
portnum=portnum + '$N'
print "snmp_port", portnum
next
}
}
/^[ \t#]*icp_port[ \t]/ {
#only first squid listens on icp port
portnum=0
if (substr($0,1,1) != "#") portnum=$2
else if ("'$SNMP_PORT'" == "") {portnum=3130; print}
if (portnum != 0) {
if ("'$N'" != 0) portnum=0 # disables the icp port
print "icp_port", portnum
next
}
}
/^[ \t#]*cache_peer[ \t]/ {
#any squid after first one go only to first one as a cache_peer
parent
if ("'$N'" != 0) {
if (substr($0,1,1) == "#") {
if (!got_cache_peer) {
#insert the line after the first commented-out one
print
portnum='${HTTP_PORT:-3128}' - 1
print "cache_peer localhost parent", portnum, "0 no-query"
got_cache_peer=1
next
}
}
else {
#skip any other uncommented cache_peer options on non-first
squids
next
}
}
}
$1 == "visible_hostname" || /TAG: visible_hostname/ {
hostname=""
if (substr($0,1,1) != "#") hostname=$2
else if ("'$VISIBLE_HOSTNAME'" == "") {hostname="'$HOSTNAME'";
print}
if (hostname != "") {
print "visible_hostname",hostname "/'$N'"
}
}
{print}
' >$NEWCONF
chmod a-w $NEWCONF
let N=$N+1
done
fi
last_logline_has()
{
# if first parameter is 1, see if the last line of any of the cache
logs
# of squids contains the other parameters; if the first parameter
is 0,
# see if true for all of the cache logs
typeset ANY LDIR LFILE
ANY=$1
shift
for LDIR in $(squid_dirs $(dirname $CACHE_LOG)); do
LFILE=$LDIR/`basename $CACHE_LOG`
if tail -1 $LFILE | egrep -q "$*"; then
if [ "$ANY" = 1 ]; then
# found one match, return true
return 0
fi
else
if [ "$ANY" = 0 ]; then
# found one non-match, return false
return 1
fi
fi
done
# return true for ANY=0 (all matched), false for ANY=1 (none matched)
return $ANY
}
start_squid()
{
echo "Starting $NUMSQUIDS Squid${PLURAL}... "
$SQUID -DFS
# it can take a while to check out a large cache
LIMIT=60 # Number of Tries
a=1
while [ $a -le "$LIMIT" ]
do
a=$(($a+1))
sleep 10
$SQUID -k check
RETVAL=$?
if [ $RETVAL != 0 ] ; then
echo "Squid start failed!!!"
start_squid_fail
RETVAL=$?
break
else
if last_logline_has 0 "storeLateRelease: released" ;then
break
RETVAL=0
fi
fi
done
sleep 2
echo "done."
[ $RETVAL == 0 ]
}
start_squid_fail()
{
RETVAL=0
$SQUID -k shutdown 2>/dev/null|| true # in case all are not shut down
if last_logline_has 1 "(store_errors|faults)" ;then
echo "Clearing Caches and Restarting $NUMSQUIDS Squid${PLURAL}... "
for D in `squid_dirs $CACHE_DIR`; do
rm -rf $D/*
done
$SQUID -z
sleep 3
$SQUID -DFS
sleep 3
$SQUID -k check
RETVAL=$?
if [ $RETVAL != 0 ] ; then
echo "Squid start failed!!!"
$SQUID -k shutdown 2>/dev/null || true # in case all are not shut
down
fi
else
echo "Script Can't Fix Squid Problem"
RETVAL=1
fi
[ $RETVAL == 0 ]
}
stop_squid()
{
echo -n "Stopping $NUMSQUIDS Squid${PLURAL}... "
$SQUID -k shutdown
RETVAL=$?
if [ $RETVAL != 0 ] ; then
echo "Squid stop failed!!!"
else
sleep 30
echo "done."
fi
[ $RETVAL == 0 ]
}
cleancache()
{
echo "Clearing Caches of $NUMSQUIDS Squid${PLURAL}... "
$SQUID -k shutdown 2>/dev/null||true # in case all are not shut down
for D in `squid_dirs $CACHE_DIR`; do
rm -rf $D/*
done
$SQUID -z
}
rotate()
{
echo "Rotating Caches of $NUMSQUIDS Squid${PLURAL}... "
$SQUID -k rotate
}
rotateiflarge()
{
for LDIR in $(squid_dirs $(dirname $ACCESS_LOG)); do
LFILE=$LDIR/`basename $ACCESS_LOG`
if [ "$(stat -c %s $LFILE)" -gt $LARGE_ACCESS_LOG ]; then
rotate
break
fi
done
}
start()
{
start_squid
[ $RETVAL == 0 ]
}
stop()
{
stop_squid
[ $RETVAL == 0 ]
}
case "$1" in
start)
cleancache
start
;;
stop)
stop
;;
status)
$SQUID -k check 2>/dev/null
exit $?
;;
reload)
$SQUID -k reconfigure
;;
restart)
stop
start
;;
cleancache)
cleancache
;;
rotate)
rotate
;;
rotateiflarge)
rotateiflarge
;;
*)
echo $"Usage: $0 {start|stop|status|restart|reload|cleancache|
rotate|rotateiflarge}"
exit 1
esac
