Add cpu_affinity_map configuration option to bind workers to CPU cores.
Allows the admin to place all workers on individual cores without
writing a lot of if-statements. For example,
cpu_affinity_map process_numbers=1,2,3,4 cores=1,3,5,7
will have an effect for kids 1 through 4 only and will place them on
even cores starting with core #1.
If there are conflicts for a given process, the latest option wins and
a warning is printed. If the number of specified processes do not match
the number of specified cores, Squid quits with an error. Multiple
cpu_affinity_map options are merged.
Squid builds on systems without Linux CPU affinity calls, but setting
affinity only works if there are sched_getaffinity(2) and
sched_setaffinity(2) available. If there is no OS support but
cpu_affinity options are configured, Squid quits with an error. If there
is OS support but calls fail, Squid prints an error but does not quit.
Add cpu_affinity_map configuration option to bind workers to CPU cores.
Allows the admin to place all workers on individual cores without writing a
lot of if-statements. For example,
cpu_affinity_map process_numbers=1,2,3,4 cores=1,3,5,7
will have an effect for kids 1 through 4 only and will place them on even
cores starting with core #1.
If there are conflicts for a given process, the latest option wins and a
warning is printed. If the number of specified processes do not match the
number of specified cores, Squid quits with an error. Multiple
cpu_affinity_map options are merged.
Squid builds on systems without Linux CPU affinity calls, but setting affinity
only works if there are sched_getaffinity(2) and sched_setaffinity(2)
available. If there is no OS support but cpu_affinity options are configured,
Squid quits with an error. If there is OS support but calls fail, Squid prints
an error but does not quit.
=== modified file 'configure.in'
--- configure.in 2010-09-10 15:41:41 +0000
+++ configure.in 2010-09-10 23:57:42 +0000
@@ -2917,40 +2917,42 @@ AC_CHECK_FUNCS(\
memmove \
memset \
mkstemp \
mktime \
mstats \
poll \
prctl \
pthread_attr_setschedparam \
pthread_attr_setscope \
pthread_setschedparam \
pthread_sigmask \
putenv \
random \
regcomp \
regexec \
regfree \
res_init \
__res_init \
rint \
sbrk \
+ sched_getaffinity \
+ sched_setaffinity \
select \
seteuid \
setgroups \
setpgrp \
setrlimit \
setsid \
sigaction \
snprintf \
socketpair \
srand48 \
srandom \
statfs \
sysconf \
syslog \
timegm \
vsnprintf \
)
dnl ... and some we provide local replacements for
AC_REPLACE_FUNCS(\
drand48 \
@@ -2991,40 +2993,44 @@ else
fi
AC_MSG_NOTICE([Using ${squid_opt_io_loop_engine} for the IO loop.])
AM_CONDITIONAL([USE_POLL], [test $squid_opt_io_loop_engine = poll])
AM_CONDITIONAL([USE_EPOLL], [test $squid_opt_io_loop_engine = epoll])
AM_CONDITIONAL([USE_SELECT], [test $squid_opt_io_loop_engine = select])
AM_CONDITIONAL([USE_SELECT_SIMPLE], [test $squid_opt_io_loop_engine = select_simple])
AM_CONDITIONAL([USE_SELECT_WIN32], [test $squid_opt_io_loop_engine = select_win32])
AM_CONDITIONAL([USE_KQUEUE], [test $squid_opt_io_loop_engine = kqueue])
AM_CONDITIONAL([USE_DEVPOLL], [test $squid_opt_io_loop_engine = devpoll])
case $squid_opt_io_loop_engine in
epoll) AC_DEFINE(USE_EPOLL,1,[Use epoll() for the IO loop]) ;;
poll) AC_DEFINE(USE_POLL,1,[Use poll() for the IO loop]) ;;
kqueue) AC_DEFINE(USE_KQUEUE,1,[Use kqueue() for the IO loop]) ;;
select_win32) AC_DEFINE(USE_SELECT_WIN32,1,[Use Winsock select() for the IO loop]) ;;
select) AC_DEFINE(USE_SELECT,1,[Use select() for the IO loop]) ;;
esac
+if test "x$ac_cv_func_sched_getaffinity" = "xyes" -a "x$ac_cv_func_sched_setaffinity" = "xyes" ; then
+ AC_DEFINE(HAVE_CPU_AFFINITY,1,[Support setting CPU affinity for workers])
+fi
+
SQUID_CHECK_SETRESUID_WORKS
if test "x$squid_cv_resuid_works" = "xyes" ; then
AC_DEFINE(HAVE_SETRESUID,1,[Yay! Another Linux brokenness. Knowing that setresuid() exists is not enough, because RedHat 5.0 declares setresuid() but does not implement it.])
fi
SQUID_CHECK_FUNC_STRNSTR
SQUID_CHECK_FUNC_VACOPY
SQUID_CHECK_FUNC___VACOPY
dnl IP-Filter support requires ipf header files. These aren't
dnl installed by default, so we need to check for them
if test "x$enable_ipf_transparent" != "xno" ; then
AC_MSG_CHECKING(for availability of IP-Filter header files)
# hold on to your hats...
if test "x$ac_cv_header_ip_compat_h" = "xyes" -o \
"x$ac_cv_header_ip_fil_compat_h" = "xyes" -o \
"x$ac_cv_header_netinet_ip_compat_h" = "xyes" -o \
"x$ac_cv_header_netinet_ip_fil_compat_h" = "xyes" ; then
have_ipfilter_compat_header="yes"
=== added file 'src/CpuAffinity.cc'
--- src/CpuAffinity.cc 1970-01-01 00:00:00 +0000
+++ src/CpuAffinity.cc 2010-09-11 05:24:09 +0000
@@ -0,0 +1,57 @@
+/*
+ * $Id$
+ *
+ * DEBUG: section 54 Interprocess Communication
+ *
+ */
+
+#include "config.h"
+#include "base/TextException.h"
+#include "CpuAffinity.h"
+#include "CpuAffinityMap.h"
+#include "CpuAffinitySet.h"
+#include "structs.h"
+
+#include <algorithm>
+
+static CpuAffinitySet *TheCpuAffinitySet = NULL;
+
+
+void
+CpuAffinityInit()
+{
+ Must(!TheCpuAffinitySet);
+ if (Config.cpuAffinityMap) {
+ const int processNumber = InDaemonMode() ? KidIdentifier : 1;
+ TheCpuAffinitySet = Config.cpuAffinityMap->calculateSet(processNumber);
+ if (TheCpuAffinitySet)
+ TheCpuAffinitySet->apply();
+ }
+}
+
+void
+CpuAffinityReconfigure()
+{
+ if (TheCpuAffinitySet) {
+ TheCpuAffinitySet->undo();
+ delete TheCpuAffinitySet;
+ TheCpuAffinitySet = NULL;
+ }
+ CpuAffinityInit();
+}
+
+void
+CpuAffinityCheck()
+{
+ if (Config.cpuAffinityMap) {
+ Must(!Config.cpuAffinityMap->processes().empty());
+ const int maxProcess =
+ *std::max_element(Config.cpuAffinityMap->processes().begin(),
+ Config.cpuAffinityMap->processes().end());
+ const int numberOfProcesses = InDaemonMode() ? NumberOfKids() : 1;
+ if (maxProcess > numberOfProcesses) {
+ debugs(54, DBG_IMPORTANT, "WARNING: 'cpu_affinity_map' has "
+ "non-existing process number(s)");
+ }
+ }
+}
=== added file 'src/CpuAffinity.h'
--- src/CpuAffinity.h 1970-01-01 00:00:00 +0000
+++ src/CpuAffinity.h 2010-09-10 23:31:04 +0000
@@ -0,0 +1,20 @@
+/*
+ * $Id$
+ *
+ */
+
+#ifndef SQUID_CPU_AFFINITY_H
+#define SQUID_CPU_AFFINITY_H
+
+
+/// set CPU affinity for this process on startup
+SQUIDCEXTERN void CpuAffinityInit();
+
+/// reconfigure CPU affinity for this process
+SQUIDCEXTERN void CpuAffinityReconfigure();
+
+/// check CPU affinity configuration and print warnings if needed
+SQUIDCEXTERN void CpuAffinityCheck();
+
+
+#endif // SQUID_CPU_AFFINITY_H
=== added file 'src/CpuAffinityMap.cc'
--- src/CpuAffinityMap.cc 1970-01-01 00:00:00 +0000
+++ src/CpuAffinityMap.cc 2010-09-11 00:28:06 +0000
@@ -0,0 +1,58 @@
+/*
+ * $Id$
+ *
+ * DEBUG: section 54 Interprocess Communication
+ *
+ */
+
+#include "config.h"
+#include "base/TextException.h"
+#include "CpuAffinityMap.h"
+#include "CpuAffinitySet.h"
+#include "Debug.h"
+
+
+bool
+CpuAffinityMap::add(const Vector<int> &aProcesses, const Vector<int> &aCores)
+{
+ bool error = (aProcesses.size() != aCores.size());
+ for (size_t i = 0; !error && i < aProcesses.size(); ++i) {
+ const int process = aProcesses[i];
+ const int core = aCores[i];
+ if (process <= 0 || core <= 0)
+ error = true;
+ theProcesses.push_back(process);
+ theCores.push_back(core);
+ }
+ return !error;
+}
+
+CpuAffinitySet *
+CpuAffinityMap::calculateSet(const int targetProcess) const
+{
+ Must(theProcesses.size() == theCores.size());
+ int core = 0;
+ for (size_t i = 0; i < theProcesses.size(); ++i) {
+ const int process = theProcesses[i];
+ if (process == targetProcess)
+ {
+ if (core > 0) {
+ debugs(54, DBG_CRITICAL, "WARNING: conflicting "
+ "'cpu_affinity_map' for process number " << process <<
+ ", using the last core seen: " << theCores[i]);
+ }
+ core = theCores[i];
+ }
+ }
+ CpuAffinitySet *cpuAffinitySet = NULL;
+#if HAVE_CPU_AFFINITY
+ if (core > 0) {
+ cpuAffinitySet = new CpuAffinitySet;
+ cpu_set_t cpuSet;
+ CPU_ZERO(&cpuSet);
+ CPU_SET(core - 1, &cpuSet);
+ cpuAffinitySet->set(cpuSet);
+ }
+#endif
+ return cpuAffinitySet;
+}
=== added file 'src/CpuAffinityMap.h'
--- src/CpuAffinityMap.h 1970-01-01 00:00:00 +0000
+++ src/CpuAffinityMap.h 2010-09-10 23:32:15 +0000
@@ -0,0 +1,35 @@
+/*
+ * $Id$
+ *
+ */
+
+#ifndef SQUID_IPC_CPU_AFFINITY_MAP_H
+#define SQUID_IPC_CPU_AFFINITY_MAP_H
+
+#include "Array.h"
+
+class CpuAffinitySet;
+
+
+/// stores cpu_affinity_map configuration
+class CpuAffinityMap
+{
+public:
+ /// append cpu_affinity_map option
+ bool add(const Vector<int> &aProcesses, const Vector<int> &aCores);
+
+ /// calculate CPU set for this process
+ CpuAffinitySet *calculateSet(const int targetProcess) const;
+
+ /// returns list of process numbers
+ const Vector<int> &processes() const { return theProcesses; }
+
+ /// returns list of cores
+ const Vector<int> &cores() const { return theCores; }
+
+private:
+ Vector<int> theProcesses; ///< list of process numbers
+ Vector<int> theCores; ///< list of cores
+};
+
+#endif // SQUID_IPC_CPU_AFFINITY_MAP_H
=== added file 'src/CpuAffinitySet.cc'
--- src/CpuAffinitySet.cc 1970-01-01 00:00:00 +0000
+++ src/CpuAffinitySet.cc 2010-09-11 04:53:29 +0000
@@ -0,0 +1,85 @@
+/*
+ * $Id$
+ *
+ * DEBUG: section 54 Interprocess Communication
+ *
+ */
+
+#include "config.h"
+#include "base/TextException.h"
+#include "CpuAffinitySet.h"
+#include "Debug.h"
+#include "util.h"
+
+
+CpuAffinitySet::CpuAffinitySet()
+{
+#if HAVE_CPU_AFFINITY
+ CPU_ZERO(&theCpuSet);
+ CPU_ZERO(&theOrigCpuSet);
+#endif
+}
+
+void
+CpuAffinitySet::apply()
+{
+#if HAVE_CPU_AFFINITY
+ Must(CPU_COUNT(&theCpuSet) > 0); // CPU affinity mask set
+ Must(!applied());
+
+ bool success = false;
+ if (sched_getaffinity(0, sizeof(theOrigCpuSet), &theOrigCpuSet)) {
+ debugs(54, DBG_IMPORTANT, "ERROR: failed to get CPU affinity for "
+ "process PID " << getpid() << ", ignoring CPU affinity for "
+ "this process: " << xstrerror());
+ } else {
+ cpu_set_t cpuSet;
+ xmemcpy(&cpuSet, &theCpuSet, sizeof(cpuSet));
+ CPU_AND(&cpuSet, &cpuSet, &theOrigCpuSet);
+ if (CPU_COUNT(&cpuSet) <= 0) {
+ debugs(54, DBG_IMPORTANT, "ERROR: invalid CPU affinity for process "
+ "PID " << getpid() << ", may be caused by an invalid core in "
+ "'cpu_affinity_map' or by external affinity restrictions");
+ } else if (sched_setaffinity(0, sizeof(cpuSet), &cpuSet)) {
+ debugs(54, DBG_IMPORTANT, "ERROR: failed to set CPU affinity for "
+ "process PID " << getpid() << ": " << xstrerror());
+ } else
+ success = true;
+ }
+ if (!success)
+ CPU_ZERO(&theOrigCpuSet);
+#endif
+}
+
+void
+CpuAffinitySet::undo()
+{
+#if HAVE_CPU_AFFINITY
+ if (applied()) {
+ if (sched_setaffinity(0, sizeof(theOrigCpuSet), &theOrigCpuSet)) {
+ debugs(54, DBG_IMPORTANT, "ERROR: failed to restore original CPU "
+ "affinity for process PID " << getpid() << ": " <<
+ xstrerror());
+ }
+ CPU_ZERO(&theOrigCpuSet);
+ }
+#endif
+}
+
+bool
+CpuAffinitySet::applied() const
+{
+#if HAVE_CPU_AFFINITY
+ return (CPU_COUNT(&theOrigCpuSet) > 0);
+#else
+ return false;
+#endif
+}
+
+#if HAVE_CPU_AFFINITY
+void
+CpuAffinitySet::set(const cpu_set_t &aCpuSet)
+{
+ xmemcpy(&theCpuSet, &aCpuSet, sizeof(theCpuSet));
+}
+#endif
=== added file 'src/CpuAffinitySet.h'
--- src/CpuAffinitySet.h 1970-01-01 00:00:00 +0000
+++ src/CpuAffinitySet.h 2010-09-11 00:22:30 +0000
@@ -0,0 +1,41 @@
+/*
+ * $Id$
+ *
+ */
+
+#ifndef SQUID_IPC_CPU_AFFINITY_SET_H
+#define SQUID_IPC_CPU_AFFINITY_SET_H
+
+#include "SquidString.h"
+
+#if HAVE_CPU_AFFINITY && HAVE_SCHED_H
+#include <sched.h>
+#endif
+
+
+/// cpu affinity management for a single process
+class CpuAffinitySet
+{
+public:
+ CpuAffinitySet();
+
+ /// set CPU affinity for this process
+ void apply();
+
+ /// undo CPU affinity changes for this process
+ void undo();
+
+ /// whether apply() was called and was not undone
+ bool applied() const;
+
+#if HAVE_CPU_AFFINITY
+ /// set CPU affinity mask
+ void set(const cpu_set_t &aCpuSet);
+
+private:
+ cpu_set_t theCpuSet; ///< configured CPU affinity for this process
+ cpu_set_t theOrigCpuSet; ///< CPU affinity for this process before apply()
+#endif
+};
+
+#endif // SQUID_IPC_CPU_AFFINITY_SET_H
=== modified file 'src/Makefile.am'
--- src/Makefile.am 2010-09-11 00:52:48 +0000
+++ src/Makefile.am 2010-09-11 00:53:18 +0000
@@ -274,40 +274,46 @@ squid_SOURCES = \
client_side_reply.cc \
client_side_reply.h \
client_side_request.cc \
client_side_request.h \
ClientInfo.h \
BodyPipe.cc \
BodyPipe.h \
ClientInfo.h \
ClientRequestContext.h \
clientStream.cc \
clientStream.h \
CommIO.h \
CompletionDispatcher.cc \
CompletionDispatcher.h \
$(squid_COMMSOURCES) \
CommRead.h \
ConfigOption.cc \
ConfigParser.cc \
ConfigParser.h \
ConnectionDetail.h \
+ CpuAffinity.cc \
+ CpuAffinity.h \
+ CpuAffinityMap.cc \
+ CpuAffinityMap.h \
+ CpuAffinitySet.cc \
+ CpuAffinitySet.h \
debug.cc \
Debug.h \
defines.h \
$(DELAY_POOL_SOURCE) \
disk.cc \
$(DISKIO_SOURCE) \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
enums.h \
err_type.h \
errorpage.cc \
errorpage.h \
ETag.cc \
event.cc \
event.h \
EventLoop.h \
EventLoop.cc \
external_acl.cc \
ExternalACL.h \
@@ -1118,40 +1124,44 @@ tests_testCacheManager_SOURCES = \
tests/stub_main_cc.cc \
time.cc \
BodyPipe.cc \
cache_manager.cc \
cache_cf.cc \
ProtoPort.cc \
ProtoPort.h \
CacheDigest.cc \
carp.cc \
cbdata.cc \
ChunkedCodingParser.cc \
client_db.cc \
client_side.cc \
client_side_reply.cc \
client_side_request.cc \
ClientInfo.h \
clientStream.cc \
$(squid_COMMSOURCES) \
ConfigOption.cc \
ConfigParser.cc \
+ CpuAffinityMap.cc \
+ CpuAffinityMap.h \
+ CpuAffinitySet.cc \
+ CpuAffinitySet.h \
$(DELAY_POOL_SOURCE) \
disk.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
errorpage.cc \
ETag.cc \
external_acl.cc \
ExternalACLEntry.cc \
fd.cc \
fde.cc \
forward.cc \
fqdncache.cc \
ftp.cc \
gopher.cc \
hier_code.h \
helper.cc \
HelperChildConfig.h \
HelperChildConfig.cc \
$(HTCPSOURCE) \
@@ -1303,40 +1313,44 @@ tests_testEvent_SOURCES = \
tests/stub_main_cc.cc \
time.cc \
BodyPipe.cc \
cache_manager.cc \
cache_cf.cc \
ProtoPort.cc \
ProtoPort.h \
CacheDigest.cc \
carp.cc \
cbdata.cc \
ChunkedCodingParser.cc \
client_db.cc \
client_side.cc \
client_side_reply.cc \
client_side_request.cc \
ClientInfo.h \
clientStream.cc \
$(squid_COMMSOURCES) \
ConfigOption.cc \
ConfigParser.cc \
+ CpuAffinityMap.cc \
+ CpuAffinityMap.h \
+ CpuAffinitySet.cc \
+ CpuAffinitySet.h \
$(DELAY_POOL_SOURCE) \
disk.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
errorpage.cc \
ETag.cc \
external_acl.cc \
ExternalACLEntry.cc \
fd.cc \
fde.cc \
forward.cc \
fqdncache.cc \
ftp.cc \
gopher.cc \
hier_code.h \
helper.cc \
HelperChildConfig.h \
HelperChildConfig.cc \
$(HTCPSOURCE) \
@@ -1459,40 +1473,44 @@ tests_testEventLoop_SOURCES = \
tests/stub_main_cc.cc \
time.cc \
BodyPipe.cc \
cache_manager.cc \
cache_cf.cc \
ProtoPort.cc \
ProtoPort.h \
CacheDigest.cc \
carp.cc \
cbdata.cc \
ChunkedCodingParser.cc \
client_db.cc \
client_side.cc \
client_side_reply.cc \
client_side_request.cc \
ClientInfo.h \
clientStream.cc \
$(squid_COMMSOURCES) \
ConfigOption.cc \
ConfigParser.cc \
+ CpuAffinityMap.cc \
+ CpuAffinityMap.h \
+ CpuAffinitySet.cc \
+ CpuAffinitySet.h \
$(DELAY_POOL_SOURCE) \
disk.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
errorpage.cc \
ETag.cc \
external_acl.cc \
ExternalACLEntry.cc \
fd.cc \
fde.cc \
forward.cc \
fqdncache.cc \
ftp.cc \
gopher.cc \
helper.cc \
HelperChildConfig.h \
HelperChildConfig.cc \
hier_code.h \
$(HTCPSOURCE) \
@@ -1602,40 +1620,44 @@ tests_test_http_range_SOURCES = \
$(ACL_REGISTRATION_SOURCES) \
tests/test_http_range.cc \
BodyPipe.cc \
cache_cf.cc \
ProtoPort.cc \
ProtoPort.h \
cache_manager.cc \
CacheDigest.cc \
carp.cc \
cbdata.cc \
ChunkedCodingParser.cc \
client_db.cc \
client_side.cc \
client_side_reply.cc \
client_side_request.cc \
ClientInfo.h \
clientStream.cc \
$(squid_COMMSOURCES) \
ConfigOption.cc \
ConfigParser.cc \
+ CpuAffinityMap.cc \
+ CpuAffinityMap.h \
+ CpuAffinitySet.cc \
+ CpuAffinitySet.h \
tests/stub_main_cc.cc \
debug.cc \
$(DELAY_POOL_SOURCE) \
disk.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
errorpage.cc \
ETag.cc \
external_acl.cc \
ExternalACLEntry.cc \
fd.cc \
fde.cc \
forward.cc \
fqdncache.cc \
ftp.cc \
gopher.cc \
helper.cc \
HelperChildConfig.h \
HelperChildConfig.cc \
@@ -1765,40 +1787,44 @@ tests_testHttpRequest_SOURCES = \
tests/stub_main_cc.cc \
time.cc \
BodyPipe.cc \
cache_manager.cc \
cache_cf.cc \
ProtoPort.cc \
ProtoPort.h \
CacheDigest.cc \
carp.cc \
cbdata.cc \
ChunkedCodingParser.cc \
client_db.cc \
client_side.cc \
client_side_reply.cc \
client_side_request.cc \
ClientInfo.h \
clientStream.cc \
$(squid_COMMSOURCES) \
ConfigOption.cc \
ConfigParser.cc \
+ CpuAffinityMap.cc \
+ CpuAffinityMap.h \
+ CpuAffinitySet.cc \
+ CpuAffinitySet.h \
$(DELAY_POOL_SOURCE) \
disk.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
errorpage.cc \
ETag.cc \
external_acl.cc \
ExternalACLEntry.cc \
fd.cc \
fde.cc \
forward.cc \
fqdncache.cc \
ftp.cc \
gopher.cc \
helper.cc \
HelperChildConfig.h \
HelperChildConfig.cc \
hier_code.h \
$(HTCPSOURCE) \
@@ -2141,40 +2167,44 @@ tests_testURL_SOURCES = \
tests/stub_main_cc.cc \
time.cc \
BodyPipe.cc \
cache_manager.cc \
cache_cf.cc \
ProtoPort.cc \
ProtoPort.h \
CacheDigest.cc \
carp.cc \
cbdata.cc \
ChunkedCodingParser.cc \
client_db.cc \
client_side.cc \
client_side_reply.cc \
client_side_request.cc \
ClientInfo.h \
clientStream.cc \
$(squid_COMMSOURCES) \
ConfigOption.cc \
ConfigParser.cc \
+ CpuAffinityMap.cc \
+ CpuAffinityMap.h \
+ CpuAffinitySet.cc \
+ CpuAffinitySet.h \
$(DELAY_POOL_SOURCE) \
disk.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
errorpage.cc \
ETag.cc \
external_acl.cc \
ExternalACLEntry.cc \
fd.cc \
fde.cc \
forward.cc \
fqdncache.cc \
ftp.cc \
gopher.cc \
helper.cc \
HelperChildConfig.h \
HelperChildConfig.cc \
hier_code.h \
$(HTCPSOURCE) \
=== modified file 'src/cache_cf.cc'
--- src/cache_cf.cc 2010-08-25 03:08:37 +0000
+++ src/cache_cf.cc 2010-09-10 23:11:43 +0000
@@ -33,40 +33,41 @@
*/
#include "squid.h"
#include "acl/Acl.h"
#include "acl/Gadgets.h"
#include "acl/MethodData.h"
#if USE_ADAPTATION
#include "adaptation/Config.h"
#endif
#if ICAP_CLIENT
#include "adaptation/icap/Config.h"
#endif
#if USE_ECAP
#include "adaptation/ecap/Config.h"
#endif
#include "auth/Config.h"
#include "auth/Scheme.h"
#include "CacheManager.h"
#include "ConfigParser.h"
+#include "CpuAffinityMap.h"
#include "eui/Config.h"
#if USE_SQUID_ESI
#include "esi/Parser.h"
#endif
#include "HttpRequestMethod.h"
#include "ident/Config.h"
#include "ip/Intercept.h"
#include "ip/QosConfig.h"
#include "ip/tools.h"
#include "log/Config.h"
#include "MemBuf.h"
#include "Parsing.h"
#include "ProtoPort.h"
#include "rfc1738.h"
#if SQUID_SNMP
#include "snmp.h"
#endif
#include "Store.h"
#include "StoreFileSystem.h"
#include "SwapDir.h"
@@ -164,40 +165,45 @@ static void free_IpAddress_list(Ip::Addr
static int check_null_IpAddress_list(const Ip::Address_list *);
#endif /* CURRENTLY_UNUSED */
#endif /* USE_WCCPv2 */
static void parse_http_port_list(http_port_list **);
static void dump_http_port_list(StoreEntry *, const char *, const http_port_list *);
static void free_http_port_list(http_port_list **);
#if USE_SSL
static void parse_https_port_list(https_port_list **);
static void dump_https_port_list(StoreEntry *, const char *, const https_port_list *);
static void free_https_port_list(https_port_list **);
#if 0
static int check_null_https_port_list(const https_port_list *);
#endif
#endif /* USE_SSL */
static void parse_b_size_t(size_t * var);
static void parse_b_int64_t(int64_t * var);
+static bool parseNamedIntList(const char *data, const String &name, Vector<int> &list);
+static void parse_CpuAffinityMap(CpuAffinityMap **const cpuAffinityMap);
+static void dump_CpuAffinityMap(StoreEntry *const entry, const char *const name, const CpuAffinityMap *const cpuAffinityMap);
+static void free_CpuAffinityMap(CpuAffinityMap **const cpuAffinityMap);
+
static int parseOneConfigFile(const char *file_name, unsigned int depth);
/*
* LegacyParser is a parser for legacy code that uses the global
* approach. This is static so that it is only exposed to cache_cf.
* Other modules needing access to a ConfigParser should have it
* provided to them in their parserFOO methods.
*/
static ConfigParser LegacyParser = ConfigParser();
void
self_destruct(void)
{
LegacyParser.destruct();
}
static void
update_maxobjsize(void)
{
int i;
@@ -3825,40 +3831,104 @@ free_logformat(logformat ** definitions)
static void
free_access_log(customlog ** definitions)
{
while (*definitions) {
customlog *log = *definitions;
*definitions = log->next;
log->logFormat = NULL;
log->type = CLF_UNKNOWN;
if (log->aclList)
aclDestroyAclList(&log->aclList);
safe_free(log->filename);
xfree(log);
}
}
+/// parses list of integers form name=N1,N2,N3,...
+static bool
+parseNamedIntList(const char *data, const String &name, Vector<int> &list)
+{
+ if (data && (strncmp(data, name.rawBuf(), name.size()) == 0)) {
+ data += name.size();
+ if (*data == '=') {
+ while (true) {
+ ++data;
+ int value = 0;
+ if (!StringToInt(data, value, &data, 10))
+ break;
+ list.push_back(value);
+ if (*data == '\0' || *data != ',')
+ break;
+ }
+ }
+ }
+ return *data == '\0';
+}
+
+static void
+parse_CpuAffinityMap(CpuAffinityMap **const cpuAffinityMap) {
+#if HAVE_CPU_AFFINITY
+ if (!*cpuAffinityMap)
+ *cpuAffinityMap = new CpuAffinityMap;
+
+ const char *const pToken = strtok(NULL, w_space);
+ const char *const cToken = strtok(NULL, w_space);
+ Vector<int> processes, cores;
+ if (!parseNamedIntList(pToken, "process_numbers", processes) ||
+ !parseNamedIntList(cToken, "cores", cores) ||
+ !(*cpuAffinityMap)->add(processes, cores))
+ self_destruct();
+#else
+ debugs(3, DBG_CRITICAL, "ERROR: Squid built with no CPU affinity support, "
+ "don't set 'cpu_affinity_map'!");
+ self_destruct();
+#endif
+}
+
+static void
+dump_CpuAffinityMap(StoreEntry *const entry, const char *const name, const CpuAffinityMap *const cpuAffinityMap) {
+ if (cpuAffinityMap) {
+ storeAppendPrintf(entry, "%s process_numbers=", name);
+ for (size_t i = 0; i < cpuAffinityMap->processes().size(); ++i) {
+ storeAppendPrintf(entry, "%s%i", (i ? "," : ""),
+ cpuAffinityMap->processes()[i]);
+ }
+ storeAppendPrintf(entry, " cores=");
+ for (size_t i = 0; i < cpuAffinityMap->processes().size(); ++i) {
+ storeAppendPrintf(entry, "%s%i", (i ? "," : ""),
+ cpuAffinityMap->cores()[i]);
+ }
+ storeAppendPrintf(entry, "\n");
+ }
+}
+
+static void
+free_CpuAffinityMap(CpuAffinityMap **const cpuAffinityMap) {
+ delete *cpuAffinityMap;
+ *cpuAffinityMap = 0;
+}
+
#if USE_ADAPTATION
static void
parse_adaptation_service_set_type()
{
Adaptation::Config::ParseServiceSet();
}
static void
parse_adaptation_service_chain_type()
{
Adaptation::Config::ParseServiceChain();
}
static void
parse_adaptation_access_type()
{
Adaptation::Config::ParseAccess(LegacyParser);
}
=== modified file 'src/cf.data.depend'
--- src/cf.data.depend 2010-04-05 10:29:50 +0000
+++ src/cf.data.depend 2010-09-10 19:55:03 +0000
@@ -1,33 +1,34 @@
# type dependencies
access_log acl logformat
acl external_acl_type auth_param
acl_access acl
acl_address acl
acl_b_size_t acl
acl_tos acl
address
authparam
b_int64_t
b_size_t
cachedir cache_replacement_policy
cachemgrpasswd
+CpuAffinityMap
debug
delay_pool_access acl delay_class
delay_pool_class delay_pools
delay_pool_count
delay_pool_rates delay_class
denyinfo acl
eol
externalAclHelper auth_param
HelperChildConfig
hostdomain cache_peer
hostdomaintype cache_peer
http_header_access acl
http_header_replace
http_port_list
https_port_list
adaptation_access_type adaptation_service_set adaptation_service_chain acl icap_service icap_class
adaptation_service_set_type icap_service ecap_service
adaptation_service_chain_type icap_service ecap_service
icap_access_type icap_class acl
icap_class_type icap_service
=== modified file 'src/cf.data.pre'
--- src/cf.data.pre 2010-09-10 20:56:24 +0000
+++ src/cf.data.pre 2010-09-11 04:53:59 +0000
@@ -7055,21 +7055,43 @@ DOC_START
The default "0" means Squid inherits the current ulimit setting.
Note: Changing this requires a restart of Squid. Also
not all comm loops supports large values.
DOC_END
NAME: workers
TYPE: int
LOC: Config.workers
DEFAULT: 1
DOC_START
Number of main Squid processes or "workers" to fork and maintain.
0: "no daemon" mode, like running "squid -N ..."
1: "no SMP" mode, start one main Squid process daemon (default)
N: start N main Squid process daemons (i.e., SMP mode)
In SMP mode, each worker does nearly all what a single Squid daemon
does (e.g., listen on http_port and forward HTTP requests).
DOC_END
+NAME: cpu_affinity_map
+TYPE: CpuAffinityMap
+LOC: Config.cpuAffinityMap
+DEFAULT: none
+DOC_START
+ Usage: cpu_affinity_map process_numbers=P1,P2,... cores=C1,C2,...
+
+ Set 1:1 mapping between Squid "worker" processes and CPU cores. E.g.
+
+ cpu_affinity_map process_numbers=1,2,3,4 cores=1,3,5,7
+
+ This will have an effect for kids 1 through 4 only and place them on
+ even cores starting with core #1.
+
+ CPU cores are numbered starting from 1. Requires sched_getaffinity(2)
+ and sched_setaffinity(2) system calls.
+
+ Multiple cpu_affinity_map options are merged.
+
+ See also: workers
+DOC_END
+
EOF
=== modified file 'src/main.cc'
--- src/main.cc 2010-09-08 14:46:42 +0000
+++ src/main.cc 2010-09-11 05:26:20 +0000
@@ -23,40 +23,41 @@
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
*
*/
#include "squid.h"
#include "AccessLogEntry.h"
#if ICAP_CLIENT
#include "adaptation/icap/icap_log.h"
#endif
#include "auth/Gadgets.h"
#include "base/TextException.h"
#include "ConfigParser.h"
+#include "CpuAffinity.h"
#include "errorpage.h"
#include "event.h"
#include "EventLoop.h"
#include "ExternalACL.h"
#include "Store.h"
#include "ICP.h"
#include "ident/Ident.h"
#include "HttpReply.h"
#include "pconn.h"
#include "Mem.h"
#include "acl/Asn.h"
#include "acl/Acl.h"
#include "htcp.h"
#include "StoreFileSystem.h"
#include "DiskIO/DiskIOModule.h"
#include "comm.h"
#include "ipc/Kids.h"
#include "ipc/Coordinator.h"
#include "ipc/Strand.h"
#include "ip/tools.h"
@@ -753,40 +754,44 @@ mainReconfigureFinish(void *)
errorClean();
enter_suid(); /* root to read config file */
// we may have disabled the need for PURGE
if (Config2.onoff.enable_purge)
Config2.onoff.enable_purge = 2;
// parse the config returns a count of errors encountered.
const int oldWorkers = Config.workers;
if ( parseConfigFile(ConfigFile) != 0) {
// for now any errors are a fatal condition...
self_destruct();
}
if (oldWorkers != Config.workers) {
debugs(1, DBG_CRITICAL, "WARNING: Changing 'workers' (from " <<
oldWorkers << " to " << Config.workers <<
") is not supported and ignored");
Config.workers = oldWorkers;
}
+ if (IamPrimaryProcess())
+ CpuAffinityCheck();
+ CpuAffinityReconfigure();
+
setUmask(Config.umask);
Mem::Report();
setEffectiveUser();
_db_init(Debug::cache_log, Debug::debugOptions);
ipcache_restart(); /* clear stuck entries */
fqdncache_restart(); /* sigh, fqdncache too */
parseEtcHosts();
errorInitialize(); /* reload error pages */
accessLogInit();
#if USE_LOADABLE_MODULES
LoadableModulesConfigure(Config.loadable_module_names);
#endif
#if USE_ADAPTATION
bool enableAdaptation = false;
#if ICAP_CLIENT
Adaptation::Icap::TheConfig.finalize();
enableAdaptation = Adaptation::Icap::TheConfig.onoff || enableAdaptation;
#endif
@@ -1393,40 +1398,44 @@ SquidMain(int argc, char **argv)
}
sendSignal();
/* NOTREACHED */
}
if (opt_create_swap_dirs) {
/* chroot if configured to run inside chroot */
if (Config.chroot_dir && chroot(Config.chroot_dir)) {
fatal("failed to chroot");
}
setEffectiveUser();
debugs(0, 0, "Creating Swap Directories");
Store::Root().create();
return 0;
}
+ if (IamPrimaryProcess())
+ CpuAffinityCheck();
+ CpuAffinityInit();
+
if (!opt_no_daemon && Config.workers > 0)
watch_child(argv);
setMaxFD();
/* init comm module */
comm_init();
comm_select_init();
if (opt_no_daemon) {
/* we have to init fdstat here. */
fd_open(0, FD_LOG, "stdin");
fd_open(1, FD_LOG, "stdout");
fd_open(2, FD_LOG, "stderr");
}
#if USE_WIN32_SERVICE
WIN32_svcstatusupdate(SERVICE_START_PENDING, 10000);
=== modified file 'src/protos.h'
--- src/protos.h 2010-08-24 10:35:03 +0000
+++ src/protos.h 2010-09-11 05:33:24 +0000
@@ -566,42 +566,46 @@ SQUIDCEXTERN void enter_suid(void);
SQUIDCEXTERN void no_suid(void);
SQUIDCEXTERN void writePidFile(void);
SQUIDCEXTERN void setSocketShutdownLifetimes(int);
SQUIDCEXTERN void setMaxFD(void);
SQUIDCEXTERN void setSystemLimits(void);
SQUIDCEXTERN void squid_signal(int sig, SIGHDLR *, int flags);
SQUIDCEXTERN pid_t readPidFile(void);
SQUIDCEXTERN void keepCapabilities(void);
SQUIDCEXTERN void BroadcastSignalIfAny(int& sig);
/// whether the current process is the parent of all other Squid processes
SQUIDCEXTERN bool IamMasterProcess();
/**
whether the current process is dedicated to doing things that only
a single process should do, such as PID file maintenance and WCCP
*/
SQUIDCEXTERN bool IamPrimaryProcess();
/// whether the current process coordinates worker processes
SQUIDCEXTERN bool IamCoordinatorProcess();
/// whether the current process handles HTTP transactions and such
SQUIDCEXTERN bool IamWorkerProcess();
+/// Whether we are running in daemon mode
+SQUIDCEXTERN bool InDaemonMode(); // try using specific Iam*() checks above first
/// Whether there should be more than one worker process running
SQUIDCEXTERN bool UsingSmp(); // try using specific Iam*() checks above first
+/// returns the number of kids
+SQUIDCEXTERN int NumberOfKids();
SQUIDCEXTERN int DebugSignal;
/* AYJ debugs function to show locations being reset with memset() */
SQUIDCEXTERN void *xmemset(void *dst, int, size_t);
SQUIDCEXTERN void debug_trap(const char *);
SQUIDCEXTERN void logsFlush(void);
SQUIDCEXTERN const char *checkNullString(const char *p);
SQUIDCEXTERN void squid_getrusage(struct rusage *r);
SQUIDCEXTERN double rusage_cputime(struct rusage *r);
SQUIDCEXTERN int rusage_maxrss(struct rusage *r);
SQUIDCEXTERN int rusage_pagefaults(struct rusage *r);
SQUIDCEXTERN void releaseServerSockets(void);
SQUIDCEXTERN void PrintRusage(void);
SQUIDCEXTERN void dumpMallocStats(void);
=== modified file 'src/structs.h'
--- src/structs.h 2010-09-10 20:56:24 +0000
+++ src/structs.h 2010-09-10 23:57:42 +0000
@@ -113,40 +113,41 @@ struct ushortlist {
};
struct relist {
char *pattern;
regex_t regex;
relist *next;
};
#if DELAY_POOLS
#include "DelayConfig.h"
#endif
#if USE_ICMP
#include "icmp/IcmpConfig.h"
#endif
#include "HelperChildConfig.h"
/* forward decl for SquidConfig, see RemovalPolicy.h */
+class CpuAffinityMap;
class RemovalPolicySettings;
class external_acl;
class Store;
struct SquidConfig {
struct {
/* These should be for the Store::Root instance.
* this needs pluggable parsing to be done smoothly.
*/
int highWaterMark;
int lowWaterMark;
} Swap;
size_t memMaxSize;
struct {
int64_t min;
int pct;
int64_t max;
} quickAbort;
@@ -594,40 +595,41 @@ struct SquidConfig {
struct {
char *cert;
char *key;
int version;
char *options;
char *cipher;
char *cafile;
char *capath;
char *crlfile;
char *flags;
acl_access *cert_error;
SSL_CTX *sslContext;
} ssl_client;
#endif
char *accept_filter;
int umask;
int max_filedescriptors;
int workers;
+ CpuAffinityMap *cpuAffinityMap;
#if USE_LOADABLE_MODULES
wordlist *loadable_module_names;
#endif
int client_ip_max_connections;
};
SQUIDCEXTERN SquidConfig Config;
struct SquidConfig2 {
struct {
int enable_purge;
int mangle_request_headers;
} onoff;
uid_t effectiveUserID;
gid_t effectiveGroupID;
};
SQUIDCEXTERN SquidConfig2 Config2;
=== modified file 'src/tools.cc'
--- src/tools.cc 2010-08-09 11:06:36 +0000
+++ src/tools.cc 2010-09-11 05:27:48 +0000
@@ -807,67 +807,87 @@ no_suid(void)
#endif
}
bool
IamMasterProcess()
{
return KidIdentifier == 0;
}
bool
IamWorkerProcess()
{
// when there is only one process, it has to be the worker
if (opt_no_daemon || Config.workers == 0)
return true;
return 0 < KidIdentifier && KidIdentifier <= Config.workers;
}
bool
+InDaemonMode()
+{
+ return !opt_no_daemon && Config.workers > 0;
+}
+
+bool
UsingSmp()
{
return !opt_no_daemon && Config.workers > 1;
}
bool
IamCoordinatorProcess()
{
return UsingSmp() && KidIdentifier == Config.workers + 1;
}
bool
IamPrimaryProcess()
{
// when there is only one process, it has to be primary
if (opt_no_daemon || Config.workers == 0)
return true;
// when there is a master and worker process, the master delegates
// primary functions to its only kid
if (Config.workers == 1)
return IamWorkerProcess();
// in SMP mode, multiple kids delegate primary functions to the coordinator
return IamCoordinatorProcess();
}
+int
+NumberOfKids()
+{
+ // no kids in non-daemon mode
+ if (!InDaemonMode())
+ return 0;
+
+ // workers + the coordinator process
+ if (UsingSmp())
+ return Config.workers + 1;
+
+ return Config.workers;
+}
+
void
writePidFile(void)
{
int fd;
const char *f = NULL;
mode_t old_umask;
char buf[32];
if (!IamPrimaryProcess())
return;
if ((f = Config.pidFilename) == NULL)
return;
if (!strcmp(Config.pidFilename, "none"))
return;
enter_suid();
old_umask = umask(022);
