This is an automated email from the ASF dual-hosted git repository. bschuchardt pushed a commit to branch feature/GEODE-6035 in repository https://gitbox.apache.org/repos/asf/geode.git
commit 6a876c79e8248c4d226f9b6d0b6c76e6cdf783b6 Author: Bruce Schuchardt <bschucha...@pivotal.io> AuthorDate: Mon Nov 26 14:12:16 2018 -0800 GEODE-6035 Increase backlog for peer-to-peer connection formation Adding statistics for Linux to track TCP/IP backlog issues. tcpExtSynCookiesRecv The number of TCP/IP SYN cookies received due to a full server socket backlog. If this is non-zero consider disabling SYN cookies because they form sub-optimal connections. units: cookies received tcpExtSynCookiesSent The number of TCP/IP SYN cookies sent due to a full server socket backlog. If this is non-zero consider disabling SYN cookies because they form sub-optimal connections. units: cookies sent tcpExtListenDrops The number of TCP/IP connection requests that have been dropped due to a full backlog. If this is large increase the OS SOMAXCONN setting and increase socket backlog settings. units: requests tcpExtListenOverflows The number of TCP/IP connection requests that could not be queued due to a small backlog. These are either dropped (tcpExtListenDrops) or handled via cookies (tcpSynCookiesSent). In either case you should consider increasing SOMAXCONN and increasing backlog settings. units: requests soMaxConn Maximum TCP/IP server socket connection request backlog units: connection requests --- .../statistics/platform/LinuxProcFsStatistics.java | 51 ++++++++++++++++++++++ .../statistics/platform/LinuxSystemStats.java | 31 +++++++++++++ 2 files changed, 82 insertions(+) diff --git a/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxProcFsStatistics.java b/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxProcFsStatistics.java index 395ce0f..56c0068 100644 --- a/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxProcFsStatistics.java +++ b/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxProcFsStatistics.java @@ -200,6 +200,7 @@ public class LinuxProcFsStatistics { getMemInfo(ints); getDiskStats(longs); getNetStats(longs); + getNetStatStats(longs, ints); if (hasProcVmStat) { getVmStats(longs); } @@ -334,6 +335,56 @@ public class LinuxProcFsStatistics { } } +/* + * TcpExt:=0 SyncookiesSent=1 + * ListenOverflows=20 ListenDrops=21 + */ + private static void getNetStatStats(long[] longs, int[] ints) { + InputStreamReader isr; + BufferedReader br = null; + try { + isr = new InputStreamReader(new FileInputStream("/proc/net/netstat")); + br = new BufferedReader(isr); + String line; + do { + br.readLine(); // header + line = br.readLine(); + } while (line != null && !line.startsWith("TcpExt:")); + + st.setString(line); + st.skipTokens(1); + long tcpSyncookiesSent = st.nextTokenAsLong(); + long tcpSyncookiesRecv = st.nextTokenAsLong(); + st.skipTokens(17); + long tcpListenOverflows = st.nextTokenAsLong(); + long tcpListenDrops = st.nextTokenAsLong(); + + longs[LinuxSystemStats.tcpExtSynCookiesRecvLONG] = tcpSyncookiesRecv; + longs[LinuxSystemStats.tcpExtSynCookiesSentLONG] = tcpSyncookiesSent; + longs[LinuxSystemStats.tcpExtListenDropsLONG] = tcpListenDrops; + longs[LinuxSystemStats.tcpExtListenOverflowsLONG] = tcpListenOverflows; + + br.close(); + isr = new InputStreamReader(new FileInputStream("/proc/sys/net/core/somaxconn")); + br = new BufferedReader(isr); + line = br.readLine(); + st.setString(line); + int soMaxConn = st.nextTokenAsInt(); + + ints[LinuxSystemStats.tcpSOMaxConnINT] = soMaxConn; + + } catch (NoSuchElementException nsee) { + } catch (IOException ioe) { + } finally { + st.releaseResources(); + if (br != null) + try { + br.close(); + } catch (IOException ignore) { + } + } + } + /* * Inter-| Receive | Transmit face |bytes packets errs drop fifo frame compressed multicast|bytes * packets errs drop fifo colls carrier compressed lo:1908275823 326949246 0 0 0 0 0 0 1908275823 diff --git a/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxSystemStats.java b/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxSystemStats.java index 662c95f..9da3715 100644 --- a/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxSystemStats.java +++ b/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxSystemStats.java @@ -49,6 +49,7 @@ public class LinuxSystemStats { static final int dirtyMemoryINT = 17; static final int cpuNonUserINT = 18; static final int cpuStealINT = 19; + static final int tcpSOMaxConnINT = 20; static final int loopbackPacketsLONG = 0; static final int loopbackBytesLONG = 1; @@ -78,6 +79,11 @@ public class LinuxSystemStats { static final int iosInProgressLONG = 25; static final int timeIosInProgressLONG = 26; static final int ioTimeLONG = 27; + static final int tcpExtSynCookiesRecvLONG = 28; + static final int tcpExtSynCookiesSentLONG = 29; + static final int tcpExtListenDropsLONG = 30; + static final int tcpExtListenOverflowsLONG = 31; + static final int loadAverage1DOUBLE = 0; static final int loadAverage15DOUBLE = 1; @@ -149,6 +155,9 @@ public class LinuxSystemStats { f.createIntGauge("cpuSteal", "Steal time is the amount of time the operating system wanted to execute, but was not allowed to by the hypervisor.", "%"), + f.createIntGauge("soMaxConn", + "Maximum TCP/IP server socket connection request backlog", + "connection requests"), f.createLongCounter("loopbackPackets", "The number of network packets sent (or received) on the loopback interface", @@ -219,6 +228,23 @@ public class LinuxSystemStats { f.createLongCounter("diskTime", "The total number of milliseconds that measures both completed disk operations and any accumulating backlog of in progress ops.", "milliseconds"), + f.createLongCounter("tcpExtSynCookiesRecv", + "The number of TCP/IP SYN cookies received due to a full server socket backlog. " + + "If this is non-zero consider disabling SYN cookies because they form sub-optimal connections.", + "cookies received"), + f.createLongCounter("tcpExtSynCookiesSent", + "The number of TCP/IP SYN cookies sent due to a full server socket backlog. " + + "If this is non-zero consider disabling SYN cookies because they form sub-optimal connections.", + "cookies sent"), + f.createLongCounter("tcpExtListenDrops", + "The number of TCP/IP connection requests that have been dropped due to a full backlog. " + + "If this is large increase the OS SOMAXCONN setting and increase socket backlog settings", + "requests"), + f.createLongCounter("tcpExtListenOverflows", + "The number of TCP/IP connection requests that could not be queued due to a small backlog. " + + "These are either dropped (tcpExtListenDrops) or handled via cookies (tcpSynCookiesSent). " + + "In either case you should consider increasing SOMAXCONN and increasing backlog settings.", + "requests"), f.createDoubleGauge("loadAverage1", @@ -251,6 +277,7 @@ public class LinuxSystemStats { checkOffset("dirtyMemory", dirtyMemoryINT); checkOffset("cpuNonUser", cpuNonUserINT); checkOffset("cpuSteal", cpuStealINT); + checkOffset("soMaxConn", tcpSOMaxConnINT); checkOffset("loopbackPackets", loopbackPacketsLONG); checkOffset("loopbackBytes", loopbackBytesLONG); @@ -280,6 +307,10 @@ public class LinuxSystemStats { checkOffset("diskOpsInProgress", iosInProgressLONG); checkOffset("diskTimeInProgress", timeIosInProgressLONG); checkOffset("diskTime", ioTimeLONG); + checkOffset("tcpExtSynCookiesRecv", tcpExtSynCookiesRecvLONG); + checkOffset("tcpExtSynCookiesSent", tcpExtSynCookiesSentLONG); + checkOffset("tcpExtListenDrops", tcpExtListenDropsLONG); + checkOffset("tcpExtListenOverflows", tcpExtListenOverflowsLONG); checkOffset("loadAverage1", loadAverage1DOUBLE); checkOffset("loadAverage15", loadAverage15DOUBLE);