Hi, we started noticing that HttpServletRequest.getRemoteAddr() was
sometimes returning NULL (which is invalid according to the servlet
spec), about 20-50 times per day (we have high-load servers which
routinely handle over 100 requests per second). This only happens on
secure HTTP/2 requests.
We noticed this while running the latest 9.0.30, but looking at our logs
for the past few months, this was also happening on 9.0.21 as well as
9.0.22 which are the previous versions we had deployed. This issue was
not present in 9.0.17, which was the version we were using before 9.0.21.
Before anyone cries out "there is a bug in your code", Tomcat itself
reports a NULL remoteAddr via AccessLogValve in these cases. It is also
interesting that the remotePort is reported as "-1".
We looked through the changelog, but we can't figure out what change
might have triggered this (presumably some change between 9.0.17 and
9.0.21). It looks like some kind of async race condition during the
HTTP/2 upgrade, but this is only an educated guess.
Our server.xml is below (confidential data has been modified). We are
using the NIO connector, Tomcat Native + APR libraries (but NOT the APR
connector). The useAsyncIO flag is disabled because of server lockups
we've experienced with this flag enabled on our production servers. Any
ideas?
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE Server>
<Server port="8005" shutdown="SHUTDOWN">
<Listener className="org.apache.catalina.startup.VersionLoggerListener"
/>
<Listener className="org.apache.catalina.core.AprLifecycleListener" />
<Listener
className="org.apache.catalina.core.ThreadLocalLeakPreventionListener" />
<Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener"
AWTThreadProtection="true" />
<Service name="Catalina">
<!-- Thread pool -->
<Executor name="threadPool" namePrefix="tomcat-http-"
maxThreads="1000" minSpareThreads="100"
threadPriority="10" />
<!-- HTTP connector on port 80 -->
<Connector protocol="org.apache.coyote.http11.Http11NioProtocol"
executor="threadPool" port="80" redirectPort="443"
acceptCount="100" maxHttpHeaderSize="32768"
enableLookups="false" useAsyncIO="false"
connectionTimeout="30000" maxConnections="-1" />
<!-- HTTPS connector on port 443 -->
<Connector protocol="org.apache.coyote.http11.Http11NioProtocol"
executor="threadPool" port="443"
acceptCount="100" maxHttpHeaderSize="32768"
enableLookups="false" useAsyncIO="false"
connectionTimeout="30000" maxConnections="-1"
scheme="https" secure="true" SSLEnabled="true"
defaultSSLHostConfigName="*.mydomain.com">
<UpgradeProtocol
className="org.apache.coyote.http2.Http2Protocol" />
<SSLHostConfig hostName="*.mydomain.com" protocols="all"
certificateVerification="none">
<Certificate certificateKeyAlias="server"
certificateKeystoreFile="/path/keystore_star_mydomain_com.jks"
certificateKeystorePassword="CHANGEIT"
/>
</SSLHostConfig>
<SSLHostConfig hostName="myotherdomain.com" protocols="all"
certificateVerification="none">
<Certificate certificateKeyAlias="server"
certificateKeystoreFile="/path/keystore_myotherdomain_com.jks"
certificateKeystorePassword="CHANGEIT"
/>
</SSLHostConfig>
</Connector>
<!-- Engine -->
<Engine name="Catalina" defaultHost="localhost"
backgroundProcessorDelay="1" startStopThreads="0">
<Realm className="org.apache.catalina.realm.MemoryRealm"
/>
<Host name="localhost" appBase="webapps"
unpackWARs="true"
autoDeploy="false" deployOnStartup="true"
deployIgnore="manager|host-manager"
deployXML="true" copyXML="false">
<Context docBase="manager" path="/manager"
reloadable="false" privileged="true">
<Valve
className="org.apache.catalina.valves.RemoteCIDRValve" allow="127.0.0.1, 10.0.0.0/8,
172.16.0.0/12, 192.168.0.0/16" />
</Context>
<Context docBase="host-manager" path="/host-manager"
reloadable="false" privileged="true">
<Valve
className="org.apache.catalina.valves.RemoteCIDRValve" allow="127.0.0.1, 10.0.0.0/8,
172.16.0.0/12, 192.168.0.0/16" />
</Context>
</Host>
</Engine>
</Service>
</Server>
- Manuel Dominguez Sarmiento
