I have following http client code(HttpUtils and HttpClientManager)with
HttpClient 4.3.6, and a httpclient code in 3.1, and a test case as well. I
observed that HttpClient 4.3.6 is very much slowly than HttpClient 3.1 code. In
HttpClient 4.3.6 version code, there are a lot
socket time out error(the socket time is 10 seconds).
I pasted the related classes and code below. It is kind of long code, but I
still would ask you do me a favor to review the http client configuration that
may cause the problem. Many Thanks in advance.
Following codes include:
1. HttpClient 4.3.6 code to issue HttpGet request
2. HttpClient 3.1 code to issue HttpGet request
3. Test Case that demontrate the problem.
############################################Http Client 4.3.6 Code goes
here############################################
///////////////////////////HttpUtils//////////////////////////////////////////
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.http.Consts;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.message.BasicNameValuePair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HttpUtils {
///This is the util method that will be used to issue http request.
public static String httpInvoke(String httpUrl, Map<String, Object> parameters,
RequestConfig config){
HttpClientManager httpClientConnectionManager =
HttpClientManager.getHttpClientConnectionManagerInstance();
HttpGet httpGet = new HttpGet(httpUrl);
if(config != null ){
httpGet.setConfig(config);
}
String result = httpClientConnectionManager.execute(httpGet);
return result;
}
}
//////////////////////////////////HttpClientManager/////////////////////////////////
import java.nio.charset.CodingErrorAction;
import java.util.ArrayList;
import java.util.Collection;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.Consts;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.config.ConnectionConfig;
import org.apache.http.config.MessageConstraints;
import org.apache.http.config.SocketConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
///Main class that encapsulate the Http Client
public class HttpClientManager {
private static Logger logger =
LoggerFactory.getLogger(HttpClientManager.class);
private static int defaultConnectionTimeout = 10*1000; //connection timeout
private static int defaultSocketTimeout = 10*1000; //socket time out
private static int connectionRequestTimeout = 10*1000; //connection request
timeout
private static int defaultMaxRouteConnections = 128;
private static int defaultMaxTotalConnections = 1024;
private static int defaultMaxHeaderCount = 200;
private static int defaultMaxLineLength = 2000;
private static String Charset = "utf-8";
private String proxyHost = null;
private String proxyPort =null;
private PoolingHttpClientConnectionManager cm;
private final static HttpClientManager httpClientConnectionManager = new
HttpClientManager();
private HttpClientManager() {
logger.info("HttpClientManager initial!");
cm = new PoolingHttpClientConnectionManager();
cm.setMaxTotal(defaultMaxTotalConnections);
cm.setDefaultMaxPerRoute(defaultMaxRouteConnections);
}
private CloseableHttpClient getHttpclient(){
HttpClientBuilder httpClientBuilder = HttpClients.custom();
httpClientBuilder.setConnectionManager(cm);
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
.setConnectTimeout(defaultConnectionTimeout)
.setSocketTimeout(defaultSocketTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout)
.setExpectContinueEnabled(false)
.setStaleConnectionCheckEnabled(true);
if(StringUtils.isNotBlank(proxyHost) && StringUtils.isNotBlank(proxyPort)){
try{
logger.info("using proxy, proxyHost:{}, proxyPort:{}", proxyHost, proxyPort);
int proxyPortInt = Integer.parseInt(proxyPort);
requestConfigBuilder.setProxy(new HttpHost(proxyHost, proxyPortInt));
} catch(Exception e){
logger.error("parseInt proxyPort:{}", proxyPort, e);
}
}
SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build();
MessageConstraints messageConstraints =
MessageConstraints.custom().setMaxHeaderCount(defaultMaxHeaderCount).setMaxLineLength(defaultMaxLineLength).build();
ConnectionConfig connectionConfig = ConnectionConfig.custom()
.setMalformedInputAction(CodingErrorAction.IGNORE)
.setUnmappableInputAction(CodingErrorAction.IGNORE)
.setCharset(Consts.UTF_8)
.setMessageConstraints(messageConstraints).build();
Collection<BasicHeader> collection = new ArrayList<BasicHeader>();
collection.add(new BasicHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows
NT 5.1; zh-CN; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3"));
collection.add(new BasicHeader("Accept-Language", "zh-cn,zh,en-US,en;q=0.5"));
collection.add(new BasicHeader("Accept-Charset", Charset));
collection.add(new BasicHeader("Accept-Encoding", "gzip"));
httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());
httpClientBuilder.setDefaultSocketConfig(socketConfig);
httpClientBuilder.setDefaultConnectionConfig(connectionConfig);
httpClientBuilder.setDefaultHeaders(collection);
return httpClientBuilder.build();
}
///This is the method that will be call the execute the HttpGet request
public String execute(HttpGet httpGet) {
String body = "";
try{
CloseableHttpClient httpclient = this.getHttpclient();
CloseableHttpResponse response = httpclient.execute(httpGet);
int status = response.getStatusLine().getStatusCode();
try {
if (status == HttpStatus.SC_OK) {
HttpEntity entity = response.getEntity();
if (entity != null) {
Header header = (Header) entity.getContentEncoding();
if(header != null && "gzip".equals(header.getValue())){
body = EntityUtils.toString(new GzipDecompressingEntity(entity), Charset);
} else {
body = EntityUtils.toString(entity, Charset);
}
}
} else {
logger.error("[httpClientManager] [fail] [httpGet:{}] [status:{}]", httpGet,
status);
}
} finally {
response.close();
}
} catch(Exception e) {
logger.error("[module:httpClientManager] [action:execute] [httpGet:{}]
[error:{}] ", httpGet, e.getMessage(), e);
}
return body;
}
////Singleton object that will be used to access httpClientConnectionManager
public static HttpClientManager getHttpClientConnectionManagerInstance(){
return httpClientConnectionManager;
}
}
##################################################My HttpClient 1.3 code goes
here###########################################
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
public class HttpClientUtil {
private static final HttpClientUtil INSTANCE = new HttpClientUtil();
private static final int TIMEOUT = 10 * 1000; //10s
private HttpClientUtil() {
}
///Actually, this class is more like a static class than Singleton
public static HttpClientUtil getInstance() {
return INSTANCE;
}
//brand new http client per request
private HttpClient newHttpClient() {
HttpClient client = new HttpClient();
client.getHttpConnectionManager().getParams().setConnectionTimeout(TIMEOUT);
client.getHttpConnectionManager().getParams().setSoTimeout(TIMEOUT);
return client;
}
//FIXME The encoding should be provided when convert the response binary stream
into string
public static String responseBodyAsString(HttpMethod method) throws IOException
{
BufferedReader br = null;
String lsr = System.getProperty("line.separator");
try {
InputStream in = method.getResponseBodyAsStream();
br = new BufferedReader(new InputStreamReader(in, "UTF-8"));
StringBuffer sb = new StringBuffer();
String line;
while ((line = br.readLine()) != null) {
sb.append(line).append(lsr);
}
return sb.toString();
} finally {
if (br != null) {
br.close();
}
}
}
/////This is the method that will HttpGet to the url
public String get(String url) throws IOException {
GetMethod pm = new GetMethod(url);
pm.setRequestHeader("Connection", "close");
HttpClient client = newHttpClient();
try {
client.executeMethod(pm);
String response = responseBodyAsString(pm);
return response;
} finally {
pm.releaseConnection();
}
}
}
##########################################Test
Case##########################################
The following code runs in the one thread in the junit test
while (i++ < 5000) {
long start = System.currentTimeMillis();
HttpUtils.httpInvoke(TEST_URL, null, null);
long a = System.currentTimeMillis() - start;
String timeSpent = "" + (a >= 1000 ? a + "(>1000)" : a);
writeToFile(timeSpent + "\n", logFile);
Thread.sleep(loopInterval);
}
[email protected]
