Greeting from Estonia
I'm trying to pull data from an online price quotation system
using HTTPClient's POST method. SOMETIMES (rather rarely) the thing
works, mostly not. Is something really wrong with the server or
undesigned goofed in the RTFM stage ?
The URL is accessible and the form works fine under my Mozilla browser.
Many thanks for your effort
Pavel
+++ Java prog +++
/*
* Created on 26.09.2004
*
*/
package ee.skandiska.webBotTest;
import java.io.*;
import org.apache.commons.httpclient.*;
import org.apache.commons.httpclient.methods.*;
import com.quiotix.html.parser.*;
/**
* @author Pavel
*
*/
public class rapBot {
public void run() {
System.setProperty("org.apache.commons.logging.Log",
"org.apache.commons.logging.impl.SimpleLog");
System.setProperty("org.apache.commons.logging.simplelog.showdatetime",
"true");
System.setProperty("org.apache.commons.logging.simplelog.log.httpclient
.wire", "debug");
System.setProperty("org.apache.commons.logging.simplelog.log.org.apache
.commons.httpclient", "debug");
Credentials creds = null;
creds = new UsernamePasswordCredentials("sorry",
"cant_disclose_this");
//create a singular HttpClient object
HttpClient client = new HttpClient();
//set all timeouts to 20 seconds
client.setConnectionTimeout(20000);
client.setTimeout(20000);
client.setHttpConnectionFactoryTimeout(20000);
//set the default credentials
if (creds != null) {
client.getState().
setCredentials("INDEX Member Area", "www.diamonds.net", creds);
}
String url = "http://www.diamonds.net/rapnet/prices/prcwks.asp";;
//create a method object
PostMethod postMethod = new PostMethod(url);
NameValuePair[] data = {
new NameValuePair("submit2", "Calculate"),
new NameValuePair("shape", "Round"),
new NameValuePair("size", "1.0"),
new NameValuePair("color", "F"),
new NameValuePair("clar", "VVS1")
};
postMethod.setRequestBody(data);
postMethod.setFollowRedirects(true);
DefaultMethodRetryHandler retryHandler = new
DefaultMethodRetryHandler();
retryHandler.setRetryCount(5);
retryHandler.setRequestSentRetryEnabled(true);
postMethod.setMethodRetryHandler(retryHandler);
//execute the method
String responseBody = null;
try{
int statusCode = client.executeMethod(postMethod);
Reader reader = new
InputStreamReader(postMethod.getResponseBodyAsStream());
HtmlDocument document = new HtmlParser(reader).HtmlDocument();
document.accept(new HtmlScrubber(HtmlScrubber.DEFAULT_OPTIONS
| HtmlScrubber.TRIM_SPACES));
System.out.println("+++ parsed HTML body +++");
document.accept(new HtmlDumper(System.out));
reader.close();
// responseBody = postMethod.getResponseBodyAsString();
} catch (HttpException he) {
System.err.println("Http error connecting to '" + url + "'");
System.err.println(he.getMessage());
System.exit(-4);
} catch (IOException ioe){
System.err.println("Unable to connect to '" + url + "'");
System.exit(-3);
} catch (ParseException e) {
e.printStackTrace();
System.exit(-5);
}
//write out the request headers
System.out.println("*** Request ***");
System.out.println("Request Path: " + postMethod.getPath());
System.out.println("Request Query: " + postMethod.getQueryString());
Header[] requestHeaders = postMethod.getRequestHeaders();
for (int i=0; i<requestHeaders.length; i++){
System.out.print(requestHeaders[i]);
}
//write out the response headers
System.out.println("*** Response ***");
System.out.println("Status Line: " + postMethod.getStatusLine());
Header[] responseHeaders = postMethod.getResponseHeaders();
for (int i=0; i<responseHeaders.length; i++){
System.out.print(responseHeaders[i]);
}
//clean up the connection resources
postMethod.releaseConnection();
System.exit(0);
}
public static void main(String[] args){
rapBot bot = new rapBot();
bot.run();
}
}
++++++ log +++++
2004/10/02 23:19:55:562 EEST [DEBUG] HttpClient - Java version: 1.4.2_05
2004/10/02 23:19:55:593 EEST [DEBUG] HttpClient - Java vendor: Sun
Microsystems Inc.
2004/10/02 23:19:55:593 EEST [DEBUG] HttpClient - Java class path:
C:\eclipse\workspace\Rapoport;C:\Documents and Settings\admin\My
Documents\PM\jakarta\commons-logging-1.0.4\commons-logging-1.0.4\commons-log
ging.jar;C:\Documents and Settings\admin\My
Documents\PM\jakarta\commons-httpclient-2.0.1\commons-httpclient-2.0.1.jar;C
:\Documents and Settings\admin\My
Documents\PM\jakarta\commons-httpclient-3.0-alpha2\commons-httpclient-3.0-al
pha2.jar;C:\Documents and Settings\admin\My
Documents\PM\jakarta\quotix-html-parser.jar
2004/10/02 23:19:55:640 EEST [DEBUG] HttpClient - Operating system name:
Windows XP
2004/10/02 23:19:55:640 EEST [DEBUG] HttpClient - Operating system
architecture: x86
2004/10/02 23:19:55:640 EEST [DEBUG] HttpClient - Operating system version:
5.1
2004/10/02 23:19:56:281 EEST [DEBUG] HttpClient - SUN 1.42: SUN (DSA
key/parameter generation; DSA signing; SHA-1, MD5 digests; SecureRandom;
X.509 certificates; JKS keystore; PKIX CertPathValidator; PKIX
CertPathBuilder; LDAP, Collection CertStores)
2004/10/02 23:19:56:281 EEST [DEBUG] HttpClient - SunJSSE 1.42: Sun JSSE
provider(implements RSA Signatures, PKCS12, SunX509 key/trust factories,
SSLv3, TLSv1)
2004/10/02 23:19:56:281 EEST [DEBUG] HttpClient - SunRsaSign 1.42: SUN's
provider for RSA signatures
2004/10/02 23:19:56:312 EEST [DEBUG] HttpClient - SunJCE 1.42: SunJCE
Provider (implements DES, Triple DES, AES, Blowfish, PBE, Diffie-Hellman,
HMAC-MD5, HMAC-SHA1)
2004/10/02 23:19:56:312 EEST [DEBUG] HttpClient - SunJGSS 1.0: Sun (Kerberos
v5)
2004/10/02 23:19:57:187 EEST [DEBUG] HttpConnection -
HttpConnection.setSoTimeout(20000)
2004/10/02 23:19:57:187 EEST [DEBUG] HttpMethodBase - Execute loop try 1
2004/10/02 23:19:57:187 EEST [DEBUG] header - >> "POST
/rapnet/prices/prcwks.asp HTTP/1.1[\r][\n]"
2004/10/02 23:19:57:203 EEST [DEBUG] HttpMethodBase - Adding Host request
header
2004/10/02 23:19:57:218 EEST [DEBUG] HttpMethodBase - Default charset used:
ISO-8859-1
2004/10/02 23:19:57:250 EEST [DEBUG] header - >> "User-Agent: Jakarta
Commons-HttpClient/2.0.1[\r][\n]"
2004/10/02 23:19:57:250 EEST [DEBUG] header - >> "Host:
www.diamonds.net[\r][\n]"
2004/10/02 23:19:57:250 EEST [DEBUG] header - >> "Content-Length:
56[\r][\n]"
2004/10/02 23:19:57:250 EEST [DEBUG] header - >> "Content-Type:
application/x-www-form-urlencoded[\r][\n]"
2004/10/02 23:19:57:265 EEST [DEBUG] header - >> "[\r][\n]"
2004/10/02 23:19:57:265 EEST [DEBUG] EntityEnclosingMethod - Using buffered
request body
2004/10/02 23:19:57:265 EEST [DEBUG] content - >>
"submit2=Calculate&shape=Round&size=1.0&color=F&clar=VVS1"
2004/10/02 23:19:57:265 EEST [DEBUG] EntityEnclosingMethod - Request body
sent
2004/10/02 23:19:57:562 EEST [DEBUG] HttpMethodBase - Closing the
connection.
2004/10/02 23:19:57:562 EEST [INFO] HttpMethodBase - Recoverable exception
caught when processing request
2004/10/02 23:19:57:562 EEST [DEBUG] HttpMethodBase - Opening the
connection.
2004/10/02 23:19:57:718 EEST [DEBUG] header - >> "POST
/rapnet/prices/prcwks.asp HTTP/1.1[\r][\n]"
2004/10/02 23:19:57:718 EEST [DEBUG] HttpMethodBase - Request to add Host
header ignored: header already added
2004/10/02 23:19:57:734 EEST [DEBUG] header - >> "User-Agent: Jakarta
Commons-HttpClient/2.0.1[\r][\n]"
2004/10/02 23:19:57:734 EEST [DEBUG] header - >> "Host:
www.diamonds.net[\r][\n]"
2004/10/02 23:19:57:734 EEST [DEBUG] header - >> "Content-Length:
56[\r][\n]"
2004/10/02 23:19:57:734 EEST [DEBUG] header - >> "Content-Type:
application/x-www-form-urlencoded[\r][\n]"
2004/10/02 23:19:57:765 EEST [DEBUG] header - >> "[\r][\n]"
2004/10/02 23:19:57:765 EEST [DEBUG] EntityEnclosingMethod - Using buffered
request body
2004/10/02 23:19:57:765 EEST [DEBUG] content - >>
"submit2=Calculate&shape=Round&size=1.0&color=F&clar=VVS1"
2004/10/02 23:19:57:765 EEST [DEBUG] EntityEnclosingMethod - Request body
sent
2004/10/02 23:19:58:031 EEST [DEBUG] HttpMethodBase - Closing the
connection.
2004/10/02 23:19:58:031 EEST [INFO] HttpMethodBase - Recoverable exception
caught when processing request
2004/10/02 23:19:58:031 EEST [DEBUG] HttpMethodBase - Opening the
connection.
2004/10/02 23:19:58:203 EEST [DEBUG] header - >> "POST
/rapnet/prices/prcwks.asp HTTP/1.1[\r][\n]"
2004/10/02 23:19:58:203 EEST [DEBUG] HttpMethodBase - Request to add Host
header ignored: header already added
2004/10/02 23:19:58:203 EEST [DEBUG] header - >> "User-Agent: Jakarta
Commons-HttpClient/2.0.1[\r][\n]"
2004/10/02 23:19:58:203 EEST [DEBUG] header - >> "Host:
www.diamonds.net[\r][\n]"
2004/10/02 23:19:58:203 EEST [DEBUG] header - >> "Content-Length:
56[\r][\n]"
2004/10/02 23:19:58:203 EEST [DEBUG] header - >> "Content-Type:
application/x-www-form-urlencoded[\r][\n]"
2004/10/02 23:19:58:203 EEST [DEBUG] header - >> "[\r][\n]"
2004/10/02 23:19:58:203 EEST [DEBUG] EntityEnclosingMethod - Using buffered
request body
2004/10/02 23:19:58:203 EEST [DEBUG] content - >>
"submit2=Calculate&shape=Round&size=1.0&color=F&clar=VVS1"
2004/10/02 23:19:58:203 EEST [DEBUG] EntityEnclosingMethod - Request body
sent
2004/10/02 23:19:58:500 EEST [DEBUG] HttpMethodBase - Closing the
connection.
2004/10/02 23:19:58:500 EEST [INFO] HttpMethodBase - Recoverable exception
caught when processing request
2004/10/02 23:19:58:500 EEST [DEBUG] HttpMethodBase - Opening the
connection.
2004/10/02 23:19:58:656 EEST [DEBUG] header - >> "POST
/rapnet/prices/prcwks.asp HTTP/1.1[\r][\n]"
2004/10/02 23:19:58:656 EEST [DEBUG] HttpMethodBase - Request to add Host
header ignored: header already added
2004/10/02 23:19:58:656 EEST [DEBUG] header - >> "User-Agent: Jakarta
Commons-HttpClient/2.0.1[\r][\n]"
2004/10/02 23:19:58:656 EEST [DEBUG] header - >> "Host:
www.diamonds.net[\r][\n]"
2004/10/02 23:19:58:656 EEST [DEBUG] header - >> "Content-Length:
56[\r][\n]"
2004/10/02 23:19:58:656 EEST [DEBUG] header - >> "Content-Type:
application/x-www-form-urlencoded[\r][\n]"
2004/10/02 23:19:58:656 EEST [DEBUG] header - >> "[\r][\n]"
2004/10/02 23:19:58:656 EEST [DEBUG] EntityEnclosingMethod - Using buffered
request body
2004/10/02 23:19:58:656 EEST [DEBUG] content - >>
"submit2=Calculate&shape=Round&size=1.0&color=F&clar=VVS1"
2004/10/02 23:19:58:656 EEST [DEBUG] EntityEnclosingMethod - Request body
sent
2004/10/02 23:19:58:953 EEST [DEBUG] HttpMethodBase - Closing the
connection.
2004/10/02 23:19:58:953 EEST [INFO] HttpMethodBase - Recoverable exception
caught when processing request
2004/10/02 23:19:58:953 EEST [DEBUG] HttpMethodBase - Opening the
connection.
2004/10/02 23:19:59:125 EEST [DEBUG] header - >> "POST
/rapnet/prices/prcwks.asp HTTP/1.1[\r][\n]"
2004/10/02 23:19:59:125 EEST [DEBUG] HttpMethodBase - Request to add Host
header ignored: header already added
2004/10/02 23:19:59:125 EEST [DEBUG] header - >> "User-Agent: Jakarta
Commons-HttpClient/2.0.1[\r][\n]"
2004/10/02 23:19:59:125 EEST [DEBUG] header - >> "Host:
www.diamonds.net[\r][\n]"
2004/10/02 23:19:59:125 EEST [DEBUG] header - >> "Content-Length:
56[\r][\n]"
2004/10/02 23:19:59:125 EEST [DEBUG] header - >> "Content-Type:
application/x-www-form-urlencoded[\r][\n]"
2004/10/02 23:19:59:125 EEST [DEBUG] header - >> "[\r][\n]"
2004/10/02 23:19:59:125 EEST [DEBUG] EntityEnclosingMethod - Using buffered
request body
2004/10/02 23:19:59:125 EEST [DEBUG] content - >>
"submit2=Calculate&shape=Round&size=1.0&color=F&clar=VVS1"
2004/10/02 23:19:59:125 EEST [DEBUG] EntityEnclosingMethod - Request body
sent
2004/10/02 23:19:59:421 EEST [DEBUG] HttpMethodBase - Closing the
connection.
2004/10/02 23:19:59:421 EEST [INFO] HttpMethodBase - Recoverable exception
caught when processing request
2004/10/02 23:19:59:421 EEST [DEBUG] HttpMethodBase - Opening the
connection.
2004/10/02 23:19:59:578 EEST [DEBUG] header - >> "POST
/rapnet/prices/prcwks.asp HTTP/1.1[\r][\n]"
2004/10/02 23:19:59:578 EEST [DEBUG] HttpMethodBase - Request to add Host
header ignored: header already added
2004/10/02 23:19:59:578 EEST [DEBUG] header - >> "User-Agent: Jakarta
Commons-HttpClient/2.0.1[\r][\n]"
2004/10/02 23:19:59:578 EEST [DEBUG] header - >> "Host:
www.diamonds.net[\r][\n]"
2004/10/02 23:19:59:578 EEST [DEBUG] header - >> "Content-Length:
56[\r][\n]"
2004/10/02 23:19:59:578 EEST [DEBUG] header - >> "Content-Type:
application/x-www-form-urlencoded[\r][\n]"
2004/10/02 23:19:59:593 EEST [DEBUG] header - >> "[\r][\n]"
2004/10/02 23:19:59:593 EEST [DEBUG] EntityEnclosingMethod - Using buffered
request body
2004/10/02 23:19:59:593 EEST [DEBUG] content - >>
"submit2=Calculate&shape=Round&size=1.0&color=F&clar=VVS1"
2004/10/02 23:19:59:593 EEST [DEBUG] EntityEnclosingMethod - Request body
sent
2004/10/02 23:19:59:875 EEST [DEBUG] HttpMethodBase - Closing the
connection.
2004/10/02 23:19:59:890 EEST [INFO] HttpMethodBase - Recoverable exception
caught when processing request
2004/10/02 23:19:59:890 EEST [WARN] HttpMethodBase - Recoverable exception
caught but MethodRetryHandler.retryMethod() returned false, rethrowing
exception
Http error connecting to 'http://www.diamonds.net/rapnet/prices/prcwks.asp'
java.net.SocketException: Connection reset
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
