Hi All,
We recently upgraded from commons-httpclient (3.1) to httpcomponents-client
(4.3.6) in order to support proxies with NTLM authentication. We also
converted our custom upload servlet into a mutli-part form upload. After these
changes, we've noticed that upload times for very large files have
significantly increased. For example, a 21GB file used to take ~12 minutes to
upload, now it takes ~34 minutes. Our application is for scientific data, so
datasets with very large files and/or numbers of files are common (although
usually the latter). I have attached our abstract web service client class
which includes how we are configuring our http client, connection manager,
proxies, ssl, authentication, etc. Below is an excerpt that performs the
multi-part call. Please let me know if something stands out that is a blatant
API no-no, as I may have missed something from the documentation and mailing
lists. If not, do you have any suggestions on the best approach to remedy this
problem? Has anyone encountered this same issue?
Many Thanks!
Carina
private class MultipartFormUploadRunnable implements Runnable {
private Map<File, CmsPath> fileUploads;
private IProgressMonitor monitor;
private int totalFiles;
private int workedByteBatchSize;
private File globalMetadataFile;
private Map<String, File> fileSpecificMetadataFiles;
public MultipartFormUploadRunnable(Map<File, CmsPath> fileUploads, File
globalMetadataFile,
Map<String, File> fileSpecificMetadataFiles, IProgressMonitor monitor,
int totalFiles, int workedByteBatchSize) {
super();
this.fileUploads = fileUploads;
this.monitor = monitor;
this.totalFiles = totalFiles;
this.workedByteBatchSize = workedByteBatchSize;
this.globalMetadataFile = globalMetadataFile;
this.fileSpecificMetadataFiles = fileSpecificMetadataFiles;
}
/* (non-Javadoc)
* @see java.lang.Thread#run()
*/
@Override
public void run() {
StringBuilder url = getCatWebScriptUrl();
WebServiceUrlUtility.appendPaths(url, "upload");
WebServiceUrlUtility.appendParameter(url, "enableNotififcations", "true");
WebServiceUrlUtility.appendParameter(url, "enablePipeline", "true");
WebServiceUrlUtility.appendParameter(url, "batchSize", "20");
String encodedUrl = WebServiceUrlUtility.encode(url);
CloseableHttpResponse response = null;
try {
MultipartEntityBuilder reqBuilder = MultipartEntityBuilder.create();
for(File fileToUpload : fileUploads.keySet()) {
// File form field has path for key
// where path = path on server where the file will be located
CmsPath filePath = fileUploads.get(fileToUpload);
FileBody file = new FileBody(fileToUpload);
reqBuilder = reqBuilder.addPart(filePath.toAssociationNamePath(),
file);
}
// Then create global metadata file form field
if(globalMetadataFile != null) {
FileBody globalMetadata = new FileBody(globalMetadataFile);
reqBuilder = reqBuilder.addPart("globalMetadataFile", globalMetadata);
}
// then create file-specific metadata file
if(fileSpecificMetadataFiles != null) {
for(String regex : fileSpecificMetadataFiles.keySet()) {
File metadataFile = fileSpecificMetadataFiles.get(regex);
FileBody fileBody = new FileBody(metadataFile);
String fileKey = "metadataFile_" + regex;
reqBuilder = reqBuilder.addPart(fileKey, fileBody);
}
}
HttpEntity wrappedEntity = reqBuilder.build();
// decorate http entity with progress monitor
HttpEntity reqEntity = new HttpEntityWithProgress(wrappedEntity,
monitor, workedByteBatchSize);
HttpPost httppost = new HttpPost(encodedUrl);
httppost.setEntity((HttpEntity)reqEntity);
response = executeMethod(httpClient, httppost);
EntityUtils.consumeQuietly(response.getEntity());
} catch (Throwable e) {
handleException("Failed to execute method.", e);
} finally {
closeQuietly(response);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
