Has the buffer size been changed on either client or server?
> Den 27. jan. 2015 kl. 03.46 skrev Lansing, Carina S <[email protected]>: > > Hi All, > > We recently upgraded from commons-httpclient (3.1) to httpcomponents-client > (4.3.6) in order to support proxies with NTLM authentication. We also > converted our custom upload servlet into a mutli-part form upload. After > these changes, we’ve noticed that upload times for very large files have > significantly increased. For example, a 21GB file used to take ~12 minutes > to upload, now it takes ~34 minutes. Our application is for scientific data, > so datasets with very large files and/or numbers of files are common > (although usually the latter). I have attached our abstract web service > client class which includes how we are configuring our http client, > connection manager, proxies, ssl, authentication, etc. Below is an excerpt > that performs the multi-part call. Please let me know if something stands > out that is a blatant API no-no, as I may have missed something from the > documentation and mailing lists. If not, do you have any suggestions on the > best approach to remedy this problem? Has anyone encountered this same > issue? > > Many Thanks! > Carina > > > private class MultipartFormUploadRunnable implements Runnable { > private Map<File, CmsPath> fileUploads; > private IProgressMonitor monitor; > private int totalFiles; > private int workedByteBatchSize; > private File globalMetadataFile; > private Map<String, File> fileSpecificMetadataFiles; > > > public MultipartFormUploadRunnable(Map<File, CmsPath> fileUploads, File > globalMetadataFile, > Map<String, File> fileSpecificMetadataFiles, IProgressMonitor > monitor, int totalFiles, int workedByteBatchSize) { > super(); > this.fileUploads = fileUploads; > this.monitor = monitor; > this.totalFiles = totalFiles; > this.workedByteBatchSize = workedByteBatchSize; > this.globalMetadataFile = globalMetadataFile; > this.fileSpecificMetadataFiles = fileSpecificMetadataFiles; > } > > /* (non-Javadoc) > * @see java.lang.Thread#run() > */ > @Override > public void run() { > StringBuilder url = getCatWebScriptUrl(); > WebServiceUrlUtility.appendPaths(url, "upload"); > WebServiceUrlUtility.appendParameter(url, "enableNotififcations", > "true"); > WebServiceUrlUtility.appendParameter(url, "enablePipeline", "true"); > WebServiceUrlUtility.appendParameter(url, "batchSize", "20"); > String encodedUrl = WebServiceUrlUtility.encode(url); > > > CloseableHttpResponse response = null; > > try { > > MultipartEntityBuilder reqBuilder = MultipartEntityBuilder.create(); > for(File fileToUpload : fileUploads.keySet()) { > // File form field has path for key > // where path = path on server where the file will be located > CmsPath filePath = fileUploads.get(fileToUpload); > FileBody file = new FileBody(fileToUpload); > reqBuilder = reqBuilder.addPart(filePath.toAssociationNamePath(), > file); > } > > // Then create global metadata file form field > if(globalMetadataFile != null) { > FileBody globalMetadata = new FileBody(globalMetadataFile); > reqBuilder = reqBuilder.addPart("globalMetadataFile", > globalMetadata); > } > > // then create file-specific metadata file > if(fileSpecificMetadataFiles != null) { > for(String regex : fileSpecificMetadataFiles.keySet()) { > File metadataFile = fileSpecificMetadataFiles.get(regex); > FileBody fileBody = new FileBody(metadataFile); > > String fileKey = "metadataFile_" + regex; > reqBuilder = reqBuilder.addPart(fileKey, fileBody); > } > } > > HttpEntity wrappedEntity = reqBuilder.build(); > // decorate http entity with progress monitor > HttpEntity reqEntity = new HttpEntityWithProgress(wrappedEntity, > monitor, workedByteBatchSize); > > HttpPost httppost = new HttpPost(encodedUrl); > httppost.setEntity((HttpEntity)reqEntity); > response = executeMethod(httpClient, httppost); > EntityUtils.consumeQuietly(response.getEntity()); > > } catch (Throwable e) { > handleException("Failed to execute method.", e); > } finally { > closeQuietly(response); > } > } > > } > > > --------------------------------------------------------------------- > To unsubscribe, e-mail: [email protected] > For additional commands, e-mail: [email protected]
