Author: jerome Date: Thu Mar 2 14:38:40 2006 New Revision: 382535 URL: http://svn.apache.org/viewcvs?rev=382535&view=rev Log: Fix content.limit inconsistency in http, ftp and file
Modified: lucene/nutch/trunk/conf/nutch-default.xml lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java Modified: lucene/nutch/trunk/conf/nutch-default.xml URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/conf/nutch-default.xml?rev=382535&r1=382534&r2=382535&view=diff ============================================================================== --- lucene/nutch/trunk/conf/nutch-default.xml (original) +++ lucene/nutch/trunk/conf/nutch-default.xml Thu Mar 2 14:38:40 2006 @@ -13,8 +13,8 @@ <name>file.content.limit</name> <value>65536</value> <description>The length limit for downloaded content, in bytes. - If this value is larger than zero, content longer than it will be - truncated; otherwise (zero or negative), no truncation at all. + If this value is nonnegative (>=0), content longer than it will be truncated; + otherwise, no truncation at all. </description> </property> @@ -150,11 +150,11 @@ <name>ftp.content.limit</name> <value>65536</value> <description>The length limit for downloaded content, in bytes. - If this value is larger than zero, content longer than it is truncated; - otherwise (zero or negative), no truncation at all. Caution: classical - ftp RFCs never defines partial transfer and, in fact, some ftp servers - out there do not handle client side forced close-down very well. - Our implementation tries its best to handle such situations smoothly. + If this value is nonnegative (>=0), content longer than it will be truncated; + otherwise, no truncation at all. + Caution: classical ftp RFCs never defines partial transfer and, in fact, + some ftp servers out there do not handle client side forced close-down very + well. Our implementation tries its best to handle such situations smoothly. </description> </property> Modified: lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java?rev=382535&r1=382534&r2=382535&view=diff ============================================================================== --- lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java (original) +++ lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java Thu Mar 2 14:38:40 2006 @@ -167,7 +167,7 @@ // capture content int len = (int) size; - if (this.file.maxContentLength > 0 && len > this.file.maxContentLength) + if (this.file.maxContentLength >= 0 && len > this.file.maxContentLength) len = this.file.maxContentLength; this.content = new byte[len]; Modified: lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java?rev=382535&r1=382534&r2=382535&view=diff ============================================================================== --- lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java (original) +++ lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java Thu Mar 2 14:38:40 2006 @@ -344,9 +344,9 @@ } entries.add(ftpFile); count += line.length(); - // impose download limit if limit > 0, otherwise no limit + // impose download limit if limit >= 0, otherwise no limit // here, cut off is up to the line when total bytes is just over limit - if (limit > 0 && count > limit) { + if (limit >= 0 && count > limit) { mandatory_close = true; break; } @@ -409,9 +409,9 @@ new byte[org.apache.commons.net.io.Util.DEFAULT_COPY_BUFFER_SIZE]; while((len=input.read(buf,0,buf.length)) != -1){ count += len; - // impose download limit if limit > 0, otherwise no limit + // impose download limit if limit >= 0, otherwise no limit // here, cut off is exactly of limit bytes - if (limit > 0 && count > limit) { + if (limit >= 0 && count > limit) { os.write(buf,0,len-(count-limit)); mandatory_close = true; break;