Author: ab Date: Thu Mar 22 03:08:00 2007 New Revision: 521182 URL: http://svn.apache.org/viewvc?view=rev&rev=521182 Log: NUTCH-246 - incorrect segment size being generated due to time synchronization issue.
Modified: lucene/nutch/trunk/CHANGES.txt lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java Modified: lucene/nutch/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?view=diff&rev=521182&r1=521181&r2=521182 ============================================================================== --- lucene/nutch/trunk/CHANGES.txt (original) +++ lucene/nutch/trunk/CHANGES.txt Thu Mar 22 03:08:00 2007 @@ -166,6 +166,9 @@ 56. Upgrade to Hadoop 0.12.1 release. (ab) +57. NUTCH-246 - Incorrect segment size being generated due to time + synchronization issue (Stefan Groschupf via ab) + Release 0.8 - 2006-07-25 Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java?view=diff&rev=521182&r1=521181&r2=521182 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java Thu Mar 22 03:08:00 2007 @@ -51,6 +51,7 @@ private JobConf jobConf; private URLFilters filters; private ScoringFilters scfilters; + private long curTime; public void configure(JobConf job) { this.jobConf = job; @@ -59,6 +60,7 @@ filters = new URLFilters(jobConf); scfilters = new ScoringFilters(jobConf); scoreInjected = jobConf.getFloat("db.score.injected", 1.0f); + curTime = job.getLong("injector.current.time", System.currentTimeMillis()); } public void close() {} @@ -79,6 +81,7 @@ if (url != null) { // if it passes value.set(url); // collect it CrawlDatum datum = new CrawlDatum(CrawlDatum.STATUS_INJECTED, interval); + datum.setFetchTime(curTime); datum.setScore(scoreInjected); try { scfilters.injectedScore(value, datum); @@ -96,7 +99,7 @@ /** Combine multiple new entries for a url. */ public static class InjectReducer implements Reducer { - public void configure(JobConf job) {} + public void configure(JobConf job) {} public void close() {} public void reduce(WritableComparable key, Iterator values, @@ -155,6 +158,7 @@ sortJob.setOutputFormat(SequenceFileOutputFormat.class); sortJob.setOutputKeyClass(Text.class); sortJob.setOutputValueClass(CrawlDatum.class); + sortJob.setLong("injector.current.time", System.currentTimeMillis()); JobClient.runJob(sortJob); // merge with existing crawl db