Author: jnioche Date: Fri Jan 8 12:01:46 2010 New Revision: 897180 URL: http://svn.apache.org/viewvc?rev=897180&view=rev Log: NUTCH-269 : OOME because no upper-bound on inlinks count (stack + jnioche)
Modified: lucene/nutch/trunk/CHANGES.txt lucene/nutch/trunk/conf/nutch-default.xml lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java Modified: lucene/nutch/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=897180&r1=897179&r2=897180&view=diff ============================================================================== --- lucene/nutch/trunk/CHANGES.txt (original) +++ lucene/nutch/trunk/CHANGES.txt Fri Jan 8 12:01:46 2010 @@ -2,6 +2,8 @@ Unreleased Changes +* NUTCH-269 CrawlDbReducer: OOME because no upper-bound on inlinks count (stack + jnioche) + * NUTCH-655 Injecting Crawl metadata (jnioche) * NUTCH-658 Use counters to report fetching and parsing status (jnioche) Modified: lucene/nutch/trunk/conf/nutch-default.xml URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/conf/nutch-default.xml?rev=897180&r1=897179&r2=897180&view=diff ============================================================================== --- lucene/nutch/trunk/conf/nutch-default.xml (original) +++ lucene/nutch/trunk/conf/nutch-default.xml Fri Jan 8 12:01:46 2010 @@ -384,6 +384,14 @@ </property> <property> + <name>db.update.max.inlinks</name> + <value>10000</value> + <description>Maximum number of inlinks to take into account when updating + a URL score in the crawlDB. Only the best scoring inlinks are kept. + </description> +</property> + +<property> <name>db.ignore.internal.links</name> <value>true</value> <description>If true, when adding new links to a page, links from Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java?rev=897180&r1=897179&r2=897180&view=diff ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java Fri Jan 8 12:01:46 2010 @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.io.IOException; // Commons Logging imports @@ -27,6 +28,7 @@ import org.apache.hadoop.io.*; import org.apache.hadoop.mapred.*; +import org.apache.hadoop.util.PriorityQueue; import org.apache.nutch.metadata.Nutch; import org.apache.nutch.scoring.ScoringFilterException; import org.apache.nutch.scoring.ScoringFilters; @@ -37,7 +39,7 @@ private int retryMax; private CrawlDatum result = new CrawlDatum(); - private ArrayList<CrawlDatum> linked = new ArrayList<CrawlDatum>(); + private InlinkPriorityQueue linked = null; private ScoringFilters scfilters = null; private boolean additionsAllowed; private int maxInterval; @@ -51,6 +53,8 @@ maxInterval = job.getInt("db.fetch.interval.max", 0 ); if (oldMaxInterval > 0 && maxInterval == 0) maxInterval = oldMaxInterval * FetchSchedule.SECONDS_PER_DAY; schedule = FetchScheduleFactory.getFetchSchedule(job); + int maxLinks = job.getInt("db.update.max.inlinks", 10000); + linked = new InlinkPriorityQueue(maxLinks); } public void close() {} @@ -111,7 +115,7 @@ } else { link = datum; } - linked.add(link); + linked.insert(link); break; case CrawlDatum.STATUS_SIGNATURE: signature = datum.getSignature(); @@ -120,13 +124,21 @@ LOG.warn("Unknown status, key: " + key + ", datum: " + datum); } } - + + // copy the content of the queue into a List + // in reversed order + int numLinks = linked.size(); + List<CrawlDatum> linkList = new ArrayList<CrawlDatum>(numLinks); + for (int i = numLinks - 1; i >= 0; i--) { + linkList.add(linked.pop()); + } + // if it doesn't already exist, skip it if (!oldSet && !additionsAllowed) return; // if there is no fetched datum, perhaps there is a link - if (!fetchSet && linked.size() > 0) { - fetch = linked.get(0); + if (!fetchSet && linkList.size() > 0) { + fetch = linkList.get(0); fetchSet = true; } @@ -260,7 +272,7 @@ } try { - scfilters.updateDbScore((Text)key, oldSet ? old : null, result, linked); + scfilters.updateDbScore((Text)key, oldSet ? old : null, result, linkList); } catch (Exception e) { if (LOG.isWarnEnabled()) { LOG.warn("Couldn't update score, key=" + key + ": " + e); @@ -270,5 +282,20 @@ result.getMetaData().remove(Nutch.WRITABLE_GENERATE_TIME_KEY); output.collect(key, result); } + +} +class InlinkPriorityQueue extends PriorityQueue<CrawlDatum> { + + public InlinkPriorityQueue(int maxSize) { + initialize(maxSize); + } + + /** Determines the ordering of objects in this priority queue. **/ + protected boolean lessThan(Object arg0, Object arg1) { + CrawlDatum candidate = (CrawlDatum) arg0; + CrawlDatum least = (CrawlDatum) arg1; + return candidate.getScore() > least.getScore(); + } + }