--- /home/kangas/nutch-cvs/nutch/src/java/net/nutch/fetcher/Fetcher.java	Wed Dec 22 14:30:56 2004
+++ -	Mon Jan  3 13:52:58 2005
@@ -15,6 +15,7 @@
 import net.nutch.protocol.*;
 import net.nutch.parse.*;
 import net.nutch.plugin.*;
+import net.nutch.net.URLFilter;
 
 import java.util.logging.*;
 
@@ -56,6 +57,8 @@
   private int threadCount =                       // max number of threads
     NutchConf.getInt("fetcher.threads.fetch", 10);
 
+  private URLFilter urlFilter;
+
   // All threads (FetcherThread or thread started by it) belong to
   // group "fetcher". Each FetcherThread is named as "fetcherXX",
   // where XX is the order it's started.
@@ -94,7 +97,9 @@
 
           url = fle.getPage().getURL().toString();
 
-          if (!fle.getFetch()) {                  // should we fetch this page?
+	  boolean urlReject = urlFilter != null && urlFilter.filter(url) == null; 
+
+          if (!fle.getFetch() || urlReject) {     // should we fetch this page?
             if (LOG.isLoggable(Level.FINE))
               LOG.fine("not fetching " + url);
             handleNoFetch(fle, FetcherOutput.SUCCESS);
@@ -267,6 +272,11 @@
     LOG.info("logging at " + level);
   }
 
+  /** Set a prefetch URLFilter. This should not be synchronized */ 
+  public void setUrlFilter(URLFilter urlFilter) {
+    this.urlFilter = urlFilter;
+  }
+  
   /** Runs the fetcher. */
   public void run() throws IOException, InterruptedException {
     start = System.currentTimeMillis();