Author: ab Date: Wed Jan 17 11:55:07 2007 New Revision: 497141 URL: http://svn.apache.org/viewvc?view=rev&rev=497141 Log: NUTCH-68 - ported to use map-reduce.
Added: lucene/nutch/trunk/src/java/org/apache/nutch/tools/FreeGenerator.java (with props) Modified: lucene/nutch/trunk/CHANGES.txt lucene/nutch/trunk/bin/nutch Modified: lucene/nutch/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?view=diff&rev=497141&r1=497140&r2=497141 ============================================================================== --- lucene/nutch/trunk/CHANGES.txt (original) +++ lucene/nutch/trunk/CHANGES.txt Wed Jan 17 11:55:07 2007 @@ -137,6 +137,8 @@ 44. NUTCH-430 - Integer overflow in HashComparator.compare (siren) +45. NUTCH-68 - Add a tool to generate arbitrary fetchlists. (ab) + Release 0.8 - 2006-07-25 Modified: lucene/nutch/trunk/bin/nutch URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/bin/nutch?view=diff&rev=497141&r1=497140&r2=497141 ============================================================================== --- lucene/nutch/trunk/bin/nutch (original) +++ lucene/nutch/trunk/bin/nutch Wed Jan 17 11:55:07 2007 @@ -38,7 +38,8 @@ echo " mergedb merge crawldb-s, with optional filtering" echo " readlinkdb read / dump link db" echo " inject inject new urls into the database" - echo " generate generate new segments to fetch" + echo " generate generate new segments to fetch from crawl db" + echo " freegen generate new segments to fetch from text files" echo " fetch fetch a segment's pages" echo " parse parse a segment's pages" echo " readseg read / dump segment data" @@ -172,6 +173,8 @@ CLASS=org.apache.nutch.crawl.Injector elif [ "$COMMAND" = "generate" ] ; then CLASS=org.apache.nutch.crawl.Generator +elif [ "$COMMAND" = "freegen" ] ; then + CLASS=org.apache.nutch.tools.FreeGenerator elif [ "$COMMAND" = "fetch" ] ; then CLASS=org.apache.nutch.fetcher.Fetcher elif [ "$COMMAND" = "parse" ] ; then Added: lucene/nutch/trunk/src/java/org/apache/nutch/tools/FreeGenerator.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/tools/FreeGenerator.java?view=auto&rev=497141 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/tools/FreeGenerator.java (added) +++ lucene/nutch/trunk/src/java/org/apache/nutch/tools/FreeGenerator.java Wed Jan 17 11:55:07 2007 @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.tools; + +import java.io.IOException; +import java.util.Iterator; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MapReduceBase; +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reducer; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.SequenceFileOutputFormat; +import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.crawl.Generator; +import org.apache.nutch.net.URLFilters; +import org.apache.nutch.net.URLNormalizers; +import org.apache.nutch.scoring.ScoringFilters; +import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.NutchJob; + +/** + * This tool generates fetchlists (segments to be fetched) from plain text + * files containing one URL per line. It's useful when arbitrary URL-s need to + * be fetched without adding them first to the CrawlDb, or during testing. + * + * @author Andrzej Bialecki + */ +public class FreeGenerator extends ToolBase { + private static final Log LOG = LogFactory.getLog(FreeGenerator.class); + + private static final String FILTER_KEY = "free.generator.filter"; + private static final String NORMALIZE_KEY = "free.generator.normalize"; + + public static class FG extends MapReduceBase implements Mapper, Reducer { + private URLNormalizers normalizers = null; + private URLFilters filters = null; + private ScoringFilters scfilters; + private CrawlDatum datum = new CrawlDatum(); + private Text url = new Text(); + + @Override + public void configure(JobConf job) { + super.configure(job); + scfilters = new ScoringFilters(job); + if (job.getBoolean(FILTER_KEY, false)) { + filters = new URLFilters(job); + } + if (job.getBoolean(NORMALIZE_KEY, false)) { + normalizers = new URLNormalizers(job, URLNormalizers.SCOPE_INJECT); + } + } + + public void map(WritableComparable key, Writable value, OutputCollector output, Reporter reporter) throws IOException { + // value is a line of text + String urlString = value.toString(); + try { + if (normalizers != null) { + urlString = normalizers.normalize(urlString, URLNormalizers.SCOPE_INJECT); + } + if (urlString != null && filters != null) { + urlString = filters.filter(urlString); + } + if (urlString != null) { + url.set(urlString); + scfilters.injectedScore(url, datum); + } + } catch (Exception e) { + LOG.warn("Error adding url '" + value.toString() + "', skipping: " + StringUtils.stringifyException(e)); + return; + } + if (urlString == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("- skipping " + value.toString()); + } + return; + } + output.collect(url, datum); + } + + public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter) throws IOException { + // pick just one (discard duplicates) + output.collect(key, (Writable)values.next()); + } + } + + public int run(String[] args) throws Exception { + if (args.length < 2) { + System.err.println("Usage: FreeGenerator <inputDir> <segmentsDir> [-filter] [-normalize]"); + System.err.println("\tinputDir\tinput directory containing one or more input files."); + System.err.println("\t\tEach text file contains a list of URLs, one URL per line"); + System.err.println("\tsegmentsDir\toutput directory, where new segment will be created"); + System.err.println("\t-filter\trun current URLFilters on input URLs"); + System.err.println("\t-normalize\trun current URLNormalizers on input URLs"); + return -1; + } + boolean filter = false; + boolean normalize = false; + if (args.length > 2) { + for (int i = 2; i < args.length; i++) { + if (args[i].equals("-filter")) { + filter = true; + } else if (args[i].equals("-normalize")) { + normalize = true; + } else { + LOG.fatal("Unknown argument: " + args[i] + ", exiting ..."); + return -1; + } + } + } + + JobConf job = new NutchJob(getConf()); + job.setBoolean(FILTER_KEY, filter); + job.setBoolean(NORMALIZE_KEY, normalize); + job.addInputPath(new Path(args[0])); + job.setInputFormat(TextInputFormat.class); + job.setMapperClass(FG.class); + job.setCombinerClass(FG.class); + job.setReducerClass(FG.class); + String segName = Generator.generateSegmentName(); + job.setOutputFormat(SequenceFileOutputFormat.class); + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(CrawlDatum.class); + job.setOutputPath(new Path(args[1], new Path(segName, CrawlDatum.GENERATE_DIR_NAME))); + try { + JobClient.runJob(job); + return 0; + } catch (Exception e) { + LOG.fatal("FAILED: " + StringUtils.stringifyException(e)); + return -1; + } + } + + public static void main(String[] args) throws Exception { + int res = new FreeGenerator().doMain(NutchConfiguration.create(), args); + System.exit(res); + } +} Propchange: lucene/nutch/trunk/src/java/org/apache/nutch/tools/FreeGenerator.java ------------------------------------------------------------------------------ svn:eol-style = native