Author: ab Date: Wed Mar 7 13:59:07 2007 New Revision: 515791 URL: http://svn.apache.org/viewvc?view=rev&rev=515791 Log: Upgrade to Hadoop 0.11.2 and Lucene 2.1.0 releases.
Added: lucene/nutch/trunk/lib/hadoop-0.11.2-core.jar (with props) lucene/nutch/trunk/lib/lucene-core-2.1.0.jar (with props) lucene/nutch/trunk/lib/lucene-misc-2.1.0.jar (with props) lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.1.0.jar (with props) lucene/nutch/trunk/src/plugin/summary-lucene/lib/lucene-highlighter-2.1.0.jar (with props) Removed: lucene/nutch/trunk/lib/hadoop-0.10.1-core.jar lucene/nutch/trunk/lib/lucene-core-2.0.0.jar lucene/nutch/trunk/lib/lucene-misc-2.0.0.jar lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.0.0.jar lucene/nutch/trunk/src/plugin/summary-lucene/lib/lucene-highlighter-2.0.0.jar Modified: lucene/nutch/trunk/CHANGES.txt lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.a lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1 lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1.0.0 lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Inlinks.java lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbMerger.java lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatus.java lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/plugin.xml lucene/nutch/trunk/src/plugin/summary-lucene/plugin.xml lucene/nutch/trunk/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java Modified: lucene/nutch/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/CHANGES.txt (original) +++ lucene/nutch/trunk/CHANGES.txt Wed Mar 7 13:59:07 2007 @@ -151,6 +151,8 @@ 50. NUTCH-432 - Fix a bug where platform name with spaces would break the bin/nutch script. (Brian Whitman via ab) +51. Upgrade to Hadoop 0.11.2 and Lucene 2.1.0 release. + Release 0.8 - 2006-07-25 Added: lucene/nutch/trunk/lib/hadoop-0.11.2-core.jar URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/hadoop-0.11.2-core.jar?view=auto&rev=515791 ============================================================================== Binary file - no diff available. Propchange: lucene/nutch/trunk/lib/hadoop-0.11.2-core.jar ------------------------------------------------------------------------------ svn:mime-type = application/octet-stream Added: lucene/nutch/trunk/lib/lucene-core-2.1.0.jar URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/lucene-core-2.1.0.jar?view=auto&rev=515791 ============================================================================== Binary file - no diff available. Propchange: lucene/nutch/trunk/lib/lucene-core-2.1.0.jar ------------------------------------------------------------------------------ svn:mime-type = application/octet-stream Added: lucene/nutch/trunk/lib/lucene-misc-2.1.0.jar URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/lucene-misc-2.1.0.jar?view=auto&rev=515791 ============================================================================== Binary file - no diff available. Propchange: lucene/nutch/trunk/lib/lucene-misc-2.1.0.jar ------------------------------------------------------------------------------ svn:mime-type = application/octet-stream Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.a URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.a?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== Binary files - no diff available. Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== Binary files - no diff available. Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1 URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== Binary files - no diff available. Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1.0.0 URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1.0.0?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== Binary files - no diff available. Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java Wed Mar 7 13:59:07 2007 @@ -25,7 +25,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -33,7 +32,6 @@ import org.apache.hadoop.mapred.*; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolBase; -import org.apache.nutch.net.URLFilters; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.NutchJob; Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java Wed Mar 7 13:59:07 2007 @@ -346,8 +346,6 @@ job.addInputPath(new Path(crawlDb, CrawlDb.CURRENT_NAME)); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(Text.class); - job.setInputValueClass(CrawlDatum.class); job.setOutputPath(outFolder); job.setOutputFormat(TextOutputFormat.class); @@ -396,8 +394,6 @@ job.addInputPath(tempDir); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(FloatWritable.class); - job.setInputValueClass(Text.class); job.setMapperClass(IdentityMapper.class); job.setReducerClass(CrawlDbTopNReducer.class); Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Inlinks.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Inlinks.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Inlinks.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Inlinks.java Wed Mar 7 13:59:07 2007 @@ -25,7 +25,7 @@ /** A list of [EMAIL PROTECTED] Inlink}s. */ public class Inlinks implements Writable { - private HashSet inlinks = new HashSet(1); + private HashSet<Inlink> inlinks = new HashSet<Inlink>(1); public void add(Inlink inlink) { inlinks.add(inlink); } Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbMerger.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbMerger.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbMerger.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbMerger.java Wed Mar 7 13:59:07 2007 @@ -21,7 +21,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobClient; Modified: lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java Wed Mar 7 13:59:07 2007 @@ -343,7 +343,6 @@ } public synchronized void dump() { - Iterator it = queues.keySet().iterator(); for (String id : queues.keySet()) { FetchItemQueue fiq = queues.get(id); if (fiq.getQueueSize() == 0) continue; Modified: lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java Wed Mar 7 13:59:07 2007 @@ -56,7 +56,7 @@ new Path(new Path(job.getOutputPath(), Content.DIR_NAME), name); final MapFile.Writer fetchOut = - new MapFile.Writer(fs, fetch.toString(), Text.class, CrawlDatum.class); + new MapFile.Writer(job, fs, fetch.toString(), Text.class, CrawlDatum.class); return new RecordWriter() { private MapFile.Writer contentOut; @@ -64,7 +64,7 @@ { if (Fetcher.isStoringContent(job)) { - contentOut = new MapFile.Writer(fs, content.toString(), + contentOut = new MapFile.Writer(job, fs, content.toString(), Text.class, Content.class); } Modified: lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java Wed Mar 7 13:59:07 2007 @@ -61,7 +61,7 @@ if (LOG.isInfoEnabled()) { LOG.info("merging indexes to: " + outputIndex); } - FileSystem localFs = FileSystem.getNamed("local", getConf()); + FileSystem localFs = FileSystem.getLocal(getConf()); if (localWorkingDir == null) { localWorkingDir = new Path("indexmerger-" + System.currentTimeMillis()); } @@ -102,7 +102,7 @@ // Put target back // fs.completeLocalOutput(outputIndex, tmpLocalOutput); - FileSystem.getNamed("local", conf).delete(localWorkingDir); + FileSystem.getLocal(conf).delete(localWorkingDir); if (LOG.isInfoEnabled()) { LOG.info("done merging"); } } Modified: lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java Wed Mar 7 13:59:07 2007 @@ -24,7 +24,6 @@ import org.apache.hadoop.io.*; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.nutch.crawl.CrawlDatum; -import org.apache.nutch.fetcher.Fetcher; import org.apache.hadoop.fs.*; import org.apache.hadoop.mapred.*; import org.apache.nutch.scoring.ScoringFilterException; Modified: lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java Wed Mar 7 13:59:07 2007 @@ -21,7 +21,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.nutch.crawl.SignatureFactory; -import org.apache.nutch.fetcher.Fetcher; import org.apache.hadoop.io.*; import org.apache.hadoop.mapred.*; import org.apache.hadoop.conf.*; Modified: lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatus.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatus.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatus.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatus.java Wed Mar 7 13:59:07 2007 @@ -46,7 +46,7 @@ /** Resource has moved temporarily. New url should be found in args. */ public static final int TEMP_MOVED = 13; /** Resource was not found. */ - public static final int NOTFOUND = 14; + public static final int NOTFOUND = 14; /** Temporary failure. Application may retry immediately. */ public static final int RETRY = 15; /** Unspecified exception occured. Further information may be provided in args. */ Modified: lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java (original) +++ lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java Wed Mar 7 13:59:07 2007 @@ -232,7 +232,7 @@ } else { wname = new Path(new Path(new Path(job.getOutputPath(), segmentName + "-" + slice), dirName), name); } - res = new MapFile.Writer(fs, wname.toString(), Text.class, clazz); + res = new MapFile.Writer(job, fs, wname.toString(), Text.class, clazz); sliceWriters.put(slice + dirName, res); return res; } Modified: lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java (original) +++ lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java Wed Mar 7 13:59:07 2007 @@ -20,7 +20,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.lucene.document.DateField; import org.apache.lucene.document.DateTools; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; Added: lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.1.0.jar URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.1.0.jar?view=auto&rev=515791 ============================================================================== Binary file - no diff available. Propchange: lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.1.0.jar ------------------------------------------------------------------------------ svn:mime-type = application/octet-stream Modified: lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/plugin.xml URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/plugin.xml?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/plugin.xml (original) +++ lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/plugin.xml Wed Mar 7 13:59:07 2007 @@ -29,7 +29,7 @@ provider-name="org.apache.lucene"> <runtime> - <library name="lucene-analyzers-2.0.0.jar"> + <library name="lucene-analyzers-2.1.0.jar"> <export name="*"/> </library> </runtime> Added: lucene/nutch/trunk/src/plugin/summary-lucene/lib/lucene-highlighter-2.1.0.jar URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/summary-lucene/lib/lucene-highlighter-2.1.0.jar?view=auto&rev=515791 ============================================================================== Binary file - no diff available. Propchange: lucene/nutch/trunk/src/plugin/summary-lucene/lib/lucene-highlighter-2.1.0.jar ------------------------------------------------------------------------------ svn:mime-type = application/octet-stream Modified: lucene/nutch/trunk/src/plugin/summary-lucene/plugin.xml URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/summary-lucene/plugin.xml?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/plugin/summary-lucene/plugin.xml (original) +++ lucene/nutch/trunk/src/plugin/summary-lucene/plugin.xml Wed Mar 7 13:59:07 2007 @@ -25,7 +25,7 @@ <library name="summary-lucene.jar"> <export name="*"/> </library> - <library name="lucene-highlighter-2.0.0.jar"/> + <library name="lucene-highlighter-2.1.0.jar"/> </runtime> <requires> Modified: lucene/nutch/trunk/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java (original) +++ lucene/nutch/trunk/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java Wed Mar 7 13:59:07 2007 @@ -50,11 +50,11 @@ * urls to be inserted, objects are of type URLCrawlDatum * @throws Exception */ - public static void createCrawlDb(FileSystem fs, Path crawldb, List<URLCrawlDatum> init) + public static void createCrawlDb(Configuration conf, FileSystem fs, Path crawldb, List<URLCrawlDatum> init) throws Exception { LOG.trace("* creating crawldb: " + crawldb); Path dir = new Path(crawldb, CrawlDb.CURRENT_NAME); - MapFile.Writer writer = new MapFile.Writer(fs, new Path(dir, "part-00000") + MapFile.Writer writer = new MapFile.Writer(conf, fs, new Path(dir, "part-00000") .toString(), Text.class, CrawlDatum.class); Iterator<URLCrawlDatum> it = init.iterator(); while (it.hasNext()) { @@ -85,7 +85,7 @@ * @return */ public static Configuration createConfiguration(){ - Configuration conf=new Configuration(); + Configuration conf = new Configuration(); conf.addDefaultResource("nutch-default.xml"); conf.addFinalResource("crawl-tests.xml"); return conf; Modified: lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java (original) +++ lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java Wed Mar 7 13:59:07 2007 @@ -97,8 +97,8 @@ Path crawldb1 = new Path(testDir, "crawldb1"); Path crawldb2 = new Path(testDir, "crawldb2"); Path output = new Path(testDir, "output"); - createCrawlDb(fs, crawldb1, init1, cd1); - createCrawlDb(fs, crawldb2, init2, cd2); + createCrawlDb(conf, fs, crawldb1, init1, cd1); + createCrawlDb(conf, fs, crawldb2, init2, cd2); CrawlDbMerger merger = new CrawlDbMerger(conf); LOG.fine("* merging crawldbs to " + output); merger.merge(output, new Path[]{crawldb1, crawldb2}, false, false); @@ -123,10 +123,10 @@ fs.delete(testDir); } - private void createCrawlDb(FileSystem fs, Path crawldb, TreeSet init, CrawlDatum cd) throws Exception { + private void createCrawlDb(Configuration config, FileSystem fs, Path crawldb, TreeSet init, CrawlDatum cd) throws Exception { LOG.fine("* creating crawldb: " + crawldb); Path dir = new Path(crawldb, CrawlDb.CURRENT_NAME); - MapFile.Writer writer = new MapFile.Writer(fs, new Path(dir, "part-00000").toString(), Text.class, CrawlDatum.class); + MapFile.Writer writer = new MapFile.Writer(config, fs, new Path(dir, "part-00000").toString(), Text.class, CrawlDatum.class); Iterator it = init.iterator(); while (it.hasNext()) { String key = (String)it.next(); Modified: lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java (original) +++ lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java Wed Mar 7 13:59:07 2007 @@ -330,7 +330,7 @@ fs.mkdirs(segmentsDir); // create crawldb - CrawlDBTestUtil.createCrawlDb(fs, dbDir, list); + CrawlDBTestUtil.createCrawlDb(conf, fs, dbDir, list); } /** Modified: lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java?view=diff&rev=515791&r1=515790&r2=515791 ============================================================================== --- lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java (original) +++ lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java Wed Mar 7 13:59:07 2007 @@ -109,8 +109,8 @@ Path linkdb1 = new Path(testDir, "linkdb1"); Path linkdb2 = new Path(testDir, "linkdb2"); Path output = new Path(testDir, "output"); - createLinkDb(fs, linkdb1, init1); - createLinkDb(fs, linkdb2, init2); + createLinkDb(conf, fs, linkdb1, init1); + createLinkDb(conf, fs, linkdb2, init2); LinkDbMerger merger = new LinkDbMerger(conf); LOG.fine("* merging linkdbs to " + output); merger.merge(output, new Path[]{linkdb1, linkdb2}, false, false); @@ -139,10 +139,10 @@ fs.delete(testDir); } - private void createLinkDb(FileSystem fs, Path linkdb, TreeMap init) throws Exception { + private void createLinkDb(Configuration config, FileSystem fs, Path linkdb, TreeMap init) throws Exception { LOG.fine("* creating linkdb: " + linkdb); Path dir = new Path(linkdb, LinkDb.CURRENT_NAME); - MapFile.Writer writer = new MapFile.Writer(fs, new Path(dir, "part-00000").toString(), Text.class, Inlinks.class); + MapFile.Writer writer = new MapFile.Writer(config, fs, new Path(dir, "part-00000").toString(), Text.class, Inlinks.class); Iterator it = init.keySet().iterator(); while (it.hasNext()) { String key = (String)it.next();