Author: lewismc Date: Sun Mar 18 16:46:33 2012 New Revision: 1302161 URL: http://svn.apache.org/viewvc?rev=1302161&view=rev Log: commit to address NUTCH-1273
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDatum.java nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java nutch/trunk/src/java/org/apache/nutch/crawl/NutchWritable.java nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java nutch/trunk/src/java/org/apache/nutch/protocol/Content.java nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java nutch/trunk/src/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java Modified: nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDatum.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDatum.java?rev=1302161&r1=1302160&r2=1302161&view=diff ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDatum.java (original) +++ nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDatum.java Sun Mar 18 16:46:33 2012 @@ -262,7 +262,7 @@ public class CrawlDatum implements Writa if (version > 3) { boolean hasMetadata = false; if (version < 7) { - MapWritable oldMetaData = new MapWritable(); + org.apache.hadoop.io.MapWritable oldMetaData = new org.apache.hadoop.io.MapWritable(); if (in.readBoolean()) { hasMetadata = true; metaData = new org.apache.hadoop.io.MapWritable(); Modified: nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java?rev=1302161&r1=1302160&r2=1302161&view=diff ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java (original) +++ nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java Sun Mar 18 16:46:33 2012 @@ -19,6 +19,7 @@ package org.apache.nutch.crawl; import java.io.DataOutputStream; import java.io.IOException; +import java.io.Closeable; import java.net.URL; import java.util.Date; import java.util.Iterator; @@ -35,7 +36,6 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Closeable; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.MapFile; Modified: nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java?rev=1302161&r1=1302160&r2=1302161&view=diff ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java (original) +++ nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java Sun Mar 18 16:46:33 2012 @@ -37,6 +37,7 @@ import org.apache.nutch.util.TimingUtil; import java.text.SimpleDateFormat; import java.util.Iterator; +import java.io.Closeable; /** . */ public class LinkDbReader extends Configured implements Tool, Closeable { Modified: nutch/trunk/src/java/org/apache/nutch/crawl/NutchWritable.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/NutchWritable.java?rev=1302161&r1=1302160&r2=1302161&view=diff ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/crawl/NutchWritable.java (original) +++ nutch/trunk/src/java/org/apache/nutch/crawl/NutchWritable.java Sun Mar 18 16:46:33 2012 @@ -31,12 +31,12 @@ public class NutchWritable extends Gener org.apache.hadoop.io.BytesWritable.class, org.apache.hadoop.io.FloatWritable.class, org.apache.hadoop.io.IntWritable.class, + org.apache.hadoop.io.MapWritable.class, org.apache.hadoop.io.Text.class, org.apache.hadoop.io.MD5Hash.class, org.apache.nutch.crawl.CrawlDatum.class, org.apache.nutch.crawl.Inlink.class, org.apache.nutch.crawl.Inlinks.class, - org.apache.nutch.crawl.MapWritable.class, org.apache.nutch.fetcher.FetcherOutput.class, org.apache.nutch.metadata.Metadata.class, org.apache.nutch.parse.Outlink.class, Modified: nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java?rev=1302161&r1=1302160&r2=1302161&view=diff ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java (original) +++ nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java Sun Mar 18 16:46:33 2012 @@ -75,7 +75,7 @@ public class ParseSegment extends Config OutputCollector<Text, ParseImpl> output, Reporter reporter) throws IOException { // convert on the fly from old UTF8 keys - if (key instanceof UTF8) { + if (key instanceof Text) { newKey.set(key.toString()); key = newKey; } Modified: nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java?rev=1302161&r1=1302160&r2=1302161&view=diff ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java (original) +++ nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java Sun Mar 18 16:46:33 2012 @@ -20,6 +20,7 @@ import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.net.URI; import java.util.ArrayList; import java.util.HashMap; import java.util.Locale; @@ -214,12 +215,17 @@ public class PluginDescriptor { /** * Adds a exported library with a relative path to the plugin directory. + * We automatically escape characters that are illegal in URLs. It is + * recommended that code converts an abstract pathname into a URL by + * first converting it into a URI, via the toURI method, and then + * converting the URI into a URL via the URI.toURL method. * * @param pLibPath */ public void addExportedLibRelative(String pLibPath) throws MalformedURLException { - URL url = new File(getPluginPath() + File.separator + pLibPath).toURL(); + URI uri = new File(getPluginPath() + File.separator + pLibPath).toURI(); + URL url = uri.toURL(); fExportedLibs.add(url); } @@ -242,13 +248,18 @@ public class PluginDescriptor { } /** - * Adds a not exported library with a plugin directory relative path. + * Adds a exported library with a relative path to the plugin directory. + * We automatically escape characters that are illegal in URLs. It is + * recommended that code converts an abstract pathname into a URL by + * first converting it into a URI, via the toURI method, and then + * converting the URI into a URL via the URI.toURL method. * * @param pLibPath */ public void addNotExportedLibRelative(String pLibPath) throws MalformedURLException { - URL url = new File(getPluginPath() + File.separator + pLibPath).toURL(); + URI uri = new File(getPluginPath() + File.separator + pLibPath).toURI(); + URL url = uri.toURL(); fNotExportedLibs.add(url); } @@ -279,7 +290,7 @@ public class PluginDescriptor { try { for (File file2 : file.listFiles()) { if (file2.getAbsolutePath().endsWith("properties")) - arrayList.add(file2.getParentFile().toURL()); + arrayList.add(file2.getParentFile().toURI().toURL()); } } catch (MalformedURLException e) { LOG.debug(getPluginId() + " " + e.toString()); Modified: nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java?rev=1302161&r1=1302160&r2=1302161&view=diff ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java (original) +++ nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java Sun Mar 18 16:46:33 2012 @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; +import java.net.URI; import java.net.URLDecoder; import java.util.HashMap; import java.util.Map; @@ -147,7 +148,7 @@ public class PluginManifestParser { private PluginDescriptor parseManifestFile(String pManifestPath) throws MalformedURLException, SAXException, IOException, ParserConfigurationException { - Document document = parseXML(new File(pManifestPath).toURL()); + Document document = parseXML(new File(pManifestPath).toURI().toURL()); String pPath = new File(pManifestPath).getParent(); return parsePlugin(document, pPath); } Modified: nutch/trunk/src/java/org/apache/nutch/protocol/Content.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/protocol/Content.java?rev=1302161&r1=1302160&r2=1302161&view=diff ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/protocol/Content.java (original) +++ nutch/trunk/src/java/org/apache/nutch/protocol/Content.java Sun Mar 18 16:46:33 2012 @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.ArrayFile; import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.UTF8; import org.apache.hadoop.io.VersionMismatchException; import org.apache.hadoop.io.Writable; import org.apache.hadoop.util.GenericOptionsParser; @@ -93,21 +92,21 @@ public final class Content implements Wr switch (oldVersion) { case 0: case 1: - url = UTF8.readString(in); // read url - base = UTF8.readString(in); // read base + url = Text.readString(in); // read url + base = Text.readString(in); // read base content = new byte[in.readInt()]; // read content in.readFully(content); - contentType = UTF8.readString(in); // read contentType + contentType = Text.readString(in); // read contentType // reconstruct metadata int keySize = in.readInt(); String key; for (int i = 0; i < keySize; i++) { - key = UTF8.readString(in); + key = Text.readString(in); int valueSize = in.readInt(); for (int j = 0; j < valueSize; j++) { - metadata.add(key, UTF8.readString(in)); + metadata.add(key, Text.readString(in)); } } break; Modified: nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java?rev=1302161&r1=1302160&r2=1302161&view=diff ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java (original) +++ nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java Sun Mar 18 16:46:33 2012 @@ -43,7 +43,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.MapFile; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.UTF8; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.FileInputFormat; @@ -86,8 +85,9 @@ public class SegmentReader extends Confi public void map(WritableComparable key, Writable value, OutputCollector<Text, NutchWritable> collector, Reporter reporter) throws IOException { - // convert on the fly from old formats with UTF8 keys - if (key instanceof UTF8) { + // convert on the fly from old formats with UTF8 keys. + // UTF8 deprecated and replaced by Text. + if (key instanceof Text) { newKey.set(key.toString()); key = newKey; } @@ -252,7 +252,7 @@ public class SegmentReader extends Confi writer.close(); } } - fs.delete(tempDir); + fs.delete(tempDir, true); if (LOG.isInfoEnabled()) { LOG.info("SegmentReader: done"); } } Modified: nutch/trunk/src/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java?rev=1302161&r1=1302160&r2=1302161&view=diff ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java (original) +++ nutch/trunk/src/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java Sun Mar 18 16:46:33 2012 @@ -315,7 +315,7 @@ public class ArcSegmentCreator // the arc file, TODO: currently this doesn't handle text of errors // pages (i.e. 404, etc.). We assume we won't get those. ProtocolStatus status = ProtocolStatus.STATUS_SUCCESS; - Content content = new Content(urlStr, urlStr, bytes.get(), contentType, + Content content = new Content(urlStr, urlStr, bytes.getBytes(), contentType, new Metadata(), getConf()); // set the url version into the metadata Modified: nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java?rev=1302161&r1=1302160&r2=1302161&view=diff ============================================================================== --- nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java (original) +++ nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java Sun Mar 18 16:46:33 2012 @@ -19,6 +19,7 @@ package org.apache.nutch.protocol.file; // JDK imports import java.net.URL; +import java.net.URI; import java.util.Date; import java.util.TreeMap; import java.io.IOException; @@ -151,7 +152,12 @@ public class FileResponse { if (!f.equals(f.getCanonicalFile())) { // set headers //hdrs.put("Location", f.getCanonicalFile().toURI()); - headers.set(Response.LOCATION, f.getCanonicalFile().toURL().toString()); + // + // we want to automatically escape characters that are illegal in URLs. + // It is recommended that new code convert an abstract pathname into a URL + // by first converting it into a URI, via the toURI method, and then + // converting the URI into a URL via the URI.toURL method. + headers.set(Response.LOCATION, f.getCanonicalFile().toURI().toURL().toString()); this.code = 300; // http redirect return;