Author: markus
Date: Mon Nov 21 13:42:16 2011
New Revision: 1204492

URL: http://svn.apache.org/viewvc?rev=1204492&view=rev
Log:
NUTCH-1207 ParserChecker to output signature

Modified:
    nutch/trunk/CHANGES.txt
    nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java

Modified: nutch/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1204492&r1=1204491&r2=1204492&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Mon Nov 21 13:42:16 2011
@@ -1,5 +1,7 @@
 Nutch Change Log
 
+* NUTCH-1207 ParserChecker to output signature (markus)
+
 * NUTCH-1090 InvertLinks should inform when ignoring internal links (Marek 
Backmann via markus)
 
 * NUTCH-1174 Outlinks are not properly normalized (markus)

Modified: nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java?rev=1204492&r1=1204491&r2=1204492&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java Mon Nov 21 
13:42:16 2011
@@ -24,10 +24,12 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.SignatureFactory;
 import org.apache.nutch.protocol.Content;
 import org.apache.nutch.protocol.Protocol;
 import org.apache.nutch.protocol.ProtocolFactory;
 import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.StringUtil;
 
 /**
  * Parser checker, useful for testing parser.
@@ -96,13 +98,17 @@ public class ParserChecker implements To
       return (-1);
     }
 
+    ParseResult parseResult = new ParseUtil(conf).parse(content);
+
+    // Calculate the signature
+    byte[] signature = 
SignatureFactory.getSignature(getConf()).calculate(content, parseResult.get(new 
Text(url)));
+
     if (LOG.isInfoEnabled()) {
       LOG.info("parsing: " + url);
       LOG.info("contentType: " + contentType);
+      LOG.info("signature: " + StringUtil.toHexString(signature));
     }
 
-    ParseResult parseResult = new ParseUtil(conf).parse(content);
-
     for (java.util.Map.Entry<Text, Parse> entry : parseResult) {
       Parse parse = entry.getValue();
       System.out.print("---------\nUrl\n---------------\n");


Reply via email to