Author: markus
Date: Mon Nov 21 13:42:16 2011
New Revision: 1204492
URL: http://svn.apache.org/viewvc?rev=1204492&view=rev
Log:
NUTCH-1207 ParserChecker to output signature
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1204492&r1=1204491&r2=1204492&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Mon Nov 21 13:42:16 2011
@@ -1,5 +1,7 @@
Nutch Change Log
+* NUTCH-1207 ParserChecker to output signature (markus)
+
* NUTCH-1090 InvertLinks should inform when ignoring internal links (Marek
Backmann via markus)
* NUTCH-1174 Outlinks are not properly normalized (markus)
Modified: nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java?rev=1204492&r1=1204491&r2=1204492&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java Mon Nov 21
13:42:16 2011
@@ -24,10 +24,12 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.SignatureFactory;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolFactory;
import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.StringUtil;
/**
* Parser checker, useful for testing parser.
@@ -96,13 +98,17 @@ public class ParserChecker implements To
return (-1);
}
+ ParseResult parseResult = new ParseUtil(conf).parse(content);
+
+ // Calculate the signature
+ byte[] signature =
SignatureFactory.getSignature(getConf()).calculate(content, parseResult.get(new
Text(url)));
+
if (LOG.isInfoEnabled()) {
LOG.info("parsing: " + url);
LOG.info("contentType: " + contentType);
+ LOG.info("signature: " + StringUtil.toHexString(signature));
}
- ParseResult parseResult = new ParseUtil(conf).parse(content);
-
for (java.util.Map.Entry<Text, Parse> entry : parseResult) {
Parse parse = entry.getValue();
System.out.print("---------\nUrl\n---------------\n");