[
https://issues.apache.org/jira/browse/NUTCH-2038?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14605088#comment-14605088
]
ASF GitHub Bot commented on NUTCH-2038:
---------------------------------------
Github user chrismattmann commented on a diff in the pull request:
https://github.com/apache/nutch/pull/38#discussion_r33432911
--- Diff:
src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/NaiveBayesParseFilter.java
---
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.parsefilter.naivebayes;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.DocumentFragment;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.nutch.parse.HTMLMetaTags;
+import org.apache.nutch.parse.HtmlParseFilter;
+import org.apache.nutch.parse.Outlink;
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.parse.ParseData;
+import org.apache.nutch.parse.ParseResult;
+import org.apache.nutch.parse.ParseStatus;
+import org.apache.nutch.parse.ParseText;
+import org.apache.nutch.protocol.Content;
+
+import java.io.Reader;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.util.ArrayList;
+
+/**
+ * Html Parse filter that classifies the outlinks from the parseresult as
+ * relevant or irrelevant based on the parseText's relevancy (using a
training
+ * file where you can give positive and negative example texts see the
+ * description of parsefilter.naivebayes.trainfile) and if found
irrelevent it
+ * gives the link a second chance if it contains any of the words from the
list
+ * given in parsefilter.naivebayes.wordlist. CAUTION: Set the
parser.timeout to
+ * -1 or a bigger value than 30, when using this classifier.
+ */
+public class NaiveBayesParseFilter implements HtmlParseFilter {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(NaiveBayesParseFilter.class);
+
+ public static final String TRAINFILE_MODELFILTER =
"parsefilter.naivebayes.trainfile";
+ public static final String DICTFILE_MODELFILTER =
"parsefilter.naivebayes.wordlist";
+
+ private Configuration conf;
+ private String inputFilePath;
+ private String dictionaryFile;
+ private ArrayList<String> wordlist = new ArrayList<String>();
+
+ public boolean filterParse(String text) {
+
+ try {
+ return classify(text);
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ LOG.error("Error occured while classifying:: " + text + " ::"
+ + StringUtils.stringifyException(e));
+ }
+
+ return false;
+ }
+
+ public boolean filterUrl(String url) {
+
+ return containsWord(url, wordlist);
+
+ }
+
+ public boolean classify(String text) throws IOException {
+
+ // if classified as relevent "1" then return true
+ if (NaiveBayesClassifier.classify(text).equals("1"))
+ return true;
+ return false;
+ }
+
+ public void train() throws Exception {
+ // check if the model file exists, if it does then don't train
+ if (!FileSystem.get(conf).exists(new Path("model"))) {
+ LOG.info("Training the Naive Bayes Model");
+ NaiveBayesClassifier.createModel(inputFilePath);
+ } else {
+ LOG.info("Model file already exists. Skipping training.");
+ }
+ }
+
+ public boolean containsWord(String url, ArrayList<String> wordlist) {
+ for (String word : wordlist) {
+ if (url.contains(word)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ inputFilePath = conf.get(TRAINFILE_MODELFILTER);
+ dictionaryFile = conf.get(DICTFILE_MODELFILTER);
+ if (inputFilePath == null || inputFilePath.trim().length() == 0
+ || dictionaryFile == null || dictionaryFile.trim().length() == 0) {
+ String message = "ParseFilter: NaiveBayes: trainfile or wordlist not
set in the parsefilte.naivebayes.trainfile or parsefilte.naivebayes.wordlist";
+ if (LOG.isErrorEnabled()) {
+ LOG.error(message);
+ }
+ throw new IllegalArgumentException(message);
+ }
+ try {
+ if ((FileSystem.get(conf).exists(new Path(inputFilePath)))
+ || (FileSystem.get(conf).exists(new Path(dictionaryFile)))) {
+ String message = "ParseFilter: NaiveBayes: " + inputFilePath + "
or "
+ + dictionaryFile + " not found!";
+ if (LOG.isErrorEnabled()) {
+ LOG.error(message);
+ }
+ throw new IllegalArgumentException(message);
+ }
+
+ BufferedReader br = null;
+
+ String CurrentLine;
+ Reader reader = conf.getConfResourceAsReader(dictionaryFile);
+ br = new BufferedReader(reader);
+ while ((CurrentLine = br.readLine()) != null) {
+ wordlist.add(CurrentLine);
+ }
+
+ } catch (IOException e) {
+ LOG.error(StringUtils.stringifyException(e));
+
+ }
+
+ try {
+
+ train();
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
--- End diff --
are you seeing my comments? I asked for this to be removed.
> Naive Bayes classifier based html Parse filter (for filtering outlinks)
> -----------------------------------------------------------------------
>
> Key: NUTCH-2038
> URL: https://issues.apache.org/jira/browse/NUTCH-2038
> Project: Nutch
> Issue Type: New Feature
> Components: fetcher, injector, parser
> Reporter: Asitang Mishra
> Assignee: Chris A. Mattmann
> Labels: memex, nutch
> Fix For: 1.11
>
>
> A html parse filter that will filter out the outlinks in two stages.
> Classify the parse text and decide if the parent page is relevant. If
> relevant then don't filter the outlinks. If irrelevant then go thru each
> outlink and see if the url contains any of the important words from a list.
> If it does then let it pass.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
