This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new d877b42aa NUTCH-3215 Remove trailing whitespace reported by Yetus
blanks on master (#974)
d877b42aa is described below
commit d877b42aa75ae27f180c68999fc11dab89952f41
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Mon Oct 12 12:08:33 2026 -0700
NUTCH-3215 Remove trailing whitespace reported by Yetus blanks on master
(#974)
---
.yetus/blanks-tabs.txt | 8 +-
conf/parse-plugins.xml.template | 148 ++++++-------
conf/subcollections.xml.template | 12 +-
ivy/mvn.template | 2 +-
licenses-binary/LICENSE-bouncy-castle-licence.txt | 6 +-
licenses-binary/LICENSE-bsd.txt | 22 +-
.../LICENSE-eclipse-distribution-license-v1.0.txt | 40 ++--
licenses-binary/LICENSE-unicode-icu-license.txt | 2 +-
src/java/org/apache/nutch/crawl/CrawlDb.java | 10 +-
.../apache/nutch/fetcher/FetcherThreadEvent.java | 12 +-
.../nutch/fetcher/FetcherThreadPublisher.java | 4 +-
src/java/org/apache/nutch/indexer/IndexingJob.java | 4 +-
src/java/org/apache/nutch/metadata/Nutch.java | 56 ++---
src/java/org/apache/nutch/parse/ParseSegment.java | 2 +-
.../apache/nutch/protocol/RobotRulesParser.java | 6 +-
.../apache/nutch/publisher/NutchPublishers.java | 4 +-
.../apache/nutch/scoring/webgraph/NodeDumper.java | 6 +-
.../org/apache/nutch/segment/SegmentChecker.java | 2 +-
.../org/apache/nutch/tools/CommonCrawlConfig.java | 240 ++++++++++-----------
.../nutch/tools/CommonCrawlFormatFactory.java | 24 +--
.../nutch/tools/CommonCrawlFormatJackson.java | 144 ++++++-------
.../nutch/tools/CommonCrawlFormatJettinson.java | 170 +++++++--------
.../nutch/tools/CommonCrawlFormatSimple.java | 186 ++++++++--------
.../apache/nutch/util/CrawlCompletionStats.java | 6 +-
src/java/org/apache/nutch/util/DumpFileUtil.java | 84 ++++----
.../nutch/util/ProtocolStatusStatistics.java | 10 +-
src/plugin/build-plugin.xml | 2 +-
src/plugin/feed/build.xml | 28 +--
src/plugin/feed/plugin.xml | 30 +--
src/plugin/feed/sample/rsstest.rss | 28 +--
.../indexer/arbitrary/ArbitraryIndexingFilter.java | 54 ++---
.../apache/nutch/indexer/arbitrary/Multiplier.java | 4 +-
.../nutch/indexer/arbitrary/PopularityGauge.java | 10 +-
src/plugin/index-replace/build.xml | 68 +++---
.../nutch/indexer/replace/ReplaceIndexer.java | 2 +-
src/plugin/indexer-cloudsearch/ivy.xml | 2 +-
src/plugin/indexer-solr/ivy.xml | 88 ++++----
src/plugin/indexer-solr/plugin.xml | 96 ++++-----
src/plugin/indexer-solr/schema.xml | 48 ++---
.../analysis/lang/LanguageIndexingFilter.java | 2 +-
.../nutch/protocol/htmlunit/HtmlUnitWebDriver.java | 40 ++--
src/plugin/lib-xml/build.xml | 20 +-
src/plugin/parse-metatags/build.xml | 26 +--
src/plugin/parse-tika/sample/nutch.html | 24 +--
src/plugin/parse-tika/sample/rsstest.rss | 28 +--
.../parse/tika/BoilerpipeExtractorRepository.java | 2 +-
.../nutch/scoring/depth/DepthScoringFilter.java | 2 +-
.../org/apache/nutch/parse/parse-plugin-test.xml | 8 +-
48 files changed, 916 insertions(+), 906 deletions(-)
diff --git a/.yetus/blanks-tabs.txt b/.yetus/blanks-tabs.txt
index 080c828d1..dee87e8e8 100644
--- a/.yetus/blanks-tabs.txt
+++ b/.yetus/blanks-tabs.txt
@@ -4,4 +4,10 @@
CHANGES.md
# some configuration files define key-value mappings using tabs
^conf/.*\.txt\.template
-^src/test/host-protocol-mapping\.txt
\ No newline at end of file
+^src/test/host-protocol-mapping\.txt
+# Tab-delimited data or a documented tab layout, not indentation.
+^src/plugin/urlnormalizer-protocol/data/protocols\.txt
+^src/plugin/parsefilter-regex/data/regex-parsefilter\.txt
+^src/plugin/parsefilter-regex/README\.txt
+# Fixture bytes, including a leading BOM.
+^src/plugin/parse-tika/sample/ootest\.txt
diff --git a/conf/parse-plugins.xml.template b/conf/parse-plugins.xml.template
index f085084d9..f900e630a 100644
--- a/conf/parse-plugins.xml.template
+++ b/conf/parse-plugins.xml.template
@@ -1,96 +1,96 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
- Author : mattmann
- Description: This xml file represents a natural ordering for which
parsing
- plugin should get called for a particular mimeType.
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ Author : mattmann
+ Description: This xml file represents a natural ordering for which parsing
+ plugin should get called for a particular mimeType.
-->
<parse-plugins>
<!-- by default if the mimeType is set to *, or
if it can't be determined, use parse-tika -->
- <mimeType name="*">
- <plugin id="parse-tika" />
- </mimeType>
+ <mimeType name="*">
+ <plugin id="parse-tika" />
+ </mimeType>
- <mimeType name="application/rss+xml">
- <plugin id="parse-tika" />
- <plugin id="feed" />
- </mimeType>
+ <mimeType name="application/rss+xml">
+ <plugin id="parse-tika" />
+ <plugin id="feed" />
+ </mimeType>
- <mimeType name="application/x-bzip2">
- <!-- try and parse it with the zip parser -->
- <plugin id="parse-zip" />
- </mimeType>
+ <mimeType name="application/x-bzip2">
+ <!-- try and parse it with the zip parser -->
+ <plugin id="parse-zip" />
+ </mimeType>
- <mimeType name="application/x-gzip">
- <!-- try and parse it with the zip parser -->
- <plugin id="parse-zip" />
- </mimeType>
+ <mimeType name="application/x-gzip">
+ <!-- try and parse it with the zip parser -->
+ <plugin id="parse-zip" />
+ </mimeType>
- <mimeType name="application/javascript">
- <plugin id="parse-js" />
- </mimeType>
+ <mimeType name="application/javascript">
+ <plugin id="parse-js" />
+ </mimeType>
- <mimeType name="application/x-javascript">
- <plugin id="parse-js" />
- </mimeType>
+ <mimeType name="application/x-javascript">
+ <plugin id="parse-js" />
+ </mimeType>
- <mimeType name="application/zip">
- <plugin id="parse-zip" />
- </mimeType>
+ <mimeType name="application/zip">
+ <plugin id="parse-zip" />
+ </mimeType>
- <mimeType name="text/html">
- <plugin id="parse-html" />
- </mimeType>
+ <mimeType name="text/html">
+ <plugin id="parse-html" />
+ </mimeType>
<mimeType name="application/xhtml+xml">
- <plugin id="parse-html" />
- </mimeType>
+ <plugin id="parse-html" />
+ </mimeType>
- <mimeType name="text/xml">
- <plugin id="parse-tika" />
- <plugin id="feed" />
- </mimeType>
+ <mimeType name="text/xml">
+ <plugin id="parse-tika" />
+ <plugin id="feed" />
+ </mimeType>
<!-- Types for parse-ext plugin: required for unit tests to pass. -->
- <mimeType name="application/vnd.nutch.example.cat">
- <plugin id="parse-ext" />
- </mimeType>
-
- <mimeType name="application/vnd.nutch.example.md5sum">
- <plugin id="parse-ext" />
- </mimeType>
-
- <!-- alias mappings for parse-xxx names to the actual extension
implementation
- ids described in each plugin's plugin.xml file -->
- <aliases>
- <alias name="parse-tika"
- extension-id="org.apache.nutch.parse.tika.TikaParser" />
- <alias name="parse-ext" extension-id="ExtParser" />
- <alias name="parse-html"
- extension-id="org.apache.nutch.parse.html.HtmlParser" />
- <alias name="parse-js" extension-id="JSParser" />
- <alias name="feed"
- extension-id="org.apache.nutch.parse.feed.FeedParser" />
- <alias name="parse-zip"
- extension-id="org.apache.nutch.parse.zip.ZipParser" />
- </aliases>
-
+ <mimeType name="application/vnd.nutch.example.cat">
+ <plugin id="parse-ext" />
+ </mimeType>
+
+ <mimeType name="application/vnd.nutch.example.md5sum">
+ <plugin id="parse-ext" />
+ </mimeType>
+
+ <!-- alias mappings for parse-xxx names to the actual extension
implementation
+ ids described in each plugin's plugin.xml file -->
+ <aliases>
+ <alias name="parse-tika"
+ extension-id="org.apache.nutch.parse.tika.TikaParser" />
+ <alias name="parse-ext" extension-id="ExtParser" />
+ <alias name="parse-html"
+ extension-id="org.apache.nutch.parse.html.HtmlParser" />
+ <alias name="parse-js" extension-id="JSParser" />
+ <alias name="feed"
+ extension-id="org.apache.nutch.parse.feed.FeedParser" />
+ <alias name="parse-zip"
+ extension-id="org.apache.nutch.parse.zip.ZipParser" />
+ </aliases>
+
</parse-plugins>
diff --git a/conf/subcollections.xml.template b/conf/subcollections.xml.template
index 7b8805d50..dca67ef0f 100644
--- a/conf/subcollections.xml.template
+++ b/conf/subcollections.xml.template
@@ -16,13 +16,13 @@
limitations under the License.
-->
<subcollections>
- <subcollection>
- <name>nutch</name>
- <id>nutch</id>
- <whitelist>
+ <subcollection>
+ <name>nutch</name>
+ <id>nutch</id>
+ <whitelist>
http://lucene.apache.org/nutch/
http://wiki.apache.org/nutch/
</whitelist>
- <blacklist />
- </subcollection>
+ <blacklist />
+ </subcollection>
</subcollections>
diff --git a/ivy/mvn.template b/ivy/mvn.template
index f4653f739..73fe239c8 100644
--- a/ivy/mvn.template
+++ b/ivy/mvn.template
@@ -72,7 +72,7 @@
<id>markus</id>
<name>Markus Jelsma</name>
<email>[email protected]</email>
- </developer>
+ </developer>
<developer>
<id>fenglu</id>
<name>Feng Lu</name>
diff --git a/licenses-binary/LICENSE-bouncy-castle-licence.txt
b/licenses-binary/LICENSE-bouncy-castle-licence.txt
index 16db5c525..2fe6b7bda 100644
--- a/licenses-binary/LICENSE-bouncy-castle-licence.txt
+++ b/licenses-binary/LICENSE-bouncy-castle-licence.txt
@@ -11,7 +11,7 @@ Please note this should be read in the same way as the MIT
license.
Please also note this licensing model is made possible through funding from
donations and the sale of support contracts.
License
Copyright (c) 2000 - 2021 The Legion of the Bouncy Castle Inc.
(https://www.bouncycastle.org)
- Permission is hereby granted, free of charge, to any
person obtaining a copy of this software and associated documentation files
(the "Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom
the Software is furnished to do so, subject to the following conditions:
- The above copyright notice and this permission notice
shall be included in all copies or substantial portions of the Software.
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES
OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
+ Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
+ The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
diff --git a/licenses-binary/LICENSE-bsd.txt b/licenses-binary/LICENSE-bsd.txt
index c2e90c495..e28c5645b 100644
--- a/licenses-binary/LICENSE-bsd.txt
+++ b/licenses-binary/LICENSE-bsd.txt
@@ -6,19 +6,19 @@
BSD license definition
-The BSD license is a class of extremely simple and very liberal licenses for
computer software that was originally developed at the University of California
at Berkeley (UCB). It was first used in 1980 for the Berkeley Source
Distribution (BSD), also known as BSD UNIX, an enhanced version of the original
UNIX operating system that was first written in 1969 by Ken Thompson at Bell
Labs.
+The BSD license is a class of extremely simple and very liberal licenses for
computer software that was originally developed at the University of California
at Berkeley (UCB). It was first used in 1980 for the Berkeley Source
Distribution (BSD), also known as BSD UNIX, an enhanced version of the original
UNIX operating system that was first written in 1969 by Ken Thompson at Bell
Labs.
-The only restrictions placed on users of software released under a typical BSD
license are that if they redistribute such software in any form, with or
without modification, they must include in the redistribution (1) the original
copyright notice, (2) a list of two simple restrictions and (3) a disclaimer of
liability. These restrictions can be summarized as (1) one should not claim
that they wrote the software if they did not write it and (2) one should not
sue the developer if the sof [...]
+The only restrictions placed on users of software released under a typical BSD
license are that if they redistribute such software in any form, with or
without modification, they must include in the redistribution (1) the original
copyright notice, (2) a list of two simple restrictions and (3) a disclaimer of
liability. These restrictions can be summarized as (1) one should not claim
that they wrote the software if they did not write it and (2) one should not
sue the developer if the sof [...]
-The most basic definition of a derivative work is a product that is based on,
or incorporates, one or more already existing works. This can become a complex
issue, particularly with regard to software, but the primary indicator that a
software program is a derivative of another program is if it includes source
code from the original program, even if the source code has been modified,
including improving, extending, reordering or translating it into another
programming language.
+The most basic definition of a derivative work is a product that is based on,
or incorporates, one or more already existing works. This can become a complex
issue, particularly with regard to software, but the primary indicator that a
software program is a derivative of another program is if it includes source
code from the original program, even if the source code has been modified,
including improving, extending, reordering or translating it into another
programming language.
Source code is the version of software (usually an application program or an
operating system) as it is originally written (i.e., typed into a computer) by
a human in plain text (i.e., human readable alphanumeric characters). Source
code can be written in any of hundreds of programming languages, some of the
most popular of which are C, C++ and Java.
-Due to the extremely minimal restrictions of BSD-style licenses, software
released under such licenses can be freely modified and used in proprietary
(i.e., commercial) software for which the source code is kept secret.
+Due to the extremely minimal restrictions of BSD-style licenses, software
released under such licenses can be freely modified and used in proprietary
(i.e., commercial) software for which the source code is kept secret.
It is possible for a product to be distributed under a BSD-style license and
for some other license to apply as well. This was, in fact, the case with very
early versions of BSD UNIX, which included both new code written at UCB and
code from the original versions of UNIX written at Bell Labs.
@@ -29,8 +29,8 @@ BSD-style licenses have been very successful, and they are
now widely used for a
BSD Licenses Versus the GPL
-
-The GPL (GNU General Public License) is by far the most widely used license
for free software (i.e., software whose source code is available at no cost for
anyone to use for any purpose). The Linux kernel (i.e., the core of the
operating system) as well as much of the other software generally included in
Linux distributions have been released under the terms of the GPL.
+
+The GPL (GNU General Public License) is by far the most widely used license
for free software (i.e., software whose source code is available at no cost for
anyone to use for any purpose). The Linux kernel (i.e., the core of the
operating system) as well as much of the other software generally included in
Linux distributions have been released under the terms of the GPL.
Although far fewer programs are released under BSD-style licenses, this class
of licenses is disproportionately important because of the widespread use of
BSD-licensed code in both free and proprietary operating systems.
@@ -69,7 +69,7 @@ The original version of the BSD license contained the so
called advertising clau
One of the problems with this clause arose from the fact that people who made
changes to the source code often wanted to have their names added to the
acknowledgment. This could easily result in large and cumbersome
acknowledgments for products with numerous contributors and for software
distributions consisting of multiple individual projects.
-A second problem was legal incompatibility with the terms of the GPL. This is
because the GPL prohibits the addition of restrictions beyond those that it
already imposes. Thus it was necessary to segregate GPL and BSD-licensed
software within projects.
+A second problem was legal incompatibility with the terms of the GPL. This is
because the GPL prohibits the addition of restrictions beyond those that it
already imposes. Thus it was necessary to segregate GPL and BSD-licensed
software within projects.
Initially, the "obnoxious BSD advertising clause," as it was referred to by
GPL advocates, was used only for the BSD UNIX license. That did not cause any
major problems because it was only necessary to include a single sentence of
acknowledgment in any advertisement.
@@ -78,7 +78,7 @@ Initially, the "obnoxious BSD advertising clause," as it was
referred to by GPL
However, the fact that other software developers did not copy the clause
verbatim, but replaced the phrase "University of California" with the name of
their own organization or persons involved in it, resulted in a proliferation
of slightly different licenses and a consequently serious problem when many
such programs were assembled to form a larger work or an operating system. For
example, if an operating system or other program required fifty slightly
different acknowledgment sentences [...]
-In June 1999, after two years of discussion, the Office of Technology
Licensing at UCB finally proclaimed: "Effective immediately, licensees and
distributors are no longer required to include the acknowledgment within
advertising materials. Accordingly, the foregoing paragraph of those BSD Unix
files containing it is hereby deleted in its entirety."
+In June 1999, after two years of discussion, the Office of Technology
Licensing at UCB finally proclaimed: "Effective immediately, licensees and
distributors are no longer required to include the acknowledgment within
advertising materials. Accordingly, the foregoing paragraph of those BSD Unix
files containing it is hereby deleted in its entirety."
This was clearly very useful. However, it could not eliminate the legacy of
the advertising clause, as similar clauses still exist in the licenses of many
programs that followed the old BSD license; only the developers of such
packages can change them.
@@ -96,7 +96,7 @@ Below are three examples of BSD-style licenses: (1) the BSD
license as it is use
Copyright 1994-2004 The FreeBSD Project. All rights reserved.
-
+
Redistribution and use in source and binary forms, with or
without modification, are permitted provided that the
following conditions are met:
@@ -140,7 +140,7 @@ or implied, of the FreeBSD Project.
-
+
Sudo License
Sudo is distributed under the following BSD-style license:
@@ -182,7 +182,7 @@ Additionally, lsearch.c, fnmatch.c, getcwd.c, snprintf.c
strcasecmp.c and fnmatc
-(3) A template for a BSD-style license. [YEAR], [COPYRIGHT OWNER] and
[LICENSOR] are to be replaced by the actual year of copyright, the owner of the
copyright and the licensor. The copyright owner and licensor may be the same,
as in the case of the license for FreeBSD (as shown above).
+(3) A template for a BSD-style license. [YEAR], [COPYRIGHT OWNER] and
[LICENSOR] are to be replaced by the actual year of copyright, the owner of the
copyright and the licensor. The copyright owner and licensor may be the same,
as in the case of the license for FreeBSD (as shown above).
diff --git a/licenses-binary/LICENSE-eclipse-distribution-license-v1.0.txt
b/licenses-binary/LICENSE-eclipse-distribution-license-v1.0.txt
index 347787f3e..83c4cd7f1 100644
--- a/licenses-binary/LICENSE-eclipse-distribution-license-v1.0.txt
+++ b/licenses-binary/LICENSE-eclipse-distribution-license-v1.0.txt
@@ -5,26 +5,26 @@ Eclipse Distribution License | The Eclipse Foundation
Eclipse Distribution License - v 1.0
-Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors.
+Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors.
All rights reserved.
-Redistribution and use in source and binary forms, with or without
modification,
- are permitted provided that the following conditions are met:
-Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
-Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the
documentation
- and/or other materials provided with the distribution.
-Neither the name of the Eclipse Foundation, Inc. nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED.
-IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY
DIRECT,
-INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT
-NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY,
-WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+Redistribution and use in source and binary forms, with or without
modification,
+ are permitted provided that the following conditions are met:
+Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+Neither the name of the Eclipse Foundation, Inc. nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
diff --git a/licenses-binary/LICENSE-unicode-icu-license.txt
b/licenses-binary/LICENSE-unicode-icu-license.txt
index 17ff27e7b..68c30f4c9 100644
--- a/licenses-binary/LICENSE-unicode-icu-license.txt
+++ b/licenses-binary/LICENSE-unicode-icu-license.txt
@@ -312,7 +312,7 @@ Lao Word Break Dictionary Data (laodict.txt)
# License: https://github.com/rober42539/lao-dictionary/LICENSE.txt
# (copied below)
#
- # This file is derived from the above dictionary version of Nov 22, 2020
+ # This file is derived from the above dictionary version of Nov 22, 2020
# ----------------------------------------------------------------------
# Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell.
# All rights reserved.
diff --git a/src/java/org/apache/nutch/crawl/CrawlDb.java
b/src/java/org/apache/nutch/crawl/CrawlDb.java
index e27fba673..98fee8cad 100644
--- a/src/java/org/apache/nutch/crawl/CrawlDb.java
+++ b/src/java/org/apache/nutch/crawl/CrawlDb.java
@@ -341,13 +341,13 @@ public class CrawlDb extends NutchTool implements Tool {
else if(args.containsKey(Nutch.ARG_SEGMENTS)) {
Object segments = args.get(Nutch.ARG_SEGMENTS);
ArrayList<String> segmentList = new ArrayList<>();
- if(segments instanceof ArrayList) {
- segmentList = (ArrayList<String>)segments;
+ if (segments instanceof ArrayList) {
+ segmentList = (ArrayList<String>)segments;
}
- else if(segments instanceof Path){
- segmentList.add(segments.toString());
+ else if (segments instanceof Path){
+ segmentList.add(segments.toString());
}
-
+
for(String segment: segmentList) {
dirs.add(new Path(segment));
}
diff --git a/src/java/org/apache/nutch/fetcher/FetcherThreadEvent.java
b/src/java/org/apache/nutch/fetcher/FetcherThreadEvent.java
index 2d1d4b558..b634b8491 100644
--- a/src/java/org/apache/nutch/fetcher/FetcherThreadEvent.java
+++ b/src/java/org/apache/nutch/fetcher/FetcherThreadEvent.java
@@ -93,15 +93,15 @@ public class FetcherThreadEvent implements Serializable {
/**
* Set URL of this event (fetched page)
- * @param url URL of the fetched page
+ * @param url URL of the fetched page
*/
public void setUrl(String url) {
this.url = url;
}
/**
* Add new data to the eventData object.
- * @param key A key to refer to the data being added to this event
- * @param value Data to be stored in the event referenced by the above
key
+ * @param key A key to refer to the data being added to this event
+ * @param value Data to be stored in the event referenced by the above key
*/
public void addEventData(String key, Object value) {
if(eventData == null) {
@@ -113,8 +113,8 @@ public class FetcherThreadEvent implements Serializable {
/**
* Given a collection of lists this method will add it
* the oultink metadata
- * @param links A collection of outlinks generating from the fetched
page
- * this event refers to
+ * @param links A collection of outlinks generating from the fetched page
+ * this event refers to
*/
public void addOutlinksToEventData(Collection<Outlink> links) {
ArrayList<Map<String, String>> outlinkList = new ArrayList<>();
@@ -137,7 +137,7 @@ public class FetcherThreadEvent implements Serializable {
/**
* Set timestamp for this event
- * @param timestamp Timestamp of the occurrence of this event
+ * @param timestamp Timestamp of the occurrence of this event
*/
public void setTimestamp(Long timestamp) {
this.timestamp = timestamp;
diff --git a/src/java/org/apache/nutch/fetcher/FetcherThreadPublisher.java
b/src/java/org/apache/nutch/fetcher/FetcherThreadPublisher.java
index 863c0c07f..5933259c1 100644
--- a/src/java/org/apache/nutch/fetcher/FetcherThreadPublisher.java
+++ b/src/java/org/apache/nutch/fetcher/FetcherThreadPublisher.java
@@ -46,8 +46,8 @@ public class FetcherThreadPublisher {
/**
* Publish event to all registered publishers
- * @param event {@link org.apache.nutch.fetcher.FetcherThreadEvent
Event} to be published
- * @param conf {@link org.apache.hadoop.conf.Configuration
Configuration} to be used
+ * @param event {@link org.apache.nutch.fetcher.FetcherThreadEvent Event}
to be published
+ * @param conf {@link org.apache.hadoop.conf.Configuration Configuration}
to be used
*/
public void publish(FetcherThreadEvent event, Configuration conf) {
if(publisher!=null) {
diff --git a/src/java/org/apache/nutch/indexer/IndexingJob.java
b/src/java/org/apache/nutch/indexer/IndexingJob.java
index cfbd75919..c65c7e71b 100644
--- a/src/java/org/apache/nutch/indexer/IndexingJob.java
+++ b/src/java/org/apache/nutch/indexer/IndexingJob.java
@@ -378,13 +378,13 @@ public class IndexingJob extends NutchTool implements
Tool {
Object segmentsFromArg = args.get(Nutch.ARG_SEGMENTS);
ArrayList<String> segmentList = new ArrayList<String>();
if(segmentsFromArg instanceof ArrayList) {
- segmentList = (ArrayList<String>)segmentsFromArg; }
+ segmentList = (ArrayList<String>)segmentsFromArg; }
else if(segmentsFromArg instanceof Path){
segmentList.add(segmentsFromArg.toString());
}
for(String segment: segmentList) {
- segments.add(new Path(segment));
+ segments.add(new Path(segment));
}
}
diff --git a/src/java/org/apache/nutch/metadata/Nutch.java
b/src/java/org/apache/nutch/metadata/Nutch.java
index 2542b35ee..53c44c8bc 100644
--- a/src/java/org/apache/nutch/metadata/Nutch.java
+++ b/src/java/org/apache/nutch/metadata/Nutch.java
@@ -81,32 +81,32 @@ public interface Nutch {
public static final Text WRITABLE_FIXED_INTERVAL_KEY = new Text(
FIXED_INTERVAL_KEY);
- /** For progress of job (programmatic / tooling). */
- public static final String STAT_PROGRESS = "progress";
- /** Crawl id key for programmatic jobs. */
- public static final String CRAWL_ID_KEY = "storage.crawl.id";
- /** Argument key for seed URL directory path. */
- public static final String ARG_SEEDDIR = "url_dir";
- /** Argument key for crawldb location in programmatic jobs. */
- public static final String ARG_CRAWLDB = "crawldb";
- /** Argument key for linkdb location in programmatic jobs. */
- public static final String ARG_LINKDB = "linkdb";
- /** Name of the key used in the result map from {@link
org.apache.nutch.util.NutchTool#run}. */
- public static final String VAL_RESULT = "result";
- /** Argument key for a directory of segments; similar to the -dir
option in bin/nutch. */
- public static final String ARG_SEGMENTDIR = "segment_dir";
- /** Argument key for one segment or a list of segments (job-dependent).
*/
- public static final String ARG_SEGMENTS = "segment";
- /** Argument key for hostdb location in programmatic jobs. */
- public static final String ARG_HOSTDB = "hostdb";
- /** Title key in the Pub/Sub event metadata for the title of the parsed
page*/
- public static final String FETCH_EVENT_TITLE = "title";
- /** Content-type key in the Pub/Sub event metadata for the content-type
of the parsed page*/
- public static final String FETCH_EVENT_CONTENTTYPE = "content-type";
- /** Score key in the Pub/Sub event metadata for the score of the parsed
page*/
- public static final String FETCH_EVENT_SCORE = "score";
- /** Fetch time key in the Pub/Sub event metadata for the fetch time of
the parsed page*/
- public static final String FETCH_EVENT_FETCHTIME = "fetchTime";
- /** Content-lanueage key in the Pub/Sub event metadata for the
content-language of the parsed page*/
- public static final String FETCH_EVENT_CONTENTLANG = "content-language";
+ /** For progress of job (programmatic / tooling). */
+ public static final String STAT_PROGRESS = "progress";
+ /** Crawl id key for programmatic jobs. */
+ public static final String CRAWL_ID_KEY = "storage.crawl.id";
+ /** Argument key for seed URL directory path. */
+ public static final String ARG_SEEDDIR = "url_dir";
+ /** Argument key for crawldb location in programmatic jobs. */
+ public static final String ARG_CRAWLDB = "crawldb";
+ /** Argument key for linkdb location in programmatic jobs. */
+ public static final String ARG_LINKDB = "linkdb";
+ /** Name of the key used in the result map from {@link
org.apache.nutch.util.NutchTool#run}. */
+ public static final String VAL_RESULT = "result";
+ /** Argument key for a directory of segments; similar to the -dir option in
bin/nutch. */
+ public static final String ARG_SEGMENTDIR = "segment_dir";
+ /** Argument key for one segment or a list of segments (job-dependent). */
+ public static final String ARG_SEGMENTS = "segment";
+ /** Argument key for hostdb location in programmatic jobs. */
+ public static final String ARG_HOSTDB = "hostdb";
+ /** Title key in the Pub/Sub event metadata for the title of the parsed
page*/
+ public static final String FETCH_EVENT_TITLE = "title";
+ /** Content-type key in the Pub/Sub event metadata for the content-type of
the parsed page*/
+ public static final String FETCH_EVENT_CONTENTTYPE = "content-type";
+ /** Score key in the Pub/Sub event metadata for the score of the parsed
page*/
+ public static final String FETCH_EVENT_SCORE = "score";
+ /** Fetch time key in the Pub/Sub event metadata for the fetch time of the
parsed page*/
+ public static final String FETCH_EVENT_FETCHTIME = "fetchTime";
+ /** Content-lanueage key in the Pub/Sub event metadata for the
content-language of the parsed page*/
+ public static final String FETCH_EVENT_CONTENTLANG = "content-language";
}
diff --git a/src/java/org/apache/nutch/parse/ParseSegment.java
b/src/java/org/apache/nutch/parse/ParseSegment.java
index 944d1c073..d68ff448d 100644
--- a/src/java/org/apache/nutch/parse/ParseSegment.java
+++ b/src/java/org/apache/nutch/parse/ParseSegment.java
@@ -407,7 +407,7 @@ public class ParseSegment extends NutchTool implements Tool
{
}
}
else {
- String segment_dir = crawlId+"/segments";
+ String segment_dir = crawlId+"/segments";
File segmentsDir = new File(segment_dir);
File[] segmentsList = segmentsDir.listFiles();
Arrays.sort(segmentsList, (f1, f2) -> {
diff --git a/src/java/org/apache/nutch/protocol/RobotRulesParser.java
b/src/java/org/apache/nutch/protocol/RobotRulesParser.java
index 6f3b51384..cc4bbf8be 100644
--- a/src/java/org/apache/nutch/protocol/RobotRulesParser.java
+++ b/src/java/org/apache/nutch/protocol/RobotRulesParser.java
@@ -162,8 +162,8 @@ public abstract class RobotRulesParser implements Tool {
} else {
for (int i = 0; i < confAllowList.length; i++) {
if (confAllowList[i].isEmpty()) {
- LOG.info("Empty allowlisted URL skipped!");
- continue;
+ LOG.info("Empty allowlisted URL skipped!");
+ continue;
}
allowList.add(confAllowList[i]);
}
@@ -193,7 +193,7 @@ public abstract class RobotRulesParser implements Tool {
String urlString = url.getHost();
if (matcher != null) {
- match = matcher.matches(urlString);
+ match = matcher.matches(urlString);
}
return match;
diff --git a/src/java/org/apache/nutch/publisher/NutchPublishers.java
b/src/java/org/apache/nutch/publisher/NutchPublishers.java
index 18076424c..3940afd96 100644
--- a/src/java/org/apache/nutch/publisher/NutchPublishers.java
+++ b/src/java/org/apache/nutch/publisher/NutchPublishers.java
@@ -32,7 +32,7 @@ public class NutchPublishers extends Configured implements
NutchPublisher{
private Configuration conf;
public NutchPublishers(Configuration conf) {
- this.conf = conf;
+ this.conf = conf;
this.publishers = (NutchPublisher[])PluginRepository.get(conf).
getOrderedPlugins(NutchPublisher.class,
NutchPublisher.X_POINT_ID, "publisher.order");
@@ -77,6 +77,6 @@ public class NutchPublishers extends Configured implements
NutchPublisher{
@Override
public void setConf(Configuration arg0) {
-
+
}
}
diff --git a/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
b/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
index 4f75f2ccc..1454223bf 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
@@ -100,8 +100,8 @@ public class NodeDumper extends Configured implements Tool {
@Override
public void setup(Mapper<Text, Node, FloatWritable, Text>.Context
context) {
conf = context.getConfiguration();
- inlinks = conf.getBoolean("inlinks", false);
- outlinks = conf.getBoolean("outlinks", false);
+ inlinks = conf.getBoolean("inlinks", false);
+ outlinks = conf.getBoolean("outlinks", false);
}
@Override
@@ -185,7 +185,7 @@ public class NodeDumper extends Configured implements Tool {
@Override
public void setup(Mapper<Text, Node, Text, FloatWritable>.Context
context) {
conf = context.getConfiguration();
- inlinks = conf.getBoolean("inlinks", false);
+ inlinks = conf.getBoolean("inlinks", false);
outlinks = conf.getBoolean("outlinks", false);
host = conf.getBoolean("host", false);
}
diff --git a/src/java/org/apache/nutch/segment/SegmentChecker.java
b/src/java/org/apache/nutch/segment/SegmentChecker.java
index 18cfec597..490ad8c3a 100644
--- a/src/java/org/apache/nutch/segment/SegmentChecker.java
+++ b/src/java/org/apache/nutch/segment/SegmentChecker.java
@@ -151,7 +151,7 @@ public class SegmentChecker {
throws IOException {
if (fs.exists(new Path(segment, CrawlDatum.PARSE_DIR_NAME))){
- return true;
+ return true;
}
return false;
}
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlConfig.java
b/src/java/org/apache/nutch/tools/CommonCrawlConfig.java
index 49d9c3143..26daee5ce 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlConfig.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlConfig.java
@@ -23,124 +23,124 @@ import java.util.Properties;
public class CommonCrawlConfig implements Serializable {
- /**
- * Serial version UID
- */
- private static final long serialVersionUID = 5235013733207799661L;
-
- // Prefix for key value in the output format
- private String keyPrefix = "";
-
- private boolean simpleDateFormat = false;
-
- private boolean jsonArray = false;
-
- private boolean reverseKey = false;
-
- private String reverseKeyValue = "";
-
- private boolean compressed = false;
-
- private long warcSize = 0;
-
- private String outputDir;
-
- /**
- * Default constructor
- */
- public CommonCrawlConfig() {
- // TODO
init(this.getClass().getResourceAsStream("CommonCrawlConfig.properties"));
- }
-
- public CommonCrawlConfig(InputStream stream) {
- init(stream);
- }
-
- private void init(InputStream stream) {
- if (stream == null) {
- return;
- }
- Properties properties = new Properties();
-
- try {
- properties.load(stream);
- } catch (IOException e) {
- // TODO
- } finally {
- try {
- stream.close();
- } catch (IOException e) {
- // TODO
- }
- }
-
- setKeyPrefix(properties.getProperty("keyPrefix", ""));
-
setSimpleDateFormat(Boolean.parseBoolean(properties.getProperty("simpleDateFormat",
"False")));
-
setJsonArray(Boolean.parseBoolean(properties.getProperty("jsonArray",
"False")));
-
setReverseKey(Boolean.parseBoolean(properties.getProperty("reverseKey",
"False")));
- }
-
- public void setKeyPrefix(String keyPrefix) {
- this.keyPrefix = keyPrefix;
- }
-
- public void setSimpleDateFormat(boolean simpleDateFormat) {
- this.simpleDateFormat = simpleDateFormat;
- }
-
- public void setJsonArray(boolean jsonArray) {
- this.jsonArray = jsonArray;
- }
-
- public void setReverseKey(boolean reverseKey) {
- this.reverseKey = reverseKey;
- }
-
- public void setReverseKeyValue(String reverseKeyValue) {
- this.reverseKeyValue = reverseKeyValue;
- }
-
- public String getKeyPrefix() {
- return this.keyPrefix;
- }
-
- public boolean getSimpleDateFormat() {
- return this.simpleDateFormat;
- }
-
- public boolean getJsonArray() {
- return this.jsonArray;
- }
-
- public boolean getReverseKey() {
- return this.reverseKey;
- }
-
- public String getReverseKeyValue() {
- return this.reverseKeyValue;
- }
-
- public boolean isCompressed() {
- return compressed;
- }
-
- public void setCompressed(boolean compressed) {
- this.compressed = compressed;
- }
-
- public long getWarcSize() {
- return warcSize;
- }
-
- public void setWarcSize(long warcSize) {
- this.warcSize = warcSize;
- }
-
- public String getOutputDir() {
- return outputDir;
- }
-
- public void setOutputDir(String outputDir) {
- this.outputDir = outputDir;
- }
+ /**
+ * Serial version UID
+ */
+ private static final long serialVersionUID = 5235013733207799661L;
+
+ // Prefix for key value in the output format
+ private String keyPrefix = "";
+
+ private boolean simpleDateFormat = false;
+
+ private boolean jsonArray = false;
+
+ private boolean reverseKey = false;
+
+ private String reverseKeyValue = "";
+
+ private boolean compressed = false;
+
+ private long warcSize = 0;
+
+ private String outputDir;
+
+ /**
+ * Default constructor
+ */
+ public CommonCrawlConfig() {
+ // TODO
init(this.getClass().getResourceAsStream("CommonCrawlConfig.properties"));
+ }
+
+ public CommonCrawlConfig(InputStream stream) {
+ init(stream);
+ }
+
+ private void init(InputStream stream) {
+ if (stream == null) {
+ return;
+ }
+ Properties properties = new Properties();
+
+ try {
+ properties.load(stream);
+ } catch (IOException e) {
+ // TODO
+ } finally {
+ try {
+ stream.close();
+ } catch (IOException e) {
+ // TODO
+ }
+ }
+
+ setKeyPrefix(properties.getProperty("keyPrefix", ""));
+
setSimpleDateFormat(Boolean.parseBoolean(properties.getProperty("simpleDateFormat",
"False")));
+ setJsonArray(Boolean.parseBoolean(properties.getProperty("jsonArray",
"False")));
+ setReverseKey(Boolean.parseBoolean(properties.getProperty("reverseKey",
"False")));
+ }
+
+ public void setKeyPrefix(String keyPrefix) {
+ this.keyPrefix = keyPrefix;
+ }
+
+ public void setSimpleDateFormat(boolean simpleDateFormat) {
+ this.simpleDateFormat = simpleDateFormat;
+ }
+
+ public void setJsonArray(boolean jsonArray) {
+ this.jsonArray = jsonArray;
+ }
+
+ public void setReverseKey(boolean reverseKey) {
+ this.reverseKey = reverseKey;
+ }
+
+ public void setReverseKeyValue(String reverseKeyValue) {
+ this.reverseKeyValue = reverseKeyValue;
+ }
+
+ public String getKeyPrefix() {
+ return this.keyPrefix;
+ }
+
+ public boolean getSimpleDateFormat() {
+ return this.simpleDateFormat;
+ }
+
+ public boolean getJsonArray() {
+ return this.jsonArray;
+ }
+
+ public boolean getReverseKey() {
+ return this.reverseKey;
+ }
+
+ public String getReverseKeyValue() {
+ return this.reverseKeyValue;
+ }
+
+ public boolean isCompressed() {
+ return compressed;
+ }
+
+ public void setCompressed(boolean compressed) {
+ this.compressed = compressed;
+ }
+
+ public long getWarcSize() {
+ return warcSize;
+ }
+
+ public void setWarcSize(long warcSize) {
+ this.warcSize = warcSize;
+ }
+
+ public String getOutputDir() {
+ return outputDir;
+ }
+
+ public void setOutputDir(String outputDir) {
+ this.outputDir = outputDir;
+ }
}
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlFormatFactory.java
b/src/java/org/apache/nutch/tools/CommonCrawlFormatFactory.java
index c4cb57056..f1d5db562 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlFormatFactory.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlFormatFactory.java
@@ -25,17 +25,17 @@ import org.apache.hadoop.conf.Configuration;
*
*/
public class CommonCrawlFormatFactory {
-
- // The format should not depend on variable attributes, essentially this
- // should be one for the full job
- public static CommonCrawlFormat getCommonCrawlFormat(String formatType,
Configuration nutchConf, CommonCrawlConfig config) throws IOException {
- if (formatType.equalsIgnoreCase("WARC")) {
- return new CommonCrawlFormatWARC(nutchConf, config);
- }
- if (formatType.equalsIgnoreCase("JACKSON")) {
- return new CommonCrawlFormatJackson( nutchConf, config);
- }
- return null;
- }
+ // The format should not depend on variable attributes, essentially this
+ // should be one for the full job
+ public static CommonCrawlFormat getCommonCrawlFormat(String formatType,
Configuration nutchConf, CommonCrawlConfig config) throws IOException {
+ if (formatType.equalsIgnoreCase("WARC")) {
+ return new CommonCrawlFormatWARC(nutchConf, config);
+ }
+
+ if (formatType.equalsIgnoreCase("JACKSON")) {
+ return new CommonCrawlFormatJackson( nutchConf, config);
+ }
+ return null;
+ }
}
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlFormatJackson.java
b/src/java/org/apache/nutch/tools/CommonCrawlFormatJackson.java
index 78fcd322f..8001d4a01 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlFormatJackson.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlFormatJackson.java
@@ -33,77 +33,77 @@ import com.fasterxml.jackson.core.JsonGenerator;
*/
public class CommonCrawlFormatJackson extends AbstractCommonCrawlFormat {
- private ByteArrayOutputStream out;
-
- private JsonGenerator generator;
-
- public CommonCrawlFormatJackson(Configuration nutchConf,
- CommonCrawlConfig config) throws IOException {
- super(null, null, null, nutchConf, config);
-
- JsonFactory factory = new JsonFactory();
- this.out = new ByteArrayOutputStream();
- this.generator = factory.createGenerator(out);
-
- this.generator.useDefaultPrettyPrinter(); // INDENTED OUTPUT
- }
-
- public CommonCrawlFormatJackson(String url, Content content, Metadata
metadata, Configuration nutchConf, CommonCrawlConfig config) throws IOException
{
- super(url, content, metadata, nutchConf, config);
-
- JsonFactory factory = new JsonFactory();
- this.out = new ByteArrayOutputStream();
- this.generator = factory.createGenerator(out);
-
- this.generator.useDefaultPrettyPrinter(); // INDENTED OUTPUT
- }
-
- @Override
- protected void writeKeyValue(String key, String value) throws
IOException {
- generator.writeFieldName(key);
- generator.writeString(value);
- }
-
- @Override
- protected void writeKeyNull(String key) throws IOException {
- generator.writeFieldName(key);
- generator.writeNull();
- }
-
- @Override
- protected void startArray(String key, boolean nested, boolean newline)
throws IOException {
- if (key != null) {
- generator.writeFieldName(key);
- }
- generator.writeStartArray();
- }
-
- @Override
- protected void closeArray(String key, boolean nested, boolean newline)
throws IOException {
- generator.writeEndArray();
- }
-
- @Override
- protected void writeArrayValue(String value) throws IOException {
- generator.writeString(value);
- }
-
- @Override
- protected void startObject(String key) throws IOException {
- if (key != null) {
- generator.writeFieldName(key);
- }
- generator.writeStartObject();
- }
-
- @Override
- protected void closeObject(String key) throws IOException {
- generator.writeEndObject();
- }
-
- @Override
- protected String generateJson() throws IOException {
- this.generator.flush();
+ private ByteArrayOutputStream out;
+
+ private JsonGenerator generator;
+
+ public CommonCrawlFormatJackson(Configuration nutchConf,
+ CommonCrawlConfig config) throws IOException {
+ super(null, null, null, nutchConf, config);
+
+ JsonFactory factory = new JsonFactory();
+ this.out = new ByteArrayOutputStream();
+ this.generator = factory.createGenerator(out);
+
+ this.generator.useDefaultPrettyPrinter(); // INDENTED OUTPUT
+ }
+
+ public CommonCrawlFormatJackson(String url, Content content, Metadata
metadata, Configuration nutchConf, CommonCrawlConfig config) throws IOException
{
+ super(url, content, metadata, nutchConf, config);
+
+ JsonFactory factory = new JsonFactory();
+ this.out = new ByteArrayOutputStream();
+ this.generator = factory.createGenerator(out);
+
+ this.generator.useDefaultPrettyPrinter(); // INDENTED OUTPUT
+ }
+
+ @Override
+ protected void writeKeyValue(String key, String value) throws IOException {
+ generator.writeFieldName(key);
+ generator.writeString(value);
+ }
+
+ @Override
+ protected void writeKeyNull(String key) throws IOException {
+ generator.writeFieldName(key);
+ generator.writeNull();
+ }
+
+ @Override
+ protected void startArray(String key, boolean nested, boolean newline)
throws IOException {
+ if (key != null) {
+ generator.writeFieldName(key);
+ }
+ generator.writeStartArray();
+ }
+
+ @Override
+ protected void closeArray(String key, boolean nested, boolean newline)
throws IOException {
+ generator.writeEndArray();
+ }
+
+ @Override
+ protected void writeArrayValue(String value) throws IOException {
+ generator.writeString(value);
+ }
+
+ @Override
+ protected void startObject(String key) throws IOException {
+ if (key != null) {
+ generator.writeFieldName(key);
+ }
+ generator.writeStartObject();
+ }
+
+ @Override
+ protected void closeObject(String key) throws IOException {
+ generator.writeEndObject();
+ }
+
+ @Override
+ protected String generateJson() throws IOException {
+ this.generator.flush();
return this.out.toString(StandardCharsets.UTF_8);
- }
+ }
}
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlFormatJettinson.java
b/src/java/org/apache/nutch/tools/CommonCrawlFormatJettinson.java
index 499530f41..da08ac837 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlFormatJettinson.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlFormatJettinson.java
@@ -32,90 +32,90 @@ import org.codehaus.jettison.json.JSONObject;
*
*/
public class CommonCrawlFormatJettinson extends AbstractCommonCrawlFormat {
-
- private Deque<JSONObject> stackObjects;
-
- private Deque<JSONArray> stackArrays;
- public CommonCrawlFormatJettinson(String url, Content content, Metadata
metadata, Configuration nutchConf, CommonCrawlConfig config) throws IOException
{
- super(url, content, metadata, nutchConf, config);
-
- stackObjects = new ArrayDeque<>();
- stackArrays = new ArrayDeque<>();
- }
-
- @Override
- protected void writeKeyValue(String key, String value) throws
IOException {
- try {
- stackObjects.getFirst().put(key, value);
- } catch (JSONException jsone) {
- throw new IOException(jsone.getMessage());
- }
- }
-
- @Override
- protected void writeKeyNull(String key) throws IOException {
- try {
- stackObjects.getFirst().put(key, JSONObject.NULL);
- } catch (JSONException jsone) {
- throw new IOException(jsone.getMessage());
- }
- }
-
- @Override
- protected void startArray(String key, boolean nested, boolean newline)
throws IOException {
- JSONArray array = new JSONArray();
- stackArrays.push(array);
- }
-
- @Override
- protected void closeArray(String key, boolean nested, boolean newline)
throws IOException {
- try {
- if (stackArrays.size() > 1) {
- JSONArray array = stackArrays.pop();
- if (nested) {
- stackArrays.getFirst().put(array);
- }
- else {
- stackObjects.getFirst().put(key, array);
- }
- }
- } catch (JSONException jsone) {
- throw new IOException(jsone.getMessage());
- }
- }
-
- @Override
- protected void writeArrayValue(String value) throws IOException {
- if (stackArrays.size() > 1) {
- stackArrays.getFirst().put(value);
- }
- }
-
- @Override
- protected void startObject(String key) throws IOException {
- JSONObject object = new JSONObject();
- stackObjects.push(object);
- }
-
- @Override
- protected void closeObject(String key) throws IOException {
- try {
- if (stackObjects.size() > 1) {
- JSONObject object = stackObjects.pop();
- stackObjects.getFirst().put(key, object);
- }
- } catch (JSONException jsone) {
- throw new IOException(jsone.getMessage());
- }
- }
-
- @Override
- protected String generateJson() throws IOException {
- try {
- return stackObjects.getFirst().toString(2);
- } catch (JSONException jsone) {
- throw new IOException(jsone.getMessage());
- }
- }
+ private Deque<JSONObject> stackObjects;
+
+ private Deque<JSONArray> stackArrays;
+
+ public CommonCrawlFormatJettinson(String url, Content content, Metadata
metadata, Configuration nutchConf, CommonCrawlConfig config) throws IOException
{
+ super(url, content, metadata, nutchConf, config);
+
+ stackObjects = new ArrayDeque<>();
+ stackArrays = new ArrayDeque<>();
+ }
+
+ @Override
+ protected void writeKeyValue(String key, String value) throws IOException {
+ try {
+ stackObjects.getFirst().put(key, value);
+ } catch (JSONException jsone) {
+ throw new IOException(jsone.getMessage());
+ }
+ }
+
+ @Override
+ protected void writeKeyNull(String key) throws IOException {
+ try {
+ stackObjects.getFirst().put(key, JSONObject.NULL);
+ } catch (JSONException jsone) {
+ throw new IOException(jsone.getMessage());
+ }
+ }
+
+ @Override
+ protected void startArray(String key, boolean nested, boolean newline)
throws IOException {
+ JSONArray array = new JSONArray();
+ stackArrays.push(array);
+ }
+
+ @Override
+ protected void closeArray(String key, boolean nested, boolean newline)
throws IOException {
+ try {
+ if (stackArrays.size() > 1) {
+ JSONArray array = stackArrays.pop();
+ if (nested) {
+ stackArrays.getFirst().put(array);
+ }
+ else {
+ stackObjects.getFirst().put(key, array);
+ }
+ }
+ } catch (JSONException jsone) {
+ throw new IOException(jsone.getMessage());
+ }
+ }
+
+ @Override
+ protected void writeArrayValue(String value) throws IOException {
+ if (stackArrays.size() > 1) {
+ stackArrays.getFirst().put(value);
+ }
+ }
+
+ @Override
+ protected void startObject(String key) throws IOException {
+ JSONObject object = new JSONObject();
+ stackObjects.push(object);
+ }
+
+ @Override
+ protected void closeObject(String key) throws IOException {
+ try {
+ if (stackObjects.size() > 1) {
+ JSONObject object = stackObjects.pop();
+ stackObjects.getFirst().put(key, object);
+ }
+ } catch (JSONException jsone) {
+ throw new IOException(jsone.getMessage());
+ }
+ }
+
+ @Override
+ protected String generateJson() throws IOException {
+ try {
+ return stackObjects.getFirst().toString(2);
+ } catch (JSONException jsone) {
+ throw new IOException(jsone.getMessage());
+ }
+ }
}
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlFormatSimple.java
b/src/java/org/apache/nutch/tools/CommonCrawlFormatSimple.java
index 4a592a08d..fac100dc4 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlFormatSimple.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlFormatSimple.java
@@ -28,95 +28,95 @@ import org.apache.nutch.protocol.Content;
*
*/
public class CommonCrawlFormatSimple extends AbstractCommonCrawlFormat {
-
- private StringBuilder sb;
-
- private int tabCount;
-
- public CommonCrawlFormatSimple(String url, Content content, Metadata
metadata, Configuration nutchConf, CommonCrawlConfig config) throws IOException
{
- super(url, content, metadata, nutchConf, config);
-
- this.sb = new StringBuilder();
- this.tabCount = 0;
- }
-
- @Override
- protected void writeKeyValue(String key, String value) throws
IOException {
- sb.append(printTabs() + "\"" + key + "\": " + quote(value) +
",\n");
- }
-
- @Override
- protected void writeKeyNull(String key) throws IOException {
- sb.append(printTabs() + "\"" + key + "\": null,\n");
- }
-
- @Override
- protected void startArray(String key, boolean nested, boolean newline)
throws IOException {
- String name = (key != null) ? "\"" + key + "\": " : "";
- String nl = (newline) ? "\n" : "";
- sb.append(printTabs() + name + "[" + nl);
- if (newline) {
- this.tabCount++;
- }
- }
-
- @Override
- protected void closeArray(String key, boolean nested, boolean newline)
throws IOException {
- if (sb.charAt(sb.length()-1) == ',') {
- sb.deleteCharAt(sb.length()-1); // delete comma
- }
- else if (sb.charAt(sb.length()-2) == ',') {
- sb.deleteCharAt(sb.length()-2); // delete comma
- }
- String nl = (newline) ? printTabs() : "";
- if (newline) {
- this.tabCount++;
- }
- sb.append(nl + "],\n");
- }
-
- @Override
- protected void writeArrayValue(String value) {
- sb.append("\"" + value + "\",");
- }
-
- @Override
+
+ private StringBuilder sb;
+
+ private int tabCount;
+
+ public CommonCrawlFormatSimple(String url, Content content, Metadata
metadata, Configuration nutchConf, CommonCrawlConfig config) throws IOException
{
+ super(url, content, metadata, nutchConf, config);
+
+ this.sb = new StringBuilder();
+ this.tabCount = 0;
+ }
+
+ @Override
+ protected void writeKeyValue(String key, String value) throws IOException {
+ sb.append(printTabs() + "\"" + key + "\": " + quote(value) + ",\n");
+ }
+
+ @Override
+ protected void writeKeyNull(String key) throws IOException {
+ sb.append(printTabs() + "\"" + key + "\": null,\n");
+ }
+
+ @Override
+ protected void startArray(String key, boolean nested, boolean newline)
throws IOException {
+ String name = (key != null) ? "\"" + key + "\": " : "";
+ String nl = (newline) ? "\n" : "";
+ sb.append(printTabs() + name + "[" + nl);
+ if (newline) {
+ this.tabCount++;
+ }
+ }
+
+ @Override
+ protected void closeArray(String key, boolean nested, boolean newline)
throws IOException {
+ if (sb.charAt(sb.length()-1) == ',') {
+ sb.deleteCharAt(sb.length()-1); // delete comma
+ }
+ else if (sb.charAt(sb.length()-2) == ',') {
+ sb.deleteCharAt(sb.length()-2); // delete comma
+ }
+ String nl = (newline) ? printTabs() : "";
+ if (newline) {
+ this.tabCount++;
+ }
+ sb.append(nl + "],\n");
+ }
+
+ @Override
+ protected void writeArrayValue(String value) {
+ sb.append("\"" + value + "\",");
+ }
+
+ @Override
protected void startObject(String key) throws IOException {
- String name = "";
- if (key != null) {
- name = "\"" + key + "\": ";
- }
- sb.append(printTabs() + name + "{\n");
- this.tabCount++;
- }
-
- @Override
+ String name = "";
+ if (key != null) {
+ name = "\"" + key + "\": ";
+ }
+ sb.append(printTabs() + name + "{\n");
+ this.tabCount++;
+ }
+
+ @Override
protected void closeObject(String key) throws IOException {
- if (sb.charAt(sb.length()-2) == ',') {
- sb.deleteCharAt(sb.length()-2); // delete comma
- }
- this.tabCount--;
- sb.append(printTabs() + "},\n");
- }
-
- @Override
+ if (sb.charAt(sb.length()-2) == ',') {
+ sb.deleteCharAt(sb.length()-2); // delete comma
+ }
+ this.tabCount--;
+ sb.append(printTabs() + "},\n");
+ }
+
+ @Override
protected String generateJson() throws IOException {
- sb.deleteCharAt(sb.length()-1); // delete new line
- sb.deleteCharAt(sb.length()-1); // delete comma
- return sb.toString();
- }
-
- private String printTabs() {
- StringBuilder sb = new StringBuilder();
- for (int i=0; i < this.tabCount ;i++) {
- sb.append("\t");
- }
- return sb.toString();
- }
-
+ sb.deleteCharAt(sb.length()-1); // delete new line
+ sb.deleteCharAt(sb.length()-1); // delete comma
+ return sb.toString();
+ }
+
+ private String printTabs() {
+ StringBuilder sb = new StringBuilder();
+ for (int i=0; i < this.tabCount ;i++) {
+ sb.append("\t");
+ }
+ return sb.toString();
+ }
+
private static String quote(String string) throws IOException {
- StringBuilder sb = new StringBuilder();
-
+ StringBuilder sb = new StringBuilder();
+
if (string == null || string.length() == 0) {
sb.append("\"\"");
return sb.toString();
@@ -140,34 +140,34 @@ public class CommonCrawlFormatSimple extends
AbstractCommonCrawlFormat {
break;
case '/':
if (b == '<') {
- sb.append('\\');
+ sb.append('\\');
}
sb.append(c);
break;
case '\b':
- sb.append("\\b");
+ sb.append("\\b");
break;
case '\t':
- sb.append("\\t");
+ sb.append("\\t");
break;
case '\n':
- sb.append("\\n");
+ sb.append("\\n");
break;
case '\f':
- sb.append("\\f");
+ sb.append("\\f");
break;
case '\r':
- sb.append("\\r");
+ sb.append("\\r");
break;
default:
if (c < ' ' || (c >= '\u0080' && c < '\u00a0')
|| (c >= '\u2000' && c < '\u2100')) {
- sb.append("\\u");
+ sb.append("\\u");
hhhh = Integer.toHexString(c);
sb.append("0000", 0, 4 - hhhh.length());
sb.append(hhhh);
} else {
- sb.append(c);
+ sb.append(c);
}
}
}
diff --git a/src/java/org/apache/nutch/util/CrawlCompletionStats.java
b/src/java/org/apache/nutch/util/CrawlCompletionStats.java
index ac815f74f..79e8bd87e 100644
--- a/src/java/org/apache/nutch/util/CrawlCompletionStats.java
+++ b/src/java/org/apache/nutch/util/CrawlCompletionStats.java
@@ -52,8 +52,10 @@ import org.slf4j.LoggerFactory;
* Extracts some simple crawl completion stats from the crawldb
*
* Stats will be sorted by host/domain and will be of the form:
- * 1 www.spitzer.caltech.edu FETCHED
- * 50 www.spitzer.caltech.edu UNFETCHED
+ * <pre>
+ * 1 www.spitzer.caltech.edu FETCHED
+ * 50 www.spitzer.caltech.edu UNFETCHED
+ * </pre>
*
*/
public class CrawlCompletionStats extends Configured implements Tool {
diff --git a/src/java/org/apache/nutch/util/DumpFileUtil.java
b/src/java/org/apache/nutch/util/DumpFileUtil.java
index 9461a0177..b9a064d99 100644
--- a/src/java/org/apache/nutch/util/DumpFileUtil.java
+++ b/src/java/org/apache/nutch/util/DumpFileUtil.java
@@ -30,8 +30,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DumpFileUtil {
- private static final Logger LOG = LoggerFactory
- .getLogger(MethodHandles.lookup().lookupClass());
+ private static final Logger LOG = LoggerFactory
+ .getLogger(MethodHandles.lookup().lookupClass());
private final static String DIR_PATTERN = "%s/%s/%s";
private final static String FILENAME_PATTERN = "%s_%s.%s";
@@ -57,12 +57,12 @@ public class DumpFileUtil {
firstLevelDirName, secondLevelDirName);
if (makeDir) {
- try {
- FileUtils.forceMkdir(new File(fullDirPath));
- } catch (IOException e) {
- LOG.error("Failed to create dir: {}", fullDirPath);
- fullDirPath = null;
- }
+ try {
+ FileUtils.forceMkdir(new File(fullDirPath));
+ } catch (IOException e) {
+ LOG.error("Failed to create dir: {}", fullDirPath);
+ fullDirPath = null;
+ }
}
return fullDirPath;
@@ -83,7 +83,7 @@ public class DumpFileUtil {
fileExtension = StringUtils.substring(fileExtension, 0,
MAX_LENGTH_OF_EXTENSION);
}
- // Added to prevent FileNotFoundException (Invalid Argument) - in *nix
environment
+ // Added to prevent FileNotFoundException (Invalid Argument) - in *nix
environment
fileBaseName = fileBaseName.replaceAll("\\?", "");
fileExtension = fileExtension.replaceAll("\\?", "");
@@ -119,38 +119,38 @@ public class DumpFileUtil {
return outputFullPath;
}
- public static String displayFileTypes(Map<String, Integer> typeCounts,
Map<String, Integer> filteredCounts) {
- StringBuilder builder = new StringBuilder();
- // print total stats
- builder.append("\nTOTAL Stats:\n");
- builder.append("[\n");
- int mimetypeCount = 0;
- for (String mimeType : typeCounts.keySet()) {
- builder.append(" {\"mimeType\":\"");
- builder.append(mimeType);
- builder.append("\",\"count\":\"");
- builder.append(typeCounts.get(mimeType));
- builder.append("\"}\n");
- mimetypeCount += typeCounts.get(mimeType);
- }
- builder.append("]\n");
- builder.append("Total count: " + mimetypeCount + "\n");
- // filtered types stats
- mimetypeCount = 0;
- if (!filteredCounts.isEmpty()) {
- builder.append("\nFILTERED Stats:\n");
- builder.append("[\n");
- for (String mimeType : filteredCounts.keySet()) {
- builder.append(" {\"mimeType\":\"");
- builder.append(mimeType);
- builder.append("\",\"count\":\"");
- builder.append(filteredCounts.get(mimeType));
- builder.append("\"}\n");
- mimetypeCount += filteredCounts.get(mimeType);
- }
- builder.append("]\n");
- builder.append("Total filtered count: " + mimetypeCount
+ "\n");
- }
- return builder.toString();
+ public static String displayFileTypes(Map<String, Integer> typeCounts,
Map<String, Integer> filteredCounts) {
+ StringBuilder builder = new StringBuilder();
+ // print total stats
+ builder.append("\nTOTAL Stats:\n");
+ builder.append("[\n");
+ int mimetypeCount = 0;
+ for (String mimeType : typeCounts.keySet()) {
+ builder.append(" {\"mimeType\":\"");
+ builder.append(mimeType);
+ builder.append("\",\"count\":\"");
+ builder.append(typeCounts.get(mimeType));
+ builder.append("\"}\n");
+ mimetypeCount += typeCounts.get(mimeType);
+ }
+ builder.append("]\n");
+ builder.append("Total count: " + mimetypeCount + "\n");
+ // filtered types stats
+ mimetypeCount = 0;
+ if (!filteredCounts.isEmpty()) {
+ builder.append("\nFILTERED Stats:\n");
+ builder.append("[\n");
+ for (String mimeType : filteredCounts.keySet()) {
+ builder.append(" {\"mimeType\":\"");
+ builder.append(mimeType);
+ builder.append("\",\"count\":\"");
+ builder.append(filteredCounts.get(mimeType));
+ builder.append("\"}\n");
+ mimetypeCount += filteredCounts.get(mimeType);
+ }
+ builder.append("]\n");
+ builder.append("Total filtered count: " + mimetypeCount + "\n");
+ }
+ return builder.toString();
}
}
diff --git a/src/java/org/apache/nutch/util/ProtocolStatusStatistics.java
b/src/java/org/apache/nutch/util/ProtocolStatusStatistics.java
index 3c37887cc..e4edb4ce6 100644
--- a/src/java/org/apache/nutch/util/ProtocolStatusStatistics.java
+++ b/src/java/org/apache/nutch/util/ProtocolStatusStatistics.java
@@ -50,10 +50,12 @@ import org.apache.nutch.metadata.Nutch;
* An example output run showing the number of encountered status
* codes such as 200, 300, and a count of un-fetched record.
*
- * 38 200
- * 19 301
- * 2 302
- * 665 UNFETCHED
+ * <pre>
+ * 38 200
+ * 19 301
+ * 2 302
+ * 665 UNFETCHED
+ * </pre>
*
*/
public class ProtocolStatusStatistics extends Configured implements Tool {
diff --git a/src/plugin/build-plugin.xml b/src/plugin/build-plugin.xml
index 0dd4da342..d8964c171 100755
--- a/src/plugin/build-plugin.xml
+++ b/src/plugin/build-plugin.xml
@@ -185,7 +185,7 @@
<fileset refid="lib.jars"/>
</copy>
</target>
-
+
<target name="copy-generated-lib" if="lib-available">
<copy file="${build.dir}/${name}.jar" todir="${deploy.dir}"
failonerror="false"/>
</target>
diff --git a/src/plugin/feed/build.xml b/src/plugin/feed/build.xml
index 250156137..8f7c4d99d 100644
--- a/src/plugin/feed/build.xml
+++ b/src/plugin/feed/build.xml
@@ -1,19 +1,19 @@
<?xml version="1.0"?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
-->
<project name="feed" default="jar-core">
diff --git a/src/plugin/feed/plugin.xml b/src/plugin/feed/plugin.xml
index 342bb28c7..eed25b5c4 100644
--- a/src/plugin/feed/plugin.xml
+++ b/src/plugin/feed/plugin.xml
@@ -1,22 +1,22 @@
<?xml version="1.0"?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
-->
<plugin id="feed" name="Feed Parse/Index/Query Plug-in" version="1.0.0"
- provider-name="nutch.org">
+ provider-name="nutch.org">
<runtime>
<library name="feed.jar">
<export name="*" />
diff --git a/src/plugin/feed/sample/rsstest.rss
b/src/plugin/feed/sample/rsstest.rss
index 83caae152..cd6a1a12d 100644
--- a/src/plugin/feed/sample/rsstest.rss
+++ b/src/plugin/feed/sample/rsstest.rss
@@ -1,19 +1,19 @@
<?xml version="1.0" encoding="ISO-8859-1" ?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
-->
<rss version="0.91">
<channel>
diff --git
a/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.java
b/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.java
index 901364985..4693a0286 100644
---
a/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.java
+++
b/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.java
@@ -188,39 +188,39 @@ public class ArbitraryIndexingFilter implements
IndexingFilter {
}
if (allFieldsAccess) {
theConstructor = theClass.getDeclaredConstructor(String[].class,
- NutchDocument.class,
- Parse.class,
- Text.class,
- CrawlDatum.class,
- Inlinks.class);
- } else {
+ NutchDocument.class,
+ Parse.class,
+ Text.class,
+ CrawlDatum.class,
+ Inlinks.class);
+ } else {
theConstructor = theClass.getDeclaredConstructor(String[].class);
- }
+ }
} catch (NoSuchMethodException nme) {
LOG.error("Exception preparing reflection for constructor. className
was {}",
- String.valueOf(className));
+ String.valueOf(className));
nme.printStackTrace();
continue;
} catch (Exception e) {
LOG.error("Exception preparing reflection tasks. className was {}",
- String.valueOf(className));
+ String.valueOf(className));
e.printStackTrace();
continue;
}
try {
constrArgs = new String[userConstrArgs.length + 1];
System.arraycopy(userConstrArgs,0,constrArgs,1,userConstrArgs.length);
- if (allFieldsAccess) {
+ if (allFieldsAccess) {
instance = theConstructor.newInstance(constrArgs,
- doc,
- parse,
- url,
- datum,
- inlinks);
- } else {
+ doc,
+ parse,
+ url,
+ datum,
+ inlinks);
+ } else {
constrArgs[0] = url.toString();
instance = theConstructor.newInstance(new Object[]{constrArgs});
- }
+ }
if (methodArgs.length > 0) {
result = theMethod.invoke(instance, new Object[]{methodArgs});
@@ -229,8 +229,8 @@ public class ArbitraryIndexingFilter implements
IndexingFilter {
}
} catch (Exception e) {
LOG.error("Exception in reflection trying to instantiate/invoke. "
- + "url was {} & className was {}",
- String.valueOf(url), String.valueOf(className));
+ + "url was {} & className was {}",
+ String.valueOf(url), String.valueOf(className));
if (constrArgs.length > 1) {
LOG.error("constrArgs[1] was {}", String.valueOf(constrArgs[1]));
}
@@ -239,19 +239,19 @@ public class ArbitraryIndexingFilter implements
IndexingFilter {
LOG.error("methodArgs[0] was {}", String.valueOf(methodArgs[0]));
}
e.printStackTrace();
- continue;
+ continue;
}
LOG.debug("{}.{}() returned {} for field {}.", className,
- methodName, String.valueOf(result), String.valueOf(fieldName));
+ methodName, String.valueOf(result), String.valueOf(fieldName));
// If user chose to overwrite, remove existing value
if (overwrite) {
- LOG.debug("overwrite == true for fieldName == {} ", fieldName);
- if (doc.getFieldNames().contains(fieldName)) {
- LOG.debug("Removing field '{}' from doc for overwrite", fieldName);
- doc.removeField(fieldName);
- }
+ LOG.debug("overwrite == true for fieldName == {} ", fieldName);
+ if (doc.getFieldNames().contains(fieldName)) {
+ LOG.debug("Removing field '{}' from doc for overwrite", fieldName);
+ doc.removeField(fieldName);
+ }
}
if (result == null) {
LOG.debug("Call to {}.{} returned null", className, methodName);
@@ -288,7 +288,7 @@ public class ArbitraryIndexingFilter implements
IndexingFilter {
LOG.debug("In setIndexedConf() where ndx was passed in as {}",
String.valueOf(ndx));
fieldName =
conf.get("index.arbitrary.fieldName.".concat(String.valueOf(ndx)));
LOG.debug("Looking now for index.arbitrary.fieldname.{} which was: {}",
- String.valueOf(ndx),String.valueOf(fieldName));
+ String.valueOf(ndx),String.valueOf(fieldName));
if (fieldName == null || fieldName == "") {
throw new RuntimeException ("Problem in configuration where the
index.arbitrary.fieldName."
diff --git
a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/Multiplier.java
b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/Multiplier.java
index 749be508d..51ca6dc8c 100644
---
a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/Multiplier.java
+++
b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/Multiplier.java
@@ -41,7 +41,7 @@ public class Multiplier {
}
public static void main(String[] args) {
- Multiplier mp = new Multiplier(args);
- out.println(mp.getProduct(args));
+ Multiplier mp = new Multiplier(args);
+ out.println(mp.getProduct(args));
}
}
diff --git
a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.java
b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.java
index 4b8df2124..da282a3a7 100644
---
a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.java
+++
b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.java
@@ -39,7 +39,7 @@ public class PopularityGauge {
private double popularityBoost;
public PopularityGauge(String args[],
- NutchDocument docIn,
+ NutchDocument docIn,
Parse parseIn,
Text urlIn,
CrawlDatum datumIn,
@@ -57,24 +57,24 @@ public class PopularityGauge {
if(anchorSet.contains("dinosaur")){
popularityBoost = popularityBoost + 0.50;
} else {
- popularityBoost = popularityBoost - 0.20;
+ popularityBoost = popularityBoost - 0.20;
}
if (anchorSet.contains("baseball")) {
popularityBoost = popularityBoost + 0.25;
} else {
- popularityBoost = popularityBoost - 0.15;
+ popularityBoost = popularityBoost - 0.15;
}
if (anchorSet.contains("source code")) {
popularityBoost = popularityBoost + 0.25;
} else {
- popularityBoost = popularityBoost - 0.15;
+ popularityBoost = popularityBoost - 0.15;
}
}
return popularityBoost;
}
public static void main(String[] args,
- NutchDocument doc,
+ NutchDocument doc,
Parse parse,
Text url,
CrawlDatum datum,
diff --git a/src/plugin/index-replace/build.xml
b/src/plugin/index-replace/build.xml
index ea8c95d9f..e0e4a41cd 100644
--- a/src/plugin/index-replace/build.xml
+++ b/src/plugin/index-replace/build.xml
@@ -17,39 +17,39 @@
-->
<project name="index-replace" default="jar-core">
- <import file="../build-plugin.xml" />
-
- <!-- Add compilation dependencies to classpath -->
- <path id="plugin.deps">
- <fileset dir="${nutch.root}/build">
- <include name="**/index-basic/*.jar" />
- <include name="**/index-metadata/*.jar" />
- </fileset>
- <pathelement
location="${nutch.root}/build/lib-regex-filter/test"/>
- </path>
-
- <!-- Compile Unit test dependencies -->
- <target name="deps-test-compile">
- <ant target="compile-test" inheritall="false"
dir="../index-basic"/>
- <ant target="compile-test" inheritall="false"
dir="../index-metadata"/>
- </target>
-
- <!-- Deploy Unit test dependencies -->
- <target name="deps-test">
- <ant target="deploy" inheritall="false"
dir="../nutch-extensionpoints" />
- <ant target="deploy" inheritall="false" dir="../protocol-file"
/>
- <ant target="deploy" inheritall="false" dir="../parse-html" />
- <ant target="deploy" inheritall="false" dir="../parse-metatags"
/>
- <ant target="deploy" inheritall="false" dir="../index-basic" />
- <ant target="deploy" inheritall="false" dir="../index-metadata"
/>
- </target>
-
- <!-- Copy test file for junit test -->
- <mkdir dir="${build.test}/data" />
- <copy todir="${build.test}/data">
- <fileset dir="sample">
- <include name="*.html" />
- </fileset>
- </copy>
+ <import file="../build-plugin.xml" />
+
+ <!-- Add compilation dependencies to classpath -->
+ <path id="plugin.deps">
+ <fileset dir="${nutch.root}/build">
+ <include name="**/index-basic/*.jar" />
+ <include name="**/index-metadata/*.jar" />
+ </fileset>
+ <pathelement location="${nutch.root}/build/lib-regex-filter/test"/>
+ </path>
+
+ <!-- Compile Unit test dependencies -->
+ <target name="deps-test-compile">
+ <ant target="compile-test" inheritall="false" dir="../index-basic"/>
+ <ant target="compile-test" inheritall="false" dir="../index-metadata"/>
+ </target>
+
+ <!-- Deploy Unit test dependencies -->
+ <target name="deps-test">
+ <ant target="deploy" inheritall="false" dir="../nutch-extensionpoints" />
+ <ant target="deploy" inheritall="false" dir="../protocol-file" />
+ <ant target="deploy" inheritall="false" dir="../parse-html" />
+ <ant target="deploy" inheritall="false" dir="../parse-metatags" />
+ <ant target="deploy" inheritall="false" dir="../index-basic" />
+ <ant target="deploy" inheritall="false" dir="../index-metadata" />
+ </target>
+
+ <!-- Copy test file for junit test -->
+ <mkdir dir="${build.test}/data" />
+ <copy todir="${build.test}/data">
+ <fileset dir="sample">
+ <include name="*.html" />
+ </fileset>
+ </copy>
</project>
diff --git
a/src/plugin/index-replace/src/java/org/apache/nutch/indexer/replace/ReplaceIndexer.java
b/src/plugin/index-replace/src/java/org/apache/nutch/indexer/replace/ReplaceIndexer.java
index c21ae242d..a882df516 100644
---
a/src/plugin/index-replace/src/java/org/apache/nutch/indexer/replace/ReplaceIndexer.java
+++
b/src/plugin/index-replace/src/java/org/apache/nutch/indexer/replace/ReplaceIndexer.java
@@ -85,7 +85,7 @@ import org.apache.nutch.parse.Parse;
public class ReplaceIndexer implements IndexingFilter {
private static final Logger LOG = LoggerFactory
- .getLogger(MethodHandles.lookup().lookupClass());
+ .getLogger(MethodHandles.lookup().lookupClass());
/** Special field name signifying the start of a host-specific match set */
private static final String HOSTMATCH = "hostmatch";
diff --git a/src/plugin/indexer-cloudsearch/ivy.xml
b/src/plugin/indexer-cloudsearch/ivy.xml
index a9af5f6c8..0b1018e13 100644
--- a/src/plugin/indexer-cloudsearch/ivy.xml
+++ b/src/plugin/indexer-cloudsearch/ivy.xml
@@ -37,7 +37,7 @@
</publications>
<dependencies>
- <dependency org="com.amazonaws" name="aws-java-sdk-cloudsearch"
rev="1.10.0"/>
+ <dependency org="com.amazonaws" name="aws-java-sdk-cloudsearch"
rev="1.10.0"/>
</dependencies>
</ivy-module>
diff --git a/src/plugin/indexer-solr/ivy.xml b/src/plugin/indexer-solr/ivy.xml
index 5b2642ad7..0b0410bff 100644
--- a/src/plugin/indexer-solr/ivy.xml
+++ b/src/plugin/indexer-solr/ivy.xml
@@ -17,52 +17,52 @@
-->
<ivy-module xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
-
xsi:noNamespaceSchemaLocation="http://ant.apache.org/ivy/schemas/ivy.xsd";
- xmlns:ns0="http://ant.apache.org/ivy/maven"; version="2.0">
- <info organisation="org.apache.nutch" module="${ant.project.name}">
- <license name="Apache 2.0" />
- <ivyauthor name="Apache Nutch Team"
- url="https://nutch.apache.org/"; />
- <description>
- Apache Nutch
- </description>
- </info>
+ xsi:noNamespaceSchemaLocation="http://ant.apache.org/ivy/schemas/ivy.xsd";
+ xmlns:ns0="http://ant.apache.org/ivy/maven"; version="2.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0" />
+ <ivyauthor name="Apache Nutch Team"
+ url="https://nutch.apache.org/"; />
+ <description>
+ Apache Nutch
+ </description>
+ </info>
- <configurations>
- <include file="../../..//ivy/ivy-configurations.xml" />
- </configurations>
+ <configurations>
+ <include file="../../..//ivy/ivy-configurations.xml" />
+ </configurations>
- <publications>
- <!--get the artifact from our module name -->
- <artifact conf="master" />
- </publications>
+ <publications>
+ <!--get the artifact from our module name -->
+ <artifact conf="master" />
+ </publications>
- <dependencies>
- <dependency org="org.testcontainers" name="testcontainers-solr"
rev="2.0.3" conf="test->default"/>
- <dependency org="org.apache.solr" name="solr-solrj"
- rev="8.11.4" conf="*->default">
- <!-- exclusions of dependencies provided by Nutch core
-->
- <exclude org="commons-codec" name="commons-codec" />
- <exclude org="commons-logging" name="commons-logging" />
- <exclude org="org.slf4j" name="slf4j-api" />
- </dependency>
- <dependency org="org.apache.httpcomponents" name="httpclient"
- rev="4.5.14" conf="*->default">
- <!-- exclusions of dependencies provided by Nutch core
-->
- <exclude org="commons-codec" name="commons-codec" />
- <exclude org="commons-logging" name="commons-logging" />
- </dependency>
- <dependency org="org.apache.httpcomponents" name="httpmime"
- rev="4.5.14" conf="*->default">
- <!-- exclusions of dependencies provided by Nutch core
-->
- <exclude org="commons-codec" name="commons-codec" />
- <exclude org="commons-logging" name="commons-logging" />
- </dependency>
- <dependency org="org.apache.httpcomponents" name="httpcore"
- rev="4.4.16" conf="*->default">
- <!-- exclusions of dependencies provided by Nutch core
-->
- <exclude org="commons-logging" name="commons-logging" />
- </dependency>
- </dependencies>
+ <dependencies>
+ <dependency org="org.testcontainers" name="testcontainers-solr"
rev="2.0.3" conf="test->default"/>
+ <dependency org="org.apache.solr" name="solr-solrj"
+ rev="8.11.4" conf="*->default">
+ <!-- exclusions of dependencies provided by Nutch core -->
+ <exclude org="commons-codec" name="commons-codec" />
+ <exclude org="commons-logging" name="commons-logging" />
+ <exclude org="org.slf4j" name="slf4j-api" />
+ </dependency>
+ <dependency org="org.apache.httpcomponents" name="httpclient"
+ rev="4.5.14" conf="*->default">
+ <!-- exclusions of dependencies provided by Nutch core -->
+ <exclude org="commons-codec" name="commons-codec" />
+ <exclude org="commons-logging" name="commons-logging" />
+ </dependency>
+ <dependency org="org.apache.httpcomponents" name="httpmime"
+ rev="4.5.14" conf="*->default">
+ <!-- exclusions of dependencies provided by Nutch core -->
+ <exclude org="commons-codec" name="commons-codec" />
+ <exclude org="commons-logging" name="commons-logging" />
+ </dependency>
+ <dependency org="org.apache.httpcomponents" name="httpcore"
+ rev="4.4.16" conf="*->default">
+ <!-- exclusions of dependencies provided by Nutch core -->
+ <exclude org="commons-logging" name="commons-logging" />
+ </dependency>
+ </dependencies>
</ivy-module>
diff --git a/src/plugin/indexer-solr/plugin.xml
b/src/plugin/indexer-solr/plugin.xml
index be9813141..6b9a359b5 100644
--- a/src/plugin/indexer-solr/plugin.xml
+++ b/src/plugin/indexer-solr/plugin.xml
@@ -16,56 +16,56 @@
limitations under the License.
-->
<plugin id="indexer-solr" name="SolrIndexWriter" version="1.0.0"
- provider-name="nutch.apache.org">
+ provider-name="nutch.apache.org">
- <runtime>
- <library name="indexer-solr.jar">
- <export name="*" />
- </library>
- <!-- Solr dependencies -->
- <library name="commons-io-2.11.0.jar"/>
- <library name="commons-lang-2.6.jar"/>
- <library name="commons-math3-3.6.1.jar"/>
- <library name="http2-client-9.4.53.v20231009.jar"/>
- <library name="http2-common-9.4.53.v20231009.jar"/>
- <library name="http2-hpack-9.4.53.v20231009.jar"/>
- <library
name="http2-http-client-transport-9.4.53.v20231009.jar"/>
- <library name="httpclient-4.5.14.jar"/>
- <library name="httpcore-4.4.16.jar"/>
- <library name="httpmime-4.5.14.jar"/>
- <library name="jcl-over-slf4j-1.7.36.jar"/>
- <library name="jetty-alpn-client-9.4.53.v20231009.jar"/>
- <library name="jetty-alpn-java-client-9.4.53.v20231009.jar"/>
- <library name="jetty-client-9.4.53.v20231009.jar"/>
- <library name="jetty-http-9.4.53.v20231009.jar"/>
- <library name="jetty-io-9.4.53.v20231009.jar"/>
- <library name="jetty-util-9.4.53.v20231009.jar"/>
- <library name="netty-buffer-4.1.99.Final.jar"/>
- <library name="netty-codec-4.1.99.Final.jar"/>
- <library name="netty-common-4.1.99.Final.jar"/>
- <library name="netty-handler-4.1.99.Final.jar"/>
- <library name="netty-resolver-4.1.99.Final.jar"/>
- <library name="netty-transport-4.1.99.Final.jar"/>
- <library name="netty-transport-classes-epoll-4.1.99.Final.jar"/>
- <library name="netty-transport-native-epoll-4.1.99.Final.jar"/>
- <library
name="netty-transport-native-unix-common-4.1.99.Final.jar"/>
- <library name="snappy-java-1.1.10.1.jar"/>
- <library name="solr-solrj-8.11.4.jar"/>
- <library name="stax2-api-4.2.1.jar"/>
- <library name="woodstox-core-6.5.1.jar"/>
- <library name="zookeeper-3.6.2.jar"/>
- <library name="zookeeper-jute-3.6.2.jar"/>
- <!-- end of Solr dependencies -->
- </runtime>
+ <runtime>
+ <library name="indexer-solr.jar">
+ <export name="*" />
+ </library>
+ <!-- Solr dependencies -->
+ <library name="commons-io-2.11.0.jar"/>
+ <library name="commons-lang-2.6.jar"/>
+ <library name="commons-math3-3.6.1.jar"/>
+ <library name="http2-client-9.4.53.v20231009.jar"/>
+ <library name="http2-common-9.4.53.v20231009.jar"/>
+ <library name="http2-hpack-9.4.53.v20231009.jar"/>
+ <library name="http2-http-client-transport-9.4.53.v20231009.jar"/>
+ <library name="httpclient-4.5.14.jar"/>
+ <library name="httpcore-4.4.16.jar"/>
+ <library name="httpmime-4.5.14.jar"/>
+ <library name="jcl-over-slf4j-1.7.36.jar"/>
+ <library name="jetty-alpn-client-9.4.53.v20231009.jar"/>
+ <library name="jetty-alpn-java-client-9.4.53.v20231009.jar"/>
+ <library name="jetty-client-9.4.53.v20231009.jar"/>
+ <library name="jetty-http-9.4.53.v20231009.jar"/>
+ <library name="jetty-io-9.4.53.v20231009.jar"/>
+ <library name="jetty-util-9.4.53.v20231009.jar"/>
+ <library name="netty-buffer-4.1.99.Final.jar"/>
+ <library name="netty-codec-4.1.99.Final.jar"/>
+ <library name="netty-common-4.1.99.Final.jar"/>
+ <library name="netty-handler-4.1.99.Final.jar"/>
+ <library name="netty-resolver-4.1.99.Final.jar"/>
+ <library name="netty-transport-4.1.99.Final.jar"/>
+ <library name="netty-transport-classes-epoll-4.1.99.Final.jar"/>
+ <library name="netty-transport-native-epoll-4.1.99.Final.jar"/>
+ <library name="netty-transport-native-unix-common-4.1.99.Final.jar"/>
+ <library name="snappy-java-1.1.10.1.jar"/>
+ <library name="solr-solrj-8.11.4.jar"/>
+ <library name="stax2-api-4.2.1.jar"/>
+ <library name="woodstox-core-6.5.1.jar"/>
+ <library name="zookeeper-3.6.2.jar"/>
+ <library name="zookeeper-jute-3.6.2.jar"/>
+ <!-- end of Solr dependencies -->
+ </runtime>
- <requires>
- <import plugin="nutch-extensionpoints" />
- </requires>
+ <requires>
+ <import plugin="nutch-extensionpoints" />
+ </requires>
- <extension id="org.apache.nutch.indexer.solr"
- name="Solr Index Writer"
point="org.apache.nutch.indexer.IndexWriter">
- <implementation id="SolrIndexWriter"
-
class="org.apache.nutch.indexwriter.solr.SolrIndexWriter" />
- </extension>
+ <extension id="org.apache.nutch.indexer.solr"
+ name="Solr Index Writer" point="org.apache.nutch.indexer.IndexWriter">
+ <implementation id="SolrIndexWriter"
+ class="org.apache.nutch.indexwriter.solr.SolrIndexWriter" />
+ </extension>
</plugin>
diff --git a/src/plugin/indexer-solr/schema.xml
b/src/plugin/indexer-solr/schema.xml
index cf7759d92..67e44217e 100644
--- a/src/plugin/indexer-solr/schema.xml
+++ b/src/plugin/indexer-solr/schema.xml
@@ -111,9 +111,9 @@
<!-- A general text field that has reasonable, generic
cross-language defaults: it tokenizes with StandardTokenizer,
- removes stop words from case-insensitive "stopwords.txt"
- (empty by default), and down cases. At query time only, it
- also applies synonyms. -->
+ removes stop words from case-insensitive "stopwords.txt"
+ (empty by default), and down cases. At query time only, it
+ also applies synonyms. -->
<fieldType name="text_general" class="solr.TextField"
positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.StandardTokenizerFactory"/>
@@ -147,11 +147,11 @@
words="stopwords.txt"
/>
<filter class="solr.LowerCaseFilterFactory"/>
- <filter class="solr.EnglishPossessiveFilterFactory"/>
+ <filter class="solr.EnglishPossessiveFilterFactory"/>
<filter class="solr.KeywordMarkerFilterFactory"
protected="protwords.txt"/>
- <!-- Optionally you may want to use this less aggressive stemmer
instead of PorterStemFilterFactory:
+ <!-- Optionally you may want to use this less aggressive stemmer instead of
PorterStemFilterFactory:
<filter class="solr.EnglishMinimalStemFilterFactory"/>
- -->
+ -->
<filter class="solr.PorterStemFilterFactory"/>
</analyzer>
<analyzer type="query">
@@ -162,26 +162,26 @@
words="stopwords.txt"
/>
<filter class="solr.LowerCaseFilterFactory"/>
- <filter class="solr.EnglishPossessiveFilterFactory"/>
+ <filter class="solr.EnglishPossessiveFilterFactory"/>
<filter class="solr.KeywordMarkerFilterFactory"
protected="protwords.txt"/>
- <!-- Optionally you may want to use this less aggressive stemmer
instead of PorterStemFilterFactory:
+ <!-- Optionally you may want to use this less aggressive stemmer instead of
PorterStemFilterFactory:
<filter class="solr.EnglishMinimalStemFilterFactory"/>
- -->
+ -->
<filter class="solr.PorterStemFilterFactory"/>
</analyzer>
</fieldType>
<!-- A text field with defaults appropriate for English, plus
- aggressive word-splitting and autophrase features enabled.
- This field is just like text_en, except it adds
- WordDelimiterFilter to enable splitting and matching of
- words on case-change, alpha numeric boundaries, and
- non-alphanumeric chars. This means certain compound word
- cases will work, for example query "wi fi" will match
- document "WiFi" or "wi-fi". However, other cases will still
- not match, for example if the query is "wifi" and the
- document is "wi fi" or if the query is "wi-fi" and the
- document is "wifi".
+ aggressive word-splitting and autophrase features enabled.
+ This field is just like text_en, except it adds
+ WordDelimiterFilter to enable splitting and matching of
+ words on case-change, alpha numeric boundaries, and
+ non-alphanumeric chars. This means certain compound word
+ cases will work, for example query "wi fi" will match
+ document "WiFi" or "wi-fi". However, other cases will still
+ not match, for example if the query is "wifi" and the
+ document is "wi fi" or if the query is "wi-fi" and the
+ document is "wifi".
-->
<fieldType name="text_en_splitting" class="solr.TextField"
positionIncrementGap="100" autoGeneratePhraseQueries="true">
<analyzer type="index">
@@ -230,7 +230,7 @@
</fieldType>
<!-- Just like text_general except it reverses the characters of
- each token, to enable more efficient leading wildcard queries. -->
+ each token, to enable more efficient leading wildcard queries. -->
<fieldType name="text_general_rev" class="solr.TextField"
positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.StandardTokenizerFactory"/>
@@ -262,10 +262,10 @@
a token of "foo|1.4" would be indexed as "foo" with a payload of 1.4f
Attributes of the DelimitedPayloadTokenFilterFactory :
"delimiter" - a one character delimiter. Default is | (pipe)
- "encoder" - how to encode the following value into a playload
- float -> org.apache.lucene.analysis.payloads.FloatEncoder,
- integer -> o.a.l.a.p.IntegerEncoder
- identity -> o.a.l.a.p.IdentityEncoder
+ "encoder" - how to encode the following value into a playload
+ float -> org.apache.lucene.analysis.payloads.FloatEncoder,
+ integer -> o.a.l.a.p.IntegerEncoder
+ identity -> o.a.l.a.p.IdentityEncoder
Fully Qualified class name implementing PayloadEncoder, Encoder
must have a no arg constructor.
-->
<filter class="solr.DelimitedPayloadTokenFilterFactory"
encoder="float"/>
diff --git
a/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java
b/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java
index 754b92c7b..02eade209 100644
---
a/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java
+++
b/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java
@@ -75,7 +75,7 @@ public class LanguageIndexingFilter implements IndexingFilter
{
}
if (!indexLangs.isEmpty() && !indexLangs.contains(lang)) {
- return null;
+ return null;
}
doc.add("lang", lang);
diff --git
a/src/plugin/lib-htmlunit/src/java/org/apache/nutch/protocol/htmlunit/HtmlUnitWebDriver.java
b/src/plugin/lib-htmlunit/src/java/org/apache/nutch/protocol/htmlunit/HtmlUnitWebDriver.java
index 9b63a4fdd..9f26c207b 100644
---
a/src/plugin/lib-htmlunit/src/java/org/apache/nutch/protocol/htmlunit/HtmlUnitWebDriver.java
+++
b/src/plugin/lib-htmlunit/src/java/org/apache/nutch/protocol/htmlunit/HtmlUnitWebDriver.java
@@ -56,7 +56,7 @@ public class HtmlUnitWebDriver extends HtmlUnitDriver {
client.getOptions().setThrowExceptionOnScriptError(false);
if(enableRedirect)
client.addWebWindowListener(new
HtmlUnitWebWindowListener(maxRedirects));
- return client;
+ return client;
}
public static WebDriver getDriverForPage(String url, Configuration conf) {
@@ -67,9 +67,9 @@ public class HtmlUnitWebDriver extends HtmlUnitDriver {
int redirects = Integer.parseInt(conf.get("http.redirect.max", "0"));
enableRedirect = redirects > 0;
maxRedirects = redirects;
-
+
WebDriver driver = null;
-
+
try {
driver = new HtmlUnitWebDriver();
driver.manage().timeouts().pageLoadTimeout(Duration.of(pageLoadTimout,
@@ -77,8 +77,8 @@ public class HtmlUnitWebDriver extends HtmlUnitDriver {
driver.get(url);
} catch(Exception e) {
if(e instanceof TimeoutException) {
- LOG.debug("HtmlUnit WebDriver: Timeout Exception: Capturing
whatever loaded so far...");
- return driver;
+ LOG.debug("HtmlUnit WebDriver: Timeout Exception: Capturing whatever
loaded so far...");
+ return driver;
}
cleanUpDriver(driver);
throw new RuntimeException(e);
@@ -91,19 +91,19 @@ public class HtmlUnitWebDriver extends HtmlUnitDriver {
try {
if (conf.getBoolean("take.screenshot", false))
takeScreenshot(driver, conf);
-
+
String innerHtml = "";
if(enableJavascript) {
- WebElement body = driver.findElement(By.tagName("body"));
- innerHtml =
(String)((JavascriptExecutor)driver).executeScript("return
arguments[0].innerHTML;", body);
+ WebElement body = driver.findElement(By.tagName("body"));
+ innerHtml = (String)((JavascriptExecutor)driver).executeScript("return
arguments[0].innerHTML;", body);
}
else
- innerHtml = driver.getPageSource().replaceAll("&", "&");
+ innerHtml = driver.getPageSource().replaceAll("&", "&");
return innerHtml;
} catch(Exception e) {
- TemporaryFilesystem.getDefaultTmpFS().deleteTemporaryFiles();
- cleanUpDriver(driver);
- throw new RuntimeException(e);
+ TemporaryFilesystem.getDefaultTmpFS().deleteTemporaryFiles();
+ cleanUpDriver(driver);
+ throw new RuntimeException(e);
}
}
@@ -135,19 +135,19 @@ public class HtmlUnitWebDriver extends HtmlUnitDriver {
try {
if (conf.getBoolean("take.screenshot", false))
- takeScreenshot(driver, conf);
+ takeScreenshot(driver, conf);
String innerHtml = "";
if(enableJavascript) {
- WebElement body = driver.findElement(By.tagName("body"));
- innerHtml =
(String)((JavascriptExecutor)driver).executeScript("return
arguments[0].innerHTML;", body);
+ WebElement body = driver.findElement(By.tagName("body"));
+ innerHtml = (String)((JavascriptExecutor)driver).executeScript("return
arguments[0].innerHTML;", body);
}
else
- innerHtml = driver.getPageSource().replaceAll("&", "&");
+ innerHtml = driver.getPageSource().replaceAll("&", "&");
return innerHtml;
} catch (Exception e) {
- TemporaryFilesystem.getDefaultTmpFS().deleteTemporaryFiles();
+ TemporaryFilesystem.getDefaultTmpFS().deleteTemporaryFiles();
throw new RuntimeException(e);
} finally {
cleanUpDriver(driver);
@@ -161,7 +161,7 @@ public class HtmlUnitWebDriver extends HtmlUnitDriver {
LOG.debug("In-memory screenshot taken of: {}", url);
FileSystem fs = FileSystem.get(conf);
if (conf.get("screenshot.location") != null) {
- Path screenshotPath = new Path(conf.get("screenshot.location") + "/"
+ srcFile.getName());
+ Path screenshotPath = new Path(conf.get("screenshot.location") + "/" +
srcFile.getName());
OutputStream os = null;
if (!fs.exists(screenshotPath)) {
LOG.debug("No existing screenshot already exists... creating new
file at {} {}.", screenshotPath, srcFile.getName());
@@ -175,8 +175,8 @@ public class HtmlUnitWebDriver extends HtmlUnitDriver {
+ "'screenshot.location' is absent from nutch-site.xml.", url);
}
} catch (Exception e) {
- cleanUpDriver(driver);
- throw new RuntimeException(e);
+ cleanUpDriver(driver);
+ throw new RuntimeException(e);
}
}
}
diff --git a/src/plugin/lib-xml/build.xml b/src/plugin/lib-xml/build.xml
index 0f87c073e..189f1ff32 100644
--- a/src/plugin/lib-xml/build.xml
+++ b/src/plugin/lib-xml/build.xml
@@ -17,20 +17,20 @@
-->
<project name="lib-xml" default="jar">
- <import file="../build-plugin.xml" />
+ <import file="../build-plugin.xml" />
- <!--
+ <!--
! Override the compile and jar targets,
! since there is nothing to compile here.
! -->
- <target name="compile" depends="init, resolve-default" />
+ <target name="compile" depends="init, resolve-default" />
- <!--
- <target name="jar" depends="compile">
- <copy todir="${build.dir}" verbose="true">
- <fileset dir="./lib" includes="**/*.jar" />
- </copy>
- </target>
- -->
+ <!--
+ <target name="jar" depends="compile">
+ <copy todir="${build.dir}" verbose="true">
+ <fileset dir="./lib" includes="**/*.jar" />
+ </copy>
+ </target>
+ -->
</project>
diff --git a/src/plugin/parse-metatags/build.xml
b/src/plugin/parse-metatags/build.xml
index e30292d92..412ba3f29 100644
--- a/src/plugin/parse-metatags/build.xml
+++ b/src/plugin/parse-metatags/build.xml
@@ -17,21 +17,21 @@
-->
<project name="parse-metatags" default="jar-core">
- <import file="../build-plugin.xml" />
+ <import file="../build-plugin.xml" />
- <!-- Deploy Unit test dependencies -->
- <target name="deps-test">
- <ant target="deploy" inheritall="false"
dir="../nutch-extensionpoints" />
- <ant target="deploy" inheritall="false" dir="../protocol-file"
/>
- </target>
+ <!-- Deploy Unit test dependencies -->
+ <target name="deps-test">
+ <ant target="deploy" inheritall="false" dir="../nutch-extensionpoints" />
+ <ant target="deploy" inheritall="false" dir="../protocol-file" />
+ </target>
- <!-- for junit test -->
- <mkdir dir="${build.test}/data" />
- <copy todir="${build.test}/data">
- <fileset dir="sample">
- <include name="*.html" />
- </fileset>
- </copy>
+ <!-- for junit test -->
+ <mkdir dir="${build.test}/data" />
+ <copy todir="${build.test}/data">
+ <fileset dir="sample">
+ <include name="*.html" />
+ </fileset>
+ </copy>
</project>
diff --git a/src/plugin/parse-tika/sample/nutch.html
b/src/plugin/parse-tika/sample/nutch.html
index ec7a19691..2353896a6 100644
--- a/src/plugin/parse-tika/sample/nutch.html
+++ b/src/plugin/parse-tika/sample/nutch.html
@@ -237,7 +237,7 @@ document.write("Last Published: " + document.lastModified);
</li>
<li>
<a
href="#09+February+2009+-+Lucene+at+ApacheCon+Europe+2009+in%0A%09%09%09Amsterdam">09
February 2009 - Lucene at ApacheCon Europe 2009 in
- Amsterdam</a>
+ Amsterdam</a>
</li>
<li>
<a href="#2+April+2007%3A+Nutch+0.9+Released">2 April 2007: Nutch 0.9
Released</a>
@@ -391,23 +391,23 @@ document.write("Last Published: " +
document.lastModified);
<a href="http://lucene.apache.org/nutch/release/";>here</a>.</p>
<a name="N100A3"></a><a
name="09+February+2009+-+Lucene+at+ApacheCon+Europe+2009+in%0A%09%09%09Amsterdam"></a>
<h3 class="h4">09 February 2009 - Lucene at ApacheCon Europe 2009 in
- Amsterdam</h3>
+ Amsterdam</h3>
<p>
-
+
<a href="http://www.eu.apachecon.com/c/aceu2009/"; title="ApacheCon EU 2009">
- <img alt="ApacheCon EU 2009 Logo"
class="float-right"
src="http://www.eu.apachecon.com/page_attachments/0000/0115/125x125_basic.gif";>
- </a>
+ <img alt="ApacheCon EU 2009 Logo" class="float-right"
src="http://www.eu.apachecon.com/page_attachments/0000/0115/125x125_basic.gif";>
+ </a>
- Lucene will be extremely well represented at
- <a
href="http://www.eu.apachecon.com/c/aceu2009/";>ApacheCon EU 2009</a>
- in Amsterdam, Netherlands this March 23-27, 2009:
- </p>
+ Lucene will be extremely well represented at
+ <a href="http://www.eu.apachecon.com/c/aceu2009/";>ApacheCon EU 2009</a>
+ in Amsterdam, Netherlands this March 23-27, 2009:
+ </p>
<ul>
-
+
<li>
-
+
<a href="http://eu.apachecon.com/c/aceu2009/sessions/197";>Lucene Boot Camp</a>
- - A two day training session, March 23 &
24th</li>
+ - A two day training session, March 23 & 24th</li>
<li>
<a href="http://eu.apachecon.com/c/aceu2009/sessions/201";>Solr Boot Camp</a> -
A one day training session, March 24th</li>
diff --git a/src/plugin/parse-tika/sample/rsstest.rss
b/src/plugin/parse-tika/sample/rsstest.rss
index 8fb241d62..538452639 100644
--- a/src/plugin/parse-tika/sample/rsstest.rss
+++ b/src/plugin/parse-tika/sample/rsstest.rss
@@ -1,19 +1,19 @@
<?xml version="1.0" encoding="ISO-8859-1" ?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
-->
<rss version="0.91">
<channel>
diff --git
a/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/BoilerpipeExtractorRepository.java
b/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/BoilerpipeExtractorRepository.java
index 5d65318b6..1403546af 100644
---
a/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/BoilerpipeExtractorRepository.java
+++
b/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/BoilerpipeExtractorRepository.java
@@ -26,7 +26,7 @@ import de.l3s.boilerpipe.BoilerpipeExtractor;
class BoilerpipeExtractorRepository {
private static final Logger LOG = LoggerFactory
- .getLogger(MethodHandles.lookup().lookupClass());
+ .getLogger(MethodHandles.lookup().lookupClass());
public static final HashMap<String, BoilerpipeExtractor> extractorRepository
= new HashMap<>();
/**
diff --git
a/src/plugin/scoring-depth/src/java/org/apache/nutch/scoring/depth/DepthScoringFilter.java
b/src/plugin/scoring-depth/src/java/org/apache/nutch/scoring/depth/DepthScoringFilter.java
index 561bdca1f..88bdf9a57 100644
---
a/src/plugin/scoring-depth/src/java/org/apache/nutch/scoring/depth/DepthScoringFilter.java
+++
b/src/plugin/scoring-depth/src/java/org/apache/nutch/scoring/depth/DepthScoringFilter.java
@@ -48,7 +48,7 @@ import org.apache.nutch.scoring.ScoringFilterException;
public class DepthScoringFilter extends Configured implements ScoringFilter {
private static final Logger LOG = LoggerFactory
- .getLogger(MethodHandles.lookup().lookupClass());
+ .getLogger(MethodHandles.lookup().lookupClass());
public static final String DEPTH_KEY = "_depth_";
public static final Text DEPTH_KEY_W = new Text(DEPTH_KEY);
diff --git a/src/test/org/apache/nutch/parse/parse-plugin-test.xml
b/src/test/org/apache/nutch/parse/parse-plugin-test.xml
index f6a426776..2eeb48391 100644
--- a/src/test/org/apache/nutch/parse/parse-plugin-test.xml
+++ b/src/test/org/apache/nutch/parse/parse-plugin-test.xml
@@ -26,7 +26,7 @@
<mimeType name="*">
<plugin id="parse-tika" />
</mimeType>
-
+
<!-- test these 4 plugins -->
<mimeType name="text/html">
<!--
@@ -35,11 +35,11 @@
!-->
<plugin id="parse-plugin-that-not-exist"/>
</mimeType>
-
+
<mimeType name="application/x-javascript">
<plugin id="parse-js"/>
</mimeType>
-
+
<mimeType name="text/rss">
<!-- Test that an extension-id can be directly used here -->
<plugin id="org.apache.nutch.parse.rss.RSSParser"/>
@@ -53,6 +53,6 @@
<alias name="parse-rss"
extension-id="org.apache.nutch.parse.rss.RSSParser" />
<alias name="parse-tika"
- extension-id="org.apache.nutch.parse.tika.TikaParser" />
+ extension-id="org.apache.nutch.parse.tika.TikaParser" />
</aliases>
</parse-plugins>
![http://www.eu.apachecon.com/page_attachments/0000/0115/125x125_basic.gif"](http://www.eu.apachecon.com/page_attachments/0000/0115/125x125_basic.gif"){kind=link}