This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 22ee1a233 NUTCH-3130 Address deprecated API usage across Nutch
codebase and build (#967)
22ee1a233 is described below
commit 22ee1a2336aac115319530d0ab569af08b61a0a2
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Tue Sep 22 18:23:20 2026 -0700
NUTCH-3130 Address deprecated API usage across Nutch codebase and build
(#967)
---
.github/workflows/master-build.yml | 41 ++-
ivy/ivy.xml | 6 +
sonar-project.properties | 4 +-
src/java/org/apache/nutch/crawl/CrawlDbReader.java | 5 +-
src/java/org/apache/nutch/crawl/Generator.java | 44 ---
.../org/apache/nutch/fetcher/FetchItemQueues.java | 4 +-
src/java/org/apache/nutch/indexer/IndexWriter.java | 10 -
.../org/apache/nutch/indexer/IndexWriters.java | 1 -
.../nutch/metadata/SpellCheckedMetadata.java | 5 +-
.../nutch/net/protocols/ProtocolException.java | 46 ----
.../org/apache/nutch/plugin/PluginRepository.java | 2 +-
.../org/apache/nutch/protocol/ProtocolStatus.java | 8 +-
.../apache/nutch/protocol/RobotRulesParser.java | 2 +
.../apache/nutch/scoring/webgraph/LinkDumper.java | 25 +-
.../apache/nutch/scoring/webgraph/LinkRank.java | 25 +-
.../apache/nutch/scoring/webgraph/NodeDumper.java | 86 +++---
.../apache/nutch/scoring/webgraph/NodeReader.java | 34 +--
.../nutch/scoring/webgraph/ScoreUpdater.java | 34 +--
.../apache/nutch/scoring/webgraph/WebGraph.java | 34 +--
.../apache/nutch/tools/CommonCrawlDataDumper.java | 118 ++++----
.../nutch/tools/CommonCrawlFormatFactory.java | 32 ---
src/java/org/apache/nutch/tools/FileDumper.java | 65 +++--
src/java/org/apache/nutch/tools/ResolveUrls.java | 34 +--
.../apache/nutch/util/CrawlCompletionStats.java | 55 ++--
src/java/org/apache/nutch/util/NutchJob.java | 6 +-
.../nutch/indexer/geoip/GeoIPDocumentCreator.java | 299 +++++++++++----------
.../cloudsearch/CloudSearchIndexWriter.java | 5 -
.../nutch/indexwriter/csv/CSVIndexWriter.java | 5 -
.../nutch/indexwriter/dummy/DummyIndexWriter.java | 5 -
.../indexwriter/elastic/ElasticIndexWriter.java | 5 -
.../nutch/indexwriter/kafka/KafkaIndexWriter.java | 5 -
.../opensearch1x/OpenSearch1xIndexWriter.java | 5 -
.../indexwriter/rabbit/RabbitIndexWriter.java | 5 -
.../nutch/indexwriter/solr/SolrIndexWriter.java | 5 -
.../protocol/http/api/TestRobotRulesParser.java | 49 ----
.../indexer/filter/MimeTypeIndexingFilter.java | 26 +-
src/plugin/protocol-ftp/plugin.xml | 2 +-
.../java/org/apache/nutch/protocol/ftp/Client.java | 8 +-
.../org/apache/nutch/crawl/CrawlDBTestUtil.java | 3 +
.../org/apache/nutch/crawl/CrawlDbUpdateUtil.java | 3 +
40 files changed, 502 insertions(+), 654 deletions(-)
diff --git a/.github/workflows/master-build.yml
b/.github/workflows/master-build.yml
index 6ceda001b..27d4f49cb 100644
--- a/.github/workflows/master-build.yml
+++ b/.github/workflows/master-build.yml
@@ -264,15 +264,15 @@ jobs:
# run if the build configuration or both 'core' and 'plugins' files were
changed
- name: test all
if: ${{ steps.filter.outputs.buildconf == 'true' || (
steps.filter.outputs.core == 'true' && steps.filter.outputs.plugins == 'true'
) }}
- run: ant clean test forbidden-api-checks -buildfile build.xml
+ run: ant clean test forbidden-api-checks -buildfile build.xml | tee
build.log
# run only if 'core' files were changed
- name: test core
if: ${{ steps.filter.outputs.core == 'true' &&
steps.filter.outputs.plugins == 'false' && steps.filter.outputs.buildconf ==
'false' }}
- run: ant clean test-core forbidden-api-checks -buildfile build.xml
+ run: ant clean test-core forbidden-api-checks -buildfile build.xml |
tee build.log
# run only if 'plugins' files were changed
- name: test plugins
if: ${{ steps.filter.outputs.plugins == 'true' &&
steps.filter.outputs.core == 'false' && steps.filter.outputs.buildconf ==
'false' }}
- run: ant clean test-plugins forbidden-api-checks -buildfile build.xml
+ run: ant clean test-plugins forbidden-api-checks -buildfile build.xml
| tee build.log
# run indexer integration tests when indexer plugin files change (Docker
required, ubuntu-latest only)
- name: test indexer integration
if: ${{ steps.filter.outputs.indexer_plugins == 'true' && matrix.os ==
'ubuntu-latest' }}
@@ -295,6 +295,41 @@ jobs:
else
echo "has_results=false" >> $GITHUB_OUTPUT
fi
+ # Scenario 1: Approved deprecations are allowlisted and do not fail the
build.
+ # Scenario 2: Any other deprecation in the Nutch codebase fails the
build.
+ - name: Check for deprecation warnings
+ # Skip the step entirely when no test compile ran (path filter).
hashFiles
+ # is empty if build.log was never created; the job then shows skipped,
+ # not a green success. always() still runs this after a failed ant test
+ # so deprecation failures are not hidden by an earlier test failure.
+ # JDK 17 only: JDK 21 javac emits java.net.URL constructor (and similar
+ # platform) deprecations that are out of scope for this Nutch API gate.
+ if: always() && matrix.java == '17' && hashFiles('build.log') != ''
+ run: |
+ deprecations=$(grep -iE "warning: \[deprecation\]" build.log || true)
+ if [ -z "$deprecations" ]; then
+ echo ":white_check_mark: No Java deprecation warnings found."
+ exit 0
+ fi
+ # Remaining Nutch-owned deprecations after NUTCH-3130: deferred
+ # finalize() (Plugin, PluginRepository, Ftp) and Hadoop JobContext
+ # DistributedCache stubs in test helpers.
+ APPROVED_FILES=(
+ 'CrawlDBTestUtil\.java'
+ 'CrawlDbUpdateUtil\.java'
+ 'Ftp\.java'
+ 'Plugin\.java'
+ 'PluginRepository\.java'
+ )
+ APPROVED_PATTERN=$(IFS='|'; printf '%s' "${APPROVED_FILES[*]}")
+ unapproved=$(echo "$deprecations" | grep -v -E "$APPROVED_PATTERN"
|| true)
+ if [ -n "$unapproved" ]; then
+ echo ":x: Unapproved Java deprecation warnings detected! Failing
the build."
+ echo "$unapproved"
+ exit 1
+ fi
+ echo ":white_check_mark: Deprecation warnings only in approved
deprecated code."
+ exit 0
- name: Upload Build and Test Artifacts (Binaries)
uses: actions/upload-artifact@v7
if: always() && matrix.os == 'ubuntu-latest' && matrix.java == '17' &&
steps.check_tests.outputs.has_results == 'true'
diff --git a/ivy/ivy.xml b/ivy/ivy.xml
index 6f8b136bf..5317a29d7 100644
--- a/ivy/ivy.xml
+++ b/ivy/ivy.xml
@@ -44,6 +44,7 @@
<dependency org="org.slf4j" name="slf4j-api" rev="2.0.18" conf="*->master"
/>
<dependency org="org.apache.commons" name="commons-lang3" rev="3.20.0"
conf="*->default" />
+ <dependency org="org.apache.commons" name="commons-text" rev="1.10.0"
conf="*->default" />
<dependency org="org.apache.commons" name="commons-collections4"
rev="4.5.0" conf="*->master" />
<dependency org="org.apache.httpcomponents" name="httpclient" rev="4.5.14"
conf="*->master" />
<!-- commons-httpclient is still required -->
@@ -55,6 +56,11 @@
<dependency org="org.apache.commons" name="commons-jexl3" rev="3.6.0"
conf="*->default" />
<dependency org="com.tdunning" name="t-digest" rev="3.3" />
+ <!-- Pin to Hadoop 3.5.0's commons-cli so a newer transitive cannot
+ win at compile time and flag HelpFormatter / Option.Builder.build().
-->
+ <dependency org="commons-cli" name="commons-cli" rev="1.9.0" force="true"
+ conf="*->default" />
+
<!-- Hadoop Dependencies -->
<dependency org="org.apache.hadoop" name="hadoop-common"
rev="${hadoop.version}" conf="*->default">
<exclude org="ch.qos.reload4j" name="*"/>
diff --git a/sonar-project.properties b/sonar-project.properties
index cfa681213..c9879a45e 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -24,10 +24,10 @@ sonar.links.scm=https://github.com/apache/nutch
sonar.links.issue=https://issues.apache.org/jira/projects/NUTCH/issues
sonar.links.ci=https://github.com/apache/nutch/actions
-sonar.sources=src/java,src/plugin,src/bin
+sonar.sources=src/java,src/plugin,src/bin,docker,conf
sonar.tests=src/test,src/plugin
sonar.test.inclusions=**/src/test/**/*.java,**/Test*.java,**/*IT.java
-sonar.exclusions=**/build.xml,**/build-ivy.xml,**/build-plugin.xml,**/ivy.xml,**/plugin.xml,**/sample/**,**/data/**,**/logs/**
+sonar.exclusions=**/build.xml,**/build-ivy.xml,**/build-plugin.xml,**/ivy.xml,**/plugin.xml,**/sample/**,**/data/**,**/logs/**,src/testresources/**,src/java/overview.html
sonar.sourceEncoding=UTF-8
sonar.java.source=17
diff --git a/src/java/org/apache/nutch/crawl/CrawlDbReader.java
b/src/java/org/apache/nutch/crawl/CrawlDbReader.java
index 266c819b8..0b3e29269 100644
--- a/src/java/org/apache/nutch/crawl/CrawlDbReader.java
+++ b/src/java/org/apache/nutch/crawl/CrawlDbReader.java
@@ -79,6 +79,7 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.json.JsonWriteFeature;
import com.fasterxml.jackson.core.util.MinimalPrettyPrinter;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -265,8 +266,8 @@ public class CrawlDbReader extends AbstractChecker
implements Closeable {
public LineRecordWriter(DataOutputStream out) {
this.out = out;
- jsonMapper.getFactory()
- .configure(JsonGenerator.Feature.ESCAPE_NON_ASCII, true);
+ jsonMapper.getFactory().configure(
+ JsonWriteFeature.ESCAPE_NON_ASCII.mappedFeature(), true);
SimpleModule module = new SimpleModule();
module.addSerializer(Writable.class, new WritableSerializer());
jsonMapper.registerModule(module);
diff --git a/src/java/org/apache/nutch/crawl/Generator.java
b/src/java/org/apache/nutch/crawl/Generator.java
index c1767c9ed..d77d8fede 100644
--- a/src/java/org/apache/nutch/crawl/Generator.java
+++ b/src/java/org/apache/nutch/crawl/Generator.java
@@ -797,50 +797,6 @@ public class Generator extends NutchTool implements Tool {
false, 1, null);
}
- /**
- * This is an old signature used for compatibility - does not specify whether
- * or not to normalise and set the number of segments to 1
- *
- * @param dbDir
- * Crawl database directory
- * @param segments
- * Segments directory
- * @param numLists
- * Number of fetch lists (partitions) per segment or number of
- * fetcher map tasks. (One fetch list partition is fetched in one
- * fetcher map task.)
- * @param topN
- * Number of top URLs to be selected
- * @param curTime
- * Current time in milliseconds
- * @param filter
- * whether to apply filtering operation
- * @param force
- * if true, and the target lockfile exists, consider it valid. If
- * false and the target file exists, throw an IOException.
- * @deprecated since 1.19 use
- * {@link #generate(Path, Path, int, long, long, boolean,
boolean, boolean, int, String, String)}
- * or
- * {@link #generate(Path, Path, int, long, long, boolean,
boolean, boolean, int, String)}
- * in the instance that no hostdb is available
- * @throws IOException
- * if an I/O exception occurs.
- * @see LockUtil#createLockFile(Configuration, Path, boolean)
- * @throws InterruptedException
- * if a thread is waiting, sleeping, or otherwise occupied, and the
- * thread is interrupted, either before or during the activity.
- * @throws ClassNotFoundException
- * if runtime class(es) are not available
- * @return Path to generated segment or null if no entries were selected
- **/
- @Deprecated
- public Path[] generate(Path dbDir, Path segments, int numLists, long topN,
- long curTime, boolean filter, boolean force)
- throws IOException, InterruptedException, ClassNotFoundException {
- return generate(dbDir, segments, numLists, topN, curTime, filter, true,
- force, 1, null);
- }
-
/**
* This signature should be used in the instance that no hostdb is available.
* Generate fetchlists in one or more segments. Whether to filter URLs or not
diff --git a/src/java/org/apache/nutch/fetcher/FetchItemQueues.java
b/src/java/org/apache/nutch/fetcher/FetchItemQueues.java
index 8cf9dd6e3..d028fb5f5 100644
--- a/src/java/org/apache/nutch/fetcher/FetchItemQueues.java
+++ b/src/java/org/apache/nutch/fetcher/FetchItemQueues.java
@@ -17,12 +17,12 @@
package org.apache.nutch.fetcher;
import java.lang.invoke.MethodHandles;
+import java.time.Duration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
@@ -101,7 +101,7 @@ public class FetchItemQueues {
if (dedupRedirMaxTime > 0 && dedupRedirMaxSize > 0) {
redirectDedupCache = CacheBuilder.newBuilder()
.maximumSize(dedupRedirMaxSize)
- .expireAfterWrite(dedupRedirMaxTime, TimeUnit.SECONDS).build();
+ .expireAfterWrite(Duration.ofSeconds(dedupRedirMaxTime)).build();
}
}
diff --git a/src/java/org/apache/nutch/indexer/IndexWriter.java
b/src/java/org/apache/nutch/indexer/IndexWriter.java
index 43d4a48c7..6cf4d5d2b 100644
--- a/src/java/org/apache/nutch/indexer/IndexWriter.java
+++ b/src/java/org/apache/nutch/indexer/IndexWriter.java
@@ -17,7 +17,6 @@
package org.apache.nutch.indexer;
import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.plugin.Pluggable;
import java.io.IOException;
@@ -30,15 +29,6 @@ public interface IndexWriter extends Pluggable, Configurable
{
*/
final static String X_POINT_ID = IndexWriter.class.getName();
- /**
- * @param conf Nutch configuration
- * @param name target name of the {@link IndexWriter} to be opened
- * @throws IOException Some exception thrown by some writer.
- * @deprecated use {@link #open(IndexWriterParams)}} instead.
- */
- @Deprecated
- public void open(Configuration conf, String name) throws IOException;
-
/**
* Initializes the internal variables from a given index writer
configuration.
*
diff --git a/src/java/org/apache/nutch/indexer/IndexWriters.java
b/src/java/org/apache/nutch/indexer/IndexWriters.java
index 4b809d1e4..e5cd2771f 100644
--- a/src/java/org/apache/nutch/indexer/IndexWriters.java
+++ b/src/java/org/apache/nutch/indexer/IndexWriters.java
@@ -212,7 +212,6 @@ public class IndexWriters {
public void open(Configuration conf, String name) throws IOException {
for (Map.Entry<String, IndexWriterWrapper> entry : this.indexWriters
.entrySet()) {
- entry.getValue().getIndexWriter().open(conf, name);
entry.getValue().getIndexWriter()
.open(entry.getValue().getIndexWriterConfig().getParams());
}
diff --git a/src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java
b/src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java
index 0547f4f43..70f9e7163 100644
--- a/src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java
+++ b/src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java
@@ -21,7 +21,7 @@ import java.lang.reflect.Modifier;
import java.util.HashMap;
import java.util.Map;
-import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.similarity.LevenshteinDistance;
/**
* A decorator to Metadata that adds spellchecking capabilities to property
@@ -115,7 +115,8 @@ public class SpellCheckedMetadata extends
CaseInsensitiveMetadata {
if ((value == null) && (normalized != null)) {
int threshold = Math.min(3, searched.length() / TRESHOLD_DIVIDER);
for (int i = 0; i < normalized.length && value == null; i++) {
- if (StringUtils.getLevenshteinDistance(searched, normalized[i]) <
threshold) {
+ if (LevenshteinDistance.getDefaultInstance().apply(searched,
+ normalized[i]) < threshold) {
value = NAMES_IDX.get(normalized[i]);
}
}
diff --git a/src/java/org/apache/nutch/net/protocols/ProtocolException.java
b/src/java/org/apache/nutch/net/protocols/ProtocolException.java
deleted file mode 100644
index 97d1f7fe5..000000000
--- a/src/java/org/apache/nutch/net/protocols/ProtocolException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.net.protocols;
-
-import java.io.Serializable;
-
-/**
- * Base exception for all protocol handlers
- *
- * @deprecated Use {@link org.apache.nutch.protocol.ProtocolException} instead.
- */
-@Deprecated
-@SuppressWarnings("serial")
-public class ProtocolException extends Exception implements Serializable {
-
- public ProtocolException() {
- super();
- }
-
- public ProtocolException(String message) {
- super(message);
- }
-
- public ProtocolException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public ProtocolException(Throwable cause) {
- super(cause);
- }
-
-}
diff --git a/src/java/org/apache/nutch/plugin/PluginRepository.java
b/src/java/org/apache/nutch/plugin/PluginRepository.java
index bec062521..6da593592 100644
--- a/src/java/org/apache/nutch/plugin/PluginRepository.java
+++ b/src/java/org/apache/nutch/plugin/PluginRepository.java
@@ -98,7 +98,7 @@ public class PluginRepository implements
URLStreamHandlerFactory {
try {
installExtensions(this.fRegisteredPlugins);
} catch (PluginRuntimeException e) {
- LOG.error("Could not install extensions.", e.toString());
+ LOG.error("Could not install extensions: {}", e.toString());
throw new RuntimeException(e.getMessage());
}
diff --git a/src/java/org/apache/nutch/protocol/ProtocolStatus.java
b/src/java/org/apache/nutch/protocol/ProtocolStatus.java
index 1659fda40..f8785fc0d 100644
--- a/src/java/org/apache/nutch/protocol/ProtocolStatus.java
+++ b/src/java/org/apache/nutch/protocol/ProtocolStatus.java
@@ -67,10 +67,16 @@ public class ProtocolStatus implements Writable {
* Request was refused by protocol plugins, because it would block. The
* expected number of milliseconds to wait before retry may be provided in
* args.
+ * @deprecated unused internally; retained for CrawlDatum / ProtocolStatus
+ * compatibility. Do not introduce new uses.
*/
@Deprecated
public static final int WOULDBLOCK = 22;
- /** Thread was blocked http.max.delays times during fetching. */
+ /**
+ * Thread was blocked http.max.delays times during fetching.
+ * @deprecated unused internally; retained for CrawlDatum / ProtocolStatus
+ * compatibility. Do not introduce new uses.
+ */
@Deprecated
public static final int BLOCKED = 23;
diff --git a/src/java/org/apache/nutch/protocol/RobotRulesParser.java
b/src/java/org/apache/nutch/protocol/RobotRulesParser.java
index 3efa0fc3b..6f3b51384 100644
--- a/src/java/org/apache/nutch/protocol/RobotRulesParser.java
+++ b/src/java/org/apache/nutch/protocol/RobotRulesParser.java
@@ -212,6 +212,8 @@ public abstract class RobotRulesParser implements Tool {
* @param robotName
* A string containing all the robots agent names used by parser for
* matching
+ * @deprecated since 1.15 use {@link #parseRules(String, byte[], String,
java.util.Collection)}
+ * instead.
* @return BaseRobotRules object
*/
@Deprecated
diff --git a/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java
b/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java
index 52211dcde..898495f57 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java
@@ -27,10 +27,9 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.time.StopWatch;
import org.slf4j.Logger;
@@ -417,24 +416,26 @@ public class LinkDumper extends Configured implements
Tool {
public int run(String[] args) throws Exception {
Options options = new Options();
- OptionBuilder.withArgName("help");
- OptionBuilder.withDescription("show this help message");
- Option helpOpts = OptionBuilder.create("help");
+ Option helpOpts = Option.builder("help")
+ .argName("help")
+ .desc("show this help message")
+ .build();
options.addOption(helpOpts);
- OptionBuilder.withArgName("webgraphdb");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("the web graph database to use");
- Option webGraphDbOpts = OptionBuilder.create("webgraphdb");
+ Option webGraphDbOpts = Option.builder("webgraphdb")
+ .argName("webgraphdb")
+ .hasArg()
+ .desc("the web graph database to use")
+ .build();
options.addOption(webGraphDbOpts);
- CommandLineParser parser = new GnuParser();
+ CommandLineParser parser = new DefaultParser();
try {
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || !line.hasOption("webgraphdb")) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("LinkDumper", options);
+ HelpFormatter formatter = HelpFormatter.builder().get();
+ formatter.printHelp("LinkDumper", "", options, "", false);
return -1;
}
diff --git a/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
b/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
index de15e3d66..6c4872f29 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
@@ -31,10 +31,9 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.hadoop.conf.Configuration;
@@ -725,24 +724,26 @@ public class LinkRank extends Configured implements Tool {
public int run(String[] args) throws Exception {
Options options = new Options();
- OptionBuilder.withArgName("help");
- OptionBuilder.withDescription("show this help message");
- Option helpOpts = OptionBuilder.create("help");
+ Option helpOpts = Option.builder("help")
+ .argName("help")
+ .desc("show this help message")
+ .build();
options.addOption(helpOpts);
- OptionBuilder.withArgName("webgraphdb");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("the web graph db to use");
- Option webgraphOpts = OptionBuilder.create("webgraphdb");
+ Option webgraphOpts = Option.builder("webgraphdb")
+ .argName("webgraphdb")
+ .hasArg()
+ .desc("the web graph db to use")
+ .build();
options.addOption(webgraphOpts);
- CommandLineParser parser = new GnuParser();
+ CommandLineParser parser = new DefaultParser();
try {
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || !line.hasOption("webgraphdb")) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("LinkRank", options);
+ HelpFormatter formatter = HelpFormatter.builder().get();
+ formatter.printHelp("LinkRank", "", options, "", false);
return -1;
}
diff --git a/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
b/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
index a8a8e7fa1..e88ef7986 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
@@ -22,10 +22,9 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.time.StopWatch;
import org.slf4j.Logger;
@@ -373,67 +372,76 @@ public class NodeDumper extends Configured implements
Tool {
public int run(String[] args) throws Exception {
Options options = new Options();
- OptionBuilder.withArgName("help");
- OptionBuilder.withDescription("show this help message");
- Option helpOpts = OptionBuilder.create("help");
+ Option helpOpts = Option.builder("help")
+ .argName("help")
+ .desc("show this help message")
+ .build();
options.addOption(helpOpts);
- OptionBuilder.withArgName("webgraphdb");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("the web graph database to use");
- Option webGraphDbOpts = OptionBuilder.create("webgraphdb");
+ Option webGraphDbOpts = Option.builder("webgraphdb")
+ .argName("webgraphdb")
+ .hasArg()
+ .desc("the web graph database to use")
+ .build();
options.addOption(webGraphDbOpts);
- OptionBuilder.withArgName("inlinks");
- OptionBuilder.withDescription("show highest inlinks");
- Option inlinkOpts = OptionBuilder.create("inlinks");
+ Option inlinkOpts = Option.builder("inlinks")
+ .argName("inlinks")
+ .desc("show highest inlinks")
+ .build();
options.addOption(inlinkOpts);
- OptionBuilder.withArgName("outlinks");
- OptionBuilder.withDescription("show highest outlinks");
- Option outlinkOpts = OptionBuilder.create("outlinks");
+ Option outlinkOpts = Option.builder("outlinks")
+ .argName("outlinks")
+ .desc("show highest outlinks")
+ .build();
options.addOption(outlinkOpts);
- OptionBuilder.withArgName("scores");
- OptionBuilder.withDescription("show highest scores");
- Option scoreOpts = OptionBuilder.create("scores");
+ Option scoreOpts = Option.builder("scores")
+ .argName("scores")
+ .desc("show highest scores")
+ .build();
options.addOption(scoreOpts);
- OptionBuilder.withArgName("topn");
- OptionBuilder.hasOptionalArg();
- OptionBuilder.withDescription("show topN scores");
- Option topNOpts = OptionBuilder.create("topn");
+ Option topNOpts = Option.builder("topn")
+ .argName("topn")
+ .optionalArg(true)
+ .desc("show topN scores")
+ .build();
options.addOption(topNOpts);
- OptionBuilder.withArgName("output");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("the output directory to use");
- Option outputOpts = OptionBuilder.create("output");
+ Option outputOpts = Option.builder("output")
+ .argName("output")
+ .hasArg()
+ .desc("the output directory to use")
+ .build();
options.addOption(outputOpts);
- OptionBuilder.withArgName("asEff");
- OptionBuilder
- .withDescription("Solr ExternalFileField compatible output format");
- Option effOpts = OptionBuilder.create("asEff");
+ Option effOpts = Option.builder("asEff")
+ .argName("asEff")
+ .desc("Solr ExternalFileField compatible output format")
+ .build();
options.addOption(effOpts);
- OptionBuilder.hasArgs(2);
- OptionBuilder.withDescription("group <host|domain> <sum|max>");
- Option groupOpts = OptionBuilder.create("group");
+ Option groupOpts = Option.builder("group")
+ .numberOfArgs(2)
+ .desc("group <host|domain> <sum|max>")
+ .build();
options.addOption(groupOpts);
- OptionBuilder.withArgName("asSequenceFile");
- OptionBuilder.withDescription("whether to output as a sequencefile");
- Option sequenceFileOpts = OptionBuilder.create("asSequenceFile");
+ Option sequenceFileOpts = Option.builder("asSequenceFile")
+ .argName("asSequenceFile")
+ .desc("whether to output as a sequencefile")
+ .build();
options.addOption(sequenceFileOpts);
- CommandLineParser parser = new GnuParser();
+ CommandLineParser parser = new DefaultParser();
try {
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || !line.hasOption("webgraphdb")) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("NodeDumper", options);
+ HelpFormatter formatter = HelpFormatter.builder().get();
+ formatter.printHelp("NodeDumper", "", options, "", false);
return -1;
}
diff --git a/src/java/org/apache/nutch/scoring/webgraph/NodeReader.java
b/src/java/org/apache/nutch/scoring/webgraph/NodeReader.java
index d6fd9d05b..855b7d878 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/NodeReader.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/NodeReader.java
@@ -20,10 +20,9 @@ import java.io.IOException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
@@ -91,32 +90,35 @@ public class NodeReader extends Configured {
public static void main(String[] args) throws Exception {
Options options = new Options();
- OptionBuilder.withArgName("help");
- OptionBuilder.withDescription("show this help message");
- Option helpOpts = OptionBuilder.create("help");
+ Option helpOpts = Option.builder("help")
+ .argName("help")
+ .desc("show this help message")
+ .build();
options.addOption(helpOpts);
- OptionBuilder.withArgName("webgraphdb");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("the webgraphdb to use");
- Option webGraphOpts = OptionBuilder.create("webgraphdb");
+ Option webGraphOpts = Option.builder("webgraphdb")
+ .argName("webgraphdb")
+ .hasArg()
+ .desc("the webgraphdb to use")
+ .build();
options.addOption(webGraphOpts);
- OptionBuilder.withArgName("url");
- OptionBuilder.hasOptionalArg();
- OptionBuilder.withDescription("the url to dump");
- Option urlOpts = OptionBuilder.create("url");
+ Option urlOpts = Option.builder("url")
+ .argName("url")
+ .optionalArg(true)
+ .desc("the url to dump")
+ .build();
options.addOption(urlOpts);
- CommandLineParser parser = new GnuParser();
+ CommandLineParser parser = new DefaultParser();
try {
// command line must take a webgraphdb and a url
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || !line.hasOption("webgraphdb")
|| !line.hasOption("url")) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("WebGraphReader", options);
+ HelpFormatter formatter = HelpFormatter.builder().get();
+ formatter.printHelp("WebGraphReader", "", options, "", false);
return;
}
diff --git a/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
b/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
index a595d4bf3..89d61055a 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
@@ -23,10 +23,9 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.time.StopWatch;
import org.slf4j.Logger;
@@ -229,31 +228,34 @@ public class ScoreUpdater extends Configured implements
Tool{
public int run(String[] args) throws Exception {
Options options = new Options();
- OptionBuilder.withArgName("help");
- OptionBuilder.withDescription("show this help message");
- Option helpOpts = OptionBuilder.create("help");
+ Option helpOpts = Option.builder("help")
+ .argName("help")
+ .desc("show this help message")
+ .build();
options.addOption(helpOpts);
- OptionBuilder.withArgName("crawldb");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("the crawldb to use");
- Option crawlDbOpts = OptionBuilder.create("crawldb");
+ Option crawlDbOpts = Option.builder("crawldb")
+ .argName("crawldb")
+ .hasArg()
+ .desc("the crawldb to use")
+ .build();
options.addOption(crawlDbOpts);
- OptionBuilder.withArgName("webgraphdb");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("the webgraphdb to use");
- Option webGraphOpts = OptionBuilder.create("webgraphdb");
+ Option webGraphOpts = Option.builder("webgraphdb")
+ .argName("webgraphdb")
+ .hasArg()
+ .desc("the webgraphdb to use")
+ .build();
options.addOption(webGraphOpts);
- CommandLineParser parser = new GnuParser();
+ CommandLineParser parser = new DefaultParser();
try {
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || !line.hasOption("webgraphdb")
|| !line.hasOption("crawldb")) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("ScoreUpdater", options);
+ HelpFormatter formatter = HelpFormatter.builder().get();
+ formatter.printHelp("ScoreUpdater", "", options, "", false);
return -1;
}
diff --git a/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
b/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
index fee0921d0..f3999848e 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
@@ -29,10 +29,9 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.time.StopWatch;
import org.slf4j.Logger;
@@ -751,19 +750,20 @@ public class WebGraph extends Configured implements Tool {
"whether to use URLFilters on the URL's in the segment");
// argument options
- @SuppressWarnings("static-access")
- Option graphOpt = OptionBuilder
- .withArgName("webgraphdb")
+ Option graphOpt = Option.builder("webgraphdb")
+ .argName("webgraphdb")
.hasArg()
- .withDescription(
- "the web graph database to create (if none exists) or use if one
does")
- .create("webgraphdb");
- @SuppressWarnings("static-access")
- Option segOpt = OptionBuilder.withArgName("segment").hasArgs()
- .withDescription("the segment(s) to use").create("segment");
- @SuppressWarnings("static-access")
- Option segDirOpt = OptionBuilder.withArgName("segmentDir").hasArgs()
- .withDescription("the segment directory to use").create("segmentDir");
+ .desc(
+ "the web graph database to create (if none exists) or use if one does")
+ .build();
+ Option segOpt = Option.builder("segment")
+ .argName("segment").hasArgs()
+ .desc("the segment(s) to use")
+ .build();
+ Option segDirOpt = Option.builder("segmentDir")
+ .argName("segmentDir").hasArgs()
+ .desc("the segment directory to use")
+ .build();
// create the options
Options options = new Options();
@@ -774,13 +774,13 @@ public class WebGraph extends Configured implements Tool {
options.addOption(segOpt);
options.addOption(segDirOpt);
- CommandLineParser parser = new GnuParser();
+ CommandLineParser parser = new DefaultParser();
try {
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || !line.hasOption("webgraphdb")
|| (!line.hasOption("segment") && !line.hasOption("segmentDir"))) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("WebGraph", options, true);
+ HelpFormatter formatter = HelpFormatter.builder().get();
+ formatter.printHelp("WebGraph", "", options, "", true);
return -1;
}
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
index e97fac03b..2bba97918 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
@@ -40,10 +40,9 @@ import java.util.regex.Pattern;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
@@ -574,64 +573,65 @@ public class CommonCrawlDataDumper extends NutchTool
implements Tool {
public int run(String[] args) throws Exception {
Option helpOpt = new Option("h", "help", false, "show this help message.");
// argument options
- @SuppressWarnings("static-access")
- Option outputOpt = OptionBuilder.withArgName("outputDir").hasArg()
- .withDescription(
- "output directory (which will be created) to host the CBOR data.")
- .create("outputDir");
+ Option outputOpt = Option.builder("outputDir")
+ .argName("outputDir").hasArg()
+ .desc(
+ "output directory (which will be created) to host the CBOR data.")
+ .build();
// WARC format
Option warcOpt = new Option("warc", "export to a WARC file");
- @SuppressWarnings("static-access")
- Option segOpt = OptionBuilder.withArgName("segment").hasArgs()
- .withDescription("the segment or directory containing segments to
use").create("segment");
+ Option segOpt = Option.builder("segment")
+ .argName("segment").hasArgs()
+ .desc("the segment or directory containing segments to use")
+ .build();
// create mimetype and gzip options
- @SuppressWarnings("static-access")
- Option mimeOpt = OptionBuilder.isRequired(false).withArgName("mimetype")
- .hasArgs().withDescription(
- "an optional list of mimetypes to dump, excluding all others.
Defaults to all.")
- .create("mimetype");
- @SuppressWarnings("static-access")
- Option gzipOpt = OptionBuilder.withArgName("gzip").hasArg(false)
- .withDescription(
- "an optional flag indicating whether to additionally gzip the
data.")
- .create("gzip");
- @SuppressWarnings("static-access")
- Option keyPrefixOpt = OptionBuilder.withArgName("keyPrefix").hasArg(true)
- .withDescription("an optional prefix for key in the output format.")
- .create("keyPrefix");
- @SuppressWarnings("static-access")
- Option simpleDateFormatOpt = OptionBuilder.withArgName("SimpleDateFormat")
- .hasArg(false).withDescription(
- "an optional format for timestamp in GMT epoch milliseconds.")
- .create("SimpleDateFormat");
- @SuppressWarnings("static-access")
- Option epochFilenameOpt = OptionBuilder.withArgName("epochFilename")
+ Option mimeOpt = Option.builder("mimetype")
+ .required(false).argName("mimetype")
+ .hasArgs().desc(
+ "an optional list of mimetypes to dump, excluding all others. Defaults
to all.")
+ .build();
+ Option gzipOpt = Option.builder("gzip")
+ .argName("gzip").hasArg(false)
+ .desc(
+ "an optional flag indicating whether to additionally gzip the data.")
+ .build();
+ Option keyPrefixOpt = Option.builder("keyPrefix")
+ .argName("keyPrefix").hasArg(true)
+ .desc("an optional prefix for key in the output format.")
+ .build();
+ Option simpleDateFormatOpt = Option.builder("SimpleDateFormat")
+ .argName("SimpleDateFormat")
+ .hasArg(false).desc(
+ "an optional format for timestamp in GMT epoch milliseconds.")
+ .build();
+ Option epochFilenameOpt = Option.builder("epochFilename")
+ .argName("epochFilename")
.hasArg(false)
- .withDescription("an optional format for output filename.")
- .create("epochFilename");
- @SuppressWarnings("static-access")
- Option jsonArrayOpt = OptionBuilder.withArgName("jsonArray").hasArg(false)
- .withDescription("an optional format for JSON output.")
- .create("jsonArray");
- @SuppressWarnings("static-access")
- Option reverseKeyOpt =
OptionBuilder.withArgName("reverseKey").hasArg(false)
- .withDescription("an optional format for key value in JSON output.")
- .create("reverseKey");
- @SuppressWarnings("static-access")
- Option extensionOpt = OptionBuilder.withArgName("extension").hasArg(true)
- .withDescription("an optional file extension for output documents.")
- .create("extension");
- @SuppressWarnings("static-access")
- Option sizeOpt = OptionBuilder.withArgName("warcSize").hasArg(true)
- .withType(Number.class)
- .withDescription("an optional file size in bytes for the WARC file(s)")
- .create("warcSize");
- @SuppressWarnings("static-access")
- Option linkDbOpt = OptionBuilder.withArgName("linkdb").hasArg(true)
- .withDescription("an optional linkdb parameter to include inlinks in
dump files")
- .isRequired(false)
- .create("linkdb");
+ .desc("an optional format for output filename.")
+ .build();
+ Option jsonArrayOpt = Option.builder("jsonArray")
+ .argName("jsonArray").hasArg(false)
+ .desc("an optional format for JSON output.")
+ .build();
+ Option reverseKeyOpt = Option.builder("reverseKey")
+ .argName("reverseKey").hasArg(false)
+ .desc("an optional format for key value in JSON output.")
+ .build();
+ Option extensionOpt = Option.builder("extension")
+ .argName("extension").hasArg(true)
+ .desc("an optional file extension for output documents.")
+ .build();
+ Option sizeOpt = Option.builder("warcSize")
+ .argName("warcSize").hasArg(true)
+ .type(Number.class)
+ .desc("an optional file size in bytes for the WARC file(s)")
+ .build();
+ Option linkDbOpt = Option.builder("linkdb")
+ .argName("linkdb").hasArg(true)
+ .desc("an optional linkdb parameter to include inlinks in dump files")
+ .required(false)
+ .build();
// create the options
Options options = new Options();
@@ -653,14 +653,14 @@ public class CommonCrawlDataDumper extends NutchTool
implements Tool {
options.addOption(sizeOpt);
options.addOption(linkDbOpt);
- CommandLineParser parser = new GnuParser();
+ CommandLineParser parser = new DefaultParser();
try {
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || !line.hasOption("outputDir") || (!line
.hasOption("segment"))) {
- HelpFormatter formatter = new HelpFormatter();
- formatter
- .printHelp(CommonCrawlDataDumper.class.getName(), options, true);
+ HelpFormatter formatter = HelpFormatter.builder().get();
+ formatter.printHelp(CommonCrawlDataDumper.class.getName(), "", options,
+ "", true);
return 0;
}
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlFormatFactory.java
b/src/java/org/apache/nutch/tools/CommonCrawlFormatFactory.java
index e46853261..78239a854 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlFormatFactory.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlFormatFactory.java
@@ -19,8 +19,6 @@ package org.apache.nutch.tools;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.metadata.Metadata;
-import org.apache.nutch.protocol.Content;
/**
* Factory class that creates new {@link
org.apache.nutch.tools.CommonCrawlFormat CommonCrawlFormat} objects (a.k.a.
formatter) that map crawled files to CommonCrawl format.
@@ -28,36 +26,6 @@ import org.apache.nutch.protocol.Content;
*/
public class CommonCrawlFormatFactory {
- /**
- * Returns a new instance of a {@link
org.apache.nutch.tools.CommonCrawlFormat CommonCrawlFormat} object specifying
the type of formatter.
- * @param formatType the type of formatter to be created.
- * @param url the url.
- * @param content the content.
- * @param metadata the metadata.
- * @param nutchConf the configuration.
- * @param config the CommonCrawl output configuration.
- * @return the new {@link org.apache.nutch.tools.CommonCrawlFormat
CommonCrawlFormat} object.
- * @throws IOException If any I/O error occurs.
- * @deprecated
- */
- public static CommonCrawlFormat getCommonCrawlFormat(String formatType,
String url, Content content, Metadata metadata, Configuration nutchConf,
CommonCrawlConfig config) throws IOException {
- if (formatType == null) {
- return null;
- }
-
- if (formatType.equalsIgnoreCase("jackson")) {
- return new CommonCrawlFormatJackson(url, content,
metadata, nutchConf, config);
- }
- else if (formatType.equalsIgnoreCase("jettinson")) {
- return new CommonCrawlFormatJettinson(url, content,
metadata, nutchConf, config);
- }
- else if (formatType.equalsIgnoreCase("simple")) {
- return new CommonCrawlFormatSimple(url, content,
metadata, nutchConf, config);
- }
-
- return null;
- }
-
// The format should not depend on variable attributes, essentially this
// should be one for the full job
public static CommonCrawlFormat getCommonCrawlFormat(String formatType,
Configuration nutchConf, CommonCrawlConfig config) throws IOException {
diff --git a/src/java/org/apache/nutch/tools/FileDumper.java
b/src/java/org/apache/nutch/tools/FileDumper.java
index 35d096351..88ef2bece 100644
--- a/src/java/org/apache/nutch/tools/FileDumper.java
+++ b/src/java/org/apache/nutch/tools/FileDumper.java
@@ -28,10 +28,9 @@ import java.util.Map;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FilenameUtils;
@@ -307,41 +306,37 @@ public class FileDumper {
// boolean options
Option helpOpt = new Option("h", "help", false, "show this help message");
// argument options
- @SuppressWarnings("static-access")
- Option outputOpt = OptionBuilder
- .withArgName("outputDir")
- .hasArg()
- .withDescription(
+ Option outputOpt = Option.builder("outputDir")
+ .argName("outputDir")
+ .hasArg()
+ .desc(
"output directory (which will be created) to host the raw data")
- .create("outputDir");
- @SuppressWarnings("static-access")
- Option segOpt = OptionBuilder.withArgName("segment").hasArgs()
- .withDescription("the segment(s) to use").create("segment");
- @SuppressWarnings("static-access")
- Option mimeOpt = OptionBuilder
- .withArgName("mimetype")
- .hasArgs()
- .withDescription(
+ .build();
+ Option segOpt = Option.builder("segment")
+ .argName("segment").hasArgs()
+ .desc("the segment(s) to use")
+ .build();
+ Option mimeOpt = Option.builder("mimetype")
+ .argName("mimetype")
+ .hasArgs()
+ .desc(
"an optional list of mimetypes to dump, excluding all others. Defaults
to all.")
- .create("mimetype");
- @SuppressWarnings("static-access")
- Option mimeStat = OptionBuilder
- .withArgName("mimeStats")
- .withDescription(
+ .build();
+ Option mimeStat = Option.builder("mimeStats")
+ .argName("mimeStats")
+ .desc(
"only display mimetype stats for the segment(s) instead of dumping
file.")
- .create("mimeStats");
- @SuppressWarnings("static-access")
- Option dirStructureOpt = OptionBuilder
- .withArgName("flatdir")
- .withDescription(
+ .build();
+ Option dirStructureOpt = Option.builder("flatdir")
+ .argName("flatdir")
+ .desc(
"optionally specify that the output directory should only contain
files.")
- .create("flatdir");
- @SuppressWarnings("static-access")
- Option reverseURLOutput = OptionBuilder
- .withArgName("reverseUrlDirs")
- .withDescription(
+ .build();
+ Option reverseURLOutput = Option.builder("reverseUrlDirs")
+ .argName("reverseUrlDirs")
+ .desc(
"optionally specify to use reverse URL folders for output structure.")
- .create("reverseUrlDirs");
+ .build();
// create the options
Options options = new Options();
@@ -353,13 +348,13 @@ public class FileDumper {
options.addOption(dirStructureOpt);
options.addOption(reverseURLOutput);
- CommandLineParser parser = new GnuParser();
+ CommandLineParser parser = new DefaultParser();
try {
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || !line.hasOption("outputDir")
|| (!line.hasOption("segment"))) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("FileDumper", options, true);
+ HelpFormatter formatter = HelpFormatter.builder().get();
+ formatter.printHelp("FileDumper", "", options, "", true);
return;
}
diff --git a/src/java/org/apache/nutch/tools/ResolveUrls.java
b/src/java/org/apache/nutch/tools/ResolveUrls.java
index ddfa6cf21..78f4ec3e0 100644
--- a/src/java/org/apache/nutch/tools/ResolveUrls.java
+++ b/src/java/org/apache/nutch/tools/ResolveUrls.java
@@ -30,10 +30,9 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.hadoop.util.StringUtils;
import org.apache.nutch.util.URLUtil;
@@ -167,30 +166,33 @@ public class ResolveUrls {
public static void main(String[] args) {
Options options = new Options();
- OptionBuilder.withArgName("help");
- OptionBuilder.withDescription("show this help message");
- Option helpOpts = OptionBuilder.create("help");
+ Option helpOpts = Option.builder("help")
+ .argName("help")
+ .desc("show this help message")
+ .build();
options.addOption(helpOpts);
- OptionBuilder.withArgName("urls");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("the urls file to check");
- Option urlOpts = OptionBuilder.create("urls");
+ Option urlOpts = Option.builder("urls")
+ .argName("urls")
+ .hasArg()
+ .desc("the urls file to check")
+ .build();
options.addOption(urlOpts);
- OptionBuilder.withArgName("numThreads");
- OptionBuilder.hasArgs();
- OptionBuilder.withDescription("the number of threads to use");
- Option numThreadOpts = OptionBuilder.create("numThreads");
+ Option numThreadOpts = Option.builder("numThreads")
+ .argName("numThreads")
+ .hasArgs()
+ .desc("the number of threads to use")
+ .build();
options.addOption(numThreadOpts);
- CommandLineParser parser = new GnuParser();
+ CommandLineParser parser = new DefaultParser();
try {
// parse out common line arguments
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || !line.hasOption("urls")) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("ResolveUrls", options);
+ HelpFormatter formatter = HelpFormatter.builder().get();
+ formatter.printHelp("ResolveUrls", "", options, "", false);
return;
}
diff --git a/src/java/org/apache/nutch/util/CrawlCompletionStats.java
b/src/java/org/apache/nutch/util/CrawlCompletionStats.java
index d806b9295..46d1505b6 100644
--- a/src/java/org/apache/nutch/util/CrawlCompletionStats.java
+++ b/src/java/org/apache/nutch/util/CrawlCompletionStats.java
@@ -24,11 +24,10 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.MissingOptionException;
import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.hadoop.conf.Configuration;
@@ -68,33 +67,29 @@ public class CrawlCompletionStats extends Configured
implements Tool {
@Override
public int run(String[] args) throws Exception {
Option helpOpt = new Option("h", "help", false, "Show this message");
- @SuppressWarnings("static-access")
- Option inDirs = OptionBuilder
- .withArgName("inputDirs")
- .isRequired()
- .withDescription("Comma separated list of crawldb directories (e.g.,
\"./crawl1/crawldb,./crawl2/crawldb\")")
+ Option inDirs = Option.builder("inputDirs")
+ .argName("inputDirs")
+ .required()
+ .desc("Comma separated list of crawldb directories (e.g.,
\"./crawl1/crawldb,./crawl2/crawldb\")")
.hasArgs()
- .create("inputDirs");
- @SuppressWarnings("static-access")
- Option outDir = OptionBuilder
- .withArgName("outputDir")
- .isRequired()
- .withDescription("Output directory where results should be dumped")
+ .build();
+ Option outDir = Option.builder("outputDir")
+ .argName("outputDir")
+ .required()
+ .desc("Output directory where results should be dumped")
.hasArgs()
- .create("outputDir");
- @SuppressWarnings("static-access")
- Option modeOpt = OptionBuilder
- .withArgName("mode")
- .isRequired()
- .withDescription("Set statistics gathering mode (by 'host' or by
'domain')")
+ .build();
+ Option modeOpt = Option.builder("mode")
+ .argName("mode")
+ .required()
+ .desc("Set statistics gathering mode (by 'host' or by 'domain')")
.hasArgs()
- .create("mode");
- @SuppressWarnings("static-access")
- Option numReducers = OptionBuilder
- .withArgName("numReducers")
- .withDescription("Optional number of reduce jobs to use. Defaults to
1")
+ .build();
+ Option numReducers = Option.builder("numReducers")
+ .argName("numReducers")
+ .desc("Optional number of reduce jobs to use. Defaults to 1")
.hasArgs()
- .create("numReducers");
+ .build();
Options options = new Options();
options.addOption(helpOpt);
@@ -103,20 +98,20 @@ public class CrawlCompletionStats extends Configured
implements Tool {
options.addOption(modeOpt);
options.addOption(numReducers);
- CommandLineParser parser = new GnuParser();
+ CommandLineParser parser = new DefaultParser();
CommandLine cli;
try {
cli = parser.parse(options, args);
} catch (MissingOptionException e) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("CrawlCompletionStats", options, true);
+ HelpFormatter formatter = HelpFormatter.builder().get();
+ formatter.printHelp("CrawlCompletionStats", "", options, "", true);
return 1;
}
if (cli.hasOption("help")) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("CrawlCompletionStats", options, true);
+ HelpFormatter formatter = HelpFormatter.builder().get();
+ formatter.printHelp("CrawlCompletionStats", "", options, "", true);
return 1;
}
diff --git a/src/java/org/apache/nutch/util/NutchJob.java
b/src/java/org/apache/nutch/util/NutchJob.java
index abf5343f4..5b7af5d7e 100644
--- a/src/java/org/apache/nutch/util/NutchJob.java
+++ b/src/java/org/apache/nutch/util/NutchJob.java
@@ -37,8 +37,10 @@ public class NutchJob extends Job {
private static final String JOB_FAILURE_LOG_FORMAT = "%s job did not
succeed, job id: %s, job status: %s, reason: %s";
/**
- * @deprecated, use instead {@link #getInstance(Configuration)} or
- * {@link Job#getInstance(Configuration, String)}.
+ * @deprecated Use {@link Job#getInstance(Configuration)} or
+ * {@link Job#getInstance(Configuration, String)} instead. This
+ * constructor still initializes {@link PluginRepository} so
+ * custom URL stream handlers are registered.
*
* @param conf
* configuration for the job
diff --git
a/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java
b/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java
index e7da55f95..92ca1f4fb 100644
---
a/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java
+++
b/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java
@@ -99,6 +99,15 @@ public class GeoIPDocumentCreator {
}
}
+ /**
+ * geoip2 5.x {@code ipAddress()} returns {@link InetAddress}; city/insights
+ * paths store the lookup IP as a String. Normalize so addIfNotDuplicate
works
+ * across databases.
+ */
+ private static String ipString(InetAddress address) {
+ return address == null ? null : address.getHostAddress();
+ }
+
/**
* Populate a {@link org.apache.nutch.indexer.NutchDocument} based on lookup
* of IP in Anonymous IP database.
@@ -114,8 +123,8 @@ public class GeoIPDocumentCreator {
Optional<AnonymousIpResponse> opt =
reader.tryAnonymousIp(InetAddress.getByName(serverIp));
if (opt.isPresent()) {
AnonymousIpResponse response = opt.get();
- addIfNotDuplicate(doc, "ip", response.getIpAddress());
- addIfNotNull(doc, ANONYMOUS_NETWORK_ADDRESS,
response.getNetwork().toString());
+ addIfNotDuplicate(doc, "ip", ipString(response.ipAddress()));
+ addIfNotNull(doc, ANONYMOUS_NETWORK_ADDRESS,
response.network().toString());
addIfNotNull(doc, "isAnonymous", response.isAnonymous());
addIfNotNull(doc, "isAnonymousVpn", response.isAnonymousVpn());
addIfNotNull(doc, "isHostingProxy", response.isHostingProvider());
@@ -143,10 +152,10 @@ public class GeoIPDocumentCreator {
Optional<AsnResponse> opt = reader.tryAsn(InetAddress.getByName(serverIp));
if (opt.isPresent()) {
AsnResponse response = opt.get();
- addIfNotDuplicate(doc, "ip", response.getIpAddress());
- addIfNotNull(doc, ASN_NETWORK_ADDRESS, response.getNetwork().toString());
- addIfNotNull(doc, "autonomousSystemNumber",
response.getAutonomousSystemNumber());
- addIfNotNull(doc, "autonomousSystemOrganization",
response.getAutonomousSystemOrganization());
+ addIfNotDuplicate(doc, "ip", ipString(response.ipAddress()));
+ addIfNotNull(doc, ASN_NETWORK_ADDRESS, response.network().toString());
+ addIfNotNull(doc, "autonomousSystemNumber",
response.autonomousSystemNumber());
+ addIfNotNull(doc, "autonomousSystemOrganization",
response.autonomousSystemOrganization());
} else {
LOG.debug("'{}' IP address not found in ASN DB.", serverIp);
}
@@ -176,67 +185,67 @@ public class GeoIPDocumentCreator {
}
private static NutchDocument processCityDocument(NutchDocument doc,
CityResponse response) {
- City city = response.getCity();
- addIfNotNull(doc, "cityName", city.getName());
- addIfNotNull(doc, "cityConfidence", city.getConfidence());
- addIfNotNull(doc, "cityGeoNameId", city.getGeoNameId());
-
- Continent continent = response.getContinent();
- addIfNotNull(doc, "continentCode", continent.getCode());
- addIfNotNull(doc, "continentGeoNameId", continent.getGeoNameId());
- addIfNotNull(doc, "continentName", continent.getName());
-
- Country country = response.getRegisteredCountry();
- addIfNotNull(doc, "countryIsoCode", country.getIsoCode());
- addIfNotNull(doc, "countryName", country.getName());
- addIfNotNull(doc, "countryConfidence", country.getConfidence());
- addIfNotNull(doc, "countryGeoNameId", country.getGeoNameId());
+ City city = response.city();
+ addIfNotNull(doc, "cityName", city.name());
+ addIfNotNull(doc, "cityConfidence", city.confidence());
+ addIfNotNull(doc, "cityGeoNameId", city.geonameId());
+
+ Continent continent = response.continent();
+ addIfNotNull(doc, "continentCode", continent.code());
+ addIfNotNull(doc, "continentGeoNameId", continent.geonameId());
+ addIfNotNull(doc, "continentName", continent.name());
+
+ Country country = response.registeredCountry();
+ addIfNotNull(doc, "countryIsoCode", country.isoCode());
+ addIfNotNull(doc, "countryName", country.name());
+ addIfNotNull(doc, "countryConfidence", country.confidence());
+ addIfNotNull(doc, "countryGeoNameId", country.geonameId());
addIfNotNull(doc, "countryInEuropeanUnion", country.isInEuropeanUnion());
- Location location = response.getLocation();
- if (location.getLatitude() != null && location.getLongitude() != null) {
- addIfNotNull(doc, "latLon", location.getLatitude() + "," +
location.getLongitude());
+ Location location = response.location();
+ if (location.latitude() != null && location.longitude() != null) {
+ addIfNotNull(doc, "latLon", location.latitude() + "," +
location.longitude());
}
- addIfNotNull(doc, "accuracyRadius", location.getAccuracyRadius());
- addIfNotNull(doc, "timeZone", location.getTimeZone());
- addIfNotNull(doc, "populationDensity", location.getPopulationDensity());
-
- Postal postal = response.getPostal();
- addIfNotNull(doc, "postalCode", postal.getCode());
- addIfNotNull(doc, "postalConfidence", postal.getConfidence());
-
- RepresentedCountry rCountry = response.getRepresentedCountry();
- addIfNotNull(doc, "countryType", rCountry.getType());
-
- Subdivision mostSubdivision = response.getMostSpecificSubdivision();
- addIfNotNull(doc, "mostSpecificSubDivName", mostSubdivision.getName());
- addIfNotNull(doc, "mostSpecificSubDivIsoCode",
mostSubdivision.getIsoCode());
- addIfNotNull(doc, "mostSpecificSubDivConfidence",
mostSubdivision.getConfidence());
- addIfNotNull(doc, "mostSpecificSubDivGeoNameId",
mostSubdivision.getGeoNameId());
-
- Subdivision leastSubdivision = response.getLeastSpecificSubdivision();
- addIfNotNull(doc, "leastSpecificSubDivName", leastSubdivision.getName());
- addIfNotNull(doc, "leastSpecificSubDivIsoCode",
leastSubdivision.getIsoCode());
- addIfNotNull(doc, "leastSpecificSubDivConfidence",
leastSubdivision.getConfidence());
- addIfNotNull(doc, "leastSpecificSubDivGeoNameId",
leastSubdivision.getGeoNameId());
-
- Traits traits = response.getTraits();
- addIfNotNull(doc, "autonomousSystemNumber",
traits.getAutonomousSystemNumber());
- addIfNotNull(doc, "autonomousSystemOrganization",
traits.getAutonomousSystemOrganization());
- if (traits.getConnectionType() != null) {
- addIfNotNull(doc, "connectionType",
traits.getConnectionType().toString());
+ addIfNotNull(doc, "accuracyRadius", location.accuracyRadius());
+ addIfNotNull(doc, "timeZone", location.timeZone());
+ addIfNotNull(doc, "populationDensity", location.populationDensity());
+
+ Postal postal = response.postal();
+ addIfNotNull(doc, "postalCode", postal.code());
+ addIfNotNull(doc, "postalConfidence", postal.confidence());
+
+ RepresentedCountry rCountry = response.representedCountry();
+ addIfNotNull(doc, "countryType", rCountry.type());
+
+ Subdivision mostSubdivision = response.mostSpecificSubdivision();
+ addIfNotNull(doc, "mostSpecificSubDivName", mostSubdivision.name());
+ addIfNotNull(doc, "mostSpecificSubDivIsoCode", mostSubdivision.isoCode());
+ addIfNotNull(doc, "mostSpecificSubDivConfidence",
mostSubdivision.confidence());
+ addIfNotNull(doc, "mostSpecificSubDivGeoNameId",
mostSubdivision.geonameId());
+
+ Subdivision leastSubdivision = response.leastSpecificSubdivision();
+ addIfNotNull(doc, "leastSpecificSubDivName", leastSubdivision.name());
+ addIfNotNull(doc, "leastSpecificSubDivIsoCode",
leastSubdivision.isoCode());
+ addIfNotNull(doc, "leastSpecificSubDivConfidence",
leastSubdivision.confidence());
+ addIfNotNull(doc, "leastSpecificSubDivGeoNameId",
leastSubdivision.geonameId());
+
+ Traits traits = response.traits();
+ addIfNotNull(doc, "autonomousSystemNumber",
traits.autonomousSystemNumber());
+ addIfNotNull(doc, "autonomousSystemOrganization",
traits.autonomousSystemOrganization());
+ if (traits.connectionType() != null) {
+ addIfNotNull(doc, "connectionType", traits.connectionType().toString());
}
- addIfNotNull(doc, "domain", traits.getDomain());
- addIfNotNull(doc, "isp", traits.getIsp());
- addIfNotNull(doc, "mobileCountryCode", traits.getMobileCountryCode());
- addIfNotNull(doc, "mobileNetworkCode", traits.getMobileNetworkCode());
- if (traits.getNetwork() != null) {
- addIfNotNull(doc, CITY_NETWORK_ADDRESS, traits.getNetwork().toString());
+ addIfNotNull(doc, "domain", traits.domain());
+ addIfNotNull(doc, "isp", traits.isp());
+ addIfNotNull(doc, "mobileCountryCode", traits.mobileCountryCode());
+ addIfNotNull(doc, "mobileNetworkCode", traits.mobileNetworkCode());
+ if (traits.network() != null) {
+ addIfNotNull(doc, CITY_NETWORK_ADDRESS, traits.network().toString());
}
- addIfNotNull(doc, "organization", traits.getOrganization());
- addIfNotNull(doc, "staticIpScore", traits.getStaticIpScore());
- addIfNotNull(doc, "userCount", traits.getUserCount());
- addIfNotNull(doc, "userType", traits.getUserType());
+ addIfNotNull(doc, "organization", traits.organization());
+ addIfNotNull(doc, "staticIpScore", traits.staticIpScore());
+ addIfNotNull(doc, "userCount", traits.userCount());
+ addIfNotNull(doc, "userType", traits.userType());
addIfNotNull(doc, "isAnonymous", traits.isAnonymous());
addIfNotNull(doc, "isAnonymousVpn", traits.isAnonymousVpn());
addIfNotNull(doc, "isAnycast", traits.isAnycast());
@@ -264,12 +273,12 @@ public class GeoIPDocumentCreator {
.getByName(serverIp));
if (opt.isPresent()) {
ConnectionTypeResponse response = opt.get();
- addIfNotDuplicate(doc, "ip", response.getIpAddress());
- if (response.getConnectionType() != null) {
- addIfNotNull(doc, "connectionType",
response.getConnectionType().toString());
+ addIfNotDuplicate(doc, "ip", ipString(response.ipAddress()));
+ if (response.connectionType() != null) {
+ addIfNotNull(doc, "connectionType",
response.connectionType().toString());
}
- if (response.getNetwork() != null) {
- addIfNotNull(doc, CONNECTION_NETWORK_ADDRESS,
response.getNetwork().toString());
+ if (response.network() != null) {
+ addIfNotNull(doc, CONNECTION_NETWORK_ADDRESS,
response.network().toString());
}
} else {
LOG.debug("'{}' IP address not found in Connection DB.", serverIp);
@@ -295,24 +304,24 @@ public class GeoIPDocumentCreator {
CountryResponse response = opt.get();
addIfNotDuplicate(doc, "ip", serverIp);
- Continent continent = response.getContinent();
- addIfNotDuplicate(doc, "continentCode", continent.getCode());
- addIfNotDuplicate(doc, "continentGeoNameId", continent.getGeoNameId());
- addIfNotDuplicate(doc, "continentName", continent.getName());
+ Continent continent = response.continent();
+ addIfNotDuplicate(doc, "continentCode", continent.code());
+ addIfNotDuplicate(doc, "continentGeoNameId", continent.geonameId());
+ addIfNotDuplicate(doc, "continentName", continent.name());
- Country country = response.getRegisteredCountry();
- addIfNotDuplicate(doc, "countryIsoCode", country.getIsoCode());
- addIfNotDuplicate(doc, "countryName", country.getName());
- addIfNotDuplicate(doc, "countryConfidence", country.getConfidence());
- addIfNotDuplicate(doc, "countryGeoNameId", country.getGeoNameId());
+ Country country = response.registeredCountry();
+ addIfNotDuplicate(doc, "countryIsoCode", country.isoCode());
+ addIfNotDuplicate(doc, "countryName", country.name());
+ addIfNotDuplicate(doc, "countryConfidence", country.confidence());
+ addIfNotDuplicate(doc, "countryGeoNameId", country.geonameId());
addIfNotDuplicate(doc, "countryInEuropeanUnion",
country.isInEuropeanUnion());
- RepresentedCountry rCountry = response.getRepresentedCountry();
- addIfNotDuplicate(doc, "countryType", rCountry.getType());
+ RepresentedCountry rCountry = response.representedCountry();
+ addIfNotDuplicate(doc, "countryType", rCountry.type());
- Traits traits = response.getTraits();
- if (traits.getNetwork() != null) {
- addIfNotNull(doc, COUNTRY_NETWORK_ADDRESS,
traits.getNetwork().toString());
+ Traits traits = response.traits();
+ if (traits.network() != null) {
+ addIfNotNull(doc, COUNTRY_NETWORK_ADDRESS,
traits.network().toString());
}
addIfNotDuplicate(doc, "isAnonymous", traits.isAnonymous());
addIfNotDuplicate(doc, "isAnonymousVpn", traits.isAnonymousVpn());
@@ -343,9 +352,9 @@ public class GeoIPDocumentCreator {
Optional<DomainResponse> opt =
reader.tryDomain(InetAddress.getByName(serverIp));
if (opt.isPresent()) {
DomainResponse response = opt.get();
- addIfNotDuplicate(doc, "ip", response.getIpAddress());
- addIfNotNull(doc, "domain", response.getDomain());
- addIfNotNull(doc, DOMAIN_NETWORK_ADDRESS,
response.getNetwork().toString());
+ addIfNotDuplicate(doc, "ip", ipString(response.ipAddress()));
+ addIfNotNull(doc, "domain", response.domain());
+ addIfNotNull(doc, DOMAIN_NETWORK_ADDRESS, response.network().toString());
} else {
LOG.debug("'{}' IP address not found in Domain DB.", serverIp);
}
@@ -369,67 +378,67 @@ public class GeoIPDocumentCreator {
}
private static NutchDocument processInsightsDocument(NutchDocument doc,
InsightsResponse response) {
- City city = response.getCity();
- addIfNotNull(doc, "cityName", city.getName());
- addIfNotNull(doc, "cityConfidence", city.getConfidence());
- addIfNotNull(doc, "cityGeoNameId", city.getGeoNameId());
-
- Continent continent = response.getContinent();
- addIfNotNull(doc, "continentCode", continent.getCode());
- addIfNotNull(doc, "continentGeoNameId", continent.getGeoNameId());
- addIfNotNull(doc, "continentName", continent.getName());
-
- Country country = response.getRegisteredCountry();
- addIfNotNull(doc, "countryIsoCode", country.getIsoCode());
- addIfNotNull(doc, "countryName", country.getName());
- addIfNotNull(doc, "countryConfidence", country.getConfidence());
- addIfNotNull(doc, "countryGeoNameId", country.getGeoNameId());
+ City city = response.city();
+ addIfNotNull(doc, "cityName", city.name());
+ addIfNotNull(doc, "cityConfidence", city.confidence());
+ addIfNotNull(doc, "cityGeoNameId", city.geonameId());
+
+ Continent continent = response.continent();
+ addIfNotNull(doc, "continentCode", continent.code());
+ addIfNotNull(doc, "continentGeoNameId", continent.geonameId());
+ addIfNotNull(doc, "continentName", continent.name());
+
+ Country country = response.registeredCountry();
+ addIfNotNull(doc, "countryIsoCode", country.isoCode());
+ addIfNotNull(doc, "countryName", country.name());
+ addIfNotNull(doc, "countryConfidence", country.confidence());
+ addIfNotNull(doc, "countryGeoNameId", country.geonameId());
addIfNotNull(doc, "countryInEuropeanUnion", country.isInEuropeanUnion());
- Location location = response.getLocation();
- if (location.getLatitude() != null && location.getLongitude() != null) {
- addIfNotNull(doc, "latLon", location.getLatitude() + "," +
location.getLongitude());
+ Location location = response.location();
+ if (location.latitude() != null && location.longitude() != null) {
+ addIfNotNull(doc, "latLon", location.latitude() + "," +
location.longitude());
}
- addIfNotNull(doc, "accuracyRadius", location.getAccuracyRadius());
- addIfNotNull(doc, "timeZone", location.getTimeZone());
- addIfNotNull(doc, "populationDensity", location.getPopulationDensity());
-
- Postal postal = response.getPostal();
- addIfNotNull(doc, "postalCode", postal.getCode());
- addIfNotNull(doc, "postalConfidence", postal.getConfidence());
-
- RepresentedCountry rCountry = response.getRepresentedCountry();
- addIfNotNull(doc, "countryType", rCountry.getType());
-
- Subdivision mostSubdivision = response.getMostSpecificSubdivision();
- addIfNotNull(doc, "mostSpecificSubDivName", mostSubdivision.getName());
- addIfNotNull(doc, "mostSpecificSubDivIsoCode",
mostSubdivision.getIsoCode());
- addIfNotNull(doc, "mostSpecificSubDivConfidence",
mostSubdivision.getConfidence());
- addIfNotNull(doc, "mostSpecificSubDivGeoNameId",
mostSubdivision.getGeoNameId());
-
- Subdivision leastSubdivision = response.getLeastSpecificSubdivision();
- addIfNotNull(doc, "leastSpecificSubDivName", leastSubdivision.getName());
- addIfNotNull(doc, "leastSpecificSubDivIsoCode",
leastSubdivision.getIsoCode());
- addIfNotNull(doc, "leastSpecificSubDivConfidence",
leastSubdivision.getConfidence());
- addIfNotNull(doc, "leastSpecificSubDivGeoNameId",
leastSubdivision.getGeoNameId());
-
- Traits traits = response.getTraits();
- addIfNotNull(doc, "autonomousSystemNumber",
traits.getAutonomousSystemNumber());
- addIfNotNull(doc, "autonomousSystemOrganization",
traits.getAutonomousSystemOrganization());
- if (traits.getConnectionType() != null) {
- addIfNotNull(doc, "connectionType",
traits.getConnectionType().toString());
+ addIfNotNull(doc, "accuracyRadius", location.accuracyRadius());
+ addIfNotNull(doc, "timeZone", location.timeZone());
+ addIfNotNull(doc, "populationDensity", location.populationDensity());
+
+ Postal postal = response.postal();
+ addIfNotNull(doc, "postalCode", postal.code());
+ addIfNotNull(doc, "postalConfidence", postal.confidence());
+
+ RepresentedCountry rCountry = response.representedCountry();
+ addIfNotNull(doc, "countryType", rCountry.type());
+
+ Subdivision mostSubdivision = response.mostSpecificSubdivision();
+ addIfNotNull(doc, "mostSpecificSubDivName", mostSubdivision.name());
+ addIfNotNull(doc, "mostSpecificSubDivIsoCode", mostSubdivision.isoCode());
+ addIfNotNull(doc, "mostSpecificSubDivConfidence",
mostSubdivision.confidence());
+ addIfNotNull(doc, "mostSpecificSubDivGeoNameId",
mostSubdivision.geonameId());
+
+ Subdivision leastSubdivision = response.leastSpecificSubdivision();
+ addIfNotNull(doc, "leastSpecificSubDivName", leastSubdivision.name());
+ addIfNotNull(doc, "leastSpecificSubDivIsoCode",
leastSubdivision.isoCode());
+ addIfNotNull(doc, "leastSpecificSubDivConfidence",
leastSubdivision.confidence());
+ addIfNotNull(doc, "leastSpecificSubDivGeoNameId",
leastSubdivision.geonameId());
+
+ Traits traits = response.traits();
+ addIfNotNull(doc, "autonomousSystemNumber",
traits.autonomousSystemNumber());
+ addIfNotNull(doc, "autonomousSystemOrganization",
traits.autonomousSystemOrganization());
+ if (traits.connectionType() != null) {
+ addIfNotNull(doc, "connectionType", traits.connectionType().toString());
}
- addIfNotNull(doc, "domain", traits.getDomain());
- addIfNotNull(doc, "isp", traits.getIsp());
- addIfNotNull(doc, "mobileCountryCode", traits.getMobileCountryCode());
- addIfNotNull(doc, "mobileNetworkCode", traits.getMobileNetworkCode());
- if (traits.getNetwork() != null) {
- addIfNotNull(doc, INSIGHTS_NETWORK_ADDRESS,
traits.getNetwork().toString());
+ addIfNotNull(doc, "domain", traits.domain());
+ addIfNotNull(doc, "isp", traits.isp());
+ addIfNotNull(doc, "mobileCountryCode", traits.mobileCountryCode());
+ addIfNotNull(doc, "mobileNetworkCode", traits.mobileNetworkCode());
+ if (traits.network() != null) {
+ addIfNotNull(doc, INSIGHTS_NETWORK_ADDRESS, traits.network().toString());
}
- addIfNotNull(doc, "organization", traits.getOrganization());
- addIfNotNull(doc, "staticIpScore", traits.getStaticIpScore());
- addIfNotNull(doc, "userCount", traits.getUserCount());
- addIfNotNull(doc, "userType", traits.getUserType());
+ addIfNotNull(doc, "organization", traits.organization());
+ addIfNotNull(doc, "staticIpScore", traits.staticIpScore());
+ addIfNotNull(doc, "userCount", traits.userCount());
+ addIfNotNull(doc, "userType", traits.userType());
addIfNotNull(doc, "isAnonymous", traits.isAnonymous());
addIfNotNull(doc, "isAnonymousVpn", traits.isAnonymousVpn());
addIfNotNull(doc, "isAnycast", traits.isAnycast());
@@ -456,11 +465,11 @@ public class GeoIPDocumentCreator {
Optional<IspResponse> opt = reader.tryIsp(InetAddress.getByName(serverIp));
if (opt.isPresent()) {
IspResponse response = opt.get();
- addIfNotDuplicate(doc, "ip", response.getIpAddress());
- addIfNotNull(doc, "autonSystemNum",
response.getAutonomousSystemNumber());
- addIfNotNull(doc, "autonSystemOrg",
response.getAutonomousSystemOrganization());
- addIfNotNull(doc, "isp", response.getIsp());
- addIfNotNull(doc, "org", response.getOrganization());
+ addIfNotDuplicate(doc, "ip", ipString(response.ipAddress()));
+ addIfNotNull(doc, "autonSystemNum", response.autonomousSystemNumber());
+ addIfNotNull(doc, "autonSystemOrg",
response.autonomousSystemOrganization());
+ addIfNotNull(doc, "isp", response.isp());
+ addIfNotNull(doc, "org", response.organization());
} else {
LOG.debug("'{}' IP address not found in ISP DB.", serverIp);
}
diff --git
a/src/plugin/indexer-cloudsearch/src/java/org/apache/nutch/indexwriter/cloudsearch/CloudSearchIndexWriter.java
b/src/plugin/indexer-cloudsearch/src/java/org/apache/nutch/indexwriter/cloudsearch/CloudSearchIndexWriter.java
index 57ea13d8d..d4bf6584b 100644
---
a/src/plugin/indexer-cloudsearch/src/java/org/apache/nutch/indexwriter/cloudsearch/CloudSearchIndexWriter.java
+++
b/src/plugin/indexer-cloudsearch/src/java/org/apache/nutch/indexwriter/cloudsearch/CloudSearchIndexWriter.java
@@ -88,11 +88,6 @@ public class CloudSearchIndexWriter implements IndexWriter {
private String endpoint;
private String regionName;
- @Override
- public void open(Configuration conf, String name) throws IOException {
- //Implementation not required
- }
-
@Override
public void open(IndexWriterParams parameters) throws IOException {
// LOG.debug("CloudSearchIndexWriter.open() name={} ", name);
diff --git
a/src/plugin/indexer-csv/src/java/org/apache/nutch/indexwriter/csv/CSVIndexWriter.java
b/src/plugin/indexer-csv/src/java/org/apache/nutch/indexwriter/csv/CSVIndexWriter.java
index 765986e21..e0747a906 100644
---
a/src/plugin/indexer-csv/src/java/org/apache/nutch/indexwriter/csv/CSVIndexWriter.java
+++
b/src/plugin/indexer-csv/src/java/org/apache/nutch/indexwriter/csv/CSVIndexWriter.java
@@ -193,11 +193,6 @@ public class CSVIndexWriter implements IndexWriter {
private Path csvLocalOutFile;
- @Override
- public void open(Configuration conf, String name) throws IOException {
-
- }
-
/**
* Initializes the internal variables from a given index writer
configuration.
*
diff --git
a/src/plugin/indexer-dummy/src/java/org/apache/nutch/indexwriter/dummy/DummyIndexWriter.java
b/src/plugin/indexer-dummy/src/java/org/apache/nutch/indexwriter/dummy/DummyIndexWriter.java
index 0c8b8618b..a1a3f6cb8 100644
---
a/src/plugin/indexer-dummy/src/java/org/apache/nutch/indexwriter/dummy/DummyIndexWriter.java
+++
b/src/plugin/indexer-dummy/src/java/org/apache/nutch/indexwriter/dummy/DummyIndexWriter.java
@@ -50,11 +50,6 @@ public class DummyIndexWriter implements IndexWriter {
private boolean delete = false;
private String path;
- @Override
- public void open(Configuration conf, String name) throws IOException {
- //Implementation not required
- }
-
/**
* Initializes the internal variables from a given index writer
configuration.
*
diff --git
a/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
b/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
index 84978dbc2..0c8450e57 100644
---
a/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
+++
b/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
@@ -101,11 +101,6 @@ public class ElasticIndexWriter implements IndexWriter {
private Configuration config;
- @Override
- public void open(Configuration conf, String name) throws IOException {
- // Implementation not required
- }
-
/**
* Initializes the internal variables from a given index writer
configuration.
*
diff --git
a/src/plugin/indexer-kafka/src/java/org/apache/nutch/indexwriter/kafka/KafkaIndexWriter.java
b/src/plugin/indexer-kafka/src/java/org/apache/nutch/indexwriter/kafka/KafkaIndexWriter.java
index 2fcf6de87..5930add0b 100644
---
a/src/plugin/indexer-kafka/src/java/org/apache/nutch/indexwriter/kafka/KafkaIndexWriter.java
+++
b/src/plugin/indexer-kafka/src/java/org/apache/nutch/indexwriter/kafka/KafkaIndexWriter.java
@@ -66,11 +66,6 @@ public class KafkaIndexWriter implements IndexWriter {
private List<ProducerRecord<String, JsonNode>> inputDocs = null;
- @Override
- public void open(Configuration job, String name) throws IOException {
- //Implementation not required
- }
-
@Override
public void open(IndexWriterParams params) throws IOException {
host = params.get(KafkaConstants.HOST);
diff --git
a/src/plugin/indexer-opensearch-1x/src/java/org/apache/nutch/indexwriter/opensearch1x/OpenSearch1xIndexWriter.java
b/src/plugin/indexer-opensearch-1x/src/java/org/apache/nutch/indexwriter/opensearch1x/OpenSearch1xIndexWriter.java
index a51004ebc..62a2c8ccb 100644
---
a/src/plugin/indexer-opensearch-1x/src/java/org/apache/nutch/indexwriter/opensearch1x/OpenSearch1xIndexWriter.java
+++
b/src/plugin/indexer-opensearch-1x/src/java/org/apache/nutch/indexwriter/opensearch1x/OpenSearch1xIndexWriter.java
@@ -110,11 +110,6 @@ public class OpenSearch1xIndexWriter implements
IndexWriter {
private Configuration config;
- @Override
- public void open(Configuration conf, String name) throws IOException {
- // Implementation not required
- }
-
/**
* Initializes the internal variables from a given index writer
* configuration.
diff --git
a/src/plugin/indexer-rabbit/src/java/org/apache/nutch/indexwriter/rabbit/RabbitIndexWriter.java
b/src/plugin/indexer-rabbit/src/java/org/apache/nutch/indexwriter/rabbit/RabbitIndexWriter.java
index ddd79ece8..95918a2d1 100644
---
a/src/plugin/indexer-rabbit/src/java/org/apache/nutch/indexwriter/rabbit/RabbitIndexWriter.java
+++
b/src/plugin/indexer-rabbit/src/java/org/apache/nutch/indexwriter/rabbit/RabbitIndexWriter.java
@@ -76,11 +76,6 @@ public class RabbitIndexWriter implements IndexWriter {
config = conf;
}
- @Override
- public void open(Configuration conf, String name) throws IOException {
- //Implementation not required
- }
-
/**
* Initializes the internal variables from a given index writer
configuration.
*
diff --git
a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
index bd2a51824..09ec93acb 100644
---
a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
+++
b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
@@ -75,11 +75,6 @@ public class SolrIndexWriter implements IndexWriter {
private String authHeaderName;
private String authHeaderValue;
- @Override
- public void open(Configuration conf, String name) {
- // Implementation not required
- }
-
/**
* Initializes the internal variables from a given index writer
configuration.
*
diff --git
a/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
b/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
index c3bf16037..930179381 100644
---
a/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
+++
b/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
@@ -156,53 +156,4 @@ public class TestRobotRulesParser {
assertTrue((rules.getCrawlDelay() == Long.MIN_VALUE),
"testing crawl delay for agent " + UNKNOWN_AGENT + " : ");
}
-
- /**
- * Test that the robots rules are interpreted correctly by the robots rules
- * parser.
- */
- @Deprecated
- @Test
- public void testRobotsAgentDeprecatedAPIMethod() {
- rules = parser.parseRules("testRobotsAgent", ROBOTS_STRING.getBytes(UTF_8),
- CONTENT_TYPE, SINGLE_AGENT1);
- testRulesOnPaths(SINGLE_AGENT1, TEST_PATHS, RESULTS_AGENT1);
-
- rules = parser.parseRules("testRobotsAgent", ROBOTS_STRING.getBytes(UTF_8),
- CONTENT_TYPE, SINGLE_AGENT2);
- testRulesOnPaths(SINGLE_AGENT2, TEST_PATHS, RESULTS_AGENT2);
-
- rules = parser.parseRules("testRobotsAgent", ROBOTS_STRING.getBytes(UTF_8),
- CONTENT_TYPE, MULTIPLE_AGENTS);
- testRulesOnPaths(MULTIPLE_AGENTS, TEST_PATHS, RESULTS_AGENT1_AND_AGENT2);
- }
-
- /**
- * Test that the crawl delay is extracted from the robots file for respective
- * agent. If its not specified for a given agent, default value must be
- * returned.
- */
- @Deprecated
- @Test
- public void testCrawlDelayDeprecatedAPIMethod() {
- // for SINGLE_AGENT1, the crawl delay of 10 seconds, i.e. 10000 msec must
be
- // returned by the parser
- rules = parser.parseRules("testCrawlDelay", ROBOTS_STRING.getBytes(UTF_8),
- CONTENT_TYPE, SINGLE_AGENT1);
- assertTrue((rules.getCrawlDelay() == 10000),
- "testing crawl delay for agent " + SINGLE_AGENT1 + " : ");
-
- // for SINGLE_AGENT2, the crawl delay of 20 seconds, i.e. 20000 msec must
be
- // returned by the parser
- rules = parser.parseRules("testCrawlDelay", ROBOTS_STRING.getBytes(UTF_8),
- CONTENT_TYPE, SINGLE_AGENT2);
- assertTrue((rules.getCrawlDelay() == 20000),
- "testing crawl delay for agent " + SINGLE_AGENT2 + " : ");
-
- // for UNKNOWN_AGENT, the default crawl delay must be returned.
- rules = parser.parseRules("testCrawlDelay", ROBOTS_STRING.getBytes(UTF_8),
- CONTENT_TYPE, UNKNOWN_AGENT);
- assertTrue((rules.getCrawlDelay() == Long.MIN_VALUE),
- "testing crawl delay for agent " + UNKNOWN_AGENT + " : ");
- }
}
diff --git
a/src/plugin/mimetype-filter/src/java/org/apache/nutch/indexer/filter/MimeTypeIndexingFilter.java
b/src/plugin/mimetype-filter/src/java/org/apache/nutch/indexer/filter/MimeTypeIndexingFilter.java
index 8ce1129c1..b646193a0 100644
---
a/src/plugin/mimetype-filter/src/java/org/apache/nutch/indexer/filter/MimeTypeIndexingFilter.java
+++
b/src/plugin/mimetype-filter/src/java/org/apache/nutch/indexer/filter/MimeTypeIndexingFilter.java
@@ -27,10 +27,9 @@ import java.util.List;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.UnrecognizedOptionException;
import org.apache.hadoop.conf.Configuration;
@@ -192,34 +191,31 @@ public class MimeTypeIndexingFilter implements
IndexingFilter {
*/
public static void main(String[] args) throws IOException, IndexingException
{
Option helpOpt = new Option("h", "help", false, "show this help message");
- @SuppressWarnings("static-access")
- Option rulesOpt = OptionBuilder.withArgName("file").hasArg()
- .withDescription(
- "Rules file to be used in the tests relative to the conf
directory")
- .isRequired().create("rules");
+ Option rulesOpt = Option.builder("rules")
+ .argName("file").hasArg()
+ .desc(
+ "Rules file to be used in the tests relative to the conf directory")
+ .required()
+ .build();
Options options = new Options();
options.addOption(helpOpt).addOption(rulesOpt);
- CommandLineParser parser = new GnuParser();
- HelpFormatter formatter = new HelpFormatter();
+ CommandLineParser parser = new DefaultParser();
+ HelpFormatter formatter = HelpFormatter.builder().get();
String rulesFile;
try {
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || !line.hasOption("rules")) {
- formatter
-
.printHelp("org.apache.nutch.indexer.filter.MimeTypeIndexingFilter",
- options, true);
+
formatter.printHelp("org.apache.nutch.indexer.filter.MimeTypeIndexingFilter",
"", options, "", true);
return;
}
rulesFile = line.getOptionValue("rules");
} catch (UnrecognizedOptionException e) {
- formatter
- .printHelp("org.apache.nutch.indexer.filter.MimeTypeIndexingFilter",
- options, true);
+
formatter.printHelp("org.apache.nutch.indexer.filter.MimeTypeIndexingFilter",
"", options, "", true);
return;
} catch (Exception e) {
LOG.error(StringUtils.stringifyException(e));
diff --git a/src/plugin/protocol-ftp/plugin.xml
b/src/plugin/protocol-ftp/plugin.xml
index 1421e379a..1a4012c49 100644
--- a/src/plugin/protocol-ftp/plugin.xml
+++ b/src/plugin/protocol-ftp/plugin.xml
@@ -25,7 +25,7 @@
<library name="protocol-ftp.jar">
<export name="*"/>
</library>
- <library name="commons-net-1.2.0-dev.jar"/>
+ <library name="commons-net-3.9.0.jar"/>
</runtime>
<requires>
diff --git
a/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
b/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
index f83b805de..e698fcc70 100644
--- a/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
+++ b/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
@@ -28,7 +28,7 @@ import java.util.List;
import org.apache.commons.net.MalformedServerReplyException;
import org.apache.commons.net.ftp.FTP;
-import org.apache.commons.net.ftp.FTPCommand;
+import org.apache.commons.net.ftp.FTPCmd;
import org.apache.commons.net.ftp.FTPConnectionClosedException;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPFileEntryParser;
@@ -140,7 +140,7 @@ public class Client extends FTP {
* @throws FtpExceptionCanNotHaveDataConnection can occur if there is a
* malformed server reply
*/
- protected Socket __openPassiveDataConnection(int command, String arg)
+ protected Socket __openPassiveDataConnection(FTPCmd command, String arg)
throws IOException, FtpExceptionCanNotHaveDataConnection {
Socket socket;
@@ -330,7 +330,7 @@ public class Client extends FTP {
FTPFileEntryParser parser) throws IOException,
FtpExceptionCanNotHaveDataConnection, FtpExceptionUnknownForcedDataClose,
FtpExceptionControlClosedByForcedDataClose {
- Socket socket = __openPassiveDataConnection(FTPCommand.LIST, path);
+ Socket socket = __openPassiveDataConnection(FTPCmd.LIST, path);
if (socket == null)
throw new FtpExceptionCanNotHaveDataConnection("LIST "
@@ -411,7 +411,7 @@ public class Client extends FTP {
FtpExceptionUnknownForcedDataClose,
FtpExceptionControlClosedByForcedDataClose {
- Socket socket = __openPassiveDataConnection(FTPCommand.RETR, path);
+ Socket socket = __openPassiveDataConnection(FTPCmd.RETR, path);
if (socket == null)
throw new FtpExceptionCanNotHaveDataConnection("RETR "
diff --git a/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java
b/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java
index df8655ee2..fc67ec268 100644
--- a/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java
+++ b/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java
@@ -271,12 +271,14 @@ public class CrawlDBTestUtil {
@Override
@Deprecated
+ // Implements deprecated Hadoop DistributedCache API; required by
JobContext.
public Path[] getLocalCacheArchives() throws IOException {
return null;
}
@Override
@Deprecated
+ // Implements deprecated Hadoop DistributedCache API; required by
JobContext.
public Path[] getLocalCacheFiles() throws IOException {
return null;
}
@@ -358,6 +360,7 @@ public class CrawlDBTestUtil {
@Override
@Deprecated
+ // Implements deprecated Hadoop DistributedCache API; required by
JobContext.
public boolean getSymlink() {
return false;
}
diff --git a/src/test/org/apache/nutch/crawl/CrawlDbUpdateUtil.java
b/src/test/org/apache/nutch/crawl/CrawlDbUpdateUtil.java
index aca818c21..0555e8367 100644
--- a/src/test/org/apache/nutch/crawl/CrawlDbUpdateUtil.java
+++ b/src/test/org/apache/nutch/crawl/CrawlDbUpdateUtil.java
@@ -230,12 +230,14 @@ public class CrawlDbUpdateUtil <T extends Reducer<Text,
CrawlDatum, Text, CrawlD
@Override
@Deprecated
+ // Implements deprecated Hadoop DistributedCache API; required by
JobContext.
public Path[] getLocalCacheArchives() throws IOException {
return null;
}
@Override
@Deprecated
+ // Implements deprecated Hadoop DistributedCache API; required by
JobContext.
public Path[] getLocalCacheFiles() throws IOException {
return null;
}
@@ -317,6 +319,7 @@ public class CrawlDbUpdateUtil <T extends Reducer<Text,
CrawlDatum, Text, CrawlD
@Override
@Deprecated
+ // Implements deprecated Hadoop DistributedCache API; required by
JobContext.
public boolean getSymlink() {
return false;
}
