This is an automated email from the ASF dual-hosted git repository.
sebastian-nagel pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
from ca8538abf Bump actions/cache from 4 to 5 (#936)
add 32cee5212 NUTCH-1807 Avoid methods relying on system-specific default
locale / charset
add c180c0d29 NUTCH-1807 Avoid methods relying on system-specific default
locale / charset
add c77975cad NUTCH-1807 Avoid methods relying on system-specific default
locale / charset
add 226fa5a30 NUTCH-1807 Avoid methods relying on system-specific default
locale / charset
add e356e5c02 Remove unused test configuration file.
add c451051b9 NUTCH-1807 Avoid methods relying on system-specific default
locale / charset
add 9068999c0 NUTCH-1807 Avoid methods relying on system-specific default
locale / charset
add 04dc40e78 NUTCH-1807 Avoid methods relying on system-specific default
locale / charset
add f2c7555d0 NUTCH-1807 Avoid methods relying on system-specific default
locale / charset
add e807f54d6 NUTCH-1807 Avoid methods relying on system-specific default
locale / charset
add 4319278a4 NUTCH-1807 Avoid methods relying on system-specific default
locale / charset
add b845812d3 Address Yetus' comments: - remove tabs in Java code (added
lines) - remove @author annotations in change context
new 10b9a56e6 Merge pull request #924 from
sebastian-nagel/NUTCH-1807-forbidden-apis
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/master-build.yml | 6 +-
.yetus/blanks-tabs.txt | 3 +
build.xml | 50 ++++++-
default.properties | 3 +
.../apache/nutch/crawl/AdaptiveFetchSchedule.java | 45 +++----
src/java/org/apache/nutch/crawl/CrawlDbReader.java | 45 ++++---
.../org/apache/nutch/crawl/DeduplicationJob.java | 14 +-
src/java/org/apache/nutch/crawl/Generator.java | 62 ++++-----
src/java/org/apache/nutch/crawl/LinkDb.java | 23 ++--
src/java/org/apache/nutch/crawl/MD5Signature.java | 6 +-
.../apache/nutch/crawl/TextProfileSignature.java | 7 +-
src/java/org/apache/nutch/fetcher/FetchItem.java | 4 +-
src/java/org/apache/nutch/fetcher/Fetcher.java | 6 +-
.../org/apache/nutch/fetcher/FetcherThread.java | 59 ++++----
src/java/org/apache/nutch/hostdb/HostDatum.java | 6 +-
.../org/apache/nutch/indexer/IndexerMapReduce.java | 50 +++----
.../org/apache/nutch/parse/ParseOutputFormat.java | 85 ++++++------
.../org/apache/nutch/protocol/ProtocolFactory.java | 48 ++++---
.../apache/nutch/protocol/RobotRulesParser.java | 32 ++---
.../apache/nutch/scoring/webgraph/LinkRank.java | 53 ++++----
.../nutch/segment/ContentAsTextInputFormat.java | 10 +-
.../org/apache/nutch/segment/SegmentReader.java | 20 +--
.../nutch/tools/AbstractCommonCrawlFormat.java | 4 +-
.../apache/nutch/tools/CommonCrawlDataDumper.java | 15 +--
.../nutch/tools/CommonCrawlFormatJackson.java | 33 ++---
.../apache/nutch/tools/CommonCrawlFormatWARC.java | 9 +-
src/java/org/apache/nutch/tools/DmozParser.java | 40 +++---
src/java/org/apache/nutch/tools/FileDumper.java | 50 ++++---
src/java/org/apache/nutch/tools/ResolveUrls.java | 13 +-
.../org/apache/nutch/tools/ShowProperties.java | 22 +--
.../apache/nutch/tools/arc/ArcRecordReader.java | 43 +++---
.../apache/nutch/tools/arc/ArcSegmentCreator.java | 44 +++---
.../org/apache/nutch/tools/warc/WARCExporter.java | 27 ++--
.../org/apache/nutch/util/AbstractChecker.java | 16 +--
src/java/org/apache/nutch/util/DumpFileUtil.java | 47 ++++---
.../org/apache/nutch/util/EncodingDetector.java | 25 ++--
src/java/org/apache/nutch/util/NutchJob.java | 18 +--
.../org/apache/nutch/util/PrefixStringMatcher.java | 4 +-
.../org/apache/nutch/util/SitemapProcessor.java | 12 +-
src/java/org/apache/nutch/util/TimingUtil.java | 17 ++-
src/java/org/apache/nutch/util/URLUtil.java | 52 +++----
.../org/apache/nutch/parse/feed/FeedParser.java | 44 +++---
.../parse/headings/TestHeadingsParseFilter.java | 19 ++-
.../nutch/indexer/anchor/AnchorIndexingFilter.java | 7 +-
.../nutch/indexer/links/LinksIndexingFilter.java | 22 +--
.../cloudsearch/CloudSearchIndexWriter.java | 7 +-
.../nutch/indexwriter/csv/CSVIndexWriter.java | 12 +-
.../nutch/indexwriter/dummy/DummyIndexWriter.java | 7 +-
.../nutch/indexwriter/rabbit/RabbitDocument.java | 7 +-
.../indexwriter/rabbit/RabbitIndexWriter.java | 20 ++-
.../nutch/indexwriter/rabbit/RabbitMessage.java | 7 +-
.../nutch/analysis/lang/HTMLLanguageParser.java | 7 +-
.../analysis/lang/TestHTMLLanguageParser.java | 11 +-
.../apache/nutch/protocol/http/api/HttpBase.java | 29 ++--
.../protocol/http/api/HttpRobotRulesParser.java | 23 ++--
.../protocol/http/api/TestRobotRulesParser.java | 38 +++---
.../nutch/urlfilter/api/RegexURLFilterBase.java | 58 ++++----
.../urlfilter/api/RegexURLFilterBaseTest.java | 28 ++--
.../nutch/microformats/reltag/RelTagParser.java | 23 ++--
.../indexer/filter/MimeTypeIndexingFilter.java | 47 +++----
.../indexer/filter/MimeTypeIndexingFilterTest.java | 13 +-
.../java/org/apache/nutch/parse/ext/ExtParser.java | 46 +++----
.../apache/nutch/parse/html/DOMContentUtils.java | 55 ++++----
.../apache/nutch/parse/html/HTMLMetaProcessor.java | 16 ++-
.../nutch/parse/html/TestDOMContentUtils.java | 22 +--
.../nutch/parse/html/TestRobotsMetaProcessor.java | 34 ++---
.../org/apache/nutch/parse/js/JSParseFilter.java | 27 ++--
.../org/apache/nutch/parse/tika/DOMBuilder.java | 150 ++++++++++++---------
.../apache/nutch/parse/tika/DOMContentUtils.java | 48 +++----
.../nutch/parse/tika/TestBoilerpipeExtraction.java | 49 ++++---
.../parse/tika/TestLinkExtractionEdgeCases.java | 69 +++++-----
.../nutch/parse/tika/TestMetadataExtraction.java | 59 ++++----
.../org/apache/nutch/parse/tika/TestOOParser.java | 6 +-
.../parse/tika/TestParserFailureHandling.java | 68 +++++-----
.../nutch/parse/tika/TestRobotsMetaProcessor.java | 21 +--
.../nutch/parsefilter/naivebayes/Classify.java | 11 +-
.../apache/nutch/parsefilter/naivebayes/Train.java | 7 +-
.../nutch/parsefilter/regex/RegexParseFilter.java | 43 +++---
.../parsefilter/regex/TestRegexParseFilter.java | 24 ++--
.../java/org/apache/nutch/protocol/file/File.java | 15 +--
.../apache/nutch/protocol/file/FileResponse.java | 20 ++-
.../java/org/apache/nutch/protocol/foo/Foo.java | 11 +-
.../java/org/apache/nutch/protocol/ftp/Client.java | 63 ++++-----
.../java/org/apache/nutch/protocol/ftp/Ftp.java | 41 +++---
.../org/apache/nutch/protocol/ftp/FtpResponse.java | 32 ++---
.../nutch/protocol/ftp/FtpRobotRulesParser.java | 10 +-
.../nutch/protocol/htmlunit/HttpResponse.java | 9 +-
.../apache/nutch/protocol/http/HttpResponse.java | 12 +-
.../protocol/http/TestBadServerResponses.java | 15 ++-
.../httpclient/HttpBasicAuthentication.java | 28 ++--
.../httpclient/TestProtocolHttpClient.java | 35 +++--
.../protocol/interactiveselenium/HttpResponse.java | 18 +--
.../nutch/protocol/okhttp/IPFilterRules.java | 5 +-
.../nutch/protocol/okhttp/OkHttpResponse.java | 10 +-
.../protocol/okhttp/TestBadServerResponses.java | 33 +++--
.../protocol/okhttp/TestIPAddressFiltering.java | 13 +-
.../nutch/protocol/selenium/HttpResponse.java | 13 +-
.../publisher/rabbitmq/RabbitMQPublisherImpl.java | 3 +-
.../subcollection/SubcollectionIndexingFilter.java | 27 ++--
.../apache/nutch/collection/TestSubcollection.java | 12 +-
.../nutch/urlfilter/domain/DomainURLFilter.java | 23 ++--
.../domaindenylist/DomainDenylistURLFilter.java | 21 +--
.../apache/nutch/urlfilter/fast/FastURLFilter.java | 56 ++++----
.../nutch/urlfilter/prefix/PrefixURLFilter.java | 33 +++--
.../nutch/urlfilter/suffix/SuffixURLFilter.java | 61 +++++----
.../net/urlnormalizer/ajax/AjaxURLNormalizer.java | 9 +-
.../net/urlnormalizer/host/HostURLNormalizer.java | 15 ++-
.../protocol/ProtocolURLNormalizer.java | 18 +--
.../urlnormalizer/regex/RegexURLNormalizer.java | 24 ++--
.../regex/TestRegexURLNormalizer.java | 29 ++--
.../urlnormalizer/slash/SlashURLNormalizer.java | 32 ++---
src/test/domain-urlfilter.txt | 22 ---
src/test/host-protocol-mapping.txt | 35 +++++
.../org/apache/nutch/crawl/CrawlDBTestUtil.java | 41 +++---
src/test/org/apache/nutch/fetcher/TestFetcher.java | 22 +--
.../org/apache/nutch/plugin/TestPluginSystem.java | 26 ++--
.../apache/nutch/protocol/TestProtocolFactory.java | 30 ++++-
.../apache/nutch/segment/TestSegmentMerger.java | 9 +-
.../segment/TestSegmentMergerCrawlDatums.java | 9 +-
.../apache/nutch/util/TestEncodingDetector.java | 22 ++-
src/test/org/apache/nutch/util/TestGZIPUtils.java | 29 ++--
src/test/org/apache/nutch/util/TestMimeUtil.java | 5 +-
src/test/org/apache/nutch/util/TestNodeWalker.java | 5 +-
src/testresources/forbidden-apis-signatures.txt | 24 ++++
124 files changed, 1802 insertions(+), 1535 deletions(-)
delete mode 100644 src/test/domain-urlfilter.txt
create mode 100644 src/test/host-protocol-mapping.txt
create mode 100644 src/testresources/forbidden-apis-signatures.txt
