This is an automated email from the ASF dual-hosted git repository. sebastian-nagel pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nutch.git
commit 10b9a56e6b3f7e0924b175b025f9d7705dacc24f Merge: ca8538abf b845812d3 Author: Sebastian Nagel <[email protected]> AuthorDate: Tue Jun 30 08:44:53 2026 +0200 Merge pull request #924 from sebastian-nagel/NUTCH-1807-forbidden-apis .github/workflows/master-build.yml | 6 +- .yetus/blanks-tabs.txt | 3 + build.xml | 50 ++++++- default.properties | 3 + .../apache/nutch/crawl/AdaptiveFetchSchedule.java | 45 +++---- src/java/org/apache/nutch/crawl/CrawlDbReader.java | 45 ++++--- .../org/apache/nutch/crawl/DeduplicationJob.java | 14 +- src/java/org/apache/nutch/crawl/Generator.java | 62 ++++----- src/java/org/apache/nutch/crawl/LinkDb.java | 23 ++-- src/java/org/apache/nutch/crawl/MD5Signature.java | 6 +- .../apache/nutch/crawl/TextProfileSignature.java | 7 +- src/java/org/apache/nutch/fetcher/FetchItem.java | 4 +- src/java/org/apache/nutch/fetcher/Fetcher.java | 6 +- .../org/apache/nutch/fetcher/FetcherThread.java | 59 ++++---- src/java/org/apache/nutch/hostdb/HostDatum.java | 6 +- .../org/apache/nutch/indexer/IndexerMapReduce.java | 50 +++---- .../org/apache/nutch/parse/ParseOutputFormat.java | 85 ++++++------ .../org/apache/nutch/protocol/ProtocolFactory.java | 48 ++++--- .../apache/nutch/protocol/RobotRulesParser.java | 32 ++--- .../apache/nutch/scoring/webgraph/LinkRank.java | 53 ++++---- .../nutch/segment/ContentAsTextInputFormat.java | 10 +- .../org/apache/nutch/segment/SegmentReader.java | 20 +-- .../nutch/tools/AbstractCommonCrawlFormat.java | 4 +- .../apache/nutch/tools/CommonCrawlDataDumper.java | 15 +-- .../nutch/tools/CommonCrawlFormatJackson.java | 33 ++--- .../apache/nutch/tools/CommonCrawlFormatWARC.java | 9 +- src/java/org/apache/nutch/tools/DmozParser.java | 40 +++--- src/java/org/apache/nutch/tools/FileDumper.java | 50 ++++--- src/java/org/apache/nutch/tools/ResolveUrls.java | 13 +- .../org/apache/nutch/tools/ShowProperties.java | 22 +-- .../apache/nutch/tools/arc/ArcRecordReader.java | 43 +++--- .../apache/nutch/tools/arc/ArcSegmentCreator.java | 44 +++--- .../org/apache/nutch/tools/warc/WARCExporter.java | 27 ++-- .../org/apache/nutch/util/AbstractChecker.java | 16 +-- src/java/org/apache/nutch/util/DumpFileUtil.java | 47 ++++--- .../org/apache/nutch/util/EncodingDetector.java | 25 ++-- src/java/org/apache/nutch/util/NutchJob.java | 18 +-- .../org/apache/nutch/util/PrefixStringMatcher.java | 4 +- .../org/apache/nutch/util/SitemapProcessor.java | 12 +- src/java/org/apache/nutch/util/TimingUtil.java | 17 ++- src/java/org/apache/nutch/util/URLUtil.java | 52 +++---- .../org/apache/nutch/parse/feed/FeedParser.java | 44 +++--- .../parse/headings/TestHeadingsParseFilter.java | 19 ++- .../nutch/indexer/anchor/AnchorIndexingFilter.java | 7 +- .../nutch/indexer/links/LinksIndexingFilter.java | 22 +-- .../cloudsearch/CloudSearchIndexWriter.java | 7 +- .../nutch/indexwriter/csv/CSVIndexWriter.java | 12 +- .../nutch/indexwriter/dummy/DummyIndexWriter.java | 7 +- .../nutch/indexwriter/rabbit/RabbitDocument.java | 7 +- .../indexwriter/rabbit/RabbitIndexWriter.java | 20 ++- .../nutch/indexwriter/rabbit/RabbitMessage.java | 7 +- .../nutch/analysis/lang/HTMLLanguageParser.java | 7 +- .../analysis/lang/TestHTMLLanguageParser.java | 11 +- .../apache/nutch/protocol/http/api/HttpBase.java | 29 ++-- .../protocol/http/api/HttpRobotRulesParser.java | 23 ++-- .../protocol/http/api/TestRobotRulesParser.java | 38 +++--- .../nutch/urlfilter/api/RegexURLFilterBase.java | 58 ++++---- .../urlfilter/api/RegexURLFilterBaseTest.java | 28 ++-- .../nutch/microformats/reltag/RelTagParser.java | 23 ++-- .../indexer/filter/MimeTypeIndexingFilter.java | 47 +++---- .../indexer/filter/MimeTypeIndexingFilterTest.java | 13 +- .../java/org/apache/nutch/parse/ext/ExtParser.java | 46 +++---- .../apache/nutch/parse/html/DOMContentUtils.java | 55 ++++---- .../apache/nutch/parse/html/HTMLMetaProcessor.java | 16 ++- .../nutch/parse/html/TestDOMContentUtils.java | 22 +-- .../nutch/parse/html/TestRobotsMetaProcessor.java | 34 ++--- .../org/apache/nutch/parse/js/JSParseFilter.java | 27 ++-- .../org/apache/nutch/parse/tika/DOMBuilder.java | 150 ++++++++++++--------- .../apache/nutch/parse/tika/DOMContentUtils.java | 48 +++---- .../nutch/parse/tika/TestBoilerpipeExtraction.java | 49 ++++--- .../parse/tika/TestLinkExtractionEdgeCases.java | 69 +++++----- .../nutch/parse/tika/TestMetadataExtraction.java | 59 ++++---- .../org/apache/nutch/parse/tika/TestOOParser.java | 6 +- .../parse/tika/TestParserFailureHandling.java | 68 +++++----- .../nutch/parse/tika/TestRobotsMetaProcessor.java | 21 +-- .../nutch/parsefilter/naivebayes/Classify.java | 11 +- .../apache/nutch/parsefilter/naivebayes/Train.java | 7 +- .../nutch/parsefilter/regex/RegexParseFilter.java | 43 +++--- .../parsefilter/regex/TestRegexParseFilter.java | 24 ++-- .../java/org/apache/nutch/protocol/file/File.java | 15 +-- .../apache/nutch/protocol/file/FileResponse.java | 20 ++- .../java/org/apache/nutch/protocol/foo/Foo.java | 11 +- .../java/org/apache/nutch/protocol/ftp/Client.java | 63 ++++----- .../java/org/apache/nutch/protocol/ftp/Ftp.java | 41 +++--- .../org/apache/nutch/protocol/ftp/FtpResponse.java | 32 ++--- .../nutch/protocol/ftp/FtpRobotRulesParser.java | 10 +- .../nutch/protocol/htmlunit/HttpResponse.java | 9 +- .../apache/nutch/protocol/http/HttpResponse.java | 12 +- .../protocol/http/TestBadServerResponses.java | 15 ++- .../httpclient/HttpBasicAuthentication.java | 28 ++-- .../httpclient/TestProtocolHttpClient.java | 35 +++-- .../protocol/interactiveselenium/HttpResponse.java | 18 +-- .../nutch/protocol/okhttp/IPFilterRules.java | 5 +- .../nutch/protocol/okhttp/OkHttpResponse.java | 10 +- .../protocol/okhttp/TestBadServerResponses.java | 33 +++-- .../protocol/okhttp/TestIPAddressFiltering.java | 13 +- .../nutch/protocol/selenium/HttpResponse.java | 13 +- .../publisher/rabbitmq/RabbitMQPublisherImpl.java | 3 +- .../subcollection/SubcollectionIndexingFilter.java | 27 ++-- .../apache/nutch/collection/TestSubcollection.java | 12 +- .../nutch/urlfilter/domain/DomainURLFilter.java | 23 ++-- .../domaindenylist/DomainDenylistURLFilter.java | 21 +-- .../apache/nutch/urlfilter/fast/FastURLFilter.java | 56 ++++---- .../nutch/urlfilter/prefix/PrefixURLFilter.java | 33 +++-- .../nutch/urlfilter/suffix/SuffixURLFilter.java | 61 +++++---- .../net/urlnormalizer/ajax/AjaxURLNormalizer.java | 9 +- .../net/urlnormalizer/host/HostURLNormalizer.java | 15 ++- .../protocol/ProtocolURLNormalizer.java | 18 +-- .../urlnormalizer/regex/RegexURLNormalizer.java | 24 ++-- .../regex/TestRegexURLNormalizer.java | 29 ++-- .../urlnormalizer/slash/SlashURLNormalizer.java | 32 ++--- src/test/domain-urlfilter.txt | 22 --- src/test/host-protocol-mapping.txt | 35 +++++ .../org/apache/nutch/crawl/CrawlDBTestUtil.java | 41 +++--- src/test/org/apache/nutch/fetcher/TestFetcher.java | 22 +-- .../org/apache/nutch/plugin/TestPluginSystem.java | 26 ++-- .../apache/nutch/protocol/TestProtocolFactory.java | 30 ++++- .../apache/nutch/segment/TestSegmentMerger.java | 9 +- .../segment/TestSegmentMergerCrawlDatums.java | 9 +- .../apache/nutch/util/TestEncodingDetector.java | 22 ++- src/test/org/apache/nutch/util/TestGZIPUtils.java | 29 ++-- src/test/org/apache/nutch/util/TestMimeUtil.java | 5 +- src/test/org/apache/nutch/util/TestNodeWalker.java | 5 +- src/testresources/forbidden-apis-signatures.txt | 24 ++++ 124 files changed, 1802 insertions(+), 1535 deletions(-)
