This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 21ba9a8a8 NUTCH-3200 Close Closeable resources flagged by Sonar
java:S2095 (#948)
21ba9a8a8 is described below
commit 21ba9a8a81710f890bece5456af0bbadf8a48fa9
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Fri Aug 14 23:41:10 2026 -0600
NUTCH-3200 Close Closeable resources flagged by Sonar java:S2095 (#948)
---
.../apache/nutch/crawl/AdaptiveFetchSchedule.java | 176 ++++++------
.../apache/nutch/crawl/TextProfileSignature.java | 17 +-
.../apache/nutch/protocol/RobotRulesParser.java | 48 ++--
.../apache/nutch/scoring/webgraph/LinkRank.java | 33 +--
.../apache/nutch/tools/CommonCrawlDataDumper.java | 298 ++++++++++-----------
src/java/org/apache/nutch/tools/FileDumper.java | 169 ++++++------
src/java/org/apache/nutch/tools/ResolveUrls.java | 22 +-
.../org/apache/nutch/parse/feed/FeedParser.java | 6 +-
.../org/apache/nutch/parse/js/JSParseFilter.java | 15 +-
.../java/org/apache/nutch/parse/zip/ZipParser.java | 9 +-
.../nutch/parsefilter/naivebayes/Classify.java | 34 +--
.../apache/nutch/parsefilter/naivebayes/Train.java | 73 +++--
.../apache/nutch/protocol/file/FileResponse.java | 20 +-
13 files changed, 455 insertions(+), 465 deletions(-)
diff --git a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
index 04a3a1303..6bfb6085f 100644
--- a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
+++ b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
@@ -133,111 +133,111 @@ public class AdaptiveFetchSchedule extends
AbstractFetchSchedule {
private void setHostSpecificIntervals(String fileName,
float defaultMin, float defaultMax) throws IOException {
// Setup for reading the config file.
- Reader configReader = null;
- configReader = conf.getConfResourceAsReader(fileName);
+ Reader configReader = conf.getConfResourceAsReader(fileName);
if (configReader == null) {
configReader = new FileReader(fileName, StandardCharsets.UTF_8);
}
- BufferedReader reader = new BufferedReader(configReader);
- String line;
- int lineNo = 0;
+ try (BufferedReader reader = new BufferedReader(configReader)) {
+ String line;
+ int lineNo = 0;
- // Read the file line by line.
- while ((line = reader.readLine()) != null) {
- lineNo++;
+ // Read the file line by line.
+ while ((line = reader.readLine()) != null) {
+ lineNo++;
- // Skip blank lines and comments.
- if (StringUtils.isBlank(line) || line.startsWith("#")) {
- continue;
- }
+ // Skip blank lines and comments.
+ if (StringUtils.isBlank(line) || line.startsWith("#")) {
+ continue;
+ }
- // Trim and partition the line.
- line = line.trim();
- String[] parts = line.split("\\s+");
+ // Trim and partition the line.
+ line = line.trim();
+ String[] parts = line.split("\\s+");
- // There should be three parts.
- if (parts.length != 3) {
- LOG.error(
- "Malformed (domain, min_interval, max_interval) triplet on line {}
of the config. file: `{}`",
- lineNo, line);
- continue;
- }
+ // There should be three parts.
+ if (parts.length != 3) {
+ LOG.error(
+ "Malformed (domain, min_interval, max_interval) triplet on line
{} of the config. file: `{}`",
+ lineNo, line);
+ continue;
+ }
- // Normalize the parts.
- String host = parts[0].trim().toLowerCase(Locale.ROOT);
- String minInt = parts[1].trim();
- String maxInt = parts[2].trim();
-
- // "0" and "default" both mean `use default interval`; normalize to "0".
- if (minInt.equalsIgnoreCase("default")) { minInt = "0"; }
- if (maxInt.equalsIgnoreCase("default")) { maxInt = "0"; }
-
- // Convert intervals to float and ignore the line in case of failure.
- float m, M;
- try {
- m = Float.parseFloat(minInt);
- M = Float.parseFloat(maxInt);
- } catch (NumberFormatException e) {
- LOG.error(
- "Improper fetch intervals given on line {} in the config. file
`{}`: {}",
- lineNo, line, e.toString());
- continue;
- }
+ // Normalize the parts.
+ String host = parts[0].trim().toLowerCase(Locale.ROOT);
+ String minInt = parts[1].trim();
+ String maxInt = parts[2].trim();
+
+ // "0" and "default" both mean `use default interval`; normalize to
"0".
+ if (minInt.equalsIgnoreCase("default")) { minInt = "0"; }
+ if (maxInt.equalsIgnoreCase("default")) { maxInt = "0"; }
+
+ // Convert intervals to float and ignore the line in case of failure.
+ float m, M;
+ try {
+ m = Float.parseFloat(minInt);
+ M = Float.parseFloat(maxInt);
+ } catch (NumberFormatException e) {
+ LOG.error(
+ "Improper fetch intervals given on line {} in the config. file
`{}`: {}",
+ lineNo, line, e.toString());
+ continue;
+ }
- // If both intervals are set to default,
- // ignore the line and issue a warning.
- if (m == 0 && M == 0) {
- LOG.warn(
- "Ignoring default interval values on line {} of config. file:
`{}`",
- lineNo, line);
- continue;
- }
+ // If both intervals are set to default,
+ // ignore the line and issue a warning.
+ if (m == 0 && M == 0) {
+ LOG.warn(
+ "Ignoring default interval values on line {} of config. file:
`{}`",
+ lineNo, line);
+ continue;
+ }
- // Replace the zero with the default value.
- if (m == 0) {
- m = defaultMin;
- } else if (M == 0) {
- M = defaultMax;
- }
+ // Replace the zero with the default value.
+ if (m == 0) {
+ m = defaultMin;
+ } else if (M == 0) {
+ M = defaultMax;
+ }
- // Intervals cannot be negative and the min cannot be above the max
- // (we assume here that the default values satisfy this).
- if (m < 0 || M < 0) {
- LOG.error(
- "Improper fetch intervals given on line {} in the config. file:
`{}`: intervals cannot be negative",
- lineNo, line);
- continue;
- }
+ // Intervals cannot be negative and the min cannot be above the max
+ // (we assume here that the default values satisfy this).
+ if (m < 0 || M < 0) {
+ LOG.error(
+ "Improper fetch intervals given on line {} in the config. file:
`{}`: intervals cannot be negative",
+ lineNo, line);
+ continue;
+ }
- if (m > M) {
- LOG.error(
- "Improper fetch intervals given on line {} in the config. file:
`{}`: min. interval cannot be above max. interval",
- lineNo, line);
- continue;
- }
+ if (m > M) {
+ LOG.error(
+ "Improper fetch intervals given on line {} in the config. file:
`{}`: min. interval cannot be above max. interval",
+ lineNo, line);
+ continue;
+ }
- // The custom intervals should respect the boundaries of the default
values.
- if (m < defaultMin) {
- LOG.error(
- "Min. interval out of bounds ({}) on line {} in the config. file:
`{}`",
- defaultMin, lineNo, line);
- continue;
- }
+ // The custom intervals should respect the boundaries of the default
values.
+ if (m < defaultMin) {
+ LOG.error(
+ "Min. interval out of bounds ({}) on line {} in the config.
file: `{}`",
+ defaultMin, lineNo, line);
+ continue;
+ }
- if (M > defaultMax) {
- LOG.error(
- "Max. interval out of bounds ({}) on line {} in the config. file:
`{}`",
- defaultMax, lineNo, line);
- continue;
- }
+ if (M > defaultMax) {
+ LOG.error(
+ "Max. interval out of bounds ({}) on line {} in the config.
file: `{}`",
+ defaultMax, lineNo, line);
+ continue;
+ }
- // If all is well, store the specific intervals.
- hostSpecificMinInterval.put(host, m);
- LOG.debug("Added custom min. interval {} for host {}.", m, host);
+ // If all is well, store the specific intervals.
+ hostSpecificMinInterval.put(host, m);
+ LOG.debug("Added custom min. interval {} for host {}.", m, host);
- hostSpecificMaxInterval.put(host, M);
- LOG.debug("Added custom max. interval {} for host {}.", M, host);
+ hostSpecificMaxInterval.put(host, M);
+ LOG.debug("Added custom max. interval {} for host {}.", M, host);
+ }
}
}
diff --git a/src/java/org/apache/nutch/crawl/TextProfileSignature.java
b/src/java/org/apache/nutch/crawl/TextProfileSignature.java
index 63583a694..f54d230ae 100644
--- a/src/java/org/apache/nutch/crawl/TextProfileSignature.java
+++ b/src/java/org/apache/nutch/crawl/TextProfileSignature.java
@@ -195,17 +195,16 @@ public class TextProfileSignature extends Signature {
HashMap<String, byte[]> res = new HashMap<>();
File[] files = new File(args[0]).listFiles();
for (int i = 0; i < files.length; i++) {
- FileInputStream fis = new FileInputStream(files[i]);
- BufferedReader br = new BufferedReader(
- new InputStreamReader(fis, StandardCharsets.UTF_8));
StringBuffer text = new StringBuffer();
- String line = null;
- while ((line = br.readLine()) != null) {
- if (text.length() > 0)
- text.append("\n");
- text.append(line);
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(
+ new FileInputStream(files[i]), StandardCharsets.UTF_8))) {
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ if (text.length() > 0)
+ text.append("\n");
+ text.append(line);
+ }
}
- br.close();
byte[] signature = sig.calculate(null, new ParseImpl(text.toString(),
null));
res.put(files[i].toString(), signature);
diff --git a/src/java/org/apache/nutch/protocol/RobotRulesParser.java
b/src/java/org/apache/nutch/protocol/RobotRulesParser.java
index 5b0412910..3efa0fc3b 100644
--- a/src/java/org/apache/nutch/protocol/RobotRulesParser.java
+++ b/src/java/org/apache/nutch/protocol/RobotRulesParser.java
@@ -398,30 +398,30 @@ public abstract class RobotRulesParser implements Tool {
System.out.println("Testing robots.txt for agent names: "
+ (agentNames.isEmpty() ? "* (any other agent)" : agentNames));
- LineNumberReader testsIn = new LineNumberReader(
- new FileReader(urlFile, StandardCharsets.UTF_8));
- String testPath;
- testPath = testsIn.readLine();
- while (testPath != null) {
- testPath = testPath.trim();
- try {
- // testPath can be just a path or a complete URL
- URL url = new URL(testPath);
- String status;
- if (isAllowListed(url)) {
- status = "allowlisted";
- } else if (rules.isAllowed(testPath)) {
- status = "allowed";
- } else {
- status = "not allowed";
+ try (LineNumberReader testsIn = new LineNumberReader(
+ new FileReader(urlFile, StandardCharsets.UTF_8))) {
+ String testPath;
+ testPath = testsIn.readLine();
+ while (testPath != null) {
+ testPath = testPath.trim();
+ try {
+ // testPath can be just a path or a complete URL
+ URL url = new URL(testPath);
+ String status;
+ if (isAllowListed(url)) {
+ status = "allowlisted";
+ } else if (rules.isAllowed(testPath)) {
+ status = "allowed";
+ } else {
+ status = "not allowed";
+ }
+ System.out.println(status + ":\t" + testPath);
+ } catch (MalformedURLException e) {
+ LOG.warn("Not a valid URL: {}", testPath);
}
- System.out.println(status + ":\t" + testPath);
- } catch (MalformedURLException e) {
- LOG.warn("Not a valid URL: {}", testPath);
+ testPath = testsIn.readLine();
}
- testPath = testsIn.readLine();
}
- testsIn.close();
} catch (IOException e) {
LOG.error("Failed to run:", e);
return -1;
@@ -476,9 +476,9 @@ public abstract class RobotRulesParser implements Tool {
try {
int contentLength = url.openConnection().getContentLength();
byte[] robotsBytes = new byte[contentLength];
- InputStream openStream = url.openStream();
- openStream.read(robotsBytes);
- openStream.close();
+ try (InputStream openStream = url.openStream()) {
+ openStream.read(robotsBytes);
+ }
rules = robotParser.parseContent(url.toString(), robotsBytes,
"text/plain", agentNames);
} catch (IOException e) {
diff --git a/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
b/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
index 80d6ac7cf..de15e3d66 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
@@ -141,25 +141,26 @@ public class LinkRank extends Configured implements Tool {
}
Path numLinksFile = numLinksFiles[0].getPath();
LOG.info("Reading numlinks temp file {}", numLinksFile);
- FSDataInputStream readLinks = fs.open(numLinksFile);
CompressionCodecFactory cf = new CompressionCodecFactory(conf);
CompressionCodec codec = cf.getCodec(numLinksFiles[0].getPath());
- InputStream streamLinks;
- if (codec == null) {
- LOG.debug("No compression codec found for {}, trying uncompressed",
- numLinksFile);
- streamLinks = readLinks;
- } else {
- LOG.info("Compression codec of numlinks temp file: {}",
- codec.getDefaultExtension());
- readLinks.seek(0);
- streamLinks = codec.createInputStream(readLinks);
+ String numLinksLine;
+ try (FSDataInputStream readLinks = fs.open(numLinksFile)) {
+ InputStream streamLinks;
+ if (codec == null) {
+ LOG.debug("No compression codec found for {}, trying uncompressed",
+ numLinksFile);
+ streamLinks = readLinks;
+ } else {
+ LOG.info("Compression codec of numlinks temp file: {}",
+ codec.getDefaultExtension());
+ readLinks.seek(0);
+ streamLinks = codec.createInputStream(readLinks);
+ }
+ try (BufferedReader buffer = new BufferedReader(
+ new InputStreamReader(streamLinks, StandardCharsets.UTF_8))) {
+ numLinksLine = buffer.readLine();
+ }
}
- BufferedReader buffer = new BufferedReader(
- new InputStreamReader(streamLinks, StandardCharsets.UTF_8));
-
- String numLinksLine = buffer.readLine();
- readLinks.close();
// check if there are links to process, if none, webgraph might be empty
if (numLinksLine == null || numLinksLine.length() == 0) {
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
index 39935016d..fe714218e 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
@@ -269,182 +269,180 @@ public class CommonCrawlDataDumper extends NutchTool
implements Tool {
}
}
- LinkDbReader linkDbReader = null;
- if (linkdb != null) {
- linkDbReader = new LinkDbReader(nutchConfig, new
Path(linkdb.toString()));
- }
- if (parts == null || parts.size() == 0) {
- LOG.error( "No segment directories found in {} ",
- segmentRootDir.getAbsolutePath());
- this.errorTracker.recordError(ErrorTracker.ErrorType.OTHER);
- return;
- }
- LOG.info("Found {} segment parts", parts.size());
- if (gzip && !warc) {
- fileList = new ArrayList<>();
- constructNewStream(outputDir);
- }
+ try (LinkDbReader linkDbReader = linkdb != null
+ ? new LinkDbReader(nutchConfig, new Path(linkdb.toString()))
+ : null) {
+ if (parts == null || parts.size() == 0) {
+ LOG.error( "No segment directories found in {} ",
+ segmentRootDir.getAbsolutePath());
+ this.errorTracker.recordError(ErrorTracker.ErrorType.OTHER);
+ return;
+ }
+ LOG.info("Found {} segment parts", parts.size());
+ if (gzip && !warc) {
+ fileList = new ArrayList<>();
+ constructNewStream(outputDir);
+ }
- for (Path segmentPart : parts) {
- LOG.info("Processing segment Part : [ {} ]", segmentPart);
- try {
- SequenceFile.Reader reader = new SequenceFile.Reader(nutchConfig,
- SequenceFile.Reader.file(segmentPart));
+ for (Path segmentPart : parts) {
+ LOG.info("Processing segment Part : [ {} ]", segmentPart);
+ try (SequenceFile.Reader reader = new SequenceFile.Reader(nutchConfig,
+ SequenceFile.Reader.file(segmentPart))) {
- Writable key = (Writable)
reader.getKeyClass().getConstructor().newInstance();
+ Writable key = (Writable)
reader.getKeyClass().getConstructor().newInstance();
- Content content = null;
- while (reader.next(key)) {
- content = new Content();
- reader.getCurrentValue(content);
- Metadata metadata = content.getMetadata();
- String url = key.toString();
+ Content content = null;
+ while (reader.next(key)) {
+ content = new Content();
+ reader.getCurrentValue(content);
+ Metadata metadata = content.getMetadata();
+ String url = key.toString();
- String baseName = FilenameUtils.getBaseName(url);
- String extensionName = FilenameUtils.getExtension(url);
+ String baseName = FilenameUtils.getBaseName(url);
+ String extensionName = FilenameUtils.getExtension(url);
- if (!extension.isEmpty()) {
- extensionName = extension;
- } else if ((extensionName == null) || extensionName.isEmpty()) {
- extensionName = "html";
- }
+ if (!extension.isEmpty()) {
+ extensionName = extension;
+ } else if ((extensionName == null) || extensionName.isEmpty()) {
+ extensionName = "html";
+ }
- String outputFullPath = null;
- String outputRelativePath = null;
- String filename = null;
- String timestamp = null;
- String reverseKey = null;
+ String outputFullPath = null;
+ String outputRelativePath = null;
+ String filename = null;
+ String timestamp = null;
+ String reverseKey = null;
+
+ if (epochFilename || config.getReverseKey()) {
+ try {
+ long epoch = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z")
+ .parse(getDate(metadata.get("Date"))).getTime();
+ timestamp = String.valueOf(epoch);
+ } catch (ParseException pe) {
+ LOG.warn(pe.getMessage());
+ }
- if (epochFilename || config.getReverseKey()) {
- try {
- long epoch = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z")
- .parse(getDate(metadata.get("Date"))).getTime();
- timestamp = String.valueOf(epoch);
- } catch (ParseException pe) {
- LOG.warn(pe.getMessage());
+ reverseKey = reverseUrl(url);
+ config.setReverseKeyValue(
+ reverseKey.replace("/", "_") + "_" + DigestUtils.sha1Hex(url)
+ + "_" + timestamp);
}
- reverseKey = reverseUrl(url);
- config.setReverseKeyValue(
- reverseKey.replace("/", "_") + "_" + DigestUtils.sha1Hex(url)
- + "_" + timestamp);
- }
-
- if (!warc) {
- if (epochFilename) {
- outputFullPath = DumpFileUtil
- .createFileNameFromUrl(outputDir.getAbsolutePath(),
- reverseKey, url, timestamp, extensionName, !gzip);
- outputRelativePath = outputFullPath
- .substring(0, outputFullPath.lastIndexOf(File.separator) -
1);
- filename = content.getMetadata().get(Metadata.DATE) + "."
- + extensionName;
- } else {
- String md5Ofurl = DumpFileUtil.getUrlMD5(url);
- String fullDir = DumpFileUtil
- .createTwoLevelsDirectory(outputDir.getAbsolutePath(),
- md5Ofurl, !gzip);
- filename = DumpFileUtil
- .createFileName(md5Ofurl, baseName, extensionName);
- outputFullPath = String.format(Locale.ROOT, "%s/%s", fullDir,
filename);
-
- String[] fullPathLevels = fullDir
- .split(Pattern.quote(File.separator));
- String firstLevelDirName = fullPathLevels[fullPathLevels.length
- - 2];
- String secondLevelDirName = fullPathLevels[fullPathLevels.length
- - 1];
- outputRelativePath = firstLevelDirName + secondLevelDirName;
+ if (!warc) {
+ if (epochFilename) {
+ outputFullPath = DumpFileUtil
+ .createFileNameFromUrl(outputDir.getAbsolutePath(),
+ reverseKey, url, timestamp, extensionName, !gzip);
+ outputRelativePath = outputFullPath
+ .substring(0, outputFullPath.lastIndexOf(File.separator) -
1);
+ filename = content.getMetadata().get(Metadata.DATE) + "."
+ + extensionName;
+ } else {
+ String md5Ofurl = DumpFileUtil.getUrlMD5(url);
+ String fullDir = DumpFileUtil
+ .createTwoLevelsDirectory(outputDir.getAbsolutePath(),
+ md5Ofurl, !gzip);
+ filename = DumpFileUtil
+ .createFileName(md5Ofurl, baseName, extensionName);
+ outputFullPath = String.format(Locale.ROOT, "%s/%s", fullDir,
filename);
+
+ String[] fullPathLevels = fullDir
+ .split(Pattern.quote(File.separator));
+ String firstLevelDirName = fullPathLevels[fullPathLevels.length
+ - 2];
+ String secondLevelDirName =
fullPathLevels[fullPathLevels.length
+ - 1];
+ outputRelativePath = firstLevelDirName + secondLevelDirName;
+ }
}
- }
- // Encode all filetypes if no mimetypes have been given
- Boolean filter = (mimeTypes == null);
-
- String jsonData = "";
- try {
- String mimeType = new Tika().detect(content.getContent());
- // Maps file to JSON-based structure
-
- Set<String> inUrls = null; //there may be duplicates, so using set
- if (linkDbReader != null) {
- Inlinks inlinks = linkDbReader.getInlinks((Text) key);
- if (inlinks != null) {
- Iterator<Inlink> iterator = inlinks.iterator();
- inUrls = new LinkedHashSet<>();
- while (inUrls.size() <= MAX_INLINKS && iterator.hasNext()){
- inUrls.add(iterator.next().getFromUrl());
+ // Encode all filetypes if no mimetypes have been given
+ Boolean filter = (mimeTypes == null);
+
+ String jsonData = "";
+ try {
+ String mimeType = new Tika().detect(content.getContent());
+ // Maps file to JSON-based structure
+
+ Set<String> inUrls = null; //there may be duplicates, so using
set
+ if (linkDbReader != null) {
+ Inlinks inlinks = linkDbReader.getInlinks((Text) key);
+ if (inlinks != null) {
+ Iterator<Inlink> iterator = inlinks.iterator();
+ inUrls = new LinkedHashSet<>();
+ while (inUrls.size() <= MAX_INLINKS && iterator.hasNext()){
+ inUrls.add(iterator.next().getFromUrl());
+ }
}
}
- }
- //TODO: Make this Jackson Format implementation reusable
- try (CommonCrawlFormat format = CommonCrawlFormatFactory
- .getCommonCrawlFormat(warc ? "WARC" : "JACKSON", nutchConfig,
config)) {
- if (inUrls != null) {
- format.setInLinks(new ArrayList<>(inUrls));
+ //TODO: Make this Jackson Format implementation reusable
+ try (CommonCrawlFormat format = CommonCrawlFormatFactory
+ .getCommonCrawlFormat(warc ? "WARC" : "JACKSON",
nutchConfig, config)) {
+ if (inUrls != null) {
+ format.setInLinks(new ArrayList<>(inUrls));
+ }
+ jsonData = format.getJsonData(url, content, metadata);
}
- jsonData = format.getJsonData(url, content, metadata);
- }
- collectStats(typeCounts, mimeType);
- // collects statistics for the given mimetypes
- if ((mimeType != null) && (mimeTypes != null) && Arrays
- .asList(mimeTypes).contains(mimeType)) {
- collectStats(filteredCounts, mimeType);
- filter = true;
+ collectStats(typeCounts, mimeType);
+ // collects statistics for the given mimetypes
+ if ((mimeType != null) && (mimeTypes != null) && Arrays
+ .asList(mimeTypes).contains(mimeType)) {
+ collectStats(filteredCounts, mimeType);
+ filter = true;
+ }
+ } catch (IOException ioe) {
+ LOG.error("Fatal error in creating JSON data: {}",
ioe.getMessage());
+ return;
}
- } catch (IOException ioe) {
- LOG.error("Fatal error in creating JSON data: {}",
ioe.getMessage());
- return;
- }
- if (!warc) {
- if (filter) {
- byte[] byteData = serializeCBORData(jsonData);
-
- if (!gzip) {
- File outputFile = new File(outputFullPath);
- if (outputFile.exists()) {
- LOG.info("Skipping writing: [{}]: file already exists",
outputFullPath);
- } else {
- LOG.info("Writing: [{}]", outputFullPath);
- IOUtils.copy(new ByteArrayInputStream(byteData),
- new FileOutputStream(outputFile));
- }
- } else {
- if (fileList.contains(outputFullPath)) {
- LOG.info("Skipping compressing: [{}]: file already exists",
outputFullPath);
+ if (!warc) {
+ if (filter) {
+ byte[] byteData = serializeCBORData(jsonData);
+
+ if (!gzip) {
+ File outputFile = new File(outputFullPath);
+ if (outputFile.exists()) {
+ LOG.info("Skipping writing: [{}]: file already exists",
outputFullPath);
+ } else {
+ LOG.info("Writing: [{}]", outputFullPath);
+ IOUtils.copy(new ByteArrayInputStream(byteData),
+ new FileOutputStream(outputFile));
+ }
} else {
- fileList.add(outputFullPath);
- LOG.info("Compressing: [{}]", outputFullPath);
- //TarArchiveEntry tarEntry = new
TarArchiveEntry(firstLevelDirName + File.separator + secondLevelDirName +
File.separator + filename);
- TarArchiveEntry tarEntry = new TarArchiveEntry(
- outputRelativePath + File.separator + filename);
- tarEntry.setSize(byteData.length);
- tarOutput.putArchiveEntry(tarEntry);
- tarOutput.write(byteData);
- tarOutput.closeArchiveEntry();
+ if (fileList.contains(outputFullPath)) {
+ LOG.info("Skipping compressing: [{}]: file already
exists", outputFullPath);
+ } else {
+ fileList.add(outputFullPath);
+ LOG.info("Compressing: [{}]", outputFullPath);
+ //TarArchiveEntry tarEntry = new
TarArchiveEntry(firstLevelDirName + File.separator + secondLevelDirName +
File.separator + filename);
+ TarArchiveEntry tarEntry = new TarArchiveEntry(
+ outputRelativePath + File.separator + filename);
+ tarEntry.setSize(byteData.length);
+ tarOutput.putArchiveEntry(tarEntry);
+ tarOutput.write(byteData);
+ tarOutput.closeArchiveEntry();
+ }
}
}
}
}
+ } catch (Exception e){
+ LOG.warn("SKIPPED: {} Because : {}", segmentPart, e.getMessage());
+ } finally {
+ fs.close();
}
- reader.close();
- } catch (Exception e){
- LOG.warn("SKIPPED: {} Because : {}", segmentPart, e.getMessage());
- } finally {
- fs.close();
}
- }
- if (gzip && !warc) {
- closeStream();
- }
+ if (gzip && !warc) {
+ closeStream();
+ }
- if (!typeCounts.isEmpty()) {
- LOG.info("CommonsCrawlDataDumper File Stats: {}", DumpFileUtil
- .displayFileTypes(typeCounts, filteredCounts));
- }
+ if (!typeCounts.isEmpty()) {
+ LOG.info("CommonsCrawlDataDumper File Stats: {}", DumpFileUtil
+ .displayFileTypes(typeCounts, filteredCounts));
+ }
+ }
}
private void closeStream() {
diff --git a/src/java/org/apache/nutch/tools/FileDumper.java
b/src/java/org/apache/nutch/tools/FileDumper.java
index 8afc390c6..35d096351 100644
--- a/src/java/org/apache/nutch/tools/FileDumper.java
+++ b/src/java/org/apache/nutch/tools/FileDumper.java
@@ -169,114 +169,107 @@ public class FileDumper {
continue;
}
- SequenceFile.Reader reader = new SequenceFile.Reader(conf,
SequenceFile.Reader.file(file));
-
- Writable key = (Writable)
reader.getKeyClass().getConstructor().newInstance();
- Content content = null;
-
- while (reader.next(key)) {
- content = new Content();
- reader.getCurrentValue(content);
- String url = key.toString();
- String baseName = FilenameUtils.getBaseName(url);
- String extension = FilenameUtils.getExtension(url);
- if (extension == null || (extension != null &&
extension.equals(""))) {
- extension = "html";
- }
+ try (SequenceFile.Reader reader = new SequenceFile.Reader(conf,
+ SequenceFile.Reader.file(file))) {
+
+ Writable key = (Writable)
reader.getKeyClass().getConstructor().newInstance();
+ Content content = null;
+
+ while (reader.next(key)) {
+ content = new Content();
+ reader.getCurrentValue(content);
+ String url = key.toString();
+ String baseName = FilenameUtils.getBaseName(url);
+ String extension = FilenameUtils.getExtension(url);
+ if (extension == null || (extension != null &&
extension.equals(""))) {
+ extension = "html";
+ }
- ByteArrayInputStream bas = null;
- Boolean filter = false;
- try {
- bas = new ByteArrayInputStream(content.getContent());
- String mimeType = new Tika().detect(content.getContent());
- collectStats(typeCounts, mimeType);
- if (mimeType != null) {
- if (mimeTypes == null
- || Arrays.asList(mimeTypes).contains(mimeType)) {
- collectStats(filteredCounts, mimeType);
- filter = true;
+ ByteArrayInputStream bas = null;
+ Boolean filter = false;
+ try {
+ bas = new ByteArrayInputStream(content.getContent());
+ String mimeType = new Tika().detect(content.getContent());
+ collectStats(typeCounts, mimeType);
+ if (mimeType != null) {
+ if (mimeTypes == null
+ || Arrays.asList(mimeTypes).contains(mimeType)) {
+ collectStats(filteredCounts, mimeType);
+ filter = true;
+ }
}
- }
- } catch (Exception e) {
- e.printStackTrace();
- LOG.warn("Tika is unable to detect type for: [{}]", url);
- } finally {
- if (bas != null) {
- try {
- bas.close();
- } catch (Exception ignore) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG.warn("Tika is unable to detect type for: [{}]", url);
+ } finally {
+ if (bas != null) {
+ try {
+ bas.close();
+ } catch (Exception ignore) {
+ }
}
}
- }
- if (filter) {
- if (!mimeTypeStats) {
- String md5Ofurl = DumpFileUtil.getUrlMD5(url);
+ if (filter) {
+ if (!mimeTypeStats) {
+ String md5Ofurl = DumpFileUtil.getUrlMD5(url);
- String fullDir = outputDir.getAbsolutePath();
- if (!flatDir && !reverseURLDump) {
- fullDir = DumpFileUtil.createTwoLevelsDirectory(fullDir,
md5Ofurl);
- }
+ String fullDir = outputDir.getAbsolutePath();
+ if (!flatDir && !reverseURLDump) {
+ fullDir = DumpFileUtil.createTwoLevelsDirectory(fullDir,
md5Ofurl);
+ }
- if (!Strings.isNullOrEmpty(fullDir)) {
- String outputFullPath;
+ if (!Strings.isNullOrEmpty(fullDir)) {
+ String outputFullPath;
- if (reverseURLDump) {
- String[] reversedURL =
TableUtil.reverseUrl(url).split(":");
- reversedURL[0] = reversedURL[0].replace('.', '/');
+ if (reverseURLDump) {
+ String[] reversedURL =
TableUtil.reverseUrl(url).split(":");
+ reversedURL[0] = reversedURL[0].replace('.', '/');
- String reversedURLPath = reversedURL[0] + "/"
- + DigestUtils.sha256Hex(url).toUpperCase(Locale.ROOT);
- outputFullPath = String.format(Locale.ROOT, "%s/%s",
- fullDir, reversedURLPath);
+ String reversedURLPath = reversedURL[0] + "/"
+ +
DigestUtils.sha256Hex(url).toUpperCase(Locale.ROOT);
+ outputFullPath = String.format(Locale.ROOT, "%s/%s",
+ fullDir, reversedURLPath);
- // We'll drop the trailing file name and create the nested
structure if it doesn't already exist.
- String[] splitPath = outputFullPath.split("/");
- File fullOutputDir = new
File(org.apache.commons.lang3.StringUtils.join(Arrays.copyOf(splitPath,
splitPath.length - 1), "/"));
+ // We'll drop the trailing file name and create the
nested structure if it doesn't already exist.
+ String[] splitPath = outputFullPath.split("/");
+ File fullOutputDir = new
File(org.apache.commons.lang3.StringUtils.join(Arrays.copyOf(splitPath,
splitPath.length - 1), "/"));
- if (!fullOutputDir.exists()) {
- if(!fullOutputDir.mkdirs());
- throw new Exception("Unable to create: ["
- + fullOutputDir.getAbsolutePath() + "]");
+ if (!fullOutputDir.exists()) {
+ if(!fullOutputDir.mkdirs());
+ throw new Exception("Unable to create: ["
+ + fullOutputDir.getAbsolutePath() + "]");
+ }
+ } else {
+ outputFullPath = String.format(Locale.ROOT, "%s/%s",
+ fullDir, DumpFileUtil.createFileName(md5Ofurl,
baseName,
+ extension));
}
- } else {
- outputFullPath = String.format(Locale.ROOT, "%s/%s",
- fullDir, DumpFileUtil.createFileName(md5Ofurl,
baseName,
- extension));
- }
- filenameToUrl.put(outputFullPath, url);
- File outputFile = new File(outputFullPath);
-
- if (!outputFile.exists()) {
- LOG.info("Writing: [{}]", outputFullPath);
-
- // Modified to prevent FileNotFoundException (Invalid
Argument)
- FileOutputStream output = null;
- try {
- output = new FileOutputStream(outputFile);
- IOUtils.write(content.getContent(), output);
- } catch (Exception e) {
- LOG.warn("Write Error: [{}]", outputFullPath);
- e.printStackTrace();
- } finally {
- if (output != null) {
+ filenameToUrl.put(outputFullPath, url);
+ File outputFile = new File(outputFullPath);
+
+ if (!outputFile.exists()) {
+ LOG.info("Writing: [{}]", outputFullPath);
+
+ // Modified to prevent FileNotFoundException (Invalid
Argument)
+ try (FileOutputStream output = new FileOutputStream(
+ outputFile)) {
+ IOUtils.write(content.getContent(), output);
output.flush();
- try {
- output.close();
- } catch (Exception ignore) {
- }
+ } catch (Exception e) {
+ LOG.warn("Write Error: [{}]", outputFullPath);
+ e.printStackTrace();
}
+ fileCount++;
+ } else {
+ LOG.info("Skipping writing: [{}]: file already exists",
+ outputFullPath);
}
- fileCount++;
- } else {
- LOG.info("Skipping writing: [{}]: file already exists",
- outputFullPath);
}
}
}
}
}
- reader.close();
} finally {
if (doutputStream != null) {
try {
diff --git a/src/java/org/apache/nutch/tools/ResolveUrls.java
b/src/java/org/apache/nutch/tools/ResolveUrls.java
index ad38307a6..ddfa6cf21 100644
--- a/src/java/org/apache/nutch/tools/ResolveUrls.java
+++ b/src/java/org/apache/nutch/tools/ResolveUrls.java
@@ -107,19 +107,19 @@ public class ResolveUrls {
pool = Executors.newFixedThreadPool(numThreads);
// read in the urls file and loop through each line, one url per line
- BufferedReader buffRead = new BufferedReader(new FileReader(new File(
- urlsFile), StandardCharsets.UTF_8));
- String urlStr = null;
- while ((urlStr = buffRead.readLine()) != null) {
-
- // spin up a resolver thread per url
- LOG.info("Starting: {}", urlStr);
- pool.execute(new ResolverThread(urlStr));
+ try (BufferedReader buffRead = new BufferedReader(new FileReader(
+ new File(urlsFile), StandardCharsets.UTF_8))) {
+ String urlStr = null;
+ while ((urlStr = buffRead.readLine()) != null) {
+
+ // spin up a resolver thread per url
+ LOG.info("Starting: {}", urlStr);
+ pool.execute(new ResolverThread(urlStr));
+ }
}
- // close the file and wait for up to 60 seconds before shutting down
- // the thread pool to give urls time to finish resolving
- buffRead.close();
+ // wait for up to 60 seconds before shutting down the thread pool to
+ // give urls time to finish resolving
pool.awaitTermination(60, TimeUnit.SECONDS);
} catch (Exception e) {
diff --git
a/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java
b/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java
index a444f4312..720f5a6d1 100644
--- a/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java
+++ b/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java
@@ -198,9 +198,9 @@ public class FeedParser implements Parser {
parser.setConf(conf);
File file = new File(name);
byte[] bytes = new byte[(int) file.length()];
- DataInputStream in = new DataInputStream(new FileInputStream(file));
- in.readFully(bytes);
- in.close();
+ try (DataInputStream in = new DataInputStream(new FileInputStream(file))) {
+ in.readFully(bytes);
+ }
ParseResult parseResult = parser.getParse(new Content(url, url, bytes,
"application/rss+xml", new Metadata(), conf));
for (Entry<Text, Parse> entry : parseResult) {
diff --git
a/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java
b/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java
index 172b2030a..8e83572f3 100644
--- a/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java
+++ b/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java
@@ -18,7 +18,6 @@ package org.apache.nutch.parse.js;
import java.io.BufferedReader;
import java.io.FileInputStream;
-import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.invoke.MethodHandles;
import java.net.MalformedURLException;
@@ -278,14 +277,14 @@ public class JSParseFilter implements HtmlParseFilter,
Parser {
System.err.println(JSParseFilter.class.getName() + " file.js baseURL");
return;
}
- InputStream in = new FileInputStream(args[0]);
- BufferedReader br = new BufferedReader(
- new InputStreamReader(in, StandardCharsets.UTF_8));
StringBuffer sb = new StringBuffer();
- String line = null;
- while ((line = br.readLine()) != null)
- sb.append(line + "\n");
- br.close();
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(new FileInputStream(args[0]),
+ StandardCharsets.UTF_8))) {
+ String line = null;
+ while ((line = br.readLine()) != null)
+ sb.append(line + "\n");
+ }
JSParseFilter parseFilter = new JSParseFilter();
parseFilter.setConf(NutchConfiguration.create());
diff --git
a/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java
b/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java
index 315f7e79b..7dc08fd7b 100644
--- a/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java
+++ b/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java
@@ -124,10 +124,11 @@ public class ZipParser implements Parser {
}
File file = new File(args[0]);
String url = "file:"+file.getCanonicalPath();
- FileInputStream in = new FileInputStream(file);
- byte[] bytes = new byte[in.available()];
- in.read(bytes);
- in.close();
+ byte[] bytes;
+ try (FileInputStream in = new FileInputStream(file)) {
+ bytes = new byte[in.available()];
+ in.read(bytes);
+ }
Configuration conf = NutchConfiguration.create();
ZipParser parser = new ZipParser();
parser.setConf(conf);
diff --git
a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.java
b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.java
index 4decb7a35..362c341cc 100644
---
a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.java
+++
b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.java
@@ -70,23 +70,23 @@ public class Classify {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(configuration);
- BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(
- fs.open(new Path("naivebayes-model")), StandardCharsets.UTF_8));
-
- uniquewords_size = Integer.parseInt(bufferedReader.readLine());
- bufferedReader.readLine();
-
- numof_ir = Integer.parseInt(bufferedReader.readLine());
- numwords_ir = Integer.parseInt(bufferedReader.readLine());
- wordfreq_ir = unflattenToHashmap(bufferedReader.readLine());
- bufferedReader.readLine();
- numof_r = Integer.parseInt(bufferedReader.readLine());
- numwords_r = Integer.parseInt(bufferedReader.readLine());
- wordfreq_r = unflattenToHashmap(bufferedReader.readLine());
-
- ismodel = true;
-
- bufferedReader.close();
+ try (BufferedReader bufferedReader = new BufferedReader(
+ new InputStreamReader(fs.open(new Path("naivebayes-model")),
+ StandardCharsets.UTF_8))) {
+
+ uniquewords_size = Integer.parseInt(bufferedReader.readLine());
+ bufferedReader.readLine();
+
+ numof_ir = Integer.parseInt(bufferedReader.readLine());
+ numwords_ir = Integer.parseInt(bufferedReader.readLine());
+ wordfreq_ir = unflattenToHashmap(bufferedReader.readLine());
+ bufferedReader.readLine();
+ numof_r = Integer.parseInt(bufferedReader.readLine());
+ numwords_r = Integer.parseInt(bufferedReader.readLine());
+ wordfreq_r = unflattenToHashmap(bufferedReader.readLine());
+
+ ismodel = true;
+ }
}
diff --git
a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.java
b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.java
index ab021334b..0b23705f4 100644
---
a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.java
+++
b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.java
@@ -90,34 +90,36 @@ public class Train {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(configuration);
- BufferedReader bufferedReader = new BufferedReader(
- configuration.getConfResourceAsReader(filepath));
+ try (BufferedReader bufferedReader = new BufferedReader(
+ configuration.getConfResourceAsReader(filepath))) {
- while ((line = bufferedReader.readLine()) != null) {
+ while ((line = bufferedReader.readLine()) != null) {
- target = line.split("\t")[0];
+ target = line.split("\t")[0];
- line = replacefirstoccuranceof(target + "\t", line);
+ line = replacefirstoccuranceof(target + "\t", line);
- linearray = line.replaceAll("[^a-zA-Z ]", "").toLowerCase(Locale.ROOT)
- .split(" ");
+ linearray = line.replaceAll("[^a-zA-Z ]", "").toLowerCase(Locale.ROOT)
+ .split(" ");
- // update the data structures
- if (target.equals("0")) {
+ // update the data structures
+ if (target.equals("0")) {
- numof_ir += 1;
- numwords_ir += linearray.length;
- for (int i = 0; i < linearray.length; i++) {
- uniquewords.add(linearray[i]);
- updateHashMap(wordfreq_ir, linearray[i]);
- }
- } else {
+ numof_ir += 1;
+ numwords_ir += linearray.length;
+ for (int i = 0; i < linearray.length; i++) {
+ uniquewords.add(linearray[i]);
+ updateHashMap(wordfreq_ir, linearray[i]);
+ }
+ } else {
+
+ numof_r += 1;
+ numwords_r += linearray.length;
+ for (int i = 0; i < linearray.length; i++) {
+ uniquewords.add(linearray[i]);
+ updateHashMap(wordfreq_r, linearray[i]);
+ }
- numof_r += 1;
- numwords_r += linearray.length;
- for (int i = 0; i < linearray.length; i++) {
- uniquewords.add(linearray[i]);
- updateHashMap(wordfreq_r, linearray[i]);
}
}
@@ -128,22 +130,19 @@ public class Train {
Path path = new Path("naivebayes-model");
- Writer writer = new BufferedWriter(new OutputStreamWriter(fs.create(path,
- true), StandardCharsets.UTF_8));
-
- writer.write(String.valueOf(uniquewords.size()) + "\n");
- writer.write("0\n");
- writer.write(String.valueOf(numof_ir) + "\n");
- writer.write(String.valueOf(numwords_ir) + "\n");
- writer.write(flattenHashMap(wordfreq_ir) + "\n");
- writer.write("1\n");
- writer.write(String.valueOf(numof_r) + "\n");
- writer.write(String.valueOf(numwords_r) + "\n");
- writer.write(flattenHashMap(wordfreq_r) + "\n");
-
- writer.close();
-
- bufferedReader.close();
+ try (Writer writer = new BufferedWriter(new OutputStreamWriter(fs.create(
+ path, true), StandardCharsets.UTF_8))) {
+
+ writer.write(String.valueOf(uniquewords.size()) + "\n");
+ writer.write("0\n");
+ writer.write(String.valueOf(numof_ir) + "\n");
+ writer.write(String.valueOf(numwords_ir) + "\n");
+ writer.write(flattenHashMap(wordfreq_ir) + "\n");
+ writer.write("1\n");
+ writer.write(String.valueOf(numof_r) + "\n");
+ writer.write(String.valueOf(numwords_r) + "\n");
+ writer.write(flattenHashMap(wordfreq_r) + "\n");
+ }
}
diff --git
a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
index 266aab165..143ef3d1e 100644
---
a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
+++
b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
@@ -204,17 +204,17 @@ public class FileResponse {
this.content = new byte[len];
- java.io.InputStream is = new java.io.FileInputStream(f);
- int offset = 0;
- int n = 0;
- while (offset < len
- && (n = is.read(this.content, offset, len - offset)) >= 0) {
- offset += n;
- }
- if (offset < len) { // keep whatever already have, but issue a warning
- File.LOG.warn("not enough bytes read from file: {}", f.getPath());
+ try (java.io.InputStream is = new java.io.FileInputStream(f)) {
+ int offset = 0;
+ int n = 0;
+ while (offset < len
+ && (n = is.read(this.content, offset, len - offset)) >= 0) {
+ offset += n;
+ }
+ if (offset < len) { // keep whatever already have, but issue a warning
+ File.LOG.warn("not enough bytes read from file: {}", f.getPath());
+ }
}
- is.close();
// set headers
headers.set(Response.CONTENT_LENGTH, Long.valueOf(size).toString());
