This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 44583888f NUTCH-3209 Replace SHA-1 URL fingerprints in dump tools with
SHA-256 (#964)
44583888f is described below
commit 44583888f0946ff80cba1f06ddc1761ba19d6344
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Sun Sep 13 04:35:32 2026 -0700
NUTCH-3209 Replace SHA-1 URL fingerprints in dump tools with SHA-256 (#964)
---
.../apache/nutch/tools/CommonCrawlDataDumper.java | 2 +-
src/java/org/apache/nutch/util/DumpFileUtil.java | 42 +++---
.../org/apache/nutch/util/DumpFileUtilTest.java | 145 +++++++++++++++------
3 files changed, 127 insertions(+), 62 deletions(-)
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
index fe714218e..e97fac03b 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
@@ -324,7 +324,7 @@ public class CommonCrawlDataDumper extends NutchTool
implements Tool {
reverseKey = reverseUrl(url);
config.setReverseKeyValue(
- reverseKey.replace("/", "_") + "_" + DigestUtils.sha1Hex(url)
+ reverseKey.replace("/", "_") + "_" +
DigestUtils.sha256Hex(url)
+ "_" + timestamp);
}
diff --git a/src/java/org/apache/nutch/util/DumpFileUtil.java
b/src/java/org/apache/nutch/util/DumpFileUtil.java
index d664ac571..9461a0177 100644
--- a/src/java/org/apache/nutch/util/DumpFileUtil.java
+++ b/src/java/org/apache/nutch/util/DumpFileUtil.java
@@ -91,27 +91,33 @@ public class DumpFileUtil {
fileExtension);
}
- public static String createFileNameFromUrl(String basePath, String
reverseKey, String urlString, String epochScrapeTime, String fileExtension,
boolean makeDir) {
- String fullDirPath = basePath + File.separator + reverseKey +
File.separator + DigestUtils.sha1Hex(urlString);
-
- if (makeDir) {
- try {
- FileUtils.forceMkdir(new File(fullDirPath));
- } catch (IOException e) {
- LOG.error("Failed to create dir: {}", fullDirPath);
- fullDirPath = null;
- }
- }
+ public static String createFileNameFromUrl(String basePath,
+ String reverseKey, String urlString, String epochScrapeTime,
+ String fileExtension, boolean makeDir) {
+ String fullDirPath = basePath + File.separator + reverseKey
+ + File.separator + DigestUtils.sha256Hex(urlString);
+
+ if (makeDir) {
+ try {
+ FileUtils.forceMkdir(new File(fullDirPath));
+ } catch (IOException e) {
+ LOG.error("Failed to create dir: {}", fullDirPath);
+ fullDirPath = null;
+ }
+ }
- if (fileExtension.length() > MAX_LENGTH_OF_EXTENSION) {
- LOG.info("File extension is too long. Truncated to {}
characters.", MAX_LENGTH_OF_EXTENSION);
- fileExtension = StringUtils.substring(fileExtension, 0,
MAX_LENGTH_OF_EXTENSION);
- }
+ if (fileExtension.length() > MAX_LENGTH_OF_EXTENSION) {
+ LOG.info("File extension is too long. Truncated to {} characters.",
+ MAX_LENGTH_OF_EXTENSION);
+ fileExtension = StringUtils.substring(fileExtension, 0,
+ MAX_LENGTH_OF_EXTENSION);
+ }
- String outputFullPath = fullDirPath + File.separator +
epochScrapeTime + "." + fileExtension;
+ String outputFullPath = fullDirPath + File.separator + epochScrapeTime
+ + "." + fileExtension;
- return outputFullPath;
- }
+ return outputFullPath;
+ }
public static String displayFileTypes(Map<String, Integer> typeCounts,
Map<String, Integer> filteredCounts) {
StringBuilder builder = new StringBuilder();
diff --git a/src/test/org/apache/nutch/util/DumpFileUtilTest.java
b/src/test/org/apache/nutch/util/DumpFileUtilTest.java
index 546ea7e6b..5916901bf 100644
--- a/src/test/org/apache/nutch/util/DumpFileUtilTest.java
+++ b/src/test/org/apache/nutch/util/DumpFileUtilTest.java
@@ -16,6 +16,9 @@
*/
package org.apache.nutch.util;
+import java.io.File;
+
+import org.apache.commons.codec.digest.DigestUtils;
import org.junit.jupiter.api.Test;
import static org.hamcrest.CoreMatchers.is;
@@ -24,47 +27,103 @@ import static org.hamcrest.MatcherAssert.assertThat;
public class DumpFileUtilTest {
- @Test
- public void testGetUrlMD5() throws Exception {
- String testUrl = "http://apache.org";;
-
- String result = DumpFileUtil.getUrlMD5(testUrl);
-
- assertThat(result, is("991e599262e04ea2ec76b6c5aed499a7"));
- }
-
- @Test
- public void testCreateTwoLevelsDirectory() throws Exception {
- String testUrl = "http://apache.org";;
- String basePath = "/tmp";
- String fullDir = DumpFileUtil.createTwoLevelsDirectory(basePath,
DumpFileUtil.getUrlMD5(testUrl));
-
- assertThat(fullDir, is("/tmp/96/ea"));
-
- String basePath2 = "/this/path/is/not/existed/just/for/testing";
- String fullDir2 = DumpFileUtil.createTwoLevelsDirectory(basePath2,
DumpFileUtil.getUrlMD5(testUrl));
-
- assertThat(fullDir2, nullValue());
- }
-
- @Test
- public void testCreateFileName() throws Exception {
- String testUrl = "http://apache.org";;
- String baseName = "test";
- String extension = "html";
- String fullDir =
DumpFileUtil.createFileName(DumpFileUtil.getUrlMD5(testUrl), baseName,
extension);
-
- assertThat(fullDir, is("991e599262e04ea2ec76b6c5aed499a7_test.html"));
-
- String tooLongBaseName =
"testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest";
- String fullDir2 =
DumpFileUtil.createFileName(DumpFileUtil.getUrlMD5(testUrl), tooLongBaseName,
extension);
-
- assertThat(fullDir2,
-
is("991e599262e04ea2ec76b6c5aed499a7_testtesttesttesttesttesttesttest.html"));
-
- String tooLongExtension =
"testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest";
- String fullDir3 =
DumpFileUtil.createFileName(DumpFileUtil.getUrlMD5(testUrl), baseName,
tooLongExtension);
-
- assertThat(fullDir3,
is("991e599262e04ea2ec76b6c5aed499a7_test.testt"));
- }
+ /**
+ * Verifies that {@link DumpFileUtil#getUrlMD5} returns the lowercase hex MD5
+ * digest of a URL.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testGetUrlMD5() throws Exception {
+ String testUrl = "http://apache.org";;
+
+ String result = DumpFileUtil.getUrlMD5(testUrl);
+
+ assertThat(result, is("991e599262e04ea2ec76b6c5aed499a7"));
+ }
+
+ /**
+ * Verifies that {@link DumpFileUtil#createTwoLevelsDirectory} builds a
+ * two-level dump directory from selected characters of the URL MD5, and
+ * returns {@code null} when the directory cannot be created.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCreateTwoLevelsDirectory() throws Exception {
+ String testUrl = "http://apache.org";;
+ String basePath = "/tmp";
+ String fullDir = DumpFileUtil.createTwoLevelsDirectory(basePath,
+ DumpFileUtil.getUrlMD5(testUrl));
+
+ assertThat(fullDir, is("/tmp/96/ea"));
+
+ String basePath2 = "/this/path/is/not/existed/just/for/testing";
+ String fullDir2 = DumpFileUtil.createTwoLevelsDirectory(basePath2,
+ DumpFileUtil.getUrlMD5(testUrl));
+
+ assertThat(fullDir2, nullValue());
+ }
+
+ /**
+ * Verifies that {@link DumpFileUtil#createFileName} formats
+ * {@code md5_basename.extension} and truncates a too-long base name or
+ * extension.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCreateFileName() throws Exception {
+ String testUrl = "http://apache.org";;
+ String baseName = "test";
+ String extension = "html";
+ String fullDir = DumpFileUtil.createFileName(
+ DumpFileUtil.getUrlMD5(testUrl), baseName, extension);
+
+ assertThat(fullDir, is("991e599262e04ea2ec76b6c5aed499a7_test.html"));
+
+ String tooLongBaseName =
"testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest";
+ String fullDir2 = DumpFileUtil.createFileName(
+ DumpFileUtil.getUrlMD5(testUrl), tooLongBaseName, extension);
+
+ assertThat(fullDir2, is(
+
"991e599262e04ea2ec76b6c5aed499a7_testtesttesttesttesttesttesttest.html"));
+
+ String tooLongExtension =
"testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest";
+ String fullDir3 = DumpFileUtil.createFileName(
+ DumpFileUtil.getUrlMD5(testUrl), baseName, tooLongExtension);
+
+ assertThat(fullDir3, is("991e599262e04ea2ec76b6c5aed499a7_test.testt"));
+ }
+
+ /**
+ * Verifies that {@link DumpFileUtil#createFileNameFromUrl} fingerprints the
+ * URL with SHA-256 (not SHA-1) in the dump path, and truncates a too-long
+ * file extension.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCreateFileNameFromUrl() throws Exception {
+ String testUrl = "http://apache.org";;
+ String reverseKey = "org.apache.http";
+ String epoch = "1234567890";
+ String sha256 = DigestUtils.sha256Hex(testUrl);
+ String sha1 = DigestUtils.sha1Hex(testUrl);
+
+ String path = DumpFileUtil.createFileNameFromUrl("/tmp", reverseKey,
+ testUrl, epoch, "html", false);
+
+ assertThat(path, is("/tmp" + File.separator + reverseKey + File.separator
+ + sha256 + File.separator + epoch + ".html"));
+ assertThat(path.contains(sha1), is(false));
+
+ String tooLongExtension = "htmlhtmlhtml";
+ String pathTruncated = DumpFileUtil.createFileNameFromUrl("/tmp",
+ reverseKey, testUrl, epoch, tooLongExtension, false);
+
+ assertThat(pathTruncated,
+ is("/tmp" + File.separator + reverseKey + File.separator + sha256
+ + File.separator + epoch + ".htmlh"));
+ }
}
