This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 19c6bcf3975 HADOOP-19875. Further improve CrcComposer performance.
(#8497)
19c6bcf3975 is described below
commit 19c6bcf3975c36f63d0a3656d886fd138de0db3c
Author: Tsz-Wo Nicholas Sze <[email protected]>
AuthorDate: Thu May 14 13:07:15 2026 -0700
HADOOP-19875. Further improve CrcComposer performance. (#8497)
---
.../java/org/apache/hadoop/util/CrcComposer.java | 4 ++--
.../main/java/org/apache/hadoop/util/CrcUtil.java | 10 +++++-----
.../java/org/apache/hadoop/util/DataChecksum.java | 4 ++--
.../java/org/apache/hadoop/util/TestCrcUtil.java | 20 ++++++++++----------
4 files changed, 19 insertions(+), 19 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java
index 60549123ff3..4136091a20c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java
@@ -26,7 +26,7 @@
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.IOException;
-import java.util.function.ToIntFunction;
+import java.util.function.LongToIntFunction;
/**
* Encapsulates logic for composing multiple CRCs into one or more combined
CRCs
@@ -40,7 +40,7 @@ public final class CrcComposer {
private static final int CRC_SIZE_BYTES = 4;
private static final Logger LOG = LoggerFactory.getLogger(CrcComposer.class);
- private final ToIntFunction<Long> mod;
+ private final LongToIntFunction mod;
private final int precomputedMonomialForHint;
private final long bytesPerCrcHint;
private final long stripeLength;
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java
index 71b9e33b1fa..398e78b7461 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java
@@ -22,7 +22,7 @@
import org.apache.hadoop.classification.InterfaceStability;
import java.util.Arrays;
-import java.util.function.ToIntFunction;
+import java.util.function.LongToIntFunction;
/**
* This class provides utilities for working with CRCs.
@@ -39,7 +39,7 @@ public final class CrcUtil {
* @return a * b (mod p),
* where mod p is computed by the given mod function.
*/
- static int multiplyMod(int a, int b, ToIntFunction<Long> mod) {
+ static int multiplyMod(int a, int b, LongToIntFunction mod) {
final long left = ((long)a) << 32;
final long right = ((long)b) << 32;
@@ -98,7 +98,7 @@ private CrcUtil() {
* @param mod mod.
* @return monomial.
*/
- public static int getMonomial(long lengthBytes, ToIntFunction<Long> mod) {
+ public static int getMonomial(long lengthBytes, LongToIntFunction mod) {
if (lengthBytes == 0) {
return MULTIPLICATIVE_IDENTITY;
} else if (lengthBytes < 0) {
@@ -135,7 +135,7 @@ public static int getMonomial(long lengthBytes,
ToIntFunction<Long> mod) {
* @return compose with monomial.
*/
public static int composeWithMonomial(
- int crcA, int crcB, int monomial, ToIntFunction<Long> mod) {
+ int crcA, int crcB, int monomial, LongToIntFunction mod) {
return multiplyMod(crcA, monomial, mod) ^ crcB;
}
@@ -148,7 +148,7 @@ public static int composeWithMonomial(
* @param mod mod.
* @return compose result.
*/
- public static int compose(int crcA, int crcB, long lengthB,
ToIntFunction<Long> mod) {
+ public static int compose(int crcA, int crcB, long lengthB,
LongToIntFunction mod) {
int monomial = getMonomial(lengthB, mod);
return composeWithMonomial(crcA, crcB, monomial, mod);
}
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
index d964ea4bde1..d46e484c81d 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
@@ -22,7 +22,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.function.ToIntFunction;
+import java.util.function.LongToIntFunction;
import java.util.zip.CRC32;
import java.util.zip.CRC32C;
import java.util.zip.Checksum;
@@ -97,7 +97,7 @@ static Checksum newCrc32C() {
* @return the int representation of the polynomial associated with the
* CRC {@code type}, suitable for use with further CRC arithmetic.
*/
- static ToIntFunction<Long> getModFunction(Type type) {
+ static LongToIntFunction getModFunction(Type type) {
switch (type) {
case CRC32:
return PureJavaCrc32::mod;
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcUtil.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcUtil.java
index 94a6b724a5b..d7dadf3079a 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcUtil.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcUtil.java
@@ -19,7 +19,7 @@
import java.util.Objects;
import java.util.Random;
-import java.util.function.ToIntFunction;
+import java.util.function.LongToIntFunction;
import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
@@ -91,7 +91,7 @@ public void testComposeCrc32CZeroLength() {
*/
private static void doTestComposeCrc(
byte[] data, DataChecksum.Type type, int chunkSize, boolean useMonomial)
{
- final ToIntFunction<Long> mod = DataChecksum.getModFunction(type);
+ final LongToIntFunction mod = DataChecksum.getModFunction(type);
// Get full end-to-end CRC in a single shot first.
DataChecksum checksum = DataChecksum.newDataChecksum(
@@ -142,7 +142,7 @@ private static void
doTestComposeCrcZerolength(DataChecksum.Type type) {
// Without loss of generality, we can pick any integer as our fake crcA
// even if we don't happen to know the preimage.
int crcA = 0xCAFEBEEF;
- final ToIntFunction<Long> mod = DataChecksum.getModFunction(type);
+ final LongToIntFunction mod = DataChecksum.getModFunction(type);
DataChecksum checksum = DataChecksum.newDataChecksum(
type, Integer.MAX_VALUE);
Objects.requireNonNull(checksum, "checksum");
@@ -230,7 +230,7 @@ public void testMultiplyMod() {
private static long[] runTestMultiplyMod(int n, DataChecksum.Type type) {
System.out.printf("Run %s with %d computations%n", type, n);
final int polynomial = getCrcPolynomialForType(type);
- final ToIntFunction<Long> mod = DataChecksum.getModFunction(type);
+ final LongToIntFunction mod = DataChecksum.getModFunction(type);
final int[] p = new int[n];
final int[] q = new int[n];
@@ -256,8 +256,8 @@ private static long[] runTestMultiplyMod(int n,
DataChecksum.Type type) {
}
times[1] = System.currentTimeMillis() - t1;
final double ops1 = n * 1000.0 / times[1];
- System.out.printf("multiplyCrc32 : %.3fs (%.2f ops)%n", times[1] /
1000.0, ops1);
- System.out.printf("multiplyCrc32 is %.2f%% faster%n", (ops1 - ops0) *
100.0 / ops0);
+ System.out.printf("tableMultiply : %.3fs (%.2f ops)%n", times[1] /
1000.0, ops1);
+ System.out.printf("tableMultiply ops is %.2f%% faster%n", (ops1 - ops0) *
100.0 / ops0);
for (int i = 0; i < n; i++) {
if (expected[i] != computed[i]) {
@@ -354,11 +354,11 @@ public static void main(String[] args) throws Exception {
}
System.out.printf("%nResult) %d x %d computations:%n", m, n);
- final double ops0 = n * 1000.0 / times[0];
+ final double ops0 = m * n * 1000.0 / times[0];
System.out.printf("galoisFieldMultiply: %.3fs (%.2f ops)%n", times[0] /
1000.0, ops0);
- final double ops1 = n * 1000.0 / times[1];
- System.out.printf("multiplyCrc32 : %.3fs (%.2f ops)%n", times[1] /
1000.0, ops1);
- System.out.printf("multiplyCrc32 is %.2f%% faster%n", (ops1 - ops0) *
100.0 / ops0);
+ final double ops1 = m * n * 1000.0 / times[1];
+ System.out.printf("tableMultiply : %.3fs (%.2f ops)%n", times[1] /
1000.0, ops1);
+ System.out.printf("tableMultiply ops is %.2f%% faster%n", (ops1 - ops0)
* 100.0 / ops0);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
