This is an automated email from the ASF dual-hosted git repository.
sebastian-nagel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new c336054cf NUTCH-3187 - protocol-okhttp: IP address filter fails to
compare first byte for /32 resp. /128 CIDRs
c336054cf is described below
commit c336054cfe70f166609546086c119b7209de740f
Author: Sebastian Nagel <[email protected]>
AuthorDate: Tue Jun 16 19:11:02 2026 +0200
NUTCH-3187 - protocol-okhttp: IP address filter fails to compare first byte
for /32 resp. /128 CIDRs
---
.../org/apache/nutch/protocol/okhttp/CIDR.java | 38 +++++++++++++++++-----
.../nutch/protocol/okhttp/IPFilterRules.java | 6 ++--
.../protocol/okhttp/TestIPAddressFiltering.java | 19 +++++++++++
3 files changed, 51 insertions(+), 12 deletions(-)
diff --git
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java
index 3add082a8..1d97e6728 100644
---
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java
+++
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java
@@ -27,10 +27,16 @@ import com.google.common.net.InetAddresses;
* defined by the CIDR.
*/
public class CIDR {
- InetAddress addr;
- int mask;
- public CIDR(InetAddress address, int mask) {
+ private final InetAddress addr;
+ private final int mask;
+
+ public CIDR(InetAddress address, int mask) throws IllegalArgumentException {
+ int maxMask = address.getAddress().length * 8;
+ if (mask < 0 || mask > maxMask) {
+ throw new IllegalArgumentException(
+ "Invalid CIDR mask /" + mask + " for " + address);
+ }
this.addr = address;
this.mask = mask;
}
@@ -42,16 +48,23 @@ public class CIDR {
ipStr = cidr.substring(0, sep);
}
addr = InetAddresses.forString(ipStr);
+ int parsedMask;
if (sep > -1) {
- mask = Integer.parseInt(cidr.substring(sep + 1));
+ parsedMask = Integer.parseInt(cidr.substring(sep + 1));
} else {
- mask = addr.getAddress().length * 8;
+ parsedMask = addr.getAddress().length * 8;
}
if (cidr.indexOf(':') > -1 && addr.getAddress().length == 4) {
// IPv4-mapped IPv6 addresses are automatically converted to IPv4,
// need to shift the mask
- mask = Math.max(0, mask - 96);
+ parsedMask = Math.max(0, parsedMask - 96);
+ }
+ int maxMask = addr.getAddress().length * 8;
+ if (parsedMask < 0 || parsedMask > maxMask) {
+ throw new IllegalArgumentException(
+ "Invalid CIDR mask /" + parsedMask + " for " + ipStr);
}
+ this.mask = parsedMask;
}
public boolean contains(InetAddress address) {
@@ -63,11 +76,18 @@ public class CIDR {
}
for (int i = 0; i < addr0.length; i++) {
int remainingMaskBits = mask - (i * 8);
- if (remainingMaskBits <= 0)
+ if (remainingMaskBits <= 0) {
return true;
- int m = ~(0xff >> remainingMaskBits); // mask for byte under cursor
- if ((addr0[i] & m) != (addr1[i] & m))
+ }
+ /*
+ * keep the mask within one byte so the shift does not wrap (Java shifts
+ * mod 32)
+ */
+ int m = remainingMaskBits >= 8 ? 0xff
+ : (0xff << (8 - remainingMaskBits)) & 0xff;
+ if ((addr0[i] & m) != (addr1[i] & m)) {
return false;
+ }
}
return true;
}
diff --git
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java
index a9252ddc5..5ba0a9fd6 100644
---
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java
+++
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java
@@ -105,15 +105,15 @@ public class IPFilterRules {
switch (ipRule.toLowerCase(Locale.ROOT)) {
case "localhost":
case "loopback":
- rules.add((InetAddress a) -> a.isLoopbackAddress());
+ rules.add(InetAddress::isLoopbackAddress);
break;
case "sitelocal":
- rules.add((InetAddress a) -> a.isSiteLocalAddress());
+ rules.add(InetAddress::isSiteLocalAddress);
break;
default:
try {
CIDR cidr = new CIDR(ipRule);
- rules.add((InetAddress a) -> cidr.contains(a));
+ rules.add(cidr::contains);
} catch (IllegalArgumentException e) {
LOG.error(
"Failed to parse {} as CIDR, ignoring to configure IP rules
({})",
diff --git
a/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java
b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java
index 309c977b1..63627bc97 100644
---
a/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java
+++
b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java
@@ -28,6 +28,7 @@ import com.google.common.net.InetAddresses;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@ -63,6 +64,17 @@ public class TestIPAddressFiltering extends
AbstractHttpProtocolPluginTest {
assertFalse(c.contains(i), i + " should not be in " + c);
}
+ @Test
+ public void testInvalidCIDR() {
+ assertThrows(IllegalArgumentException.class,
+ () -> new CIDR(InetAddress.getByName("1.2.3.4"), 33));
+ assertThrows(IllegalArgumentException.class, () -> new CIDR("1.2.3.4/33"));
+ assertThrows(IllegalArgumentException.class, () -> new
CIDR("::ffff:7f00:0/129"));
+ assertThrows(IllegalArgumentException.class, () -> new CIDR("1.2.3.4/-1"));
+ // invalid rule
+ assertThrows(IllegalArgumentException.class, () -> new CIDR("foobar"));
+ }
+
/** Tests for {@link CIDR} */
@Test
public void testCIDRs() {
@@ -89,6 +101,13 @@ public class TestIPAddressFiltering extends
AbstractHttpProtocolPluginTest {
testCIDRnotContains("10.0.0.0/8", "9.255.255.255");
testCIDRnotContains("172.16.0.0/12", "172.32.0.0");
testCIDRnotContains("172.16.0.0/12", "171.255.255.255");
+
+ testCIDRnotContains("127.0.0.1/31", "0.0.0.1");
+ // NUTCH-3187 CIDR edge cases - /32, /128 - test bit shifting
+ testCIDRnotContains("127.0.0.1", "0.0.0.1");
+ testCIDRnotContains("127.0.0.1/32", "0.0.0.1");
+ testCIDRnotContains("127.0.0.1", "255.0.0.1");
+ testCIDRnotContains("fe80::2f29:b6f0:a4c:32ae/128",
"::2f29:b6f0:a4c:32ae");
}
public void testFilter(Configuration conf, String[] included, String[]
excluded) {
