This is an automated email from the ASF dual-hosted git repository. garydgregory pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/commons-codec.git
commit c11a6597405dba4c23319beff1c1a01dd30c6fd4 Author: Gary Gregory <[email protected]> AuthorDate: Sun Sep 20 18:01:19 2026 -0400 Add URLCodec.decodeUrl(BitSet, byte[]) for custom safe sets Document decoding rules and ambiguous safe sets. Add tests for round trips, malformed escapes, null and empty inputs, and CODEC-345 examples. --- src/changes/changes.xml | 1 + .../org/apache/commons/codec/net/URLCodec.java | 56 +++++++++++++----- .../org/apache/commons/codec/net/Codec345Test.java | 6 +- .../org/apache/commons/codec/net/URLCodecTest.java | 67 ++++++++++++++++++++++ 4 files changed, 113 insertions(+), 17 deletions(-) diff --git a/src/changes/changes.xml b/src/changes/changes.xml index 8afdb972..59c3aafa 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -74,6 +74,7 @@ The <action> type attribute can be add,update,fix,remove. <action type="add" dev="ggregory" due-to="Gary Gregory">Add BeiderMorseEncoder.Builder and deprecate old constructor.</action> <action type="add" dev="ggregory" due-to="Yu Bao, Gary Gregory">Add PhoneticEngine.Builder.setMaxInputLength(int).</action> <action type="add" dev="ggregory" due-to="Gary Gregory">Add Base45 support.</action> + <action type="add" dev="ggregory" due-to="Gary Gregory">Add URLCodec.decodeUrl(BitSet, byte[]) to decode with a custom safe set.</action> <!-- UPDATE --> <action type="update" dev="ggregory" due-to="Gary Gregory">Bump org.apache.commons:commons-parent from 103 to 105.</action> </release> diff --git a/src/main/java/org/apache/commons/codec/net/URLCodec.java b/src/main/java/org/apache/commons/codec/net/URLCodec.java index f48bdb95..68fbdc02 100644 --- a/src/main/java/org/apache/commons/codec/net/URLCodec.java +++ b/src/main/java/org/apache/commons/codec/net/URLCodec.java @@ -95,31 +95,37 @@ public class URLCodec implements BinaryEncoder, BinaryDecoder, StringEncoder, St } /** - * Decodes an array of URL safe 7-bit characters into an array of original bytes. Escaped characters are converted - * back to their original representation. - * + * Decodes an array of bytes using the safe set supplied to {@link #encodeUrl(BitSet, byte[])}. * <p> - * Decoding always follows {@code www-form-urlencoded} rules: {@code +} becomes a space and {@code %} starts a hexadecimal escape. - * Output from {@link #encodeUrl(BitSet, byte[])} with a custom safe set may therefore not decode back to the original input and may cause a - * {@link DecoderException}, depending on which characters were marked safe. + * A percent sign marked safe is copied literally; otherwise it starts a two-digit hexadecimal escape. A plus sign marked safe is copied literally. + * Otherwise, a plus sign becomes a space only if space is marked safe. All other bytes are copied unchanged. A {@code null} bitset selects the default + * {@code www-form-urlencoded} safe set, giving the same behavior as {@link #decodeUrl(byte[])}. + * </p> + * <p> + * Not every safe set permits a round trip. If both space and plus are marked safe, the encoder maps both to plus and this method preserves that plus. If + * percent is marked safe, literal percent signs cannot be distinguished from generated escapes, so this method preserves all percent signs, including + * generated escapes. Use a safe set that excludes percent and does not mark both space and plus safe when a round trip is required. * </p> * - * @param bytes - * array of URL safe characters. - * @return array of original bytes. - * @throws DecoderException - * Thrown if URL decoding is unsuccessful. + * @param urlsafe bitset of characters deemed URL safe during encoding, or {@code null} to use the default safe set. + * @param bytes array of encoded bytes, or {@code null}. + * @return array of decoded bytes, or {@code null} if the input is {@code null}. + * @throws DecoderException if percent is not marked safe and an escape is incomplete or contains invalid hexadecimal digits. + * @since 1.23.0 */ - public static final byte[] decodeUrl(final byte[] bytes) throws DecoderException { + public static final byte[] decodeUrl(BitSet urlsafe, final byte[] bytes) throws DecoderException { if (bytes == null) { return null; } + if (urlsafe == null) { + urlsafe = WWW_FORM_URL_SAFE; + } final ByteArrayOutputStream buffer = new ByteArrayOutputStream(); for (int i = 0; i < bytes.length; i++) { final int b = bytes[i]; - if (b == PLUS_CHAR) { + if (b == PLUS_CHAR && !urlsafe.get(PLUS_CHAR) && urlsafe.get(' ')) { buffer.write(' '); - } else if (b == ESCAPE_CHAR) { + } else if (b == ESCAPE_CHAR && !urlsafe.get(ESCAPE_CHAR)) { try { final int u = Utils.digit16(bytes[++i]); final int l = Utils.digit16(bytes[++i]); @@ -134,6 +140,26 @@ public class URLCodec implements BinaryEncoder, BinaryDecoder, StringEncoder, St return buffer.toByteArray(); } + /** + * Decodes an array of URL safe 7-bit characters into an array of original bytes. Escaped characters are converted + * back to their original representation. + * + * <p> + * Decoding always follows {@code www-form-urlencoded} rules: {@code +} becomes a space and {@code %} starts a hexadecimal escape. + * Output from {@link #encodeUrl(BitSet, byte[])} with a custom safe set may therefore not decode back to the original input and may cause a + * {@link DecoderException}, depending on which characters were marked safe. + * </p> + * + * @param bytes + * array of URL safe characters. + * @return array of original bytes. + * @throws DecoderException + * Thrown if URL decoding is unsuccessful. + */ + public static final byte[] decodeUrl(final byte[] bytes) throws DecoderException { + return decodeUrl(null, bytes); + } + /** * Encodes an array of bytes using the given set of URL safe characters. * <p> @@ -144,7 +170,7 @@ public class URLCodec implements BinaryEncoder, BinaryDecoder, StringEncoder, St * A custom bitset can produce output that {@link #decodeUrl(byte[])} and the {@code decode} methods cannot decode back to the original input. These * decoders always convert {@code +} to a space and interpret {@code %} as the start of a hexadecimal escape, regardless of the bitset used for encoding. If * the custom bitset marks either character safe, decoding can change the original data or throw {@link DecoderException}. Callers using a custom bitset - * must choose decoding rules appropriate to that bitset and the URI component being encoded. + * can use {@link #decodeUrl(BitSet, byte[])} with the same bitset, subject to its documented limitations for ambiguous safe sets. * </p> * * @param urlsafe bitset of characters deemed URL safe, or {@code null} to use the default {@code www-form-urlencoded} safe set. diff --git a/src/test/java/org/apache/commons/codec/net/Codec345Test.java b/src/test/java/org/apache/commons/codec/net/Codec345Test.java index 861ff1be..7a72a58d 100644 --- a/src/test/java/org/apache/commons/codec/net/Codec345Test.java +++ b/src/test/java/org/apache/commons/codec/net/Codec345Test.java @@ -32,7 +32,7 @@ class Codec345Test { @ParameterizedTest @ValueSource(strings = { "/pages/1/Test+Page", "/display/TST/Caf%C3%A9" }) - void testEncodeUrlWithCallerSuppliedSafeCharacters(final String input) { + void testEncodeUrlWithCallerSuppliedSafeCharacters(final String input) throws Exception { // RFC 2396 abs_path, as used by HtmlUnit's UrlUtils and HttpClient 3.x's URI. final BitSet allowed = new BitSet(256); for (int c = 'a'; c <= 'z'; c++) { @@ -51,6 +51,8 @@ class Codec345Test { for (final char c : ":@&=+$,;/".toCharArray()) { allowed.set(c); // pchar } - assertEquals(input, new String(URLCodec.encodeUrl(allowed, input.getBytes(StandardCharsets.UTF_8)), StandardCharsets.US_ASCII)); + final byte[] encoded = URLCodec.encodeUrl(allowed, input.getBytes(StandardCharsets.UTF_8)); + assertEquals(input, new String(encoded, StandardCharsets.US_ASCII)); + assertEquals(input, new String(URLCodec.decodeUrl(allowed, encoded), StandardCharsets.UTF_8)); } } diff --git a/src/test/java/org/apache/commons/codec/net/URLCodecTest.java b/src/test/java/org/apache/commons/codec/net/URLCodecTest.java index a0c8b421..86c93faa 100644 --- a/src/test/java/org/apache/commons/codec/net/URLCodecTest.java +++ b/src/test/java/org/apache/commons/codec/net/URLCodecTest.java @@ -17,6 +17,7 @@ package org.apache.commons.codec.net; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -28,6 +29,8 @@ import org.apache.commons.codec.CharEncoding; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.EncoderException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * URL codec test cases @@ -109,6 +112,70 @@ class URLCodecTest { assertNull(result, "Result should be null"); } + @Test + void testDecodeUrlWithCustomBitSetAmbiguousEscapes() throws Exception { + final BitSet safe = new BitSet(); + safe.set('%'); + final byte[] encoded = URLCodec.encodeUrl(safe, new byte[] { '%', '/' }); + assertEquals("%%2F", new String(encoded, StandardCharsets.US_ASCII)); + assertArrayEquals(encoded, URLCodec.decodeUrl(safe, encoded)); + } + + @ParameterizedTest + @ValueSource(strings = { "%", "%A", "%WW", "%0W", "%W0" }) + void testDecodeUrlWithCustomBitSetInvalidEscapes(final String input) throws Exception { + final byte[] bytes = input.getBytes(StandardCharsets.US_ASCII); + final BitSet safe = new BitSet(); + assertThrows(DecoderException.class, () -> URLCodec.decodeUrl(null, bytes)); + assertThrows(DecoderException.class, () -> URLCodec.decodeUrl(safe, bytes)); + safe.set('%'); + assertArrayEquals(bytes, URLCodec.decodeUrl(safe, bytes)); + } + + @Test + void testDecodeUrlWithCustomBitSetNullAndEmpty() throws Exception { + assertNull(URLCodec.decodeUrl(null, null)); + assertNull(URLCodec.decodeUrl(new BitSet(), null)); + assertArrayEquals(new byte[0], URLCodec.decodeUrl(null, new byte[0])); + assertArrayEquals(new byte[0], URLCodec.decodeUrl(new BitSet(), new byte[0])); + } + + @ParameterizedTest + @ValueSource(ints = { 0, 1, 2, 3 }) + void testDecodeUrlWithCustomBitSetPlusAndSpace(final int flags) throws Exception { + final BitSet safe = new BitSet(); + safe.set(' ', (flags & 1) != 0); + safe.set('+', (flags & 2) != 0); + final String expected = flags == 1 ? " +" : "+ +"; + assertEquals(expected, new String(URLCodec.decodeUrl(safe, "+%20%2b".getBytes(StandardCharsets.US_ASCII)), StandardCharsets.US_ASCII)); + if (flags == 3) { + final byte[] encoded = URLCodec.encodeUrl(safe, " +".getBytes(StandardCharsets.US_ASCII)); + assertEquals("++", new String(encoded, StandardCharsets.US_ASCII)); + assertArrayEquals(encoded, URLCodec.decodeUrl(safe, encoded)); + } + } + + @Test + void testDecodeUrlWithCustomBitSetRoundTripAllBytes() throws Exception { + final byte[] input = new byte[256]; + for (int i = 0; i < input.length; i++) { + input[i] = (byte) i; + } + final BitSet literalPlus = new BitSet(256); + literalPlus.set(0, 256); + literalPlus.clear('%'); + literalPlus.clear(' '); + final BitSet spaceAsPlus = (BitSet) literalPlus.clone(); + spaceAsPlus.clear('+'); + spaceAsPlus.set(' '); + for (final BitSet safe : new BitSet[] { null, new BitSet(), literalPlus, spaceAsPlus }) { + final BitSet original = safe == null ? null : (BitSet) safe.clone(); + final byte[] encoded = URLCodec.encodeUrl(safe, input); + assertArrayEquals(input, URLCodec.decodeUrl(safe, encoded)); + assertEquals(original, safe, "The safe set must not be modified"); + } + } + @Test void testDecodeWithNullArray() throws Exception { final byte[] plain = null;
