On 13/04/2021 11:39, Willy Tarreau wrote:
>> You can find attached the patches 0001-bis and 0002-bis modifying the
>> existing functions (introducing an url flag) to see how it looks like.
>> This solution may be cleaner (no chunk allocation and we don't loop
>> twice over input string) but has the drawbacks of being intrusive with
>> the rest of the code and less clearer imo regarding how url variant is
>> different from standard base64.
>
> I agree they're not pretty due to the change of logic around the padding,
> thanks for having tested! But then how about having just *your* functions
> without relying on the other ones ? Now that you've extended the existing
> function, you can declare it yours, remove all the configurable stuff and
> keep the simplified version as the one you need. I'm sure it will be the
> best tradeoff overall.
>
Yes that makes sense to me too, the attached patch deal with it as
suggested.
>> diff --git a/src/base64.c b/src/base64.c
>> index 53e4d65b2..f2768d980 100644
>> --- a/src/base64.c
>> +++ b/src/base64.c
>> @@ -1,5 +1,5 @@
>> /*
>> - * ASCII <-> Base64 conversion as described in RFC1421.
>> + * ASCII <-> Base64 conversion as described in RFC4648.
>> *
>> * Copyright 2006-2010 Willy Tarreau <w...@1wt.eu>
>> * Copyright 2009-2010 Krzysztof Piotr Oledzki <o...@ans.pl>
>> @@ -17,50 +17,70 @@
>> #include <haproxy/api.h>
>> #include <haproxy/base64.h>
>>
>> -#define B64BASE '#' /* arbitrary chosen base value */
>> -#define B64CMIN '+'
>> -#define B64CMAX 'z'
>> -#define B64PADV 64 /* Base64 chosen special pad value */
>> +#define B64BASE '#' /* arbitrary chosen base value */
>> +#define B64CMIN '+'
>> +#define UB64CMIN '-'
>> +#define B64CMAX 'z'
>> +#define B64PADV 64 /* Base64 chosen special pad value */
>
> Please do not needlessly reindent code parts for no reason. It seems that
> only the "-" was added there, the rest shouldn't change.
The reason is I was following the doc/coding-style where alignment
should use spaces, but since the existing bloc was aligned via tabs, I
thought about fixing that instead of repeating the issue. I understand
though that in such case better have this in separate commit, so I have
stuck with the tabs alignment.
> By the way, contrib/ was move to dev/ during your changes so if you keep
> this comment please update it.
Done.
On 13/04/2021 08:19, Jarno Huuskonen wrote:
> Could you add a cross reference from b64dec/base64 to ub64dec/ub64enc in
> configuration.txt.
Done thanks.
--
Moemen
>From b526416364b98afaa2d2b421fbf27f80bc4e8732 Mon Sep 17 00:00:00 2001
From: Moemen MHEDHBI <mmhed...@haproxy.com>
Date: Fri, 2 Apr 2021 01:05:07 +0200
Subject: [PATCH 2/2] CLEANUP: align samples list in sample.c
---
src/sample.c | 54 ++++++++++++++++++++++++++--------------------------
1 file changed, 27 insertions(+), 27 deletions(-)
diff --git a/src/sample.c b/src/sample.c
index 04635a91f..7337ba06a 100644
--- a/src/sample.c
+++ b/src/sample.c
@@ -4129,33 +4129,33 @@ INITCALL1(STG_REGISTER, sample_register_fetches, &smp_kws);
/* Note: must not be declared <const> as its list will be overwritten */
static struct sample_conv_kw_list sample_conv_kws = {ILH, {
- { "debug", sample_conv_debug, ARG2(0,STR,STR), smp_check_debug, SMP_T_ANY, SMP_T_ANY },
- { "b64dec", sample_conv_base642bin,0, NULL, SMP_T_STR, SMP_T_BIN },
- { "base64", sample_conv_bin2base64,0, NULL, SMP_T_BIN, SMP_T_STR },
- { "ub64dec", sample_conv_base64url2bin,0, NULL, SMP_T_STR, SMP_T_BIN },
- { "ub64enc", sample_conv_bin2base64url,0, NULL, SMP_T_BIN, SMP_T_STR },
- { "upper", sample_conv_str2upper, 0, NULL, SMP_T_STR, SMP_T_STR },
- { "lower", sample_conv_str2lower, 0, NULL, SMP_T_STR, SMP_T_STR },
- { "length", sample_conv_length, 0, NULL, SMP_T_STR, SMP_T_SINT },
- { "hex", sample_conv_bin2hex, 0, NULL, SMP_T_BIN, SMP_T_STR },
- { "hex2i", sample_conv_hex2int, 0, NULL, SMP_T_STR, SMP_T_SINT },
- { "ipmask", sample_conv_ipmask, ARG2(1,MSK4,MSK6), NULL, SMP_T_ADDR, SMP_T_IPV4 },
- { "ltime", sample_conv_ltime, ARG2(1,STR,SINT), NULL, SMP_T_SINT, SMP_T_STR },
- { "utime", sample_conv_utime, ARG2(1,STR,SINT), NULL, SMP_T_SINT, SMP_T_STR },
- { "crc32", sample_conv_crc32, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
- { "crc32c", sample_conv_crc32c, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
- { "djb2", sample_conv_djb2, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
- { "sdbm", sample_conv_sdbm, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
- { "wt6", sample_conv_wt6, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
- { "xxh3", sample_conv_xxh3, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
- { "xxh32", sample_conv_xxh32, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
- { "xxh64", sample_conv_xxh64, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
- { "json", sample_conv_json, ARG1(1,STR), sample_conv_json_check, SMP_T_STR, SMP_T_STR },
- { "bytes", sample_conv_bytes, ARG2(1,SINT,SINT), NULL, SMP_T_BIN, SMP_T_BIN },
- { "field", sample_conv_field, ARG3(2,SINT,STR,SINT), sample_conv_field_check, SMP_T_STR, SMP_T_STR },
- { "word", sample_conv_word, ARG3(2,SINT,STR,SINT), sample_conv_field_check, SMP_T_STR, SMP_T_STR },
- { "regsub", sample_conv_regsub, ARG3(2,REG,STR,STR), sample_conv_regsub_check, SMP_T_STR, SMP_T_STR },
- { "sha1", sample_conv_sha1, 0, NULL, SMP_T_BIN, SMP_T_BIN },
+ { "debug", sample_conv_debug, ARG2(0,STR,STR), smp_check_debug, SMP_T_ANY, SMP_T_ANY },
+ { "b64dec", sample_conv_base642bin, 0, NULL, SMP_T_STR, SMP_T_BIN },
+ { "base64", sample_conv_bin2base64, 0, NULL, SMP_T_BIN, SMP_T_STR },
+ { "ub64enc", sample_conv_bin2base64url,0, NULL, SMP_T_BIN, SMP_T_STR },
+ { "ub64dec", sample_conv_base64url2bin,0, NULL, SMP_T_STR, SMP_T_BIN },
+ { "upper", sample_conv_str2upper, 0, NULL, SMP_T_STR, SMP_T_STR },
+ { "lower", sample_conv_str2lower, 0, NULL, SMP_T_STR, SMP_T_STR },
+ { "length", sample_conv_length, 0, NULL, SMP_T_STR, SMP_T_SINT },
+ { "hex", sample_conv_bin2hex, 0, NULL, SMP_T_BIN, SMP_T_STR },
+ { "hex2i", sample_conv_hex2int, 0, NULL, SMP_T_STR, SMP_T_SINT },
+ { "ipmask", sample_conv_ipmask, ARG2(1,MSK4,MSK6), NULL, SMP_T_ADDR, SMP_T_IPV4 },
+ { "ltime", sample_conv_ltime, ARG2(1,STR,SINT), NULL, SMP_T_SINT, SMP_T_STR },
+ { "utime", sample_conv_utime, ARG2(1,STR,SINT), NULL, SMP_T_SINT, SMP_T_STR },
+ { "crc32", sample_conv_crc32, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
+ { "crc32c", sample_conv_crc32c, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
+ { "djb2", sample_conv_djb2, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
+ { "sdbm", sample_conv_sdbm, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
+ { "wt6", sample_conv_wt6, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
+ { "xxh3", sample_conv_xxh3, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
+ { "xxh32", sample_conv_xxh32, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
+ { "xxh64", sample_conv_xxh64, ARG1(0,SINT), NULL, SMP_T_BIN, SMP_T_SINT },
+ { "json", sample_conv_json, ARG1(1,STR), sample_conv_json_check, SMP_T_STR, SMP_T_STR },
+ { "bytes", sample_conv_bytes, ARG2(1,SINT,SINT), NULL, SMP_T_BIN, SMP_T_BIN },
+ { "field", sample_conv_field, ARG3(2,SINT,STR,SINT), sample_conv_field_check, SMP_T_STR, SMP_T_STR },
+ { "word", sample_conv_word, ARG3(2,SINT,STR,SINT), sample_conv_field_check, SMP_T_STR, SMP_T_STR },
+ { "regsub", sample_conv_regsub, ARG3(2,REG,STR,STR), sample_conv_regsub_check, SMP_T_STR, SMP_T_STR },
+ { "sha1", sample_conv_sha1, 0, NULL, SMP_T_BIN, SMP_T_BIN },
#ifdef USE_OPENSSL
{ "sha2", sample_conv_sha2, ARG1(0, SINT), smp_check_sha2, SMP_T_BIN, SMP_T_BIN },
#ifdef EVP_CIPH_GCM_MODE
--
2.31.1
>From ff1fffb8c9cd39229997200de990c855e15f5dc8 Mon Sep 17 00:00:00 2001
From: Moemen MHEDHBI <mmhed...@haproxy.com>
Date: Thu, 1 Apr 2021 20:53:59 +0200
Subject: [PATCH 1/2] MINOR: sample: add ub64dec and ub64enc converters
ub64dec and ub64enc are the base64url equivalent of b64dec and base64
converters. base64url encoding is the "URL and Filename Safe Alphabet"
variant of base64 encoding. It is also used in in JWT (JSON Web Token)
standard.
RFC1421 mention in base64.c file is deprecated so it was replaced with
RFC4648 to which existing converters, base64/b64dec, still apply.
Example:
HAProxy:
http-request return content-type text/plain lf-string %[req.hdr(Authorization),word(2,.),ub64dec]
Client:
Token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyIjoiZm9vIiwia2V5IjoiY2hhZTZBaFhhaTZlIn0.5VsVj7mdxVvo1wP5c0dVHnr-S_khnIdFkThqvwukmdg
$ curl -H "Authorization: Bearer ${TOKEN}" http://haproxy.local
{"user":"foo","key":"chae6AhXai6e"}
---
doc/configuration.txt | 17 +++-
include/haproxy/base64.h | 2 +
reg-tests/sample_fetches/ubase64.vtc | 51 ++++++++++++
src/base64.c | 112 +++++++++++++++++++++++++++
src/sample.c | 38 +++++++++
5 files changed, 219 insertions(+), 1 deletion(-)
create mode 100644 reg-tests/sample_fetches/ubase64.vtc
diff --git a/doc/configuration.txt b/doc/configuration.txt
index f21a29a68..f242300e7 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -15660,11 +15660,14 @@ and(<value>)
b64dec
Converts (decodes) a base64 encoded input string to its binary
representation. It performs the inverse operation of base64().
+ For base64url("URL and Filename Safe Alphabet" (RFC 4648)) variant
+ see "ub64dec".
base64
Converts a binary input sample to a base64 string. It is used to log or
transfer binary content in a way that can be reliably transferred (e.g.
- an SSL ID can be copied in a header).
+ an SSL ID can be copied in a header). For base64url("URL and Filename
+ Safe Alphabet" (RFC 4648)) variant see "ub64enc".
bool
Returns a boolean TRUE if the input value of type signed integer is
@@ -16565,6 +16568,18 @@ table_trackers(<table>)
connections there are from a given address for example. See also the
sc_trackers sample fetch keyword.
+ub64dec
+ This converter is the base64url variant of b64dec converter. base64url
+ encoding is the "URL and Filename Safe Alphabet" variant of base64 encoding.
+ It is also the encoding used in JWT (JSON Web Token) standard.
+
+ Example:
+ # Decoding a JWT payload:
+ http-request set-var(txn.token_payload) req.hdr(Authorization),word(2,.),ub64dec
+
+ub64enc
+ This converter is the base64url variant of base64 converter.
+
upper
Convert a string sample to upper case. This can only be placed after a string
sample fetch function or after a transformation keyword returning a string
diff --git a/include/haproxy/base64.h b/include/haproxy/base64.h
index 1756bc058..ace606372 100644
--- a/include/haproxy/base64.h
+++ b/include/haproxy/base64.h
@@ -17,7 +17,9 @@
#include <haproxy/api.h>
int a2base64(char *in, int ilen, char *out, int olen);
+int a2base64url(const char *in, size_t ilen, char *out, size_t olen);
int base64dec(const char *in, size_t ilen, char *out, size_t olen);
+int base64urldec(const char *in, size_t ilen, char *out, size_t olen);
const char *s30tob64(int in, char *out);
int b64tos30(const char *in);
diff --git a/reg-tests/sample_fetches/ubase64.vtc b/reg-tests/sample_fetches/ubase64.vtc
new file mode 100644
index 000000000..d6973e3b7
--- /dev/null
+++ b/reg-tests/sample_fetches/ubase64.vtc
@@ -0,0 +1,51 @@
+varnishtest "ub64dec sample fetche Test"
+
+#REQUIRE_VERSION=2.4
+
+feature ignore_unknown_macro
+
+haproxy h1 -conf {
+ defaults
+ mode http
+ timeout connect 1s
+ timeout client 1s
+ timeout server 1s
+
+ frontend fe
+ bind "fd@${fe}"
+ acl input hdr(encode) -m found
+ http-request return content-type text/plain hdr encode %[hdr(encode),ub64enc] hdr decode %[hdr(decode),ub64dec] if input
+ http-request return content-type text/plain hdr encode %[bin(14fb9c03d97f12d97e),ub64enc] hdr decode %[str(FPucA9l_Etl-),ub64dec,hex,lower] if !input
+
+} -start
+
+client c1 -connect ${h1_fe_sock} {
+ txreq -hdr "encode: f" -hdr "decode: Zg"
+ rxresp
+ expect resp.http.encode == "Zg"
+ expect resp.http.decode == "f"
+ txreq -hdr "encode: fo" -hdr "decode: Zm8"
+ rxresp
+ expect resp.http.encode == "Zm8"
+ expect resp.http.decode == "fo"
+ txreq -hdr "encode: foo" -hdr "decode: Zm9v"
+ rxresp
+ expect resp.http.encode == "Zm9v"
+ expect resp.http.decode == "foo"
+ txreq -hdr "encode: foob" -hdr "decode: Zm9vYg"
+ rxresp
+ expect resp.http.encode == "Zm9vYg"
+ expect resp.http.decode == "foob"
+ txreq -hdr "encode: fooba" -hdr "decode: Zm9vYmE"
+ rxresp
+ expect resp.http.encode == "Zm9vYmE"
+ expect resp.http.decode == "fooba"
+ txreq -hdr "encode: foobar" -hdr "decode: Zm9vYmFy"
+ rxresp
+ expect resp.http.encode == "Zm9vYmFy"
+ expect resp.http.decode == "foobar"
+ txreq
+ rxresp
+ expect resp.http.encode == "FPucA9l_Etl-"
+ expect resp.http.decode == "14fb9c03d97f12d97e"
+} -run
diff --git a/src/base64.c b/src/base64.c
index 53e4d65b2..bd975e991 100644
--- a/src/base64.c
+++ b/src/base64.c
@@ -19,11 +19,14 @@
#define B64BASE '#' /* arbitrary chosen base value */
#define B64CMIN '+'
+#define UB64CMIN '-'
#define B64CMAX 'z'
#define B64PADV 64 /* Base64 chosen special pad value */
const char base64tab[65]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
const char base64rev[]="b###cXYZ[\\]^_`a###d###$%&'()*+,-./0123456789:;<=######>?@ABCDEFGHIJKLMNOPQRSTUVW";
+const char ubase64tab[65]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
+const char ubase64rev[]="b##XYZ[\\]^_`a###c###$%&'()*+,-./0123456789:;<=####c#>?@ABCDEFGHIJKLMNOPQRSTUVW";
/* Encodes <ilen> bytes from <in> to <out> for at most <olen> chars (including
* the trailing zero). Returns the number of bytes written. No check is made
@@ -68,6 +71,48 @@ int a2base64(char *in, int ilen, char *out, int olen)
return convlen;
}
+/* url variant of a2base64 */
+int a2base64url(const char *in, size_t ilen, char *out, size_t olen)
+{
+ int convlen;
+
+ convlen = ((ilen + 2) / 3) * 4;
+
+ if (convlen >= olen)
+ return -1;
+
+ /* we don't need to check olen anymore */
+ while (ilen >= 3) {
+ out[0] = ubase64tab[(((unsigned char)in[0]) >> 2)];
+ out[1] = ubase64tab[(((unsigned char)in[0] & 0x03) << 4) | (((unsigned char)in[1]) >> 4)];
+ out[2] = ubase64tab[(((unsigned char)in[1] & 0x0F) << 2) | (((unsigned char)in[2]) >> 6)];
+ out[3] = ubase64tab[(((unsigned char)in[2] & 0x3F))];
+ out += 4;
+ in += 3;
+ ilen -= 3;
+ }
+
+ if (!ilen) {
+ out[0] = '\0';
+ return convlen;
+ }
+
+ out[0] = ubase64tab[((unsigned char)in[0]) >> 2];
+ if (ilen == 1) {
+ out[1] = ubase64tab[((unsigned char)in[0] & 0x03) << 4];
+ out[2] = '\0';
+ convlen -= 2;
+ } else {
+ out[1] = ubase64tab[(((unsigned char)in[0] & 0x03) << 4) |
+ (((unsigned char)in[1]) >> 4)];
+ out[2] = ubase64tab[((unsigned char)in[1] & 0x0F) << 2];
+ out[3] = '\0';
+ convlen -= 1;
+ }
+
+ return convlen;
+}
+
/* Decodes <ilen> bytes from <in> to <out> for at most <olen> chars.
* Returns the number of bytes converted. No check is made for
* <in> or <out> to be NULL. Returns -1 if <in> is invalid or ilen
@@ -138,6 +183,73 @@ int base64dec(const char *in, size_t ilen, char *out, size_t olen) {
return convlen;
}
+/* url variant of base64dec */
+/* The reverse tab used to decode base64 is generated via /dev/base64/base64rev-gen.c */
+int base64urldec(const char *in, size_t ilen, char *out, size_t olen)
+{
+
+ unsigned char t[4];
+ signed char b;
+ int convlen = 0, i = 0, pad = 0, padlen = 0;
+
+ if (olen < ((ilen / 4 * 3)))
+ return -2;
+
+ switch (ilen % 4) {
+ case 0:
+ break;
+ case 2:
+ padlen = pad = 2;
+ break;
+ case 3:
+ padlen = pad = 1;
+ break;
+ default:
+ return -1;
+ }
+
+ while (ilen + pad) {
+ if (ilen) {
+ /* if (*p < UB64CMIN || *p > B64CMAX) */
+ b = (signed char) * in - UB64CMIN;
+ if ((unsigned char)b > (B64CMAX - UB64CMIN))
+ return -1;
+
+ b = ubase64rev[b] - B64BASE - 1;
+ /* b == -1: invalid character */
+ if (b < 0)
+ return -1;
+
+ in++;
+ ilen--;
+
+ } else {
+ b = B64PADV;
+ pad--;
+ }
+
+ t[i++] = b;
+
+ if (i == 4) {
+ /*
+ * WARNING: we allow to write little more data than we
+ * should, but the checks from the beginning of the
+ * functions guarantee that we can safely do that.
+ */
+
+ /* xx000000 xx001111 xx111122 xx222222 */
+ out[convlen] = ((t[0] << 2) + (t[1] >> 4));
+ out[convlen + 1] = ((t[1] << 4) + (t[2] >> 2));
+ out[convlen + 2] = ((t[2] << 6) + (t[3] >> 0));
+
+ convlen += 3;
+ i = 0;
+ }
+ }
+ convlen -= padlen;
+
+ return convlen;
+}
/* Converts the lower 30 bits of an integer to a 5-char base64 string. The
* caller is responsible for ensuring that the output buffer can accept 6 bytes
diff --git a/src/sample.c b/src/sample.c
index 0c0f36ec5..04635a91f 100644
--- a/src/sample.c
+++ b/src/sample.c
@@ -1567,6 +1567,24 @@ static int sample_conv_base642bin(const struct arg *arg_p, struct sample *smp, v
return 1;
}
+static int sample_conv_base64url2bin(const struct arg *arg_p, struct sample *smp, void *private)
+{
+ struct buffer *trash = get_trash_chunk();
+ int bin_len;
+
+ trash->data = 0;
+ bin_len = base64urldec(smp->data.u.str.area, smp->data.u.str.data,
+ trash->area, trash->size);
+ if (bin_len < 0)
+ return 0;
+
+ trash->data = bin_len;
+ smp->data.u.str = *trash;
+ smp->data.type = SMP_T_BIN;
+ smp->flags &= ~SMP_F_CONST;
+ return 1;
+}
+
static int sample_conv_bin2base64(const struct arg *arg_p, struct sample *smp, void *private)
{
struct buffer *trash = get_trash_chunk();
@@ -1585,6 +1603,24 @@ static int sample_conv_bin2base64(const struct arg *arg_p, struct sample *smp, v
return 1;
}
+static int sample_conv_bin2base64url(const struct arg *arg_p, struct sample *smp, void *private)
+{
+ struct buffer *trash = get_trash_chunk();
+ int b64_len;
+
+ trash->data = 0;
+ b64_len = a2base64url(smp->data.u.str.area, smp->data.u.str.data,
+ trash->area, trash->size);
+ if (b64_len < 0)
+ return 0;
+
+ trash->data = b64_len;
+ smp->data.u.str = *trash;
+ smp->data.type = SMP_T_STR;
+ smp->flags &= ~SMP_F_CONST;
+ return 1;
+}
+
static int sample_conv_sha1(const struct arg *arg_p, struct sample *smp, void *private)
{
blk_SHA_CTX ctx;
@@ -4096,6 +4132,8 @@ static struct sample_conv_kw_list sample_conv_kws = {ILH, {
{ "debug", sample_conv_debug, ARG2(0,STR,STR), smp_check_debug, SMP_T_ANY, SMP_T_ANY },
{ "b64dec", sample_conv_base642bin,0, NULL, SMP_T_STR, SMP_T_BIN },
{ "base64", sample_conv_bin2base64,0, NULL, SMP_T_BIN, SMP_T_STR },
+ { "ub64dec", sample_conv_base64url2bin,0, NULL, SMP_T_STR, SMP_T_BIN },
+ { "ub64enc", sample_conv_bin2base64url,0, NULL, SMP_T_BIN, SMP_T_STR },
{ "upper", sample_conv_str2upper, 0, NULL, SMP_T_STR, SMP_T_STR },
{ "lower", sample_conv_str2lower, 0, NULL, SMP_T_STR, SMP_T_STR },
{ "length", sample_conv_length, 0, NULL, SMP_T_STR, SMP_T_SINT },
--
2.31.1
