commit: 60975f5f4bdc9f9c5002ac8cfaf32e0a2095b770 Author: Mike Gilbert <floppym <AT> gentoo <DOT> org> AuthorDate: Thu Dec 7 20:58:51 2023 +0000 Commit: Mike Gilbert <floppym <AT> gentoo <DOT> org> CommitDate: Thu Dec 7 21:00:02 2023 +0000 URL: https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=60975f5f
net-misc/freerdp: fix for big-endian archs Bug: https://bugs.gentoo.org/918531 Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org> ...ec-encode-messages-considering-endianness.patch | 329 +++++++++++++++++++++ net-misc/freerdp/freerdp-2.11.1.ebuild | 1 + net-misc/freerdp/freerdp-2.11.2.ebuild | 1 + 3 files changed, 331 insertions(+) diff --git a/net-misc/freerdp/files/freerdp-2.11-Revert-codec-encode-messages-considering-endianness.patch b/net-misc/freerdp/files/freerdp-2.11-Revert-codec-encode-messages-considering-endianness.patch new file mode 100644 index 000000000000..8d45331bd95b --- /dev/null +++ b/net-misc/freerdp/files/freerdp-2.11-Revert-codec-encode-messages-considering-endianness.patch @@ -0,0 +1,329 @@ +https://github.com/FreeRDP/FreeRDP/issues/9616 + +From 8b349f4f2b5d14e28a6a73e9c2b9860d7f275bc4 Mon Sep 17 00:00:00 2001 +From: Mike Gilbert <flop...@gentoo.org> +Date: Thu, 7 Dec 2023 15:54:38 -0500 +Subject: [PATCH] Revert "[codec] encode messages considering endianness" + +This reverts commit e3ef8a02ddf079bcf6dfa58888db69a709ce50c9. +--- + libfreerdp/codec/nsc_encode.c | 76 +++++++++++++---------------------- + libfreerdp/codec/rfx_encode.c | 74 +++++++++++++--------------------- + 2 files changed, 58 insertions(+), 92 deletions(-) + +diff --git a/libfreerdp/codec/nsc_encode.c b/libfreerdp/codec/nsc_encode.c +index 477318cc1..fd9626a6d 100644 +--- a/libfreerdp/codec/nsc_encode.c ++++ b/libfreerdp/codec/nsc_encode.c +@@ -119,8 +119,6 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data, UI + UINT16 rw; + BYTE ccl; + const BYTE* src; +- const UINT32* src_32; +- const UINT16* src_16; + BYTE* yplane = NULL; + BYTE* coplane = NULL; + BYTE* cgplane = NULL; +@@ -142,85 +140,69 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data, UI + coplane = context->priv->PlaneBuffers[1] + y * rw; + cgplane = context->priv->PlaneBuffers[2] + y * rw; + aplane = context->priv->PlaneBuffers[3] + y * context->width; +- src_32 = (UINT32*)src; +- src_16 = (UINT16*)src; + + for (x = 0; x < context->width; x++) + { + switch (context->format) + { + case PIXEL_FORMAT_BGRX32: +- b_val = (INT16)(*src_32 & 0xFF); +- g_val = (INT16)((*src_32 >> 8) & 0xFF); +- r_val = (INT16)((*src_32 >> 16) & 0xFF); ++ b_val = *src++; ++ g_val = *src++; ++ r_val = *src++; ++ src++; + a_val = 0xFF; +- src_32++; + break; + + case PIXEL_FORMAT_BGRA32: +- b_val = (INT16)(*src_32 & 0xFF); +- g_val = (INT16)((*src_32 >> 8) & 0xFF); +- r_val = (INT16)((*src_32 >> 16) & 0xFF); +- a_val = (INT16)((*src_32 >> 24) & 0xFF); +- src_32++; ++ b_val = *src++; ++ g_val = *src++; ++ r_val = *src++; ++ a_val = *src++; + break; + + case PIXEL_FORMAT_RGBX32: +- r_val = (INT16)(*src_32 & 0xFF); +- g_val = (INT16)((*src_32 >> 8) & 0xFF); +- b_val = (INT16)((*src_32 >> 16) & 0xFF); ++ r_val = *src++; ++ g_val = *src++; ++ b_val = *src++; ++ src++; + a_val = 0xFF; +- src_32++; + break; + + case PIXEL_FORMAT_RGBA32: +- r_val = (INT16)(*src_32 & 0xFF); +- g_val = (INT16)((*src_32 >> 8) & 0xFF); +- b_val = (INT16)((*src_32 >> 16) & 0xFF); +- a_val = (INT16)((*src_32 >> 24) & 0xFF); +- src_32++; ++ r_val = *src++; ++ g_val = *src++; ++ b_val = *src++; ++ a_val = *src++; + break; + + case PIXEL_FORMAT_BGR24: +-#ifdef __LITTLE_ENDIAN__ + b_val = *src++; + g_val = *src++; + r_val = *src++; +-#else +- r_val = *src++; +- g_val = *src++; +- b_val = *src++; +-#endif + a_val = 0xFF; + break; + + case PIXEL_FORMAT_RGB24: +-#ifdef __LITTLE_ENDIAN__ + r_val = *src++; + g_val = *src++; + b_val = *src++; +-#else +- b_val = *src++; +- g_val = *src++; +- r_val = *src++; +-#endif + a_val = 0xFF; + break; + + case PIXEL_FORMAT_BGR16: +- b_val = (INT16)((*src_16) & 0x1F); +- g_val = (INT16)((*src_16 >> 5) & 0x3F); +- r_val = (INT16)((*src_16 >> 11) & 0x1F); ++ b_val = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)); ++ g_val = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)); ++ r_val = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07)); + a_val = 0xFF; +- src_16++; ++ src += 2; + break; + + case PIXEL_FORMAT_RGB16: +- r_val = (INT16)((*src_16) & 0x1F); +- g_val = (INT16)((*src_16 >> 5) & 0x3F); +- b_val = (INT16)((*src_16 >> 11) & 0x1F); ++ r_val = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)); ++ g_val = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)); ++ b_val = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07)); + a_val = 0xFF; +- src_16++; ++ src += 2; + break; + + case PIXEL_FORMAT_A4: +@@ -228,17 +210,17 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data, UI + int shift; + BYTE idx; + shift = (7 - (x % 8)); +- idx = (BYTE)(((*src_32 & 0xFF) >> shift) & 1); +- idx |= (BYTE)(((((*src_32 >> 8) & 0xFF) >> shift) & 1) << 1); +- idx |= (BYTE)(((((*src_32 >> 16) & 0xFF) >> shift) & 1) << 2); +- idx |= (BYTE)(((((*src_32 >> 24) & 0xFF) >> shift) & 1) << 3); ++ idx = ((*src) >> shift) & 1; ++ idx |= (((*(src + 1)) >> shift) & 1) << 1; ++ idx |= (((*(src + 2)) >> shift) & 1) << 2; ++ idx |= (((*(src + 3)) >> shift) & 1) << 3; + idx *= 3; + r_val = (INT16)context->palette[idx]; + g_val = (INT16)context->palette[idx + 1]; + b_val = (INT16)context->palette[idx + 2]; + + if (shift == 0) +- src_32++; ++ src += 4; + } + + a_val = 0xFF; +diff --git a/libfreerdp/codec/rfx_encode.c b/libfreerdp/codec/rfx_encode.c +index 355f74340..b4f393605 100644 +--- a/libfreerdp/codec/rfx_encode.c ++++ b/libfreerdp/codec/rfx_encode.c +@@ -49,8 +49,6 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i + int x_exceed; + int y_exceed; + const BYTE* src; +- const UINT32* src_32; +- const UINT16* src_16; + INT16 r, g, b; + INT16 *r_last, *g_last, *b_last; + x_exceed = 64 - width; +@@ -59,8 +57,6 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i + for (y = 0; y < height; y++) + { + src = rgb_data + y * rowstride; +- src_32 = (UINT32*)src; +- src_16 = (UINT16*)src; + + switch (pixel_format) + { +@@ -68,10 +64,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i + case PIXEL_FORMAT_BGRA32: + for (x = 0; x < width; x++) + { +- *b_buf++ = (INT16)(*src_32 & 0xFF); +- *g_buf++ = (INT16)((*src_32 >> 8) & 0xFF); +- *r_buf++ = (INT16)((*src_32 >> 16) & 0xFF); +- src_32++; ++ *b_buf++ = (INT16)(*src++); ++ *g_buf++ = (INT16)(*src++); ++ *r_buf++ = (INT16)(*src++); ++ src++; + } + + break; +@@ -80,10 +76,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i + case PIXEL_FORMAT_ABGR32: + for (x = 0; x < width; x++) + { +- *b_buf++ = (INT16)((*src_32 >> 8) & 0xFF); +- *g_buf++ = (INT16)((*src_32 >> 16) & 0xFF); +- *r_buf++ = (INT16)((*src_32 >> 24) & 0xFF); +- src_32++; ++ src++; ++ *b_buf++ = (INT16)(*src++); ++ *g_buf++ = (INT16)(*src++); ++ *r_buf++ = (INT16)(*src++); + } + + break; +@@ -92,10 +88,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i + case PIXEL_FORMAT_RGBA32: + for (x = 0; x < width; x++) + { +- *r_buf++ = (INT16)(*src_32 & 0xFF); +- *g_buf++ = (INT16)((*src_32 >> 8) & 0xFF); +- *b_buf++ = (INT16)((*src_32 >> 16) & 0xFF); +- src_32++; ++ *r_buf++ = (INT16)(*src++); ++ *g_buf++ = (INT16)(*src++); ++ *b_buf++ = (INT16)(*src++); ++ src++; + } + + break; +@@ -104,10 +100,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i + case PIXEL_FORMAT_ARGB32: + for (x = 0; x < width; x++) + { +- *r_buf++ = (INT16)((*src_32 >> 8) & 0xFF); +- *g_buf++ = (INT16)((*src_32 >> 16) & 0xFF); +- *b_buf++ = (INT16)((*src_32 >> 24) & 0xFF); +- src_32++; ++ src++; ++ *r_buf++ = (INT16)(*src++); ++ *g_buf++ = (INT16)(*src++); ++ *b_buf++ = (INT16)(*src++); + } + + break; +@@ -115,15 +111,9 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i + case PIXEL_FORMAT_BGR24: + for (x = 0; x < width; x++) + { +-#ifdef __LITTLE_ENDIAN__ + *b_buf++ = (INT16)(*src++); + *g_buf++ = (INT16)(*src++); + *r_buf++ = (INT16)(*src++); +-#else +- *r_buf++ = (INT16)(*src++); +- *g_buf++ = (INT16)(*src++); +- *b_buf++ = (INT16)(*src++); +-#endif + } + + break; +@@ -131,15 +121,9 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i + case PIXEL_FORMAT_RGB24: + for (x = 0; x < width; x++) + { +-#ifdef __LITTLE_ENDIAN__ + *r_buf++ = (INT16)(*src++); + *g_buf++ = (INT16)(*src++); + *b_buf++ = (INT16)(*src++); +-#else +- *b_buf++ = (INT16)(*src++); +- *g_buf++ = (INT16)(*src++); +- *r_buf++ = (INT16)(*src++); +-#endif + } + + break; +@@ -147,10 +131,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i + case PIXEL_FORMAT_BGR16: + for (x = 0; x < width; x++) + { +- *b_buf++ = (INT16)((*src_16) & 0x1F); +- *g_buf++ = (INT16)((*src_16 >> 5) & 0x3F); +- *r_buf++ = (INT16)((*src_16 >> 11) & 0x1F); +- src_16++; ++ *b_buf++ = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)); ++ *g_buf++ = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)); ++ *r_buf++ = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07)); ++ src += 2; + } + + break; +@@ -158,10 +142,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i + case PIXEL_FORMAT_RGB16: + for (x = 0; x < width; x++) + { +- *r_buf++ = (INT16)((*src_16 & 0x1F)); +- *g_buf++ = (INT16)((*src_16 >> 5) & 0x3F); +- *b_buf++ = (INT16)((*src_16 >> 11) & 0x1F); +- src_16++; ++ *r_buf++ = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)); ++ *g_buf++ = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)); ++ *b_buf++ = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07)); ++ src += 2; + } + + break; +@@ -175,17 +159,17 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i + int shift; + BYTE idx; + shift = (7 - (x % 8)); +- idx = (BYTE)(((*src_32 & 0xFF) >> shift) & 1); +- idx |= (BYTE)(((((*src_32 >> 8) & 0xFF) >> shift) & 1) << 1); +- idx |= (BYTE)(((((*src_32 >> 16) & 0xFF) >> shift) & 1) << 2); +- idx |= (BYTE)(((((*src_32 >> 24) & 0xFF) >> shift) & 1) << 3); ++ idx = ((*src) >> shift) & 1; ++ idx |= (((*(src + 1)) >> shift) & 1) << 1; ++ idx |= (((*(src + 2)) >> shift) & 1) << 2; ++ idx |= (((*(src + 3)) >> shift) & 1) << 3; + idx *= 3; + *r_buf++ = (INT16)palette[idx]; + *g_buf++ = (INT16)palette[idx + 1]; + *b_buf++ = (INT16)palette[idx + 2]; + + if (shift == 0) +- src_32++; ++ src += 4; + } + + break; +-- +2.43.0 + diff --git a/net-misc/freerdp/freerdp-2.11.1.ebuild b/net-misc/freerdp/freerdp-2.11.1.ebuild index 83943a4cf418..5c2b19228de4 100644 --- a/net-misc/freerdp/freerdp-2.11.1.ebuild +++ b/net-misc/freerdp/freerdp-2.11.1.ebuild @@ -95,6 +95,7 @@ BDEPEND=" PATCHES=( "${FILESDIR}/freerdp-2.11.2-clang.patch" + "${FILESDIR}/freerdp-2.11-Revert-codec-encode-messages-considering-endianness.patch" ) src_configure() { diff --git a/net-misc/freerdp/freerdp-2.11.2.ebuild b/net-misc/freerdp/freerdp-2.11.2.ebuild index 86d7791ad99e..a94d9d265ffb 100644 --- a/net-misc/freerdp/freerdp-2.11.2.ebuild +++ b/net-misc/freerdp/freerdp-2.11.2.ebuild @@ -95,6 +95,7 @@ BDEPEND=" PATCHES=( "${FILESDIR}/freerdp-2.11.2-clang.patch" + "${FILESDIR}/freerdp-2.11-Revert-codec-encode-messages-considering-endianness.patch" ) src_configure() {
