From a356bad7f973fe089284ac95f0f558191ba68c24 Mon Sep 17 00:00:00 2001
From: Yonathan Randolph <yonathan@gmail.com>
Date: Fri, 17 Oct 2014 10:13:02 -0700
Subject: [PATCH 3/3] Fully zero pixels that are too transparent.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When thresholding an RGBA cursor to bitmask + RGB, zero pixels that have been rejected due to too low opacity, so that we don’t confuse them with XOR pixels.
---
 src/VBox/Main/src-client/ConsoleVRDPServer.cpp | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/src/VBox/Main/src-client/ConsoleVRDPServer.cpp b/src/VBox/Main/src-client/ConsoleVRDPServer.cpp
index ebedd1e..be03829 100644
--- a/src/VBox/Main/src-client/ConsoleVRDPServer.cpp
+++ b/src/VBox/Main/src-client/ConsoleVRDPServer.cpp
@@ -325,28 +325,24 @@ static void findTopLeftBorder(const uint8_t *pu8AndMask, const uint8_t *pu8XorMa
  * for the guest driver, but then additions reinstall would be
  * necessary, which we try to avoid.
  */
-static void mousePointerGenerateANDMask(uint8_t *pu8DstAndMask, int cbDstAndMask, const uint8_t *pu8SrcAlpha, int w, int h)
+static void mousePointerGenerateANDMask(uint8_t *pu8DstAndMask, int cbDstAndMask, uint8_t *pu8SrcAlpha, int w, int h)
 {
-    memset(pu8DstAndMask, 0xFF, cbDstAndMask);
-
     int y;
     for (y = 0; y < h; y++)
     {
         uint8_t bitmask = 0x80;
 
         int x;
-        for (x = 0; x < w; x++, bitmask >>= 1)
+        for (x = 0; x < w; x++)
         {
-            if (bitmask == 0)
-            {
-                bitmask = 0x80;
-            }
-
             /* Whether alpha channel value is not transparent enough for the pixel to be seen. */
-            if (pu8SrcAlpha[x * 4 + 3] > 0x7f)
-            {
-                pu8DstAndMask[x / 8] &= ~bitmask;
+            uint8_t opacity = pu8SrcAlpha[x * 4 + 3];
+            if (opacity <= 0x7f)
+                /* make the color completely black to distinguish it from an inverting XOR */
+                pu8DstAndMask[x / 8] |= bitmask;
+                memset(&pu8SrcAlpha[x * 4], 0, 4);
             }
+            bitmask = bitmask >> 1 | bitmask << 7;
         }
 
         /* Point to next source and dest scans. */
@@ -414,6 +410,7 @@ void ConsoleVRDPServer::onMousePointerShapeChange(BOOL visible,
         {
             pu8AndMask = (uint8_t*)alloca(cbDstAndMask);
 
+            memcpy(pu8AndMask, shape, cbDstAndMask);
             mousePointerGenerateANDMask(pu8AndMask, cbDstAndMask, pu8XorMask, width, height);
         }
 
-- 
1.9.3 (Apple Git-50)

