From 425777148f8a7753696fa7794fae99f2104794b4 Mon Sep 17 00:00:00 2001
From: Yonathan Randolph <yonathan@gmail.com>
Date: Fri, 17 Oct 2014 10:07:14 -0700
Subject: [PATCH 2/3] VNC extension: Make XOR cursors from Windows visible, and
 fix colors.

In Windows and RDP, a cursor consists of a bitmask to be ANDed with the background, followed by a color mask to be XORed. Meanwhile, a VNC cursor consists of a color image and a bitmask that indicates whether to paint the image over the background. Previously, the VNC bitmask was assumed to be simply the inverse of the RDP bitmask. This fails for text selection cursors, which invert the background instead of painting on top of it. Simulate XOR cursors in VNC by making them opaque and rendering them on a white background.

Also, fix minor color issue: aqua cursors appeared orange (such as busy cursors in Windows 7/8), and red cursors appeared blue (such as the Unavailable cursor in Windows 7/8). The VNC extension was interpreting the RDP input as RGB when it is actually BGR.
---
 src/VBox/ExtPacks/VNC/VBoxVNC.cpp | 63 ++++++++++++++++++++++++++-------------
 1 file changed, 43 insertions(+), 20 deletions(-)

diff --git a/src/VBox/ExtPacks/VNC/VBoxVNC.cpp b/src/VBox/ExtPacks/VNC/VBoxVNC.cpp
index 1014c46..9ecf184 100644
--- a/src/VBox/ExtPacks/VNC/VBoxVNC.cpp
+++ b/src/VBox/ExtPacks/VNC/VBoxVNC.cpp
@@ -868,33 +868,56 @@ DECLCALLBACK(void) VNCServerImpl::VRDEColorPointer(HVRDESERVER hServer,
     cursor->richSource = mem;
     cursor->cleanupRichSource = TRUE;  /* libvncserver will take ownership of cursor->richSource */
 
-    unsigned char *maskmem = (unsigned char *)malloc(pPointer->u16Width * pPointer->u16Height);
+    unsigned char *maskmem = (unsigned char *)calloc(pPointer->u16Width * pPointer->u16Height, 1);
     cursor->mask = maskmem;
     cursor->cleanupMask = TRUE;  /* libvncserver will take ownership of cursor->mask */
 
     unsigned char *mask = (unsigned char *)pPointer + sizeof(VRDECOLORPOINTER);
 
-    for(int i = pPointer->u16Height - 1; i >= 0 ; i--)
-    {
-        for(uint16_t j = 0; j < pPointer->u16Width/8; j ++)
-        {
-            *maskmem = ~(*(mask + i * (pPointer->u16Width / 8) + j));
-            *maskmem++;
-        }
-    }
-    unsigned char *color = (unsigned char *)pPointer + sizeof(VRDECOLORPOINTER) + pPointer->u16MaskLen;
-    for(int i = pPointer->u16Height - 1; i >= 0 ; i--)
-    {
-        for(uint16_t j = 0; j < pPointer->u16Width; j ++)
-        {
-            // put the color value;
-            *(mem++) = *(color + (i * pPointer->u16Width *3 + j * 3 + 2));
-            *(mem++) = *(color + (i * pPointer->u16Width *3 + j * 3 + 1));
-            *(mem++) = *(color + (i * pPointer->u16Width *3 + j * 3));
-            *(mem++) = 0xff;
+    /*
+     * Convert Windows and RDP AND and XOR masks to VNC opaque and paint masks.
+     * In Windows and RDP, D := D & bitmask ^ S. (see ICONINFO, bitblt, and [MS-RDPBCGR] documentation)
+     * (This is used for I-beam text selection cursors, which invert when the background is black)
+     * In VNC, D := D & ~bitmask | S & bitmask. There is no way to represent an inverting cursor.
+     */
+    const unsigned char *color = (const unsigned char *)pPointer + sizeof(VRDECOLORPOINTER) + pPointer->u16MaskLen;
+    const unsigned char *xorColorScanLine = &color[pPointer->u16Height * pPointer->u16Width * 3 - pPointer->u16Width * 3];
+    unsigned char *vncColorScanLine = &mem[0];
+    const unsigned char *andBitScanLine = &mask[pPointer->u16Height*((pPointer->u16Width+7)/8) - (pPointer->u16Width+7)/8];
+    unsigned char *vncBitScanLine = &maskmem[0];
+    for (uint16_t y = 0; y != pPointer->u16Height; y++) {
+        uint8_t currentBit = 0x80;
+        for (uint16_t x = 0; x != pPointer->u16Width; x++) {
+            bool andMask = 0 != (andBitScanLine[x/8] & currentBit);
+            unsigned char
+                b = xorColorScanLine[x*3 + 0],
+                g = xorColorScanLine[x*3 + 1],
+                r = xorColorScanLine[x*3 + 2];
+            bool isOpaque;
+            if (! andMask) {
+                isOpaque = TRUE;
+            } else if (r || g || b) {
+                /* Assume white background (ffffff), since XOR cursors are
+                 * usually used for white text fields. */
+                isOpaque = TRUE;
+                r = 0xff ^ r;
+                g = 0xff ^ g;
+                b = 0xff ^ b;
+            } else {
+                isOpaque = FALSE;
+            }
+            vncBitScanLine[x/8] |= isOpaque ? currentBit:0;
+            vncColorScanLine[x*4 + 0] = b;
+            vncColorScanLine[x*4 + 1] = g;
+            vncColorScanLine[x*4 + 2] = r;
+            vncColorScanLine[x*4 + 3] = 0xff;
+            currentBit = currentBit >> 1 | currentBit << 7;
         }
+        xorColorScanLine -= pPointer->u16Width * 3;
+        andBitScanLine -= (pPointer->u16Width+7)/8;
+        vncColorScanLine += pPointer->u16Width * 4;
+        vncBitScanLine += (pPointer->u16Width+7)/8;
     }
-
     cursor->xhot = pPointer->u16HotX;
     cursor->yhot = pPointer->u16HotY;
 
-- 
1.9.3 (Apple Git-50)