From f2beeb139850d0b05cd1b7f424399bfbdf9c5752 Mon Sep 17 00:00:00 2001 From: Jeffrey Knockel Date: Sat, 6 Dec 2025 15:50:25 -0500 Subject: [PATCH 1/3] zrle: use 2 bytes for 15bpp format This fixes the zrle encoding to use two bytes instead of one for 15bpp. Previously we would leave much of the output buffer uninitialized. --- src/libvncclient/zrle.c | 28 ++++++++++++++------------- src/libvncserver/zrleencodetemplate.c | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/libvncclient/zrle.c b/src/libvncclient/zrle.c index 6859519ff..0ce65aa0b 100644 --- a/src/libvncclient/zrle.c +++ b/src/libvncclient/zrle.c @@ -32,6 +32,8 @@ #define REALBPP BPP #endif +#define REALBytesPP ((REALBPP + 7) / 8) + #if !defined(UNCOMP) || UNCOMP==0 #define HandleZRLE CONCAT2E(HandleZRLE,REALBPP) #define HandleZRLETile CONCAT2E(HandleZRLETile,REALBPP) @@ -87,7 +89,7 @@ HandleZRLE (rfbClient* client, int rx, int ry, int rw, int rh) int remaining; int inflateResult; int toRead; - int min_buffer_size = rw * rh * (REALBPP / 8) * 2; + int min_buffer_size = rw * rh * REALBytesPP * 2; /* First make sure we have a large enough raw buffer to hold the * decompressed data. In practice, with a fixed REALBPP, fixed frame @@ -269,29 +271,29 @@ static int HandleZRLETile(rfbClient* client, #if REALBPP!=BPP int i,j; - if(1+w*h*REALBPP/8>buffer_length) { - rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBPP/8,buffer_length,w,h); + if(1+w*h*REALBytesPP>buffer_length) { + rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBytesPP,buffer_length,w,h); return -3; } for(j=y*client->width; j<(y+h)*client->width; j+=client->width) - for(i=x; iframeBuffer)[j+i] = UncompressCPixel(buffer); #else client->GotBitmap(client, buffer, x, y, w, h); - buffer+=w*h*REALBPP/8; + buffer+=w*h*REALBytesPP; #endif } else if( type == 1 ) /* solid */ { CARDBPP color = UncompressCPixel(buffer); - if(1+REALBPP/8>buffer_length) + if(1+REALBytesPP>buffer_length) return -4; client->GotFillRect(client, x, y, w, h, color); - buffer+=REALBPP/8; + buffer+=REALBytesPP; } else if( type <= 127 ) /* packed Palette */ @@ -302,11 +304,11 @@ static int HandleZRLETile(rfbClient* client, mask=(1<buffer_length) + if(1+type*REALBytesPP+((w+divider-1)/divider)*h>buffer_length) return -5; /* read palette */ - for(i=0; ibuffer_end) + if(buffer+REALBytesPP+1>buffer_end) return -7; color = UncompressCPixel(buffer); - buffer+=REALBPP/8; + buffer+=REALBytesPP; /* read run length */ length=1; while(*buffer==0xff) { @@ -368,11 +370,11 @@ static int HandleZRLETile(rfbClient* client, CARDBPP palette[128]; int i,j; - if(2+(type-128)*REALBPP/8>buffer_length) + if(2+(type-128)*REALBytesPP>buffer_length) return -9; /* read palette */ - for(i=0; i Date: Sat, 6 Dec 2025 15:51:31 -0500 Subject: [PATCH 2/3] trle: use 2 bytes for 15bpp format This fixes the trle encoding to use two bytes instead of one for 15bpp. Previously we would leave much of the output buffer uninitialized. --- src/libvncclient/trle.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/libvncclient/trle.c b/src/libvncclient/trle.c index cf06c185c..4cc63449e 100644 --- a/src/libvncclient/trle.c +++ b/src/libvncclient/trle.c @@ -30,6 +30,8 @@ #define REALBPP BPP #endif +#define REALBytesPP ((REALBPP + 7) / 8) + #if !defined(UNCOMP) || UNCOMP == 0 #define HandleTRLE CONCAT2E(HandleTRLE, REALBPP) #elif UNCOMP > 0 @@ -53,7 +55,7 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) { int x, y, w, h; uint8_t type, last_type = 0; - int min_buffer_size = 16 * 16 * (REALBPP / 8) * 2; + int min_buffer_size = 16 * 16 * REALBytesPP * 2; uint8_t *buffer; CARDBPP palette[128]; int bpp = 0, mask = 0, divider = 0; @@ -93,14 +95,14 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) { switch (type) { case 0: { - if (!ReadFromRFBServer(client, (char *)buffer, w * h * REALBPP / 8)) + if (!ReadFromRFBServer(client, (char *)buffer, w * h * REALBytesPP)) return FALSE; #if REALBPP != BPP int i, j; for (j = y * client->width; j < (y + h) * client->width; j += client->width) - for (i = x; i < x + w; i++, buffer += REALBPP / 8) + for (i = x; i < x + w; i++, buffer += REALBytesPP) ((CARDBPP *)client->frameBuffer)[j + i] = UncompressCPixel(buffer); #else client->GotBitmap(client, buffer, x, y, w, h); @@ -109,7 +111,7 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) { break; } case 1: { - if (!ReadFromRFBServer(client, (char *)buffer, REALBPP / 8)) + if (!ReadFromRFBServer(client, (char *)buffer, REALBytesPP)) return FALSE; color = UncompressCPixel(buffer); @@ -171,11 +173,11 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) { while (j < h) { int color, length, buffer_pos = 0; /* read color */ - if (!ReadFromRFBServer(client, (char*)buffer, REALBPP / 8 + 1)) + if (!ReadFromRFBServer(client, (char*)buffer, REALBytesPP + 1)) return FALSE; color = UncompressCPixel(buffer); - buffer += REALBPP / 8; - buffer_pos += REALBPP / 8; + buffer += REALBytesPP; + buffer_pos += REALBytesPP; /* read run length */ length = 1; while (*buffer == 0xff && buffer_pos < client->raw_buffer_size-1) { @@ -260,11 +262,11 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) { bpp = (type > 4 ? 4 : (type > 2 ? 2 : 1)), mask = (1 << bpp) - 1, divider = (8 / bpp); - if (!ReadFromRFBServer(client, (char *)buffer, type * REALBPP / 8)) + if (!ReadFromRFBServer(client, (char *)buffer, type * REALBytesPP)) return FALSE; /* read palette */ - for (i = 0; i < type; i++, buffer += REALBPP / 8) + for (i = 0; i < type; i++, buffer += REALBytesPP) palette[i] = UncompressCPixel(buffer); last_type = type; @@ -272,11 +274,11 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) { } else if (type >= 130) { int i; - if (!ReadFromRFBServer(client, (char *)buffer, (type - 128) * REALBPP / 8)) + if (!ReadFromRFBServer(client, (char *)buffer, (type - 128) * REALBytesPP)) return FALSE; /* read palette */ - for (i = 0; i < type - 128; i++, buffer += REALBPP / 8) + for (i = 0; i < type - 128; i++, buffer += REALBytesPP) palette[i] = UncompressCPixel(buffer); last_type = type; From 015d5862d037b3495b31fb514df3ac600c708e82 Mon Sep 17 00:00:00 2001 From: Jeffrey Knockel Date: Sat, 6 Dec 2025 15:52:20 -0500 Subject: [PATCH 3/3] libvncclient: remove unused CARDREALBPP defines These are used to define a uint$REALBPP_t type, but REALBPP can be (e.g.) 15, and there generally is no uint15_t, so it never would have worked anyways. --- src/libvncclient/trle.c | 2 -- src/libvncclient/zrle.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/libvncclient/trle.c b/src/libvncclient/trle.c index 4cc63449e..3db1a052a 100644 --- a/src/libvncclient/trle.c +++ b/src/libvncclient/trle.c @@ -40,7 +40,6 @@ #define HandleTRLE CONCAT3E(HandleTRLE, REALBPP, Up) #endif #define CARDBPP CONCAT3E(uint, BPP, _t) -#define CARDREALBPP CONCAT3E(uint, REALBPP, _t) #if REALBPP != BPP && defined(UNCOMP) && UNCOMP != 0 #if UNCOMP > 0 @@ -294,7 +293,6 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) { } #undef CARDBPP -#undef CARDREALBPP #undef HandleTRLE #undef UncompressCPixel #undef REALBPP diff --git a/src/libvncclient/zrle.c b/src/libvncclient/zrle.c index 0ce65aa0b..64fa561f6 100644 --- a/src/libvncclient/zrle.c +++ b/src/libvncclient/zrle.c @@ -45,7 +45,6 @@ #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Up) #endif #define CARDBPP CONCAT3E(uint,BPP,_t) -#define CARDREALBPP CONCAT3E(uint,REALBPP,_t) #define ENDIAN_LITTLE 0 #define ENDIAN_BIG 1 @@ -418,7 +417,6 @@ static int HandleZRLETile(rfbClient* client, } #undef CARDBPP -#undef CARDREALBPP #undef HandleZRLE #undef HandleZRLETile #undef UncompressCPixel