treewide: fix result type of BitmapBytePad() to size_t

metux · metux · commit d54797d536bb · 2025-11-06T14:42:11.000+01:00
sizes can't have negative numbers, so use the size_t here.
Also having to fix up lots of callers.

Signed-off-by: Enrico Weigelt, metux IT consult &lt;info@metux.net&gt;
diff --git a/Xext/panoramiXprocs.c b/Xext/panoramiXprocs.c
@@ -1994,7 +1994,6 @@ PanoramiXGetImage(ClientPtr client)
     int x, y, w, h, format, rc;
     Mask plane = 0, planemask;
     int linesDone, nlines, linesPerBuf;
-    long widthBytesLine;
 
     REQUEST(xGetImageReq);
 
@@ -2066,6 +2065,8 @@ PanoramiXGetImage(ClientPtr client)
     });
 
     size_t length;
+    size_t widthBytesLine;
+
     if (format == ZPixmap) {
         widthBytesLine = PixmapBytePad(w, pDraw->depth);
         length = widthBytesLine * h;
diff --git a/dix/cursor.c b/dix/cursor.c
@@ -156,7 +156,7 @@ static void
 CheckForEmptyMask(CursorBitsPtr bits)
 {
     unsigned char *msk = bits->mask;
-    int n = BitmapBytePad(bits->width) * bits->height;
+    size_t n = BitmapBytePad(bits->width) * bits->height;
 
     bits->emptyMask = FALSE;
     while (n--)
@@ -366,10 +366,9 @@ AllocGlyphCursor(Font source, unsigned short sourceChar, Font mask, unsigned sho
             return BadValue;
         }
         if (!maskfont) {
-            long n;
             unsigned char *mskptr;
 
-            n = BitmapBytePad(cm.width) * (long) cm.height;
+            size_t n = BitmapBytePad(cm.width) * (long) cm.height;
             mskptr = mskbits = calloc(1, n);
             if (!mskptr)
                 return BadAlloc;
diff --git a/dix/dispatch.c b/dix/dispatch.c
@@ -2162,14 +2162,15 @@ ProcPutImage(ClientPtr client)
 {
     GCPtr pGC;
     DrawablePtr pDraw;
-    long length;                /* length of scanline server padded */
-    long lengthProto;           /* length of scanline protocol padded */
     char *tmpImage;
 
     REQUEST(xPutImageReq);
 
     REQUEST_AT_LEAST_SIZE(xPutImageReq);
     VALIDATE_DRAWABLE_AND_GC(stuff->drawable, pDraw, DixWriteAccess);
+
+    size_t length;                /* length of scanline server padded */
+
     if (stuff->format == XYBitmap) {
         if ((stuff->depth != 1) ||
             (stuff->leftPad >= (unsigned int) screenInfo.bitmapScanlinePad))
@@ -2194,7 +2195,7 @@ ProcPutImage(ClientPtr client)
     }
 
     tmpImage = (char *) &stuff[1];
-    lengthProto = length;
+    size_t lengthProto = length; /* length of scanline protocol padded */
 
     if (stuff->height != 0 && lengthProto >= (INT32_MAX / stuff->height))
         return BadLength;
@@ -2225,7 +2226,6 @@ DoGetImage(ClientPtr client, int format, Drawable drawable,
 
     /* coordinates relative to the bounding drawable */
     int relx, rely;
-    long widthBytesLine, length;
     Mask plane = 0;
     RegionPtr pVisibleRegion = NULL;
 
@@ -2295,6 +2295,8 @@ DoGetImage(ClientPtr client, int format, Drawable drawable,
         return BadMatch;
 
     rep.depth = pDraw->depth;
+
+    size_t widthBytesLine, length;
     if (format == ZPixmap) {
         widthBytesLine = PixmapBytePad(width, pDraw->depth);
         length = widthBytesLine * height;
@@ -3092,7 +3094,6 @@ ProcCreateCursor(ClientPtr client)
     PixmapPtr msk;
     unsigned char *srcbits;
     unsigned short width, height;
-    long n;
     CursorMetricRec cm;
     int rc;
 
@@ -3137,7 +3138,8 @@ ProcCreateCursor(ClientPtr client)
     srcbits = calloc(BitmapBytePad(width), height);
     if (!srcbits)
         return BadAlloc;
-    n = BitmapBytePad(width) * height;
+
+    size_t n = BitmapBytePad(width) * height;
 
     unsigned char *mskbits = calloc(1, n);
     if (!mskbits) {
diff --git a/dix/window.c b/dix/window.c
@@ -503,7 +503,6 @@ MakeRootTile(WindowPtr pWin)
     ScreenPtr pScreen = pWin->drawable.pScreen;
     GCPtr pGC;
     unsigned char back[128];
-    int len = BitmapBytePad(sizeof(long));
     unsigned char *to;
 
     pWin->background.pixmap = (*pScreen->CreatePixmap) (pScreen, 4, 4,
@@ -530,6 +529,7 @@ MakeRootTile(WindowPtr pWin)
         = (screenInfo.bitmapBitOrder == LSBFirst) ? _back_lsb : _back_msb;
     to = back;
 
+    size_t len = BitmapBytePad(sizeof(long));
     for (int i = 4; i > 0; i--, from++)
         for (int j = len; j > 0; j--)
             *to++ = *from;
diff --git a/fb/fbimage.c b/fb/fbimage.c
@@ -34,7 +34,7 @@ fbPutImage(DrawablePtr pDrawable,
 {
     FbGCPrivPtr pPriv = fbGetGCPrivate(pGC);
     unsigned long i;
-    FbStride srcStride;
+    size_t srcStride;
     FbStip *src = (FbStip *) pImage;
 
     x += pDrawable->x;
@@ -211,7 +211,7 @@ fbGetImage(DrawablePtr pDrawable,
     int srcBpp;
     int srcXoff, srcYoff;
     FbStip *dst;
-    FbStride dstStride;
+    size_t dstStride;
 
     /*
      * XFree86 DDX empties the root borderClip when the VT is
diff --git a/hw/xnest/Cursor.c b/hw/xnest/Cursor.c
@@ -62,7 +62,7 @@ xnestRealizeCursor(DeviceIntPtr pDev, ScreenPtr pScreen, CursorPtr pCursor)
     Pixmap const mask = xcb_generate_id(xnestUpstreamInfo.conn);
     xcb_create_pixmap(xnestUpstreamInfo.conn, 1, mask, winId, pCursor->bits->width, pCursor->bits->height);
 
-    int const pixmap_len = BitmapBytePad(pCursor->bits->width) * pCursor->bits->height;
+    size_t const pixmap_len = BitmapBytePad(pCursor->bits->width) * pCursor->bits->height;
 
     xcb_put_image(xnestUpstreamInfo.conn,
                   XCB_IMAGE_FORMAT_XY_BITMAP,
diff --git a/hw/xnest/GCOps.c b/hw/xnest/GCOps.c
@@ -106,8 +106,8 @@ xnestPutImage(DrawablePtr pDrawable, GCPtr pGC, int depth, int x, int y,
                   y,
                   leftPad,
                   depth,
-                  (format == XCB_IMAGE_FORMAT_Z_PIXMAP ? PixmapBytePad(w, depth)
-                                                       : BitmapBytePad(w + leftPad)) * h,
+                  (format == XCB_IMAGE_FORMAT_Z_PIXMAP ? (unsigned)PixmapBytePad(w, depth)
+                                                       : BitmapBytePad((unsigned)(w + leftPad))) * (unsigned)h,
                   (uint8_t*)pImage);
 }
 
diff --git a/hw/xnest/Pixmap.c b/hw/xnest/Pixmap.c
@@ -94,7 +94,6 @@ RegionPtr
 xnestPixmapToRegion(PixmapPtr pPixmap)
 {
     register RegionPtr pReg, pTmpReg;
-    register int x, y;
     unsigned long previousPixel, currentPixel;
     BoxRec Box = { 0, 0, 0, 0 };
     Bool overlap;
@@ -139,13 +138,13 @@ xnestPixmapToRegion(PixmapPtr pPixmap)
     }
 
     uint8_t *image_data = xcb_get_image_data(reply);
-    for (y = 0; y < pPixmap->drawable.height; y++) {
+    for (size_t y = 0; y < pPixmap->drawable.height; y++) {
         Box.y1 = y;
         Box.y2 = y + 1;
         previousPixel = 0L;
-        const int line_start = BitmapBytePad(pPixmap->drawable.width) * y;
+        const size_t line_start = BitmapBytePad(pPixmap->drawable.width) * y;
 
-        for (x = 0; x < pPixmap->drawable.width; x++) {
+        for (size_t x = 0; x < pPixmap->drawable.width; x++) {
             currentPixel = ((image_data[line_start + (x/8)]) >> (x % 8)) & 1;
             if (previousPixel != currentPixel) {
                 if (previousPixel == 0L) {
diff --git a/hw/xnest/xcb.c b/hw/xnest/xcb.c
@@ -96,7 +96,7 @@ uint32_t xnest_create_bitmap_from_data(
     uint32_t gc = xcb_generate_id(xnestUpstreamInfo.conn);
     xcb_create_gc(conn, gc, pix, 0, NULL);
 
-    const int leftPad = 0;
+    const size_t leftPad = 0;
 
     xcb_put_image(conn,
                   XYPixmap,
@@ -108,7 +108,7 @@ uint32_t xnest_create_bitmap_from_data(
                   0 /* dst_y */,
                   leftPad,
                   1 /* depth */,
-                  BitmapBytePad(width + leftPad) * height,
+                  BitmapBytePad((size_t)(width + leftPad)) * (size_t)height,
                   (uint8_t*)data);
 
     xcb_free_gc(conn, gc);
@@ -138,7 +138,7 @@ uint32_t xnest_create_pixmap_from_bitmap_data(
 
     xcb_aux_change_gc(conn, gc, XCB_GC_FOREGROUND | XCB_GC_BACKGROUND, &gcv);
 
-    const int leftPad = 0;
+    const size_t leftPad = 0;
     xcb_put_image(conn,
                   XYBitmap,
                   pix,
@@ -149,7 +149,7 @@ uint32_t xnest_create_pixmap_from_bitmap_data(
                   0 /* dst_y */,
                   leftPad,
                   1 /* depth */,
-                  BitmapBytePad(width + leftPad) * height,
+                  BitmapBytePad((size_t)(width + leftPad)) * (size_t)height,
                   (uint8_t*)data);
 
     xcb_free_gc(conn, gc);
diff --git a/hw/xwin/wincursor.c b/hw/xwin/wincursor.c
@@ -207,7 +207,7 @@ winLoadCursor(ScreenPtr pScreen, CursorPtr pCursor, int screen)
         for (y = 0; y < nCY; ++y)
             for (x = 0; x < xmax; ++x) {
                 int nWinPix = bits_to_bytes(pScreenPriv->cursor.sm_cx) * y + x;
-                int nXPix = BitmapBytePad(pCursor->bits->width) * y + x;
+                size_t nPix = BitmapBytePad(pCursor->bits->width) * y + x;
 
                 pAnd[nWinPix] = 0;
                 if (fReverse)
@@ -222,7 +222,7 @@ winLoadCursor(ScreenPtr pScreen, CursorPtr pCursor, int screen)
         for (y = 0; y < nCY; ++y)
             for (x = 0; x < xmax; ++x) {
                 int nWinPix = bits_to_bytes(pScreenPriv->cursor.sm_cx) * y + x;
-                int nXPix = BitmapBytePad(pCursor->bits->width) * y + x;
+                size_t nPix = BitmapBytePad(pCursor->bits->width) * y + x;
 
                 unsigned char mask = pCursor->bits->mask[nXPix];
 
@@ -318,7 +318,7 @@ winLoadCursor(ScreenPtr pScreen, CursorPtr pCursor, int screen)
                         bit = pAnd[nWinPix];
                         bit = bit & (1 << (7 - (x & 7)));
                         if (!bit) {     /* Within the cursor mask? */
-                            int nXPix =
+                            size_t nXPix =
                                 BitmapBytePad(pCursor->bits->width) * y +
                                 (x / 8);
                             bit =
diff --git a/include/servermd.h b/include/servermd.h
@@ -47,6 +47,8 @@ SOFTWARE.
 #ifndef SERVERMD_H
 #define SERVERMD_H 1
 
+#include <stddef.h>
+
 #if !defined(_DIX_CONFIG_H_) && !defined(_XORG_SERVER_H_)
 #error Drivers must include xorg-server.h before any other xserver headers
 #error xserver code must include dix-config.h before any other headers
@@ -123,7 +125,8 @@ extern _X_EXPORT PaddingInfo PixmapWidthPaddingInfo[];
 #define PixmapBytePad(w, d) \
     (PixmapWidthInPadUnits(w, d) << PixmapWidthPaddingInfo[d].padBytesLog2)
 
-#define BitmapBytePad(w) \
-    (((int)((w) + BITMAP_SCANLINE_PAD - 1) >> LOG2_BITMAP_PAD) << LOG2_BYTES_PER_SCANLINE_PAD)
+static inline size_t BitmapBytePad(size_t w) {
+    return ((((w) + BITMAP_SCANLINE_PAD - 1) >> LOG2_BITMAP_PAD) << LOG2_BYTES_PER_SCANLINE_PAD);
+}
 
 #endif                          /* SERVERMD_H */
diff --git a/mi/miglblt.c b/mi/miglblt.c
@@ -85,7 +85,6 @@ miPolyGlyphBlt(DrawablePtr pDrawable, GCPtr pGC, int x, int y, unsigned int ngly
 {
     int width, height;
     PixmapPtr pPixmap;
-    int nbyLine;                /* bytes per line of padded pixmap */
     FontPtr pfont;
     GCPtr pGCtmp;
     int i;
@@ -96,7 +95,6 @@ miPolyGlyphBlt(DrawablePtr pDrawable, GCPtr pGC, int x, int y, unsigned int ngly
     unsigned char *pglyph;      /* pointer bits in glyph */
     int gWidth, gHeight;        /* width and height of glyph */
     int nbyGlyphWidth;          /* bytes per scanline of glyph */
-    int nbyPadGlyph;            /* server padded line of glyph */
 
     ChangeGCVal gcvals[3];
 
@@ -128,7 +126,7 @@ miPolyGlyphBlt(DrawablePtr pDrawable, GCPtr pGC, int x, int y, unsigned int ngly
 
     ChangeGC(NULL, pGCtmp, GCFunction | GCForeground | GCBackground, gcvals);
 
-    nbyLine = BitmapBytePad(width);
+    size_t nbyLine = BitmapBytePad(width);
     pbits = calloc(height, nbyLine);
     if (!pbits) {
         dixDestroyPixmap(pPixmap, 0);
@@ -142,7 +140,7 @@ miPolyGlyphBlt(DrawablePtr pDrawable, GCPtr pGC, int x, int y, unsigned int ngly
         gHeight = GLYPHHEIGHTPIXELS(pci);
         if (gWidth && gHeight) {
             nbyGlyphWidth = GLYPHWIDTHBYTESPADDED(pci);
-            nbyPadGlyph = BitmapBytePad(gWidth);
+            size_t nbyPadGlyph = BitmapBytePad(gWidth);
 
             if (nbyGlyphWidth == nbyPadGlyph
 #if GLYPHPADBYTES != 4
diff --git a/render/render.c b/render/render.c
@@ -1386,7 +1386,6 @@ ProcRenderCreateCursor(ClientPtr client)
     CARD32 *argb;
     unsigned char *srcline;
     unsigned char *mskline;
-    int stride;
     int x, y;
     int nbytes_mono;
     CursorMetricRec cm;
@@ -1411,7 +1410,7 @@ ProcRenderCreateCursor(ClientPtr client)
     if (!argbbits)
         return BadAlloc;
 
-    stride = BitmapBytePad(width);
+    size_t stride = BitmapBytePad(width);
     nbytes_mono = stride * height;
 
     unsigned char *srcbits = calloc(1, nbytes_mono);
diff --git a/xfixes/cursor.c b/xfixes/cursor.c
@@ -321,7 +321,7 @@ CopyCursorToImage(CursorPtr pCursor, CARD32 *image)
     {
         unsigned char *srcLine = pCursor->bits->source;
         unsigned char *mskLine = pCursor->bits->mask;
-        int stride = BitmapBytePad(width);
+        size_t stride = BitmapBytePad(width);
         int x, y;
         CARD32 fg, bg;
 
