xserver: Branch 'master'

[GitLab Mirror]

 hw/xwayland/xwayland-shm.c |   11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

New commits:
commit 079c5ccbcd07c5e8d51239b79dc3cfed46fef506
Author: Olivier Fourdan <ofourdan at redhat.com>
Date:   Thu Sep 16 10:18:03 2021 +0200

    xwayland/shm: Avoid integer overflow on large pixmaps
    
    Xwayland's xwl_shm_create_pixmap() computes the size of the shared
    memory pool to create using a size_t, yet the Wayland protocol uses an
    integer for that size.
    
    If the pool size becomes larger than INT32_MAX, we end up asking Wayland
    to create a shared memory pool of negative size which in turn will raise
    a protocol error which terminates the Wayland connection, and therefore
    Xwayland.
    
    Avoid that issue early by return a NULL pixmap in that case, which will
    trigger a BadAlloc error, but leave Xwayland alive.
    
    Signed-off-by: Olivier Fourdan <ofourdan at redhat.com>
    Reviewed-by: Jonas Ådahl <jadahl at gmail.com>

diff --git a/hw/xwayland/xwayland-shm.c b/hw/xwayland/xwayland-shm.c
index cf7e97ca3..ff128316d 100644
--- a/hw/xwayland/xwayland-shm.c
+++ b/hw/xwayland/xwayland-shm.c
@@ -234,6 +234,15 @@ xwl_shm_create_pixmap(ScreenPtr screen,
         (width == 0 && height == 0) || depth < 15)
         return fbCreatePixmap(screen, width, height, depth, hint);
 
+    stride = PixmapBytePad(width, depth);
+    size = stride * height;
+    /* Size in the protocol is an integer, make sure we don't exceed
+     * INT32_MAX or else the Wayland compositor will raise an error and
+     * kill the Wayland connection!
+     */
+    if (size > INT32_MAX)
+        return NULL;
+
     pixmap = fbCreatePixmap(screen, 0, 0, depth, hint);
     if (!pixmap)
         return NULL;
@@ -242,8 +251,6 @@ xwl_shm_create_pixmap(ScreenPtr screen,
     if (xwl_pixmap == NULL)
         goto err_destroy_pixmap;
 
-    stride = PixmapBytePad(width, depth);
-    size = stride * height;
     xwl_pixmap->buffer = NULL;
     xwl_pixmap->size = size;
     fd = os_create_anonymous_file(size);