[PATCH xserver] xwayland: rotate logical size for RRMode

[Simon Ser]

From: emersion <contact at emersion.fr>

The logical size is the size of the output in the global compositor
space. The mode width/height should be scaled as in the logical
size, but shouldn't be transformed. Thus we need to rotate back
the logical size to be able to use it as the mode width/height.

This fixes issues with pointer input on transformed outputs.

Signed-Off-By: Simon Ser <contact at emersion.fr>
---
This supersedes my previous patch [1], and rotates the logical size
instead of using the raw wl_output mode (so that scaling still
works).

[1]: https://lists.x.org/archives/xorg-devel/2018-July/057285.html

 hw/xwayland/xwayland-output.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/hw/xwayland/xwayland-output.c b/hw/xwayland/xwayland-output.c
index 379062549..26e055867 100644
--- a/hw/xwayland/xwayland-output.c
+++ b/hw/xwayland/xwayland-output.c
@@ -213,6 +213,7 @@ apply_output_change(struct xwl_output *xwl_output)
 {
     struct xwl_screen *xwl_screen = xwl_output->xwl_screen;
     struct xwl_output *it;
+    int mode_width, mode_height;
     int width = 0, height = 0, has_this_output = 0;
     RRModePtr randr_mode;
     Bool need_rotate;
@@ -224,7 +225,16 @@ apply_output_change(struct xwl_output *xwl_output)
     /* xdg-output sends output size in compositor space. so already rotated */
     need_rotate = (xwl_output->xdg_output == NULL);
 
-    randr_mode = xwayland_cvt(xwl_output->width, xwl_output->height,
+    /* We need to rotate back the logical size for the mode */
+    if (xwl_output->rotation & (RR_Rotate_0 | RR_Rotate_180)) {
+        mode_width = xwl_output->width;
+        mode_height = xwl_output->height;
+    } else {
+        mode_width = xwl_output->height;
+        mode_height = xwl_output->width;
+    }
+
+    randr_mode = xwayland_cvt(mode_width, mode_height,
                               xwl_output->refresh / 1000.0, 0, 0);
     RROutputSetModes(xwl_output->randr_output, &randr_mode, 1, 1);
     RRCrtcNotify(xwl_output->randr_crtc, randr_mode,
-- 
2.18.0