[PATCH xserver v2] xwayland: rotate logical size for RRMode

[Simon Ser]

From: emersion <contact at emersion.fr>

The logical size is the size of the output in the global compositor
space. The mode width/height should be scaled as in the logical
size, but shouldn't be transformed. Thus we need to rotate back
the logical size to be able to use it as the mode width/height.

This fixes issues with pointer input on transformed outputs.

Signed-Off-By: Simon Ser <contact at emersion.fr>
---

Changes from v1 to v2:
- Fixed rotation when xdg-output isn't available

I've made sure this works on both rootston (with xdg-output) and
Weston (without xdg-output).

 hw/xwayland/xwayland-output.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/hw/xwayland/xwayland-output.c b/hw/xwayland/xwayland-output.c
index 379062549..0d2ec7890 100644
--- a/hw/xwayland/xwayland-output.c
+++ b/hw/xwayland/xwayland-output.c
@@ -213,6 +213,7 @@ apply_output_change(struct xwl_output *xwl_output)
 {
     struct xwl_screen *xwl_screen = xwl_output->xwl_screen;
     struct xwl_output *it;
+    int mode_width, mode_height;
     int width = 0, height = 0, has_this_output = 0;
     RRModePtr randr_mode;
     Bool need_rotate;
@@ -224,7 +225,16 @@ apply_output_change(struct xwl_output *xwl_output)
     /* xdg-output sends output size in compositor space. so already rotated */
     need_rotate = (xwl_output->xdg_output == NULL);
 
-    randr_mode = xwayland_cvt(xwl_output->width, xwl_output->height,
+    /* We need to rotate back the logical size for the mode */
+    if (need_rotate || xwl_output->rotation & (RR_Rotate_0 | RR_Rotate_180)) {
+        mode_width = xwl_output->width;
+        mode_height = xwl_output->height;
+    } else {
+        mode_width = xwl_output->height;
+        mode_height = xwl_output->width;
+    }
+
+    randr_mode = xwayland_cvt(mode_width, mode_height,
                               xwl_output->refresh / 1000.0, 0, 0);
     RROutputSetModes(xwl_output->randr_output, &randr_mode, 1, 1);
     RRCrtcNotify(xwl_output->randr_crtc, randr_mode,
-- 
2.18.0