[PATCH v5 xserver] xkb: fix releasing overlay while keydown

[Mihail Konev]

Testcase:

In ~/.xbindkeysrc:
  "xterm &"
       XF86LaunchA

In ~/ov.xkb:
  xkb_keymap {
      xkb_keycodes { include "evdev" };
      xkb_types    { include "complete" };
      xkb_compat   { include "complete"
          interpret Overlay1_Enable+AnyOfOrNone(all) {
              action= SetControls(controls=Overlay1);
          };
      };
      xkb_symbols  { include "pc+inet(evdev)+us"
          key <INS> { [ Overlay1_Enable ] };
          key <AE01> { overlay1 = <AE02> }; // Insert+1 => 2
          key <TLDE> { overlay1 = <I128> }; // Insert+~ => XF86LaunchA
      };
      xkb_geometry { include "pc(pc104)" };
  };

Apply this layout: 'xkbcomp ~/ov.xkb $DISPLAY'.
Run "xbindkeys -n -v"
In the exact order:
- press Insert
- press Tilde
- release Insert
- wait
- release Tilde
Keyboard input in the new terminal window(s) would be locked
until another Insert+Tilde .

Reported-by: Mariusz Mazur <mariusz.g.mazur at gmail.com>
Signed-off-by: Mihail Konev <k.mvc at ya.ru>
---
 include/xkbsrv.h |  8 ++++++++
 xkb/xkbInit.c    |  9 +++++++++
 xkb/xkbPrKeyEv.c | 28 +++++++++++++++++++++++-----
 3 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/include/xkbsrv.h b/include/xkbsrv.h
index 6e4ad44d49f9..d19e4b933466 100644
--- a/include/xkbsrv.h
+++ b/include/xkbsrv.h
@@ -195,6 +195,14 @@ typedef struct _XkbSrvInfo {
     XkbFilterPtr filters;
 
     XkbSrvCheckRepeatPtr checkRepeat;
+
+    /* Keeps track of overlay in effect for a given key,
+     * so that if an overlay is released while key is down,
+     * the key retains overlaid until its release.
+     * Cannot be a bitmask, as needs at least three values
+     * (as overlaid keys need to generate two releases).
+     * */
+    char overlay_perkey_state[256];
 } XkbSrvInfoRec, *XkbSrvInfoPtr;
 
 #define	XkbSLI_IsDefault	(1L<<0)
diff --git a/xkb/xkbInit.c b/xkb/xkbInit.c
index 9c772f549be0..46016aba5553 100644
--- a/xkb/xkbInit.c
+++ b/xkb/xkbInit.c
@@ -505,6 +505,13 @@ XkbInitControls(DeviceIntPtr pXDev, XkbSrvInfoPtr xkbi)
     return Success;
 }
 
+static Status
+XkbInitOverlayState(XkbSrvInfoPtr xkbi)
+{
+    memset(xkbi->overlay_perkey_state, 0, sizeof(xkbi->overlay_perkey_state));
+    return Success;
+}
+
 static Bool
 InitKeyboardDeviceStructInternal(DeviceIntPtr dev, XkbRMLVOSet * rmlvo,
                                  const char *keymap, int keymap_length,
@@ -608,6 +615,8 @@ InitKeyboardDeviceStructInternal(DeviceIntPtr dev, XkbRMLVOSet * rmlvo,
 
     XkbInitIndicatorMap(xkbi);
 
+    XkbInitOverlayState(xkbi);
+
     XkbUpdateActions(dev, xkb->min_key_code, XkbNumKeys(xkb), &changes,
                      &check, &cause);
 
diff --git a/xkb/xkbPrKeyEv.c b/xkb/xkbPrKeyEv.c
index f7a6b4b14306..d68ef89ae8fd 100644
--- a/xkb/xkbPrKeyEv.c
+++ b/xkb/xkbPrKeyEv.c
@@ -121,20 +121,38 @@ XkbProcessKeyboardEvent(DeviceEvent *event, DeviceIntPtr keybd)
         case XkbKB_Overlay2:
         {
             unsigned which;
+            unsigned overlay_active_now;
+            unsigned is_keyrelease = (event->type == ET_KeyRelease) ? 1 : 0;
+            unsigned key_was_overlaid = 0;
 
             if (behavior.type == XkbKB_Overlay1)
                 which = XkbOverlay1Mask;
             else
                 which = XkbOverlay2Mask;
-            if ((xkbi->desc->ctrls->enabled_ctrls & which) == 0)
+            overlay_active_now = (xkbi->desc->ctrls->enabled_ctrls & which) ? 1 : 0;
+
+            if ((unsigned char)key == key) {
+                key_was_overlaid = xkbi->overlay_perkey_state[key];
+                if (overlay_active_now && !is_keyrelease)
+                    xkbi->overlay_perkey_state[key] = 2;
+                else if (is_keyrelease && (overlay_active_now || key_was_overlaid))
+                    xkbi->overlay_perkey_state[key] = key_was_overlaid - 1;
+                else if (key_was_overlaid && !overlay_active_now && !is_keyrelease) {
+                    /* ignore key presses after overlay is released,
+                     * as their release would have been overridden in prev branch,
+                     * and key would need another key-and-release to recover from overlay
+                     * */
+                    return;
+                } else {
+                    break;
+                }
+            }
+
+            if (!overlay_active_now && !key_was_overlaid)
                 break;
             if ((behavior.data >= xkbi->desc->min_key_code) &&
                 (behavior.data <= xkbi->desc->max_key_code)) {
                 event->detail.key = behavior.data;
-                /* 9/11/94 (ef) -- XXX! need to match release with */
-                /*                 press even if the state of the  */
-                /*                 corresponding overlay control   */
-                /*                 changes while the key is down   */
             }
         }
             break;
-- 
2.9.2