[PATCH xserver] xkb: fix releasing overlay while keydown

[Mihail Konev]

Testcase:

In ~/.xbindkeysrc:

  "xterm &"
       XF86LaunchA

In ~/ov.xkb:

  xkb_keymap {
          xkb_keycodes { include "evdev" };
          xkb_types    { include "complete" };

          xkb_compat   { include "complete"
               interpret Overlay1_Enable+AnyOfOrNone(all) {
                  action= SetControls(controls=Overlay1);
               };
          };

          xkb_symbols  { include "pc+inet(evdev)+us"
                  key <INS> { [ Overlay1_Enable ] };
                  key <AE01> { overlay1 = <AE02> }; // Insert+1 => 2
                  key <TLDE> { overlay1 = <I128> }; // Insert+~ => XF86LaunchA
          };

          xkb_geometry { include "pc(pc104)" };
  };

Apply this layout: 'xkbcomp ~/ov.xkb $DISPLAY'.
Run "xbindkeys -n -v"
In the exact order:
- press Insert
- press Tilde
- release Insert
- wait
- release Tilde
Keyboard input in the new terminal window(s) would be locked
until another Insert+Tilde .

Signed-off-by: Mihail Konev <k.mvc at ya.ru>
---

On Sun, Nov 06, 2016 at 12:03:47AM +0500, Mihail Konev wrote:
> +
> +            if (overlay_enabled)
> +                overlay_keys[which_overlay][(unsigned char)key] = behavior.data;
> +            else
> +                behavior_data = overlay_keys[which_overlay][(unsigned char)key];
> +
> +            if ((behavior_data >= xkbi->desc->min_key_code) &&
> +                (behavior_data <= xkbi->desc->max_key_code)) {
> +                event->detail.key = behavior_data;
>              }
>          }
>              break;

The v2 patch is incorrect; it does 'overlay_keys[i] = behavior.data', and then
'event->detail.key = ' to either 'overlay_keys[i]' or 'behavior.data',

I.e. it effectively just omits the 'if (!(xkbi->...->ctls & which)) break;' .
(And somehow works; must be because there are (for whatever reason) two releases
of an overlaid key being generated).

The intent was to 'if (!(xkbi->...->ctls & which) && !key_was_overlaid_before_overlay_release)'
instead.
It is implemented in this patch.
(Which does not work unless overlay_state handles two overlaid key releases instead of one,
as v2 tries/appears to).

 xkb/xkbPrKeyEv.c | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/xkb/xkbPrKeyEv.c b/xkb/xkbPrKeyEv.c
index f7a6b4b14306..73760b0b365b 100644
--- a/xkb/xkbPrKeyEv.c
+++ b/xkb/xkbPrKeyEv.c
@@ -43,6 +43,14 @@ THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 /***====================================================================***/
 
+/* Keeps track of overlay in effect for a given key,
+ * so that if an overlay is released while key is down,
+ * the key retains overlaid until its release.
+ * Cannot be a bitmask, as needs at least three values
+ * (as overlaid keys need to generate two releases).
+ * */
+static unsigned char overlay_state[256];
+
 void
 XkbProcessKeyboardEvent(DeviceEvent *event, DeviceIntPtr keybd)
 {
@@ -121,20 +129,23 @@ XkbProcessKeyboardEvent(DeviceEvent *event, DeviceIntPtr keybd)
         case XkbKB_Overlay2:
         {
             unsigned which;
+            unsigned key_is_overlaid = 0;
 
             if (behavior.type == XkbKB_Overlay1)
                 which = XkbOverlay1Mask;
             else
                 which = XkbOverlay2Mask;
-            if ((xkbi->desc->ctrls->enabled_ctrls & which) == 0)
+
+            if ((unsigned char)key == key) {
+                key_is_overlaid = overlay_state[key];
+                overlay_state[key] = (event->type == ET_KeyPress) ? 2 : (key_is_overlaid - 1);
+            }
+
+            if (!(xkbi->desc->ctrls->enabled_ctrls & which) && !key_is_overlaid)
                 break;
             if ((behavior.data >= xkbi->desc->min_key_code) &&
                 (behavior.data <= xkbi->desc->max_key_code)) {
                 event->detail.key = behavior.data;
-                /* 9/11/94 (ef) -- XXX! need to match release with */
-                /*                 press even if the state of the  */
-                /*                 corresponding overlay control   */
-                /*                 changes while the key is down   */
             }
         }
             break;
-- 
2.9.2