[PATCH] Solaris: Avoid switching to inactive VT's

[Alan Coopersmith]

From: Aaron Zang <Aaron.Zang at Sun.COM>

Fix for OpenSolaris bug 6876992: "[vconsole] Ctrl+Alt+F12 switchs to blank
console screen with hotkeys property turned-off"
http://bugs.opensolaris.org/bugdatabase/view_bug.do?bug_id=6876992

Xorg needs to do sanity test for the VT it is commanded to switch to.
If the VT is not opened by any process, discard the switching request.

The changes also contain the fix for some flaws discovered when
getting the new gdm to run.

Signed-off-by: Alan Coopersmith <alan.coopersmith at sun.com>
---
 hw/xfree86/common/xf86Events.c           |   10 +++++++++-
 hw/xfree86/os-support/solaris/sun_VTsw.c |   24 ++++++++++++++++++++----
 hw/xfree86/os-support/solaris/sun_init.c |   19 +++++++++++++------
 3 files changed, 42 insertions(+), 11 deletions(-)

diff --git a/hw/xfree86/common/xf86Events.c b/hw/xfree86/common/xf86Events.c
index 8cd765a..8e6a15b 100644
--- a/hw/xfree86/common/xf86Events.c
+++ b/hw/xfree86/common/xf86Events.c
@@ -202,8 +202,16 @@ xf86ProcessActionEvent(ActionEvent action, void *arg)
 	    vtno--;
 #endif
 #if defined(sun)
-	    if (vtno == xf86Info.vtno)
+	    if (vtno == xf86Info.vtno) {
 		break;
+	    } else {
+		struct vt_stat state;
+		if (ioctl(xf86Info.consoleFd, VT_GETSTATE, &state) < 0)
+			break;
+
+		if ((state.v_state & (1 << vtno)) == 0)
+			break;
+	    }
 
 	    xf86Info.vtRequestsPending = TRUE;
 	    xf86Info.vtPendingNum = vtno;
diff --git a/hw/xfree86/os-support/solaris/sun_VTsw.c b/hw/xfree86/os-support/solaris/sun_VTsw.c
index 0dc76b8..517d6d9 100644
--- a/hw/xfree86/os-support/solaris/sun_VTsw.c
+++ b/hw/xfree86/os-support/solaris/sun_VTsw.c
@@ -44,17 +44,27 @@
  * Handle the VT-switching interface for Solaris/OpenSolaris
  */
 
+static int xf86VTPruneDoor = 0;
+
 void
-xf86VTRequest(int sig)
+xf86VTRelease(int sig)
 {
 	if (xf86Info.vtPendingNum != -1)
 	{
-		ioctl(xf86Info.consoleFd, VT_RELDISP, 1);
-		xf86Info.vtPendingNum = -1;
-
+		xf86VTPruneDoor = 1;
+		xf86Info.vtRequestsPending = TRUE;
 		return;
 	}
 
+	ioctl(xf86Info.consoleFd, VT_RELDISP, 1);
+	xf86Info.vtPendingNum = -1;
+
+	return;
+}
+
+void
+xf86VTAcquire(int sig)
+{
 	xf86Info.vtRequestsPending = TRUE;
 	return;
 }
@@ -74,6 +84,12 @@ xf86VTSwitchAway(void)
 
 	xf86Info.vtRequestsPending = FALSE;
 
+	if (xf86VTPruneDoor) {
+		xf86VTPruneDoor = 0;
+		ioctl(xf86Info.consoleFd, VT_RELDISP, 1);
+		return (TRUE);
+	}
+
 	vt_door_arg.vt_ev = VT_EV_HOTKEYS;
 	vt_door_arg.vt_num = xf86Info.vtPendingNum;
 	door_arg.data_ptr = (char *)&vt_door_arg;
diff --git a/hw/xfree86/os-support/solaris/sun_init.c b/hw/xfree86/os-support/solaris/sun_init.c
index 2c569f0..5846866 100644
--- a/hw/xfree86/os-support/solaris/sun_init.c
+++ b/hw/xfree86/os-support/solaris/sun_init.c
@@ -39,6 +39,8 @@ static Bool Protect0 = FALSE;
 static int VTnum = -1;
 static int xf86StartVT = -1;
 static int vtEnabled = 0;
+extern void xf86VTAcquire(int);
+extern void xf86VTRelease(int);
 #endif
 
 /* Device to open as xf86Info.consoleFd */
@@ -137,7 +139,8 @@ xf86OpenConsole(void)
 	    else
 	    {
 		if ((ioctl(fd, VT_OPENQRY, &xf86Info.vtno) < 0) ||
-		    (xf86Info.vtno == -1)) {
+		    (xf86Info.vtno == -1))
+		{
 		    FatalError("xf86OpenConsole: Cannot find a free VT\n");
 		}
 	    }
@@ -146,7 +149,8 @@ xf86OpenConsole(void)
 	    snprintf(consoleDev, PATH_MAX, "/dev/vt/%d", xf86Info.vtno);
 	}
 
-	if (fd != -1) {
+	if (fd != -1)
+	{
 	    close(fd);
 	}
 
@@ -178,11 +182,12 @@ xf86OpenConsole(void)
 	    if (ioctl(xf86Info.consoleFd, VT_GETMODE, &VT) < 0)
 		FatalError("xf86OpenConsole: VT_GETMODE failed\n");
 
-	    OsSignal(SIGUSR1, xf86VTRequest);
+	    OsSignal(SIGUSR1, xf86VTAcquire);
+	    OsSignal(SIGUSR2, xf86VTRelease);
 
 	    VT.mode = VT_PROCESS;
-	    VT.relsig = SIGUSR1;
 	    VT.acqsig = SIGUSR1;
+	    VT.relsig = SIGUSR2;
 
 	    if (ioctl(xf86Info.consoleFd, VT_SETMODE, &VT) < 0)
 		FatalError("xf86OpenConsole: VT_SETMODE VT_PROCESS failed\n");
@@ -204,7 +209,8 @@ xf86OpenConsole(void)
     else /* serverGeneration != 1 */
     {
 #ifdef HAS_USL_VTS
-	if (vtEnabled) {
+	if (vtEnabled)
+	{
 	    /*
 	     * Now re-get the VT
 	     */
@@ -285,7 +291,8 @@ xf86CloseConsole(void)
 #endif
 
 #ifdef HAS_USL_VTS
-    if (vtEnabled == 1) {
+    if (vtEnabled)
+    {
 	if (ioctl(xf86Info.consoleFd, VT_GETMODE, &VT) != -1)
 	{
 	    VT.mode = VT_AUTO;		/* Set default vt handling */
-- 
1.5.6.5