[PATCH RFC]: Generic Linux multi-domain PCI handling

David S. Miller davem at davemloft.net
Tue Dec 27 22:29:27 PST 2005


I've been plagued with having a sparc64 machine which is multi-domain
PCI and my video card is on domain 1.  Instead of going off and
forking the linuxPci.c code like was done for Alpha I tried to do
something that would work on PPC and other non-sparc64 platforms.

One thing of note is that it might be a good idea to dynamically
allocate the pciBusInfo[] array after ARCH_PCI_INIT() has run
and pciMaxBusNum has it's final stable value.  With even moderate
domain values like 128 or 256, this static table gets a bit
large.

Anyways, the general idea is to run over the dirents under
/proc/bus/pci/ and if we find colon encoded directory names
of the form "%x:%x" we use that to determine which domains
exist on the machine.  This is implemented in the function
linuxPci.c:probe_domains().

For each domain found, we allocate a pciBusInfo_t and stick that into
the pciBusInfo[] entry for bus zero on the found domain.

If we don't find any domains, that's ok, we just revert to existing
behavior which is to assign &linuxPci0 to pciBusInfo[0] and set
pciNumBuses to 1.

I then set pciMaxBusNum to whatever we calculated pciNumBuses to
be, otherwise Pci.c goes absolutely bananas probing domains that
don't exist and results in an enormous number of PCI config space
accesses to every possible bus on every non-existent domain, which
makes the X server sit for a few seconds chugging and doing
nothing but stat() and open() calls. :-)

The final part of this change is to make linuxPciOpenFile() open
domain'd files correctly.  Instead of "0000" we put the actual
domain number there in the file path string, and we track the
domain of the cached file descriptor so that still works correctly
as well.

With this I have X11R7 CVS working on my sparc64 Radeon card, which
sits on domain 1.

Comments?

--- ./hw/xfree86/os-support/bus/Pci.h.~1~	2005-12-24 17:55:56.000000000 -0800
+++ ./hw/xfree86/os-support/bus/Pci.h	2005-12-24 18:01:06.000000000 -0800
@@ -124,6 +124,10 @@
 # define MAX_PCI_DOMAINS	512
 # define PCI_DOM_MASK	0x01fful
 # define MAX_PCI_BUSES	(MAX_PCI_DOMAINS*256) /* 256 per domain      */
+#elif defined(linux)
+# define MAX_PCI_DOMAINS	256
+# define PCI_DOM_MASK	0x00fful
+# define MAX_PCI_BUSES	(MAX_PCI_DOMAINS*256) /* 256 per domain      */
 #else
 # define MAX_PCI_BUSES   256	/* Max number of PCI buses           */
 #endif
--- ./hw/xfree86/os-support/bus/linuxPci.c.~1~	2005-11-08 11:04:56.000000000 -0800
+++ ./hw/xfree86/os-support/bus/linuxPci.c	2005-12-27 21:57:56.000000000 -0800
@@ -50,6 +50,8 @@
 #endif
 
 #include <stdio.h>
+#include <sys/types.h>
+#include <dirent.h>
 #include "compiler.h"
 #include "xf86.h"
 #include "xf86Priv.h"
@@ -111,18 +113,68 @@ static pciBusInfo_t linuxPci0 = {
 /* bridge      */	NULL
 };
 
+static int
+probe_domains(void)
+{
+	struct dirent *dent;
+	int found, max_bus;
+	DIR *dir;
+
+	dir = opendir("/proc/bus/pci");
+	if (!dir)
+		return -1;
+
+	found = 0;
+	max_bus = -1;
+	while ((dent = readdir(dir)) != NULL) {
+		int domain, bus, ret;
+
+		ret = sscanf(dent->d_name, "%x:%x", &domain, &bus);
+		if (ret != 2)
+			continue;
+
+		bus = PCI_MAKE_BUS(domain, bus);
+		if (bus + 1 > pciNumBuses)
+			pciNumBuses = bus + 1;
+
+		bus = PCI_MAKE_BUS(domain, 0);
+		if (pciBusInfo[bus] != NULL)
+			continue;
+
+		found++;
+
+		pciBusInfo[bus] = xnfalloc(sizeof(pciBusInfo_t));
+		memcpy(pciBusInfo[bus], &linuxPci0, sizeof(pciBusInfo_t));
+	}
+
+	xf86MsgVerb(X_INFO, 2, "PCI: Found %d domains.\n", found);
+
+	if (!found) {
+		pciNumBuses = 1;
+		pciBusInfo[0] = &linuxPci0;
+	}
+
+	pciMaxBusNum = pciNumBuses;
+
+	return 0;
+}
+
 void
-linuxPciInit()
+linuxPciInit(void)
 {
 	struct stat st;
+	int num_domains;
+
 	if ((xf86Info.pciFlags == PCIForceNone) ||
 	    (-1 == stat("/proc/bus/pci", &st))) {
 		/* when using this as default for all linux architectures,
 		   we'll need a fallback for 2.0 kernels here */
 		return;
 	}
-	pciNumBuses    = 1;
-	pciBusInfo[0]  = &linuxPci0;
+	num_domains = probe_domains();
+	if (num_domains < 0)
+		return;
+
 	pciFindFirstFP = pciGenFindFirst;
 	pciFindNextFP  = pciGenFindNext;
 }
@@ -130,31 +182,33 @@ linuxPciInit()
 static int
 linuxPciOpenFile(PCITAG tag, Bool write)
 {
-	static int	lbus,ldev,lfunc,fd = -1,is_write = 0;
-	int		bus, dev, func;
+	static int	ldom,lbus,ldev,lfunc,fd = -1,is_write = 0;
+	int		dom, bus, dev, func;
 	char		file[32];
 	struct stat	ignored;
 
 	bus  = PCI_BUS_FROM_TAG(tag);
+	dom  = PCI_DOM_FROM_BUS(bus);
+	bus  = PCI_BUS_NO_DOMAIN(bus);
 	dev  = PCI_DEV_FROM_TAG(tag);
 	func = PCI_FUNC_FROM_TAG(tag);
 	if (fd == -1 || (write && (!is_write))
-	    || bus != lbus || dev != ldev || func != lfunc) {
+	    || ldom != dom || bus != lbus || dev != ldev || func != lfunc) {
 		if (fd != -1)
 			close(fd);
 		if (bus < 256) {
 		        sprintf(file,"/proc/bus/pci/%02x",bus);
 			if (stat(file, &ignored) < 0)
-				sprintf(file, "/proc/bus/pci/0000:%02x/%02x.%1x",
-					bus, dev, func);
+				sprintf(file, "/proc/bus/pci/%04x:%02x/%02x.%1x",
+					dom, bus, dev, func);
 			else
 				sprintf(file, "/proc/bus/pci/%02x/%02x.%1x",
 					bus, dev, func);
 		} else {
 		        sprintf(file,"/proc/bus/pci/%04x",bus);
 			if (stat(file, &ignored) < 0)
-				sprintf(file, "/proc/bus/pci/0000:%04x/%02x.%1x",
-					bus, dev, func);
+				sprintf(file, "/proc/bus/pci/%04x:%04x/%02x.%1x",
+					dom, bus, dev, func);
 			else
 				sprintf(file, "/proc/bus/pci/%04x/%02x.%1x",
 					bus, dev, func);
@@ -172,6 +226,7 @@ linuxPciOpenFile(PCITAG tag, Bool write)
 			    is_write = FALSE;
 		}
 		
+		ldom  = dom;
 		lbus  = bus;
 		ldev  = dev;
 		lfunc = func;



More information about the xorg mailing list