[PATCH] ARM: 2660/2: fix ixdp2800 boot and pci init

Patch from Lennert Buytenhek

The IXDP2800 is an evalution platform for the IXP2800 processor that
has two IXP2800s connected to the same PCI bus.  This is problematic
as both CPUs will try to configure the PCI bus as they boot linux.
Contrary to on the other IXP2000 platforms, the boot loader on the
IXDP2800 doesn't configure the PCI bus properly, so we do want the
linux instance on one of the CPUs to do that.
Making one of the CPUs ignore the PCI bus (and thus act like a pure
PCI slave device) is not an option because there is a 82559 NIC on
the PCI bus for each of the CPUs.
The chosen solution is to have the master CPU configure the PCI bus
while the slave is kept in a quiescent state, and then to have the
slave CPU scan the PCI bus (without assigning resources) while the
master is kept in a quiescent state.  After this ritual, the master
deletes the slave NIC from its PCI device list, the slave deletes
the master NIC from its device list, and (almost) all is well.
There's still one little problem: each of the CPUs has a 1G SDRAM
BAR, but the IXP2000 only has 512M of outbound PCI memory window.
We solve this by hand-assigning the master and slave SDRAM BARs to
a location outside each of the IXP's outbound PCI windows, and by
having the rest of the BARs autoconfigured in the outbound PCI
windows, in the range [e0000000..ffffffff], so that there is a 1:1
pci:phys mapping between them.
Even with this patch, a number of issues still remain -- just imagine
what happens if one of the CPUs is rebooted, by watchdog or by hand,
but the other one isn't.  But those issues are not easily fixable
given the strange PCI layout of this board and the behavior of the
boot loader shipped with the platform.

Signed-off-by: Lennert Buytenhek
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>

arch/arm/configs/ixdp2800_defconfig

@@ -133,7 +133,7 @@ CONFIG_ALIGNMENT_TRAP=y
 #
 CONFIG_ZBOOT_ROM_TEXT=0x0
 CONFIG_ZBOOT_ROM_BSS=0x0
-CONFIG_CMDLINE="console=ttyS0,9600 root=/dev/nfs ip=bootp mem=64M@0x0 pci=firmware"
+CONFIG_CMDLINE="console=ttyS0,9600 root=/dev/nfs ip=bootp mem=64M@0x0"
 # CONFIG_XIP_KERNEL is not set
 
 #

arch/arm/mach-ixp2000/ixdp2800.c

@@ -65,19 +65,102 @@ static struct sys_timer ixdp2800_timer = {
 /*************************************************************************
  * IXDP2800 PCI
  *************************************************************************/
+static void __init ixdp2800_slave_disable_pci_master(void)
+{
+	*IXP2000_PCI_CMDSTAT &= ~(PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY);
+}
+
+static void __init ixdp2800_master_wait_for_slave(void)
+{
+	volatile u32 *addr;
+
+	printk(KERN_INFO "IXDP2800: waiting for slave NPU to configure "
+			 "its BAR sizes\n");
+
+	addr = ixp2000_pci_config_addr(0, IXDP2X00_SLAVE_NPU_DEVFN,
+					PCI_BASE_ADDRESS_1);
+	do {
+		*addr = 0xffffffff;
+		cpu_relax();
+	} while (*addr != 0xfe000008);
+
+	addr = ixp2000_pci_config_addr(0, IXDP2X00_SLAVE_NPU_DEVFN,
+					PCI_BASE_ADDRESS_2);
+	do {
+		*addr = 0xffffffff;
+		cpu_relax();
+	} while (*addr != 0xc0000008);
+
+	/*
+	 * Configure the slave's SDRAM BAR by hand.
+	 */
+	*addr = 0x40000008;
+}
+
+static void __init ixdp2800_slave_wait_for_master_enable(void)
+{
+	printk(KERN_INFO "IXDP2800: waiting for master NPU to enable us\n");
+
+	while ((*IXP2000_PCI_CMDSTAT & PCI_COMMAND_MASTER) == 0)
+		cpu_relax();
+}
+
 void __init ixdp2800_pci_preinit(void)
 {
 	printk("ixdp2x00_pci_preinit called\n");
 
-	*IXP2000_PCI_ADDR_EXT =  0x0000e000;
+	*IXP2000_PCI_ADDR_EXT = 0x0001e000;
+
+	if (!ixdp2x00_master_npu())
+		ixdp2800_slave_disable_pci_master();
 
-	*IXP2000_PCI_DRAM_BASE_ADDR_MASK = (0x40000000 - 1) & ~0xfffff;
 	*IXP2000_PCI_SRAM_BASE_ADDR_MASK = (0x2000000 - 1) & ~0x3ffff;
+	*IXP2000_PCI_DRAM_BASE_ADDR_MASK = (0x40000000 - 1) & ~0xfffff;
 
 	ixp2000_pci_preinit();
+
+	if (ixdp2x00_master_npu()) {
+		/*
+		 * Wait until the slave set its SRAM/SDRAM BAR sizes
+		 * correctly before we proceed to scan and enumerate
+		 * the bus.
+		 */
+		ixdp2800_master_wait_for_slave();
+
+		/*
+		 * We configure the SDRAM BARs by hand because they
+		 * are 1G and fall outside of the regular allocated
+		 * PCI address space.
+		 */
+		*IXP2000_PCI_SDRAM_BAR = 0x00000008;
+	} else {
+		/*
+		 * Wait for the master to complete scanning the bus
+		 * and assigning resources before we proceed to scan
+		 * the bus ourselves.  Set pci=firmware to honor the
+		 * master's resource assignment.
+		 */
+		ixdp2800_slave_wait_for_master_enable();
+		pcibios_setup("firmware");
+	}
+}
+
+/*
+ * We assign the SDRAM BARs for the two IXP2800 CPUs by hand, outside
+ * of the regular PCI window, because there's only 512M of outbound PCI
+ * memory window on each IXP, while we need 1G for each of the BARs.
+ */
+static void __devinit ixp2800_pci_fixup(struct pci_dev *dev)
+{
+	if (machine_is_ixdp2800()) {
+		dev->resource[2].start = 0;
+		dev->resource[2].end   = 0;
+		dev->resource[2].flags = 0;
+	}
 }
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IXP2800, ixp2800_pci_fixup);
 
-int ixdp2800_pci_setup(int nr, struct pci_sys_data *sys)
+static int __init ixdp2800_pci_setup(int nr, struct pci_sys_data *sys)
 {
 	sys->mem_offset = 0x00000000;
 
@@ -129,22 +212,47 @@ static int __init ixdp2800_pci_map_irq(struct pci_dev *dev, u8 slot, u8 pin)
 	} else return IRQ_IXP2000_PCIB; /* Slave NIC interrupt */
 }
 
-static void ixdp2800_pci_postinit(void)
+static void __init ixdp2800_master_enable_slave(void)
 {
-	struct pci_dev *dev;
+	volatile u32 *addr;
 
-	if (ixdp2x00_master_npu()) {
-		dev = pci_find_slot(1, IXDP2800_SLAVE_ENET_DEVFN);
-		pci_remove_bus_device(dev);
-	} else {
-		dev = pci_find_slot(1, IXDP2800_MASTER_ENET_DEVFN);
-		pci_remove_bus_device(dev);
+	printk(KERN_INFO "IXDP2800: enabling slave NPU\n");
+
+	addr = (volatile u32 *)ixp2000_pci_config_addr(0,
+					IXDP2X00_SLAVE_NPU_DEVFN,
+					PCI_COMMAND);
+
+	*addr |= PCI_COMMAND_MASTER;
+}
 
+static void __init ixdp2800_master_wait_for_slave_bus_scan(void)
+{
+	volatile u32 *addr;
+
+	printk(KERN_INFO "IXDP2800: waiting for slave to finish bus scan\n");
+
+	addr = (volatile u32 *)ixp2000_pci_config_addr(0,
+					IXDP2X00_SLAVE_NPU_DEVFN,
+					PCI_COMMAND);
+	while ((*addr & PCI_COMMAND_MEMORY) == 0)
+		cpu_relax();
+}
+
+static void __init ixdp2800_slave_signal_bus_scan_completion(void)
+{
+	printk(KERN_INFO "IXDP2800: bus scan done, signaling master\n");
+	*IXP2000_PCI_CMDSTAT |= PCI_COMMAND_MEMORY;
+}
+
+static void __init ixdp2800_pci_postinit(void)
+{
+	if (!ixdp2x00_master_npu()) {
 		ixdp2x00_slave_pci_postinit();
+		ixdp2800_slave_signal_bus_scan_completion();
 	}
 }
 
-struct hw_pci ixdp2800_pci __initdata = {
+struct __initdata hw_pci ixdp2800_pci __initdata = {
 	.nr_controllers	= 1,
 	.setup		= ixdp2800_pci_setup,
 	.preinit	= ixdp2800_pci_preinit,
@@ -155,8 +263,21 @@ struct hw_pci ixdp2800_pci __initdata = {
 
 int __init ixdp2800_pci_init(void)
 {
-	if (machine_is_ixdp2800())
+	if (machine_is_ixdp2800()) {
+		struct pci_dev *dev;
+
 		pci_common_init(&ixdp2800_pci);
+		if (ixdp2x00_master_npu()) {
+			dev = pci_find_slot(1, IXDP2800_SLAVE_ENET_DEVFN);
+			pci_remove_bus_device(dev);
+
+			ixdp2800_master_enable_slave();
+			ixdp2800_master_wait_for_slave_bus_scan();
+		} else {
+			dev = pci_find_slot(1, IXDP2800_MASTER_ENET_DEVFN);
+			pci_remove_bus_device(dev);
+		}
+	}
 
 	return 0;
 }