We need to disable interrupts when taking the phb->lock. Otherwise we could deadlock with pci_lock taken from an interrupt.
Signed-off-by: Michael Ellerman <mich...@ellerman.id.au> -- ========================================================= [ INFO: possible irq lock inversion dependency detected ] 3.2.0-rc5-mpe+ #17 --------------------------------------------------------- swapper/56/0 just changed the state of lock: (pci_lock){-.....}, at: [<c000000000421824>] .pci_bus_read_config_word+0x84/0x110 but this lock took another, HARDIRQ-unsafe lock in the past: (&(&phb->lock)->rlock){+.+...} and interrupts could create inverse lock ordering between them. other info that might help us debug this: Possible interrupt unsafe locking scenario: CPU0 CPU1 ---- ---- lock(&(&phb->lock)->rlock); local_irq_disable(); lock(pci_lock); lock(&(&phb->lock)->rlock); <Interrupt> lock(pci_lock); *** DEADLOCK *** 1 lock held by swapper/56/0: #0: (&(shost->host_lock)->rlock){-.-...}, at: [<c0000000005ee84c>] .ipr_isr+0x4c/0x540 the shortest dependencies between 2nd lock and 1st lock: -> (&(&phb->lock)->rlock){+.+...} ops: 622770257920 { HARDIRQ-ON-W at: [<c0000000000be848>] .lock_acquire+0xb8/0x190 [<c0000000008e1c24>] ._raw_spin_lock+0x34/0x60 [<c000000000047320>] .pnv_setup_msi_irqs+0xb0/0x320 [<c00000000002dfc4>] .arch_setup_msi_irqs+0x34/0x50 [<c000000000434160>] .pci_enable_msi_block+0x210/0x300 [<c00000000090ad90>] .ipr_probe+0x6a8/0x1560 [<c00000000042a9b0>] .pci_device_probe+0x1a0/0x1b0 [<c00000000048c7c0>] .driver_probe_device+0xb0/0x1e0 [<c00000000048c9fc>] .__driver_attach+0x10c/0x110 [<c00000000048b2f8>] .bus_for_each_dev+0x88/0xe0 [<c00000000048c2c8>] .driver_attach+0x28/0x40 [<c00000000048bd68>] .bus_add_driver+0x178/0x310 [<c00000000048cebc>] .driver_register+0x9c/0x1b0 [<c00000000042b040>] .__pci_register_driver+0x70/0x150 [<c000000000cddb80>] .ipr_init+0x4c/0x68 [<c00000000000a9f0>] .do_one_initcall+0x60/0x1e0 [<c000000000cb0b58>] .kernel_init+0x134/0x1c0 [<c0000000000215a8>] .kernel_thread+0x54/0x70 SOFTIRQ-ON-W at: [<c0000000000be848>] .lock_acquire+0xb8/0x190 [<c0000000008e1c24>] ._raw_spin_lock+0x34/0x60 [<c000000000047320>] .pnv_setup_msi_irqs+0xb0/0x320 [<c00000000002dfc4>] .arch_setup_msi_irqs+0x34/0x50 [<c000000000434160>] .pci_enable_msi_block+0x210/0x300 [<c00000000090ad90>] .ipr_probe+0x6a8/0x1560 [<c00000000042a9b0>] .pci_device_probe+0x1a0/0x1b0 [<c00000000048c7c0>] .driver_probe_device+0xb0/0x1e0 [<c00000000048c9fc>] .__driver_attach+0x10c/0x110 [<c00000000048b2f8>] .bus_for_each_dev+0x88/0xe0 [<c00000000048c2c8>] .driver_attach+0x28/0x40 [<c00000000048bd68>] .bus_add_driver+0x178/0x310 [<c00000000048cebc>] .driver_register+0x9c/0x1b0 [<c00000000042b040>] .__pci_register_driver+0x70/0x150 [<c000000000cddb80>] .ipr_init+0x4c/0x68 [<c00000000000a9f0>] .do_one_initcall+0x60/0x1e0 [<c000000000cb0b58>] .kernel_init+0x134/0x1c0 [<c0000000000215a8>] .kernel_thread+0x54/0x70 INITIAL USE at: [<c0000000000be848>] .lock_acquire+0xb8/0x190 [<c0000000008e1d64>] ._raw_spin_lock_irqsave+0x54/0x80 [<c000000000046a80>] .pnv_pci_config_check_eeh+0xe0/0x4d0 [<c000000000047044>] .pnv_pci_read_config+0xc4/0x150 [<c000000000421750>] .pci_bus_read_config_dword+0xc0/0x110 [<c0000000007a4764>] .pci_scan_single_device+0xa4/0x1d0 [<c000000000424730>] .pci_scan_slot+0x100/0x1f0 [<c000000000900990>] .pci_scan_child_bus+0x40/0x108 [<c000000000900bc0>] .pci_scan_bridge+0x168/0x4d0 [<c000000000900a0c>] .pci_scan_child_bus+0xbc/0x108 [<c0000000008fe0fc>] .pcibios_scan_phb+0xe8/0x18c [<c000000000cbb98c>] .pcibios_init+0x80/0xe4 [<c00000000000a9f0>] .do_one_initcall+0x60/0x1e0 [<c000000000cb0b58>] .kernel_init+0x134/0x1c0 [<c0000000000215a8>] .kernel_thread+0x54/0x70 } ... key at: [<c00000000144934c>] __key.22640+0x0/0x8 ... acquired at: [<c0000000008e1d64>] ._raw_spin_lock_irqsave+0x54/0x80 [<c000000000046a80>] .pnv_pci_config_check_eeh+0xe0/0x4d0 [<c000000000047044>] .pnv_pci_read_config+0xc4/0x150 [<c000000000421750>] .pci_bus_read_config_dword+0xc0/0x110 [<c0000000007a4764>] .pci_scan_single_device+0xa4/0x1d0 [<c000000000424730>] .pci_scan_slot+0x100/0x1f0 [<c000000000900990>] .pci_scan_child_bus+0x40/0x108 [<c000000000900bc0>] .pci_scan_bridge+0x168/0x4d0 [<c000000000900a0c>] .pci_scan_child_bus+0xbc/0x108 [<c0000000008fe0fc>] .pcibios_scan_phb+0xe8/0x18c [<c000000000cbb98c>] .pcibios_init+0x80/0xe4 [<c00000000000a9f0>] .do_one_initcall+0x60/0x1e0 [<c000000000cb0b58>] .kernel_init+0x134/0x1c0 [<c0000000000215a8>] .kernel_thread+0x54/0x70 -> (pci_lock){-.....} ops: 21178483736576 { IN-HARDIRQ-W at: [<c0000000000be848>] .lock_acquire+0xb8/0x190 [<c0000000008e1d64>] ._raw_spin_lock_irqsave+0x54/0x80 [<c000000000421824>] .pci_bus_read_config_word+0x84/0x110 [<c0000000005e8200>] .ipr_reset_alert+0x50/0x130 [<c0000000005e84cc>] .ipr_reset_ioa_job+0x8c/0xf0 [<c0000000005ee920>] .ipr_isr+0x120/0x540 [<c0000000000f26a8>] .handle_irq_event_percpu+0x98/0x350 [<c0000000000f29b8>] .handle_irq_event+0x58/0xa0 [<c0000000000f6be0>] .handle_fasteoi_irq+0xa0/0x1c0 [<c000000000021410>] .call_handle_irq+0x1c/0x2c [<c00000000001005c>] .do_IRQ+0x29c/0x320 [<c00000000000500c>] hardware_interrupt_entry+0x1c/0x90 [<c0000000000177ec>] .cpu_idle+0x12c/0x220 [<c0000000008fcfa4>] .start_secondary+0x360/0x368 [<c000000000009368>] .start_secondary_prolog+0x10/0x14 INITIAL USE at: [<c0000000000be848>] .lock_acquire+0xb8/0x190 [<c0000000008e1d64>] ._raw_spin_lock_irqsave+0x54/0x80 [<c000000000421714>] .pci_bus_read_config_dword+0x84/0x110 [<c0000000007a4764>] .pci_scan_single_device+0xa4/0x1d0 [<c0000000004246b8>] .pci_scan_slot+0x88/0x1f0 [<c000000000900990>] .pci_scan_child_bus+0x40/0x108 [<c0000000008fe0fc>] .pcibios_scan_phb+0xe8/0x18c [<c000000000cbb98c>] .pcibios_init+0x80/0xe4 [<c00000000000a9f0>] .do_one_initcall+0x60/0x1e0 [<c000000000cb0b58>] .kernel_init+0x134/0x1c0 [<c0000000000215a8>] .kernel_thread+0x54/0x70 } ... key at: [<c0000000012b64b8>] pci_lock+0x18/0x38 ... acquired at: [<c0000000000bc1e4>] .mark_lock+0x244/0x740 [<c0000000000bcfcc>] .__lock_acquire+0x8ec/0x1b10 [<c0000000000be848>] .lock_acquire+0xb8/0x190 [<c0000000008e1d64>] ._raw_spin_lock_irqsave+0x54/0x80 [<c000000000421824>] .pci_bus_read_config_word+0x84/0x110 [<c0000000005e8200>] .ipr_reset_alert+0x50/0x130 [<c0000000005e84cc>] .ipr_reset_ioa_job+0x8c/0xf0 [<c0000000005ee920>] .ipr_isr+0x120/0x540 [<c0000000000f26a8>] .handle_irq_event_percpu+0x98/0x350 [<c0000000000f29b8>] .handle_irq_event+0x58/0xa0 [<c0000000000f6be0>] .handle_fasteoi_irq+0xa0/0x1c0 [<c000000000021410>] .call_handle_irq+0x1c/0x2c [<c00000000001005c>] .do_IRQ+0x29c/0x320 [<c00000000000500c>] hardware_interrupt_entry+0x1c/0x90 [<c0000000000177ec>] .cpu_idle+0x12c/0x220 [<c0000000008fcfa4>] .start_secondary+0x360/0x368 [<c000000000009368>] .start_secondary_prolog+0x10/0x14 stack backtrace: Call Trace: [c000000ffbe1b490] [c000000000015838] .show_stack+0x78/0x1b0 (unreliable) [c000000ffbe1b540] [c0000000000bb4b0] .print_irq_inversion_bug+0x270/0x2c0 [c000000ffbe1b5f0] [c0000000000bb5a4] .check_usage_forwards+0xa4/0x130 [c000000ffbe1b6f0] [c0000000000bc1e4] .mark_lock+0x244/0x740 [c000000ffbe1b7b0] [c0000000000bcfcc] .__lock_acquire+0x8ec/0x1b10 [c000000ffbe1b930] [c0000000000be848] .lock_acquire+0xb8/0x190 [c000000ffbe1ba10] [c0000000008e1d64] ._raw_spin_lock_irqsave+0x54/0x80 [c000000ffbe1bab0] [c000000000421824] .pci_bus_read_config_word+0x84/0x110 [c000000ffbe1bb70] [c0000000005e8200] .ipr_reset_alert+0x50/0x130 [c000000ffbe1bc10] [c0000000005e84cc] .ipr_reset_ioa_job+0x8c/0xf0 [c000000ffbe1bca0] [c0000000005ee920] .ipr_isr+0x120/0x540 [c000000ffbe1bd80] [c0000000000f26a8] .handle_irq_event_percpu+0x98/0x350 [c000000ffbe1be70] [c0000000000f29b8] .handle_irq_event+0x58/0xa0 [c000000ffbe1bf00] [c0000000000f6be0] .handle_fasteoi_irq+0xa0/0x1c0 [c000000ffbe1bf90] [c000000000021410] .call_handle_irq+0x1c/0x2c [c0000007f2d47a70] [c00000000001005c] .do_IRQ+0x29c/0x320 [c0000007f2d47b30] [c00000000000500c] hardware_interrupt_entry+0x1c/0x90 --- Exception: 501 at .arch_local_irq_restore+0x50/0x60 LR = .cpu_idle+0x138/0x220 [c0000007f2d47e20] [c0000000000177ec] .cpu_idle+0x12c/0x220 (unreliable) [c0000007f2d47ee0] [c0000000008fcfa4] .start_secondary+0x360/0x368 [c0000007f2d47f90] [c000000000009368] .start_secondary_prolog+0x10/0x14 --- arch/powerpc/platforms/powernv/pci.c | 22 ++++++++++++++-------- 1 files changed, 14 insertions(+), 8 deletions(-) diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c index a70bc1e..f92b9ef 100644 --- a/arch/powerpc/platforms/powernv/pci.c +++ b/arch/powerpc/platforms/powernv/pci.c @@ -52,32 +52,38 @@ static int pnv_msi_check_device(struct pci_dev* pdev, int nvec, int type) static unsigned int pnv_get_one_msi(struct pnv_phb *phb) { - unsigned int id; + unsigned long flags; + unsigned int id, rc; + + spin_lock_irqsave(&phb->lock, flags); - spin_lock(&phb->lock); id = find_next_zero_bit(phb->msi_map, phb->msi_count, phb->msi_next); if (id >= phb->msi_count && phb->msi_next) id = find_next_zero_bit(phb->msi_map, phb->msi_count, 0); if (id >= phb->msi_count) { - spin_unlock(&phb->lock); - return 0; + rc = 0; + goto out; } __set_bit(id, phb->msi_map); - spin_unlock(&phb->lock); - return id + phb->msi_base; + rc = id + phb->msi_base; +out: + spin_unlock_irqrestore(&phb->lock, flags); + return rc; } static void pnv_put_msi(struct pnv_phb *phb, unsigned int hwirq) { + unsigned long flags; unsigned int id; if (WARN_ON(hwirq < phb->msi_base || hwirq >= (phb->msi_base + phb->msi_count))) return; id = hwirq - phb->msi_base; - spin_lock(&phb->lock); + + spin_lock_irqsave(&phb->lock, flags); __clear_bit(id, phb->msi_map); - spin_unlock(&phb->lock); + spin_unlock_irqrestore(&phb->lock, flags); } static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) -- 1.7.5.4 _______________________________________________ Linuxppc-dev mailing list Linuxppc-dev@lists.ozlabs.org https://lists.ozlabs.org/listinfo/linuxppc-dev