Module Name: src Committed By: msaitoh Date: Thu Jun 25 07:53:02 UTC 2020
Modified Files: src/sys/dev/pci/ixgbe: if_sriov.c ixgbe.c ixgbe.h ixgbe_netbsd.c ixgbe_osdep.h ixv.c Log Message: Reduce ixgbe's busy loop using with workqueue and kpause. - Use workqueue instead of softint to make some functions sleepable. - Use new workqueue and enqueue it in ixgbe_local_timer() and ixgbe_recovery_mode_timer() to make them sleepable. - Make new ixgbe_delay() and use it. This functions sleeps if the time is more than equals 1 tick. If it's not, do delay(). To generate a diff of this commit: cvs rdiff -u -r1.6 -r1.7 src/sys/dev/pci/ixgbe/if_sriov.c cvs rdiff -u -r1.232 -r1.233 src/sys/dev/pci/ixgbe/ixgbe.c cvs rdiff -u -r1.66 -r1.67 src/sys/dev/pci/ixgbe/ixgbe.h cvs rdiff -u -r1.14 -r1.15 src/sys/dev/pci/ixgbe/ixgbe_netbsd.c cvs rdiff -u -r1.26 -r1.27 src/sys/dev/pci/ixgbe/ixgbe_osdep.h cvs rdiff -u -r1.150 -r1.151 src/sys/dev/pci/ixgbe/ixv.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/dev/pci/ixgbe/if_sriov.c diff -u src/sys/dev/pci/ixgbe/if_sriov.c:1.6 src/sys/dev/pci/ixgbe/if_sriov.c:1.7 --- src/sys/dev/pci/ixgbe/if_sriov.c:1.6 Thu Jun 27 05:55:40 2019 +++ src/sys/dev/pci/ixgbe/if_sriov.c Thu Jun 25 07:53:01 2020 @@ -645,7 +645,6 @@ ixgbe_handle_mbx(void *context, int pend hw = &adapter->hw; - IXGBE_CORE_LOCK(adapter); for (i = 0; i < adapter->num_vfs; i++) { vf = &adapter->vfs[i]; @@ -660,7 +659,6 @@ ixgbe_handle_mbx(void *context, int pend ixgbe_process_vf_ack(adapter, vf); } } - IXGBE_CORE_UNLOCK(adapter); } /* ixgbe_handle_mbx */ int Index: src/sys/dev/pci/ixgbe/ixgbe.c diff -u src/sys/dev/pci/ixgbe/ixgbe.c:1.232 src/sys/dev/pci/ixgbe/ixgbe.c:1.233 --- src/sys/dev/pci/ixgbe/ixgbe.c:1.232 Thu Jun 18 09:00:11 2020 +++ src/sys/dev/pci/ixgbe/ixgbe.c Thu Jun 25 07:53:01 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: ixgbe.c,v 1.232 2020/06/18 09:00:11 msaitoh Exp $ */ +/* $NetBSD: ixgbe.c,v 1.233 2020/06/25 07:53:01 msaitoh Exp $ */ /****************************************************************************** @@ -179,7 +179,7 @@ static void ixgbe_media_status(struct if static int ixgbe_media_change(struct ifnet *); static int ixgbe_allocate_pci_resources(struct adapter *, const struct pci_attach_args *); -static void ixgbe_free_softint(struct adapter *); +static void ixgbe_free_workqueue(struct adapter *); static void ixgbe_get_slot_info(struct adapter *); static int ixgbe_allocate_msix(struct adapter *, const struct pci_attach_args *); @@ -189,12 +189,14 @@ static int ixgbe_configure_interrupts(st static void ixgbe_free_pciintr_resources(struct adapter *); static void ixgbe_free_pci_resources(struct adapter *); static void ixgbe_local_timer(void *); -static void ixgbe_local_timer1(void *); +static void ixgbe_handle_timer(struct work *, void *); static void ixgbe_recovery_mode_timer(void *); +static void ixgbe_handle_recovery_mode_timer(struct work *, void *); static int ixgbe_setup_interface(device_t, struct adapter *); static void ixgbe_config_gpie(struct adapter *); static void ixgbe_config_dmac(struct adapter *); static void ixgbe_config_delay_values(struct adapter *); +static void ixgbe_schedule_admin_tasklet(struct adapter *); static void ixgbe_config_link(struct adapter *); static void ixgbe_check_wol_support(struct adapter *); static int ixgbe_setup_low_power_mode(struct adapter *); @@ -262,17 +264,17 @@ static int ixgbe_legacy_irq(void *); /* The MSI/MSI-X Interrupt handlers */ static int ixgbe_msix_que(void *); -static int ixgbe_msix_link(void *); +static int ixgbe_msix_admin(void *); -/* Software interrupts for deferred work */ +/* Event handlers running on workqueue */ static void ixgbe_handle_que(void *); static void ixgbe_handle_link(void *); +static void ixgbe_handle_msf(void *); static void ixgbe_handle_mod(void *); static void ixgbe_handle_phy(void *); -static void ixgbe_handle_msf(struct work *, void *); - -/* Workqueue handler for deferred work */ +/* Deferred workqueue handlers */ +static void ixgbe_handle_admin(struct work *, void *); static void ixgbe_handle_que_work(struct work *, void *); static const ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); @@ -783,6 +785,7 @@ ixgbe_attach(device_t parent, device_t d struct pci_attach_args *pa = aux; bool unsupported_sfp = false; const char *str; + char wqname[MAXCOMLEN]; char buf[256]; INIT_DEBUGOUT("ixgbe_attach: begin"); @@ -807,11 +810,20 @@ ixgbe_attach(device_t parent, device_t d aprint_normal(": %s, Version - %s\n", ixgbe_strings[ent->index], ixgbe_driver_version); - /* Core Lock Init*/ + /* Core Lock Init */ IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); - /* Set up the timer callout */ + /* Set up the timer callout and workqueue */ callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); + snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); + error = workqueue_create(&adapter->timer_wq, wqname, + ixgbe_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, + IXGBE_TASKLET_WQ_FLAGS); + if (error) { + aprint_error_dev(dev, + "could not create timer workqueue (%d)\n", error); + goto err_out; + } /* Determine hardware revision */ id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); @@ -1100,32 +1112,13 @@ ixgbe_attach(device_t parent, device_t d goto err_late; /* Tasklets for Link, SFP, Multispeed Fiber and Flow Director */ - adapter->link_si = softint_establish(SOFTINT_NET |IXGBE_SOFTINT_FLAGS, - ixgbe_handle_link, adapter); - adapter->mod_si = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, - ixgbe_handle_mod, adapter); - adapter->phy_si = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, - ixgbe_handle_phy, adapter); - if (adapter->feat_en & IXGBE_FEATURE_FDIR) - adapter->fdir_si = - softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, - ixgbe_reinit_fdir, adapter); - if ((adapter->link_si == NULL) || (adapter->mod_si == NULL) - || (adapter->phy_si == NULL) - || ((adapter->feat_en & IXGBE_FEATURE_FDIR) - && (adapter->fdir_si == NULL))) { - aprint_error_dev(dev, - "could not establish software interrupts ()\n"); - goto err_out; - } - char wqname[MAXCOMLEN]; - snprintf(wqname, sizeof(wqname), "%s-msf", device_xname(dev)); - error = workqueue_create(&adapter->msf_wq, wqname, - ixgbe_handle_msf, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, + snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); + error = workqueue_create(&adapter->admin_wq, wqname, + ixgbe_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, IXGBE_TASKLET_WQ_FLAGS); if (error) { aprint_error_dev(dev, - "could not create MSF workqueue (%d)\n", error); + "could not create admin workqueue (%d)\n", error); goto err_out; } @@ -1235,6 +1228,14 @@ ixgbe_attach(device_t parent, device_t d /* Set up the timer callout */ callout_init(&adapter->recovery_mode_timer, IXGBE_CALLOUT_FLAGS); + error = workqueue_create(&adapter->recovery_mode_timer_wq, + wqname, ixgbe_handle_recovery_mode_timer, adapter, + IXGBE_WORKQUEUE_PRI, IPL_NET, IXGBE_TASKLET_WQ_FLAGS); + if (error) { + aprint_error_dev(dev, "could not create " + "recovery_mode_timer workqueue (%d)\n", error); + goto err_out; + } /* Start the task */ callout_reset(&adapter->recovery_mode_timer, hz, @@ -1252,7 +1253,7 @@ err_out: ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); - ixgbe_free_softint(adapter); + ixgbe_free_workqueue(adapter); ixgbe_free_pci_resources(adapter); if (adapter->mta != NULL) free(adapter->mta, M_DEVBUF); @@ -1508,13 +1509,13 @@ ixgbe_is_sfp(struct ixgbe_hw *hw) } /* ixgbe_is_sfp */ static void -ixgbe_schedule_msf_tasklet(struct adapter *adapter) +ixgbe_schedule_admin_tasklet(struct adapter *adapter) { if (adapter->schedule_wqs_ok) { - if (!adapter->msf_pending) { - adapter->msf_pending = true; - workqueue_enqueue(adapter->msf_wq, - &adapter->msf_wc, NULL); + if (!adapter->admin_pending) { + atomic_or_uint(&adapter->admin_pending, 1); + workqueue_enqueue(adapter->admin_wq, + &adapter->admin_wc, NULL); } } } @@ -1527,6 +1528,7 @@ ixgbe_config_link(struct adapter *adapte { struct ixgbe_hw *hw = &adapter->hw; u32 autoneg, err = 0; + u32 task_requests = 0; bool sfp, negotiate = false; sfp = ixgbe_is_sfp(hw); @@ -1534,11 +1536,11 @@ ixgbe_config_link(struct adapter *adapte if (sfp) { if (hw->phy.multispeed_fiber) { ixgbe_enable_tx_laser(hw); - kpreempt_disable(); - ixgbe_schedule_msf_tasklet(adapter); - kpreempt_enable(); + task_requests |= IXGBE_REQUEST_TASK_MSF; } - softint_schedule(adapter->mod_si); + task_requests |= IXGBE_REQUEST_TASK_MOD; + atomic_or_32(&adapter->task_requests, task_requests); + ixgbe_schedule_admin_tasklet(adapter); } else { struct ifmedia *ifm = &adapter->media; @@ -1766,16 +1768,16 @@ ixgbe_add_hw_stats(struct adapter *adapt NULL, xname, "Watchdog timeouts"); evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, NULL, xname, "TSO errors"); - evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_INTR, - NULL, xname, "Link MSI-X IRQ Handled"); - evcnt_attach_dynamic(&adapter->link_sicount, EVCNT_TYPE_INTR, - NULL, xname, "Link softint"); - evcnt_attach_dynamic(&adapter->mod_sicount, EVCNT_TYPE_INTR, - NULL, xname, "module softint"); - evcnt_attach_dynamic(&adapter->msf_sicount, EVCNT_TYPE_INTR, - NULL, xname, "multimode softint"); - evcnt_attach_dynamic(&adapter->phy_sicount, EVCNT_TYPE_INTR, - NULL, xname, "external PHY softint"); + evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, + NULL, xname, "Admin MSI-X IRQ Handled"); + evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, + NULL, xname, "Link event"); + evcnt_attach_dynamic(&adapter->mod_workev, EVCNT_TYPE_INTR, + NULL, xname, "SFP+ module event"); + evcnt_attach_dynamic(&adapter->msf_workev, EVCNT_TYPE_INTR, + NULL, xname, "Multispeed event"); + evcnt_attach_dynamic(&adapter->phy_workev, EVCNT_TYPE_INTR, + NULL, xname, "External PHY event"); /* Max number of traffic class is 8 */ KASSERT(IXGBE_DCB_MAX_TRAFFIC_CLASS == 8); @@ -2072,11 +2074,11 @@ ixgbe_clear_evcnt(struct adapter *adapte adapter->enomem_tx_dma_setup.ev_count = 0; adapter->tso_err.ev_count = 0; adapter->watchdog_events.ev_count = 0; - adapter->link_irq.ev_count = 0; - adapter->link_sicount.ev_count = 0; - adapter->mod_sicount.ev_count = 0; - adapter->msf_sicount.ev_count = 0; - adapter->phy_sicount.ev_count = 0; + adapter->admin_irqev.ev_count = 0; + adapter->link_workev.ev_count = 0; + adapter->mod_workev.ev_count = 0; + adapter->msf_workev.ev_count = 0; + adapter->phy_workev.ev_count = 0; for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { if (i < __arraycount(stats->mpc)) { @@ -3047,17 +3049,18 @@ invalid: } /* ixgbe_media_change */ /************************************************************************ - * ixgbe_msix_link - Link status change ISR (MSI/MSI-X) + * ixgbe_msix_admin - Link status change ISR (MSI/MSI-X) ************************************************************************/ static int -ixgbe_msix_link(void *arg) +ixgbe_msix_admin(void *arg) { struct adapter *adapter = arg; struct ixgbe_hw *hw = &adapter->hw; u32 eicr, eicr_mask; + u32 task_requests = 0; s32 retval; - ++adapter->link_irq.ev_count; + ++adapter->admin_irqev.ev_count; /* Pause other interrupts */ IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER); @@ -3092,21 +3095,21 @@ ixgbe_msix_link(void *arg) || ((hw->phy.sfp_type == ixgbe_sfp_type_not_present) && (eicr & IXGBE_EICR_LSC))) { IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask); - softint_schedule(adapter->mod_si); + task_requests |= IXGBE_REQUEST_TASK_MOD; } if ((hw->mac.type == ixgbe_mac_82599EB) && (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) { IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw)); - ixgbe_schedule_msf_tasklet(adapter); + task_requests |= IXGBE_REQUEST_TASK_MSF; } } /* Link status change */ if (eicr & IXGBE_EICR_LSC) { IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); - softint_schedule(adapter->link_si); + task_requests |= IXGBE_REQUEST_TASK_LSC; } if (adapter->hw.mac.type != ixgbe_mac_82598EB) { @@ -3117,7 +3120,7 @@ ixgbe_msix_link(void *arg) return 1; /* Disable the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR); - softint_schedule(adapter->fdir_si); + task_requests |= IXGBE_REQUEST_TASK_FDIR; } if (eicr & IXGBE_EICR_ECC) { @@ -3157,8 +3160,9 @@ ixgbe_msix_link(void *arg) /* Check for VF message */ if ((adapter->feat_en & IXGBE_FEATURE_SRIOV) && - (eicr & IXGBE_EICR_MAILBOX)) - softint_schedule(adapter->mbx_si); + (eicr & IXGBE_EICR_MAILBOX)) { + task_requests |= IXGBE_REQUEST_TASK_MBX; + } } /* Check for fan failure */ @@ -3171,13 +3175,20 @@ ixgbe_msix_link(void *arg) if ((hw->phy.type == ixgbe_phy_x550em_ext_t) && (eicr & IXGBE_EICR_GPI_SDP0_X540)) { IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540); - softint_schedule(adapter->phy_si); + task_requests |= IXGBE_REQUEST_TASK_PHY; + } + + if (task_requests != 0) { + atomic_or_32(&adapter->task_requests, task_requests); + ixgbe_schedule_admin_tasklet(adapter); + /* Re-enabling other interrupts is done in the admin task */ + } else { + /* Re-enable other interrupts */ + IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); } - /* Re-enable other interrupts */ - IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); return 1; -} /* ixgbe_msix_link */ +} /* ixgbe_msix_admin */ static void ixgbe_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) @@ -3481,7 +3492,7 @@ map_err: } /* ixgbe_allocate_pci_resources */ static void -ixgbe_free_softint(struct adapter *adapter) +ixgbe_free_workqueue(struct adapter *adapter) { struct ix_queue *que = adapter->queues; struct tx_ring *txr = adapter->tx_rings; @@ -3502,36 +3513,19 @@ ixgbe_free_softint(struct adapter *adapt if (adapter->que_wq != NULL) workqueue_destroy(adapter->que_wq); - /* Drain the Link queue */ - if (adapter->link_si != NULL) { - softint_disestablish(adapter->link_si); - adapter->link_si = NULL; - } - if (adapter->mod_si != NULL) { - softint_disestablish(adapter->mod_si); - adapter->mod_si = NULL; - } - if (adapter->msf_wq != NULL) { - workqueue_destroy(adapter->msf_wq); - adapter->msf_wq = NULL; + if (adapter->admin_wq != NULL) { + workqueue_destroy(adapter->admin_wq); + adapter->admin_wq = NULL; + } + if (adapter->timer_wq != NULL) { + workqueue_destroy(adapter->timer_wq); + adapter->timer_wq = NULL; + } + if (adapter->recovery_mode_timer_wq != NULL) { + workqueue_destroy(adapter->recovery_mode_timer_wq); + adapter->recovery_mode_timer_wq = NULL; } - if (adapter->phy_si != NULL) { - softint_disestablish(adapter->phy_si); - adapter->phy_si = NULL; - } - if (adapter->feat_en & IXGBE_FEATURE_FDIR) { - if (adapter->fdir_si != NULL) { - softint_disestablish(adapter->fdir_si); - adapter->fdir_si = NULL; - } - } - if (adapter->feat_cap & IXGBE_FEATURE_SRIOV) { - if (adapter->mbx_si != NULL) { - softint_disestablish(adapter->mbx_si); - adapter->mbx_si = NULL; - } - } -} /* ixgbe_free_softint */ +} /* ixgbe_free_workqueue */ /************************************************************************ * ixgbe_detach - Device removal routine @@ -3585,7 +3579,7 @@ ixgbe_detach(device_t dev, int flags) ether_ifdetach(adapter->ifp); - ixgbe_free_softint(adapter); + ixgbe_free_workqueue(adapter); /* let hardware know driver is unloading */ ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); @@ -3619,11 +3613,11 @@ ixgbe_detach(device_t dev, int flags) evcnt_detach(&adapter->enomem_tx_dma_setup); evcnt_detach(&adapter->watchdog_events); evcnt_detach(&adapter->tso_err); - evcnt_detach(&adapter->link_irq); - evcnt_detach(&adapter->link_sicount); - evcnt_detach(&adapter->mod_sicount); - evcnt_detach(&adapter->msf_sicount); - evcnt_detach(&adapter->phy_sicount); + evcnt_detach(&adapter->admin_irqev); + evcnt_detach(&adapter->link_workev); + evcnt_detach(&adapter->mod_workev); + evcnt_detach(&adapter->msf_workev); + evcnt_detach(&adapter->phy_workev); for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { if (i < __arraycount(stats->mpc)) { @@ -3975,6 +3969,9 @@ ixgbe_init_locked(struct adapter *adapte /* Configure RX settings */ ixgbe_initialize_receive_units(adapter); + /* Initialize variable holding task enqueue requests interrupts */ + adapter->task_requests = 0; + /* Enable SDP & MSI-X interrupts based on adapter */ ixgbe_config_gpie(adapter); @@ -4066,6 +4063,7 @@ ixgbe_init_locked(struct adapter *adapte ixgbe_enable_rx_dma(hw, rxctrl); callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); + atomic_and_uint(&adapter->timer_pending, ~1); if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) callout_reset(&adapter->recovery_mode_timer, hz, ixgbe_recovery_mode_timer, adapter); @@ -4447,15 +4445,19 @@ ixgbe_local_timer(void *arg) { struct adapter *adapter = arg; - IXGBE_CORE_LOCK(adapter); - ixgbe_local_timer1(adapter); - IXGBE_CORE_UNLOCK(adapter); + if (adapter->schedule_wqs_ok) { + if (!adapter->timer_pending) { + atomic_or_uint(&adapter->timer_pending, 1); + workqueue_enqueue(adapter->timer_wq, + &adapter->timer_wc, NULL); + } + } } static void -ixgbe_local_timer1(void *arg) +ixgbe_handle_timer(struct work *wk, void *context) { - struct adapter *adapter = arg; + struct adapter *adapter = context; device_t dev = adapter->dev; struct ix_queue *que = adapter->queues; u64 queues = 0; @@ -4463,7 +4465,7 @@ ixgbe_local_timer1(void *arg) int hung = 0; int i; - KASSERT(mutex_owned(&adapter->core_mtx)); + IXGBE_CORE_LOCK(adapter); /* Check for pluggable optics */ if (adapter->sfp_probe) @@ -4547,6 +4549,8 @@ ixgbe_local_timer1(void *arg) #endif out: + atomic_and_uint(&adapter->timer_pending, ~1); + IXGBE_CORE_UNLOCK(adapter); callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); return; @@ -4555,7 +4559,8 @@ watchdog: adapter->ifp->if_flags &= ~IFF_RUNNING; adapter->watchdog_events.ev_count++; ixgbe_init_locked(adapter); -} /* ixgbe_local_timer */ + IXGBE_CORE_UNLOCK(adapter); +} /* ixgbe_handle_timer */ /************************************************************************ * ixgbe_recovery_mode_timer - Recovery mode timer routine @@ -4564,6 +4569,18 @@ static void ixgbe_recovery_mode_timer(void *arg) { struct adapter *adapter = arg; + + if (!adapter->recovery_mode_timer_pending) { + atomic_or_uint(&adapter->recovery_mode_timer_pending, 1); + workqueue_enqueue(adapter->recovery_mode_timer_wq, + &adapter->recovery_mode_timer_wc, NULL); + } +} + +static void +ixgbe_handle_recovery_mode_timer(struct work *wk, void *context) +{ + struct adapter *adapter = context; struct ixgbe_hw *hw = &adapter->hw; IXGBE_CORE_LOCK(adapter); @@ -4578,10 +4595,11 @@ ixgbe_recovery_mode_timer(void *arg) } else atomic_cas_uint(&adapter->recovery_mode, 1, 0); + atomic_and_uint(&adapter->recovery_mode_timer_pending, ~1); callout_reset(&adapter->recovery_mode_timer, hz, ixgbe_recovery_mode_timer, adapter); IXGBE_CORE_UNLOCK(adapter); -} /* ixgbe_recovery_mode_timer */ +} /* ixgbe_handle_recovery_mode_timer */ /************************************************************************ * ixgbe_sfp_probe @@ -4628,8 +4646,7 @@ ixgbe_handle_mod(void *context) device_t dev = adapter->dev; u32 err, cage_full = 0; - IXGBE_CORE_LOCK(adapter); - ++adapter->mod_sicount.ev_count; + ++adapter->mod_workev.ev_count; if (adapter->hw.need_crosstalk_fix) { switch (hw->mac.type) { case ixgbe_mac_82599EB: @@ -4679,9 +4696,23 @@ ixgbe_handle_mod(void *context) goto out; } } - ixgbe_schedule_msf_tasklet(adapter); + out: + /* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */ + adapter->phy_layer = ixgbe_get_supported_physical_layer(hw); + + /* Adjust media types shown in ifconfig */ IXGBE_CORE_UNLOCK(adapter); + ifmedia_removeall(&adapter->media); + ixgbe_add_media_types(adapter); + ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); + IXGBE_CORE_LOCK(adapter); + + atomic_or_32(&adapter->task_requests, IXGBE_REQUEST_TASK_MSF); + /* + * Don't call ixgbe_schedule_admin_tasklet() because we are on + * the workqueue now. + */ } /* ixgbe_handle_mod */ @@ -4689,43 +4720,20 @@ out: * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts ************************************************************************/ static void -ixgbe_handle_msf(struct work *wk, void *context) +ixgbe_handle_msf(void *context) { struct adapter *adapter = context; - struct ifnet *ifp = adapter->ifp; struct ixgbe_hw *hw = &adapter->hw; u32 autoneg; bool negotiate; - /* - * Hold the IFNET_LOCK across this entire call. This will - * prevent additional changes to adapter->phy_layer - * and serialize calls to this tasklet. We cannot hold the - * CORE_LOCK while calling into the ifmedia functions as - * they may block while allocating memory. - */ - IFNET_LOCK(ifp); - - IXGBE_CORE_LOCK(adapter); - adapter->msf_pending = false; - ++adapter->msf_sicount.ev_count; - /* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */ - adapter->phy_layer = ixgbe_get_supported_physical_layer(hw); + ++adapter->msf_workev.ev_count; autoneg = hw->phy.autoneg_advertised; if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); - else - negotiate = 0; if (hw->mac.ops.setup_link) hw->mac.ops.setup_link(hw, autoneg, TRUE); - IXGBE_CORE_UNLOCK(adapter); - - /* Adjust media types shown in ifconfig */ - ifmedia_removeall(&adapter->media); - ixgbe_add_media_types(adapter); - ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); - IFNET_UNLOCK(ifp); } /* ixgbe_handle_msf */ /************************************************************************ @@ -4738,7 +4746,7 @@ ixgbe_handle_phy(void *context) struct ixgbe_hw *hw = &adapter->hw; int error; - ++adapter->phy_sicount.ev_count; + ++adapter->phy_workev.ev_count; error = hw->phy.ops.handle_lasi(hw); if (error == IXGBE_ERR_OVERTEMP) device_printf(adapter->dev, @@ -4750,6 +4758,68 @@ ixgbe_handle_phy(void *context) } /* ixgbe_handle_phy */ static void +ixgbe_handle_admin(struct work *wk, void *context) +{ + struct adapter *adapter = context; + struct ifnet *ifp = adapter->ifp; + struct ixgbe_hw *hw = &adapter->hw; + u32 req; + + /* + * Hold the IFNET_LOCK across this entire call. This will + * prevent additional changes to adapter->phy_layer + * and serialize calls to this tasklet. We cannot hold the + * CORE_LOCK while calling into the ifmedia functions as + * they call ifmedia_lock() and the lock is CORE_LOCK. + */ + IFNET_LOCK(ifp); + IXGBE_CORE_LOCK(adapter); + while ((req = adapter->task_requests) != 0) { + if ((req & IXGBE_REQUEST_TASK_LSC) != 0) { + ixgbe_handle_link(adapter); + atomic_and_32(&adapter->task_requests, + ~IXGBE_REQUEST_TASK_LSC); + } + if ((req & IXGBE_REQUEST_TASK_MOD) != 0) { + ixgbe_handle_mod(adapter); + atomic_and_32(&adapter->task_requests, + ~IXGBE_REQUEST_TASK_MOD); + } + if ((req & IXGBE_REQUEST_TASK_MSF) != 0) { + ixgbe_handle_msf(adapter); + atomic_and_32(&adapter->task_requests, + ~IXGBE_REQUEST_TASK_MSF); + } + if ((req & IXGBE_REQUEST_TASK_PHY) != 0) { + ixgbe_handle_phy(adapter); + atomic_and_32(&adapter->task_requests, + ~IXGBE_REQUEST_TASK_PHY); + } + if ((req & IXGBE_REQUEST_TASK_FDIR) != 0) { + ixgbe_reinit_fdir(adapter); + atomic_and_32(&adapter->task_requests, + ~IXGBE_REQUEST_TASK_FDIR); + } +#if 0 /* notyet */ + if ((req & IXGBE_REQUEST_TASK_MBX) != 0) { + ixgbe_handle_mbx(adapter); + atomic_and_32(&adapter->task_requests, + ~IXGBE_REQUEST_TASK_MBX); + } +#endif + } + atomic_and_uint(&adapter->admin_pending, ~1); + if ((adapter->feat_en & IXGBE_FEATURE_MSIX) != 0) { + /* Re-enable other interrupts */ + IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); + } else + ixgbe_enable_intr(adapter); + + IXGBE_CORE_UNLOCK(adapter); + IFNET_UNLOCK(ifp); +} /* ixgbe_handle_admin */ + +static void ixgbe_ifstop(struct ifnet *ifp, int disable) { struct adapter *adapter = ifp->if_softc; @@ -4758,8 +4828,10 @@ ixgbe_ifstop(struct ifnet *ifp, int disa ixgbe_stop(adapter); IXGBE_CORE_UNLOCK(adapter); - workqueue_wait(adapter->msf_wq, &adapter->msf_wc); - adapter->msf_pending = false; + workqueue_wait(adapter->admin_wq, &adapter->admin_wc); + atomic_and_uint(&adapter->admin_pending, ~1); + workqueue_wait(adapter->timer_wq, &adapter->timer_wc); + atomic_and_uint(&adapter->timer_pending, ~1); } /************************************************************************ @@ -5064,7 +5136,9 @@ ixgbe_legacy_irq(void *arg) struct ifnet *ifp = adapter->ifp; struct tx_ring *txr = adapter->tx_rings; bool more = false; + bool reenable_intr = true; u32 eicr, eicr_mask; + u32 task_requests = 0; /* Silicon errata #26 on 82598 */ IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); @@ -5110,7 +5184,7 @@ ixgbe_legacy_irq(void *arg) /* Link status change */ if (eicr & IXGBE_EICR_LSC) - softint_schedule(adapter->link_si); + task_requests |= IXGBE_REQUEST_TASK_LSC; if (ixgbe_is_sfp(hw)) { /* Pluggable optics-related interrupt */ @@ -5121,26 +5195,34 @@ ixgbe_legacy_irq(void *arg) if (eicr & eicr_mask) { IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask); - softint_schedule(adapter->mod_si); + task_requests |= IXGBE_REQUEST_TASK_MOD; } if ((hw->mac.type == ixgbe_mac_82599EB) && (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) { IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw)); - ixgbe_schedule_msf_tasklet(adapter); + task_requests |= IXGBE_REQUEST_TASK_MSF; } } /* External PHY interrupt */ if ((hw->phy.type == ixgbe_phy_x550em_ext_t) && (eicr & IXGBE_EICR_GPI_SDP0_X540)) - softint_schedule(adapter->phy_si); + task_requests |= IXGBE_REQUEST_TASK_PHY; if (more) { que->req.ev_count++; ixgbe_sched_handle_que(adapter, que); - } else + reenable_intr = false; + } + if (task_requests != 0) { + atomic_or_32(&adapter->task_requests, task_requests); + ixgbe_schedule_admin_tasklet(adapter); + reenable_intr = false; + } + + if (reenable_intr == true) ixgbe_enable_intr(adapter); return 1; @@ -6741,7 +6823,7 @@ ixgbe_allocate_msix(struct adapter *adap #endif /* Set the link handler function */ adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, - adapter->osdep.intrs[vector], IPL_NET, ixgbe_msix_link, adapter, + adapter->osdep.intrs[vector], IPL_NET, ixgbe_msix_admin, adapter, intr_xname); if (adapter->osdep.ihs[vector] == NULL) { aprint_error_dev(dev, "Failed to register LINK handler\n"); @@ -6761,19 +6843,6 @@ ixgbe_allocate_msix(struct adapter *adap else aprint_normal("\n"); - if (adapter->feat_cap & IXGBE_FEATURE_SRIOV) { - adapter->mbx_si = - softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, - ixgbe_handle_mbx, adapter); - if (adapter->mbx_si == NULL) { - aprint_error_dev(dev, - "could not establish software interrupts\n"); - - error = ENXIO; - goto err_out; - } - } - kcpuset_destroy(affinity); aprint_normal_dev(dev, "Using MSI-X interrupts with %d vectors\n", vector + 1); @@ -6782,7 +6851,7 @@ ixgbe_allocate_msix(struct adapter *adap err_out: kcpuset_destroy(affinity); - ixgbe_free_softint(adapter); + ixgbe_free_workqueue(adapter); ixgbe_free_pciintr_resources(adapter); return (error); } /* ixgbe_allocate_msix */ @@ -6906,15 +6975,12 @@ ixgbe_handle_link(void *context) struct adapter *adapter = context; struct ixgbe_hw *hw = &adapter->hw; - IXGBE_CORE_LOCK(adapter); - ++adapter->link_sicount.ev_count; + ++adapter->link_workev.ev_count; ixgbe_check_link(hw, &adapter->link_speed, &adapter->link_up, 0); ixgbe_update_link_status(adapter); /* Re-enable link interrupts */ IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_LSC); - - IXGBE_CORE_UNLOCK(adapter); } /* ixgbe_handle_link */ #if 0 Index: src/sys/dev/pci/ixgbe/ixgbe.h diff -u src/sys/dev/pci/ixgbe/ixgbe.h:1.66 src/sys/dev/pci/ixgbe/ixgbe.h:1.67 --- src/sys/dev/pci/ixgbe/ixgbe.h:1.66 Tue Jun 23 05:50:01 2020 +++ src/sys/dev/pci/ixgbe/ixgbe.h Thu Jun 25 07:53:01 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: ixgbe.h,v 1.66 2020/06/23 05:50:01 msaitoh Exp $ */ +/* $NetBSD: ixgbe.h,v 1.67 2020/06/25 07:53:01 msaitoh Exp $ */ /****************************************************************************** SPDX-License-Identifier: BSD-3-Clause @@ -475,6 +475,10 @@ struct adapter { struct ifmedia media; callout_t timer; + struct workqueue *timer_wq; + struct work timer_wc; + u_int timer_pending; + u_short if_flags; /* saved ifp->if_flags */ int ec_capenable; /* saved ec->ec_capenable */ @@ -513,18 +517,15 @@ struct adapter { /* Support for pluggable optics */ bool sfp_probe; - void *link_si; /* Link tasklet */ - void *mod_si; /* SFP tasklet */ - struct workqueue *msf_wq; /* Multispeed Fiber */ - struct work msf_wc; - bool msf_pending; - void *mbx_si; /* VF -> PF mailbox interrupt */ /* Flow Director */ int fdir_reinit; - void *fdir_si; - void *phy_si; /* PHY intr tasklet */ + /* Admin task */ + struct workqueue *admin_wq; /* Link, SFP, PHY and FDIR */ + struct work admin_wc; + u_int admin_pending; + volatile u32 task_requests; bool txrx_use_workqueue; @@ -583,7 +584,10 @@ struct adapter { /* Firmware error check */ u_int recovery_mode; - struct callout recovery_mode_timer; + callout_t recovery_mode_timer; + struct workqueue *recovery_mode_timer_wq; + struct work recovery_mode_timer_wc; + u_int recovery_mode_timer_pending; /* Misc stats maintained by the driver */ struct evcnt efbig_tx_dma_setup; @@ -595,11 +599,11 @@ struct adapter { struct evcnt enomem_tx_dma_setup; struct evcnt tso_err; struct evcnt watchdog_events; - struct evcnt link_irq; - struct evcnt link_sicount; - struct evcnt mod_sicount; - struct evcnt msf_sicount; - struct evcnt phy_sicount; + struct evcnt admin_irqev; + struct evcnt link_workev; + struct evcnt mod_workev; + struct evcnt msf_workev; + struct evcnt phy_workev; union { struct ixgbe_hw_stats pf; @@ -761,9 +765,15 @@ void ixgbe_free_receive_structures(struc bool ixgbe_txeof(struct tx_ring *); bool ixgbe_rxeof(struct ix_queue *); -const struct sysctlnode *ixgbe_sysctl_instance(struct adapter *); +#define IXGBE_REQUEST_TASK_MOD 0x01 +#define IXGBE_REQUEST_TASK_MSF 0x02 +#define IXGBE_REQUEST_TASK_MBX 0x04 +#define IXGBE_REQUEST_TASK_FDIR 0x08 +#define IXGBE_REQUEST_TASK_PHY 0x10 +#define IXGBE_REQUEST_TASK_LSC 0x20 /* For NetBSD */ +const struct sysctlnode *ixgbe_sysctl_instance(struct adapter *); void ixgbe_jcl_reinit(struct adapter *, bus_dma_tag_t, struct rx_ring *, int, size_t); void ixgbe_jcl_destroy(struct adapter *, struct rx_ring *); Index: src/sys/dev/pci/ixgbe/ixgbe_netbsd.c diff -u src/sys/dev/pci/ixgbe/ixgbe_netbsd.c:1.14 src/sys/dev/pci/ixgbe/ixgbe_netbsd.c:1.15 --- src/sys/dev/pci/ixgbe/ixgbe_netbsd.c:1.14 Fri Apr 17 02:21:25 2020 +++ src/sys/dev/pci/ixgbe/ixgbe_netbsd.c Thu Jun 25 07:53:02 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: ixgbe_netbsd.c,v 1.14 2020/04/17 02:21:25 msaitoh Exp $ */ +/* $NetBSD: ixgbe_netbsd.c,v 1.15 2020/06/25 07:53:02 msaitoh Exp $ */ /* * Copyright (c) 2011 The NetBSD Foundation, Inc. * All rights reserved. @@ -305,3 +305,19 @@ atomic_load_acq_uint(volatile u_int *p) { return atomic_load_acquire(p); } + +void +ixgbe_delay(unsigned int us) +{ + + if (__predict_false(cold)) + delay(us); + else if ((us / 1000) >= hztoms(1)) { + /* + * Wait at least two clock ticks so we know the time has + * passed. + */ + kpause("ixgdly", false, mstohz(us / 1000) + 1, NULL); + } else + delay(us); +} Index: src/sys/dev/pci/ixgbe/ixgbe_osdep.h diff -u src/sys/dev/pci/ixgbe/ixgbe_osdep.h:1.26 src/sys/dev/pci/ixgbe/ixgbe_osdep.h:1.27 --- src/sys/dev/pci/ixgbe/ixgbe_osdep.h:1.26 Thu Jun 11 05:16:22 2020 +++ src/sys/dev/pci/ixgbe/ixgbe_osdep.h Thu Jun 25 07:53:02 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: ixgbe_osdep.h,v 1.26 2020/06/11 05:16:22 msaitoh Exp $ */ +/* $NetBSD: ixgbe_osdep.h,v 1.27 2020/06/25 07:53:02 msaitoh Exp $ */ /****************************************************************************** SPDX-License-Identifier: BSD-3-Clause @@ -66,9 +66,9 @@ enum { IXGBE_ERROR_CAUTION, }; -/* The happy-fun DELAY macro is defined in /usr/src/sys/i386/include/clock.h */ -#define usec_delay(x) DELAY(x) -#define msec_delay(x) DELAY(1000*(x)) +#define usec_delay(x) ixgbe_delay(x) +#define msec_delay(x) ixgbe_delay((x) * 1000) +void ixgbe_delay(unsigned int); #define DBG 0 #define MSGOUT(S, A, B) printf(S "\n", A, B) Index: src/sys/dev/pci/ixgbe/ixv.c diff -u src/sys/dev/pci/ixgbe/ixv.c:1.150 src/sys/dev/pci/ixgbe/ixv.c:1.151 --- src/sys/dev/pci/ixgbe/ixv.c:1.150 Thu Jun 18 09:00:11 2020 +++ src/sys/dev/pci/ixgbe/ixv.c Thu Jun 25 07:53:02 2020 @@ -1,4 +1,4 @@ -/*$NetBSD: ixv.c,v 1.150 2020/06/18 09:00:11 msaitoh Exp $*/ +/*$NetBSD: ixv.c,v 1.151 2020/06/25 07:53:02 msaitoh Exp $*/ /****************************************************************************** @@ -96,13 +96,15 @@ static void ixv_media_status(struct ifne static int ixv_media_change(struct ifnet *); static int ixv_allocate_pci_resources(struct adapter *, const struct pci_attach_args *); +static void ixv_free_workqueue(struct adapter *); static int ixv_allocate_msix(struct adapter *, const struct pci_attach_args *); static int ixv_configure_interrupts(struct adapter *); static void ixv_free_pci_resources(struct adapter *); static void ixv_local_timer(void *); -static void ixv_local_timer_locked(void *); +static void ixv_handle_timer(struct work *, void *); static int ixv_setup_interface(device_t, struct adapter *); +static void ixv_schedule_admin_tasklet(struct adapter *); static int ixv_negotiate_api(struct adapter *); static void ixv_initialize_transmit_units(struct adapter *); @@ -147,11 +149,11 @@ static int ixv_sysctl_tdh_handler(SYSCTL static int ixv_msix_que(void *); static int ixv_msix_mbx(void *); -/* Deferred interrupt tasklets */ +/* Event handlers running on workqueue */ static void ixv_handle_que(void *); -static void ixv_handle_link(void *); -/* Workqueue handler for deferred work */ +/* Deferred workqueue handlers */ +static void ixv_handle_admin(struct work *, void *); static void ixv_handle_que_work(struct work *, void *); const struct sysctlnode *ixv_sysctl_instance(struct adapter *); @@ -228,10 +230,12 @@ TUNABLE_INT("hw.ixv.enable_legacy_tx", & #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE +#define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE #else #define IXGBE_CALLOUT_FLAGS 0 #define IXGBE_SOFTINT_FLAGS 0 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU +#define IXGBE_TASKLET_WQ_FLAGS 0 #endif #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET @@ -307,6 +311,7 @@ ixv_attach(device_t parent, device_t dev const struct pci_attach_args *pa = aux; const char *apivstr; const char *str; + char wqname[MAXCOMLEN]; char buf[256]; INIT_DEBUGOUT("ixv_attach: begin"); @@ -354,8 +359,17 @@ ixv_attach(device_t parent, device_t dev /* SYSCTL APIs */ ixv_add_device_sysctls(adapter); - /* Set up the timer callout */ + /* Set up the timer callout and workqueue */ callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); + snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); + error = workqueue_create(&adapter->timer_wq, wqname, + ixv_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, + IXGBE_TASKLET_WQ_FLAGS); + if (error) { + aprint_error_dev(dev, + "could not create timer workqueue (%d)\n", error); + goto err_out; + } /* Save off the information about this board */ id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); @@ -569,7 +583,6 @@ ixv_detach(device_t dev, int flags) { struct adapter *adapter = device_private(dev); struct ixgbe_hw *hw = &adapter->hw; - struct ix_queue *que = adapter->queues; struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; struct ixgbevf_hw_stats *stats = &adapter->stats.vf; @@ -593,23 +606,9 @@ ixv_detach(device_t dev, int flags) } #endif - for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { - if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) - softint_disestablish(txr->txr_si); - softint_disestablish(que->que_si); - } - if (adapter->txr_wq != NULL) - workqueue_destroy(adapter->txr_wq); - if (adapter->txr_wq_enqueued != NULL) - percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); - if (adapter->que_wq != NULL) - workqueue_destroy(adapter->que_wq); - - /* Drain the Mailbox(link) queue */ - softint_disestablish(adapter->link_si); - ether_ifdetach(adapter->ifp); callout_halt(&adapter->timer, NULL); + ixv_free_workqueue(adapter); if (adapter->feat_en & IXGBE_FEATURE_NETMAP) netmap_detach(adapter->ifp); @@ -632,7 +631,8 @@ ixv_detach(device_t dev, int flags) evcnt_detach(&adapter->enomem_tx_dma_setup); evcnt_detach(&adapter->watchdog_events); evcnt_detach(&adapter->tso_err); - evcnt_detach(&adapter->link_irq); + evcnt_detach(&adapter->admin_irqev); + evcnt_detach(&adapter->link_workev); txr = adapter->tx_rings; for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { @@ -759,6 +759,9 @@ ixv_init_locked(struct adapter *adapter) /* Configure RX settings */ ixv_initialize_receive_units(adapter); + /* Initialize variable holding task enqueue requests interrupts */ + adapter->task_requests = 0; + /* Set up VLAN offload and filter */ ixv_setup_vlan_support(adapter); @@ -784,6 +787,10 @@ ixv_init_locked(struct adapter *adapter) /* Start watchdog */ callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); + atomic_and_uint(&adapter->timer_pending, ~1); + + /* OK to schedule workqueues. */ + adapter->schedule_wqs_ok = true; /* And now turn on interrupts */ ixv_enable_intr(adapter); @@ -951,14 +958,13 @@ ixv_msix_mbx(void *arg) struct adapter *adapter = arg; struct ixgbe_hw *hw = &adapter->hw; - ++adapter->link_irq.ev_count; + ++adapter->admin_irqev.ev_count; /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ /* Link status change */ hw->mac.get_link_status = TRUE; - softint_schedule(adapter->link_si); - - IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); + atomic_or_32(&adapter->task_requests, IXGBE_REQUEST_TASK_MBX); + ixv_schedule_admin_tasklet(adapter); return 1; } /* ixv_msix_mbx */ @@ -1053,6 +1059,18 @@ ixv_media_change(struct ifnet *ifp) return (0); } /* ixv_media_change */ +static void +ixv_schedule_admin_tasklet(struct adapter *adapter) +{ + if (adapter->schedule_wqs_ok) { + if (!adapter->admin_pending) { + atomic_or_uint(&adapter->admin_pending, 1); + workqueue_enqueue(adapter->admin_wq, + &adapter->admin_wc, NULL); + } + } +} + /************************************************************************ * ixv_negotiate_api * @@ -1233,15 +1251,19 @@ ixv_local_timer(void *arg) { struct adapter *adapter = arg; - IXGBE_CORE_LOCK(adapter); - ixv_local_timer_locked(adapter); - IXGBE_CORE_UNLOCK(adapter); + if (adapter->schedule_wqs_ok) { + if (!adapter->timer_pending) { + atomic_or_uint(&adapter->timer_pending, 1); + workqueue_enqueue(adapter->timer_wq, + &adapter->timer_wc, NULL); + } + } } static void -ixv_local_timer_locked(void *arg) +ixv_handle_timer(struct work *wk, void *context) { - struct adapter *adapter = arg; + struct adapter *adapter = context; device_t dev = adapter->dev; struct ix_queue *que = adapter->queues; u64 queues = 0; @@ -1249,10 +1271,11 @@ ixv_local_timer_locked(void *arg) int hung = 0; int i; - KASSERT(mutex_owned(&adapter->core_mtx)); + IXGBE_CORE_LOCK(adapter); if (ixv_check_link(adapter)) { ixv_init_locked(adapter); + IXGBE_CORE_UNLOCK(adapter); return; } @@ -1325,17 +1348,19 @@ ixv_local_timer_locked(void *arg) } #endif + atomic_and_uint(&adapter->timer_pending, ~1); + IXGBE_CORE_UNLOCK(adapter); callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); return; watchdog: - device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); adapter->ifp->if_flags &= ~IFF_RUNNING; adapter->watchdog_events.ev_count++; ixv_init_locked(adapter); -} /* ixv_local_timer */ + IXGBE_CORE_UNLOCK(adapter); +} /* ixv_handle_timer */ /************************************************************************ * ixv_update_link_status - Update OS on link state @@ -1416,6 +1441,11 @@ ixv_ifstop(struct ifnet *ifp, int disabl IXGBE_CORE_LOCK(adapter); ixv_stop(adapter); IXGBE_CORE_UNLOCK(adapter); + + workqueue_wait(adapter->admin_wq, &adapter->admin_wc); + atomic_and_uint(&adapter->admin_pending, ~1); + workqueue_wait(adapter->timer_wq, &adapter->timer_wc); + atomic_and_uint(&adapter->timer_pending, ~1); } static void @@ -1440,6 +1470,9 @@ ixv_stop(void *arg) hw->mac.ops.stop_adapter(hw); callout_stop(&adapter->timer); + /* Don't schedule workqueues. */ + adapter->schedule_wqs_ok = false; + /* reprogram the RAR[0] in case user changed it. */ hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); @@ -1500,6 +1533,39 @@ map_err: return (0); } /* ixv_allocate_pci_resources */ +static void +ixv_free_workqueue(struct adapter *adapter) +{ + struct ix_queue *que = adapter->queues; + struct tx_ring *txr = adapter->tx_rings; + int i; + + for (i = 0; i < adapter->num_queues; i++, que++, txr++) { + if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { + if (txr->txr_si != NULL) + softint_disestablish(txr->txr_si); + } + if (que->que_si != NULL) + softint_disestablish(que->que_si); + } + if (adapter->txr_wq != NULL) + workqueue_destroy(adapter->txr_wq); + if (adapter->txr_wq_enqueued != NULL) + percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); + if (adapter->que_wq != NULL) + workqueue_destroy(adapter->que_wq); + + /* Drain the Mailbox(link) queue */ + if (adapter->admin_wq != NULL) { + workqueue_destroy(adapter->admin_wq); + adapter->admin_wq = NULL; + } + if (adapter->timer_wq != NULL) { + workqueue_destroy(adapter->timer_wq); + adapter->timer_wq = NULL; + } +} /* ixv_free_workqueue */ + /************************************************************************ * ixv_free_pci_resources ************************************************************************/ @@ -2542,8 +2608,10 @@ ixv_add_stats_sysctls(struct adapter *ad NULL, xname, "Watchdog timeouts"); evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, NULL, xname, "TSO errors"); - evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_INTR, - NULL, xname, "Link MSI-X IRQ Handled"); + evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, + NULL, xname, "Admin MSI-X IRQ Handled"); + evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, + NULL, xname, "Admin event"); for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { snprintf(adapter->queues[i].evnamebuf, @@ -2712,7 +2780,8 @@ ixv_clear_evcnt(struct adapter *adapter) adapter->enomem_tx_dma_setup.ev_count = 0; adapter->watchdog_events.ev_count = 0; adapter->tso_err.ev_count = 0; - adapter->link_irq.ev_count = 0; + adapter->admin_irqev.ev_count = 0; + adapter->link_workev.ev_count = 0; for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { adapter->queues[i].irqs.ev_count = 0; @@ -2829,8 +2898,10 @@ ixv_print_debug_info(struct adapter *ada txr->me, (long)txr->no_desc_avail.ev_count); } - device_printf(dev, "MBX IRQ Handled: %lu\n", - (long)adapter->link_irq.ev_count); + device_printf(dev, "Admin IRQ Handled: %lu\n", + (long)adapter->admin_irqev.ev_count); + device_printf(dev, "Admin work Handled: %lu\n", + (long)adapter->link_workev.ev_count); } /* ixv_print_debug_info */ /************************************************************************ @@ -3317,8 +3388,16 @@ ixv_allocate_msix(struct adapter *adapte aprint_normal("\n"); /* Tasklets for Mailbox */ - adapter->link_si = softint_establish(SOFTINT_NET |IXGBE_SOFTINT_FLAGS, - ixv_handle_link, adapter); + snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); + error = workqueue_create(&adapter->admin_wq, wqname, + ixv_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, + IXGBE_TASKLET_WQ_FLAGS); + if (error) { + aprint_error_dev(dev, + "could not create admin workqueue (%d)\n", error); + goto err_out; + } + /* * Due to a broken design QEMU will fail to properly * enable the guest for MSI-X unless the vectors in @@ -3336,6 +3415,11 @@ ixv_allocate_msix(struct adapter *adapte kcpuset_destroy(affinity); return (0); +err_out: + kcpuset_destroy(affinity); + ixv_free_workqueue(adapter); + ixv_free_pci_resources(adapter); + return (error); } /* ixv_allocate_msix */ /************************************************************************ @@ -3390,23 +3474,31 @@ ixv_configure_interrupts(struct adapter /************************************************************************ - * ixv_handle_link - Tasklet handler for MSI-X MBX interrupts + * ixv_handle_admin - Tasklet handler for MSI-X MBX interrupts * * Done outside of interrupt context since the driver might sleep ************************************************************************/ static void -ixv_handle_link(void *context) +ixv_handle_admin(struct work *wk, void *context) { struct adapter *adapter = context; + struct ixgbe_hw *hw = &adapter->hw; IXGBE_CORE_LOCK(adapter); + ++adapter->link_workev.ev_count; adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, &adapter->link_up, FALSE); ixv_update_link_status(adapter); + adapter->task_requests = 0; + atomic_and_uint(&adapter->admin_pending, ~1); + + /* Re-enable interrupts */ + IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); + IXGBE_CORE_UNLOCK(adapter); -} /* ixv_handle_link */ +} /* ixv_handle_admin */ /************************************************************************ * ixv_check_link - Used in the local timer to poll for link changes