/******************************************************************************* Intel 10 Gigabit PCI Express Linux driver Copyright(c) 1999 - 2011 Intel Corporation. This program is free software; you can redistribute it and/or modify it under the terms and conditions of the GNU General Public License, version 2, as published by the Free Software Foundation. This program is distributed in the hope it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in the file called "COPYING". Contact Information: e1000-devel Mailing List Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 *******************************************************************************/ /****************************************************************************** Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code ******************************************************************************/ #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_SCTP #include #endif #include #include #ifdef NETIF_F_TSO #include #ifdef NETIF_F_TSO6 #include #endif #endif #ifdef SIOCETHTOOL #include #endif #include "ixgbe.h" #if defined(CONFIG_IXGBE_VMDQ_NETDEV) || defined(__VMKLNX__) #include "ixgbe_vmdq.h" #endif #include "ixgbe_cna.h" #include "ixgbe_dcb_82599.h" #include "ixgbe_sriov.h" char ixgbe_driver_name[] = "ixgbe"; static const char ixgbe_driver_string[] = "Intel(R) 10 Gigabit PCI Express Network Driver"; #define DRV_HW_PERF #ifndef CONFIG_IXGBE_NAPI #define DRIVERNAPI #else #define DRIVERNAPI "-NAPI" #endif #define FPGA #define DRIVERIOV "iov" #define VMDQ_TAG #define MAJ 3 #define MIN 7 #define BUILD 13.7.14 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \ __stringify(BUILD) DRIVERIOV DRIVERNAPI \ DRV_HW_PERF FPGA VMDQ_TAG const char ixgbe_driver_version[] = DRV_VERSION; static const char ixgbe_copyright[] = "Copyright (c) 1999-2011 Intel Corporation."; /* ixgbe_pci_tbl - PCI Device ID Table * * Wildcard entries (PCI_ANY_ID) should come last * Last entry must be all 0s * * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, * Class, Class Mask, private data (not used) } */ static struct pci_device_id ixgbe_pci_tbl[] = { {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS)}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP)}, /* required last entry */ {0, } }; MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl); #ifdef __VMKLNX__ extern int vmklnx_configure_net_vf(struct pci_dev *pf_dev, void *cfg_data, int vf_idx); #endif /* __VMKLNX__ */ #if defined(CONFIG_DCA) || defined(CONFIG_DCA_MODULE) static int ixgbe_notify_dca(struct notifier_block *, unsigned long event, void *p); static struct notifier_block dca_notifier = { .notifier_call = ixgbe_notify_dca, .next = NULL, .priority = 0 }; #endif MODULE_AUTHOR("Intel Corporation, "); MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); MODULE_LICENSE("GPL"); MODULE_VERSION(DRV_VERSION); static VMK_ReturnStatus ixgbe_passthru_ops(struct net_device *netdev, vmk_NetPTOP op, void *pargs); #ifdef __VMKLNX__ #define pci_enable_sriov(dev,vfs) \ (vmklnx_enable_vfs((dev), (vfs), vmklnx_store_pt_ops, \ ixgbe_passthru_ops) != (vfs) ? -ENOTSUPP : 0) #define pci_disable_sriov(dev) \ vmklnx_disable_vfs((dev), adapter->num_vfs, NULL, NULL) #endif /* __VMKLNX__ */ #define DEFAULT_DEBUG_LEVEL_SHIFT 3 static inline void ixgbe_disable_sriov(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 gcr; u32 gpie; u32 vmdctl; /* set num VFs to 0 to prevent access to vfinfo */ adapter->num_vfs = 0; if (adapter->vfinfo) { kfree(adapter->vfinfo); adapter->vfinfo = NULL; } if (adapter->mv_list) { kfree(adapter->mv_list); adapter->mv_list = NULL; } /* disable iov and allow time for transactions to clear */ pci_disable_sriov(adapter->pdev); /* turn off device IOV mode */ gcr = IXGBE_READ_REG(hw, IXGBE_GCR_EXT); gcr &= ~(IXGBE_GCR_EXT_SRIOV); IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr); gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); gpie &= ~IXGBE_GPIE_VTMODE_MASK; IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); /* set default pool back to 0 */ vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); vmdctl &= ~IXGBE_VT_CTL_POOL_MASK; IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl); IXGBE_WRITE_FLUSH(hw); /* take a breather then clean up driver data */ msleep(100); adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED; } static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter) { if (!test_bit(__IXGBE_DOWN, &adapter->state) && !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) schedule_work(&adapter->service_task); } static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter) { BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state)); /* flush memory to make sure state is correct before next watchog */ smp_mb__before_clear_bit(); clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state); } static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter) { u32 ctrl_ext; /* Let firmware take over control of h/w */ ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD); } static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter) { u32 ctrl_ext; /* Let firmware know the driver has taken over */ ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD); } /* * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors * @adapter: pointer to adapter struct * @direction: 0 for Rx, 1 for Tx, -1 for other causes * @queue: queue to map the corresponding interrupt to * @msix_vector: the vector to map to the corresponding queue * */ static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction, u8 queue, u8 msix_vector) { u32 ivar, index; struct ixgbe_hw *hw = &adapter->hw; switch (hw->mac.type) { case ixgbe_mac_82598EB: msix_vector |= IXGBE_IVAR_ALLOC_VAL; if (direction == -1) direction = 0; index = (((direction * 64) + queue) >> 2) & 0x1F; ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); ivar &= ~(0xFF << (8 * (queue & 0x3))); ivar |= (msix_vector << (8 * (queue & 0x3))); IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: if (direction == -1) { /* other causes */ msix_vector |= IXGBE_IVAR_ALLOC_VAL; index = ((queue & 1) * 8); ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC); ivar &= ~(0xFF << index); ivar |= (msix_vector << index); IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar); break; } else { /* tx or rx causes */ msix_vector |= IXGBE_IVAR_ALLOC_VAL; index = ((16 * (queue & 1)) + (8 * direction)); ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1)); ivar &= ~(0xFF << index); ivar |= (msix_vector << index); IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar); break; } default: break; } } static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter, u64 qmask) { u32 mask; switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: mask = (IXGBE_EIMS_RTX_QUEUE & qmask); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: mask = (qmask & 0xFFFFFFFF); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); mask = (qmask >> 32); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); break; default: break; } } void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *ring, struct ixgbe_tx_buffer *tx_buffer) { if (tx_buffer->skb) { dev_kfree_skb_any(tx_buffer->skb); if (tx_buffer->length) dma_unmap_single(ring->dev, tx_buffer->dma, tx_buffer->length, DMA_TO_DEVICE); } else if (tx_buffer->length) { dma_unmap_page(ring->dev, tx_buffer->dma, tx_buffer->length, DMA_TO_DEVICE); } tx_buffer->next_to_watch = NULL; tx_buffer->skb = NULL; tx_buffer->length = 0; /* tx_buffer_info must be completely set up in the transmit path */ } static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; struct ixgbe_hw_stats *hwstats = &adapter->stats; u32 data = 0; u32 xoff[8] = {0}; int i; if ((hw->fc.current_mode == ixgbe_fc_full) || (hw->fc.current_mode == ixgbe_fc_rx_pause)) { switch (hw->mac.type) { case ixgbe_mac_82598EB: data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); break; default: data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); } hwstats->lxoffrxc += data; /* refill credits (no tx hang) if we received xoff */ if (!data) return; for (i = 0; i < adapter->num_tx_queues; i++) clear_bit(__IXGBE_HANG_CHECK_ARMED, &adapter->tx_ring[i]->state); return; } else if (!(adapter->dcb_cfg.pfc_mode_enable)) { return; } /* update stats for each tc, only valid with PFC enabled */ for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) { switch (hw->mac.type) { case ixgbe_mac_82598EB: xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); break; default: xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i)); } hwstats->pxoffrxc[i] += xoff[i]; } /* disarm tx queues that have received xoff frames */ for (i = 0; i < adapter->num_tx_queues; i++) { struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; u8 tc = tx_ring->dcb_tc; if ((tc <= 7) && (xoff[tc])) clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); } } static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring) { return ring->tx_stats.completed; } static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring) { struct ixgbe_adapter *adapter = ring->q_vector->adapter; struct ixgbe_hw *hw = &adapter->hw; u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx)); u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx)); return ((head <= tail) ? tail : tail + ring->count) - head; } static bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring) { u32 tx_done = ixgbe_get_tx_completed(tx_ring); u32 tx_done_old = tx_ring->tx_stats.tx_done_old; u32 tx_pending = ixgbe_get_tx_pending(tx_ring); bool ret = false; clear_check_for_tx_hang(tx_ring); /* * Check for a hung queue, but be thorough. This verifies * that a transmit has been completed since the previous * check AND there is at least one packet pending. The * ARMED bit is set to indicate a potential hang. The * bit is cleared if a pause frame is received to remove * false hang detection due to PFC or 802.3x frames. By * requiring this to fail twice we avoid races with * PFC clearing the ARMED bit and conditions where we * run the check_tx_hang logic with a transmit completion * pending but without time to complete it yet. */ if ((tx_done_old == tx_done) && tx_pending) { /* make sure it is true for two checks in a row */ ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); } else { /* update completed stats and continue */ tx_ring->tx_stats.tx_done_old = tx_done; /* reset the countdown */ clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); } return ret; } /** * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout * @adapter: driver private struct **/ static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter) { /* Do the reset outside of interrupt context */ if (!test_bit(__IXGBE_DOWN, &adapter->state)) { adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED; ixgbe_service_event_schedule(adapter); } } /** * ixgbe_tx_timeout - Respond to a Tx Hang * @netdev: network interface device structure **/ static void ixgbe_tx_timeout(struct net_device *netdev) { struct ixgbe_adapter *adapter = netdev_priv(netdev); bool real_tx_hang = false; int i; #define TX_TIMEO_LIMIT 16000 for (i = 0; i < adapter->num_tx_queues; i++) { struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) real_tx_hang = true; } if (real_tx_hang){ ixgbe_tx_timeout_reset(adapter); } else { e_info(drv, "Fake Tx hang detected with timeout of %d " "seconds\n", netdev->watchdog_timeo/HZ); /* fake Tx hang - increase the kernel timeout */ if (netdev->watchdog_timeo < TX_TIMEO_LIMIT) netdev->watchdog_timeo *= 2; } } /** * ixgbe_clean_tx_irq - Reclaim resources after transmit completes * @q_vector: structure containing interrupt and ring information * @tx_ring: tx ring to clean **/ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *tx_ring) { struct ixgbe_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc, *eop_desc; struct ixgbe_adapter *adapter = q_vector->adapter; unsigned int total_bytes = 0, total_packets = 0, completed = 0; unsigned int budget = q_vector->tx.work_limit; unsigned int i = tx_ring->next_to_clean; if (test_bit(__IXGBE_DOWN, &adapter->state)) return true; #ifdef __VMKLNX__ if (test_and_set_bit(__IXGBE_RING_CLEAN_BUSY, &tx_ring->state)) return true; #endif tx_buffer = &tx_ring->tx_buffer_info[i]; tx_desc = IXGBE_TX_DESC(tx_ring, i); i -= tx_ring->count; for (; budget; budget--) { eop_desc = tx_buffer->next_to_watch; /* if next_to_watch is not set then there is no work pending */ if (!eop_desc) break; /* prevent any other reads prior to eop_desc */ rmb(); /* if DD is not set pending work has not been completed */ if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) break; /* clear next_to_watch to prevent false hangs */ tx_buffer->next_to_watch = NULL; /* count the packet as being completed */ completed++; /* update the statistics for this packet */ total_bytes += tx_buffer->bytecount; total_packets += tx_buffer->gso_segs; /* free the skb */ dev_kfree_skb_any(tx_buffer->skb); tx_buffer->skb = NULL; /* unmap skb header data */ dma_unmap_single(tx_ring->dev, tx_buffer->dma, tx_buffer->length, DMA_TO_DEVICE); tx_buffer->length = 0; /* clear last DMA location and unmap remaining buffers */ while (tx_desc != eop_desc) { tx_buffer++; tx_desc++; i++; if (unlikely(!i)) { i -= tx_ring->count; tx_buffer = tx_ring->tx_buffer_info; tx_desc = IXGBE_TX_DESC(tx_ring, 0); } /* unmap any remaining paged data */ if (tx_buffer->length) { dma_unmap_page(tx_ring->dev, tx_buffer->dma, tx_buffer->length, DMA_TO_DEVICE); tx_buffer->length = 0; } } /* move us one more past the eop_desc for start of next pkt */ tx_buffer++; tx_desc++; i++; if (unlikely(!i)) { i -= tx_ring->count; tx_buffer = tx_ring->tx_buffer_info; tx_desc = IXGBE_TX_DESC(tx_ring, 0); } } i += tx_ring->count; tx_ring->next_to_clean = i; tx_ring->tx_stats.completed += completed; tx_ring->stats.bytes += total_bytes; tx_ring->stats.packets += total_packets; q_vector->tx.total_bytes += total_bytes; q_vector->tx.total_packets += total_packets; #ifdef __VMKLNX__ clear_bit(__IXGBE_RING_CLEAN_BUSY, &tx_ring->state); #endif if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) { /* schedule immediate reset if we believe we hung */ struct ixgbe_hw *hw = &adapter->hw; u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)); u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)); e_err(drv, "Detected Tx Unit Hang\n" " Tx Queue <%d>\n" " TDH, TDT <%x>, <%x>\n" " next_to_use <%x>\n" " next_to_clean <%x>\n", tx_ring->queue_index, head, tail, tx_ring->next_to_use, i); e_err(drv, "tx_buffer_info[next_to_clean]\n" " time_stamp <%lx>\n" " jiffies <%lx>\n", tx_ring->tx_buffer_info[i].time_stamp, jiffies); netif_stop_subqueue(netdev_ring(tx_ring), ring_queue_index(tx_ring)); e_info(probe, "tx hang %d detected on queue %d, resetting adapter\n", adapter->tx_timeout_count + 1, tx_ring->queue_index); ixgbe_tx_timeout_reset(adapter); /* the adapter is about to reset, no point in enabling stuff */ return true; } #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) if (unlikely(total_packets && netif_carrier_ok(netdev_ring(tx_ring)) && (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { /* Make sure that anybody stopping the queue after this * sees the new next_to_clean. */ smp_mb(); #ifdef HAVE_TX_MQ if (__netif_subqueue_stopped(netdev_ring(tx_ring), ring_queue_index(tx_ring)) && !test_bit(__IXGBE_DOWN, &q_vector->adapter->state)) { netif_wake_subqueue(netdev_ring(tx_ring), ring_queue_index(tx_ring)); ++tx_ring->tx_stats.restart_queue; } #else if (netif_queue_stopped(netdev_ring(tx_ring)) && !test_bit(__IXGBE_DOWN, &q_vector->adapter->state)) { netif_wake_queue(netdev_ring(tx_ring)); ++tx_ring->tx_stats.restart_queue; } #endif } return !!budget; } static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring, int cpu) { struct ixgbe_hw *hw = &adapter->hw; u32 rxctrl; u8 reg_idx = rx_ring->reg_idx; rxctrl = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(reg_idx)); switch (hw->mac.type) { case ixgbe_mac_82598EB: rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK; rxctrl |= dca3_get_tag(rx_ring->dev, cpu); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK_82599; rxctrl |= (dca3_get_tag(rx_ring->dev, cpu) << IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599); break; default: break; } rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN; rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN; /* just do the header data when in Packet Split mode */ if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED_DATA) || ring_is_ps_enabled(rx_ring)) rxctrl &= ~IXGBE_DCA_RXCTRL_DATA_DCA_EN; else rxctrl |= IXGBE_DCA_RXCTRL_DATA_DCA_EN; rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_RRO_EN); rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_WRO_EN | IXGBE_DCA_RXCTRL_DESC_HSRO_EN); IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl); } static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring, int cpu) { struct ixgbe_hw *hw = &adapter->hw; u32 txctrl; u8 reg_idx = tx_ring->reg_idx; switch (hw->mac.type) { case ixgbe_mac_82598EB: txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(reg_idx)); txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK; txctrl |= dca3_get_tag(tx_ring->dev, cpu); txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN; txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(reg_idx), txctrl); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx)); txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK_82599; txctrl |= (dca3_get_tag(tx_ring->dev, cpu) << IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599); txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN; txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx), txctrl); break; default: break; } } static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector) { struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_ring *ring; int cpu = get_cpu(); if (q_vector->cpu == cpu) goto out_no_update; for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next) ixgbe_update_tx_dca(adapter, ring, cpu); for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next) ixgbe_update_rx_dca(adapter, ring, cpu); q_vector->cpu = cpu; out_no_update: put_cpu(); } static void ixgbe_setup_dca(struct ixgbe_adapter *adapter) { int num_q_vectors; int i; if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED)) return; /* always use CB2 mode, difference is masked in the CB driver */ IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2); if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; else num_q_vectors = 1; for (i = 0; i < num_q_vectors; i++) { adapter->q_vector[i]->cpu = -1; ixgbe_update_dca(adapter->q_vector[i]); } } #if defined(CONFIG_DCA) || defined(CONFIG_DCA_MODULE) static int __ixgbe_notify_dca(struct device *dev, void *data) { struct ixgbe_adapter *adapter = dev_get_drvdata(dev); unsigned long event = *(unsigned long *)data; if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE)) return 0; switch (event) { case DCA_PROVIDER_ADD: /* if we're already enabled, don't do it again */ if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) break; if (dca_add_requester(dev) == 0) { adapter->flags |= IXGBE_FLAG_DCA_ENABLED; ixgbe_setup_dca(adapter); break; } /* Fall Through since DCA is disabled. */ case DCA_PROVIDER_REMOVE: if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { dca_remove_requester(dev); adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED; IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1); } break; } return 0; } #endif /* CONFIG_DCA or CONFIG_DCA_MODULE */ #ifdef NETIF_F_RXHASH static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb) { if (netdev_ring(ring)->features & NETIF_F_RXHASH) skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss); } #endif /* NETIF_F_RXHASH */ static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val) { rx_ring->next_to_use = val; /* * Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only * applicable for weak-ordered memory model archs, * such as IA-64). */ wmb(); writel(val, rx_ring->tail); } static bool ixgbe_alloc_mapped_skb(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *bi) { struct sk_buff *skb = bi->skb; dma_addr_t dma = bi->dma; if (dma) return true; if (likely(!skb)) { skb = netdev_alloc_skb_ip_align(netdev_ring(rx_ring), rx_ring->rx_buf_len); bi->skb = skb; if (!skb) { rx_ring->rx_stats.alloc_rx_buff_failed++; return false; } /* * Temporary Workaround for ESX60 issue where cb is corrupted * with FCoE. Causing PSOD HSD 4707647. Need to remove after * issue is fully root-caused and fixed. */ if (IXGBE_CB(skb)->delay_unmap) IXGBE_CB(skb)->delay_unmap = 0; /* initialize skb for ring */ skb_record_rx_queue(skb, ring_queue_index(rx_ring)); } dma = dma_map_single(rx_ring->dev, skb->data, rx_ring->rx_buf_len, DMA_FROM_DEVICE); if (dma_mapping_error(rx_ring->dev, dma)) { rx_ring->rx_stats.alloc_rx_buff_failed++; return false; } bi->dma = dma; return true; } static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *bi) { struct page *page = bi->page; dma_addr_t page_dma = bi->page_dma; #ifndef __VMKLNX__ unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2); #endif if (page_dma) return true; if (!page) { page = netdev_alloc_page(netdev_ring(rx_ring)); bi->page = page; if (unlikely(!page)) { rx_ring->rx_stats.alloc_rx_page_failed++; return false; } } page_dma = dma_map_page(rx_ring->dev, page, #ifndef __VMKLNX__ page_offset, PAGE_SIZE / 2, #else 0, PAGE_SIZE, #endif DMA_FROM_DEVICE); if (dma_mapping_error(rx_ring->dev, page_dma)) { rx_ring->rx_stats.alloc_rx_page_failed++; return false; } bi->page_dma = page_dma; #ifndef __VMKLNX__ bi->page_offset = page_offset; #else bi->page_offset = 0; #endif return true; } /** * ixgbe_alloc_rx_buffers_ps - Replace used receive buffers; packet split * @rx_ring: ring to place buffers on * @cleaned_count: number of buffers to replace **/ static void ixgbe_alloc_rx_buffers_ps(struct ixgbe_ring *rx_ring, u16 cleaned_count) { union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *bi; u16 i = rx_ring->next_to_use; rx_desc = IXGBE_RX_DESC(rx_ring, i); bi = &rx_ring->rx_buffer_info[i]; i -= rx_ring->count; while (cleaned_count--) { if (!ixgbe_alloc_mapped_skb(rx_ring, bi)) break; if (!ring_is_ps_enabled(rx_ring)) { /* Refresh the desc even if buffer_addrs didn't change * because each write-back erases this info. */ rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); } else { rx_desc->read.hdr_addr = cpu_to_le64(bi->dma); if (!ixgbe_alloc_mapped_page(rx_ring, bi)) break; rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); } rx_desc++; bi++; i++; if (unlikely(!i)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); bi = rx_ring->rx_buffer_info; i -= rx_ring->count; } /* clear the hdr_addr for the next_to_use descriptor */ rx_desc->read.hdr_addr = 0; } i += rx_ring->count; if (rx_ring->next_to_use != i) ixgbe_release_rx_desc(rx_ring, i); } static bool ixgbe_alloc_mapped_bb(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *bi) { struct page *page = bi->page; dma_addr_t page_dma; if (bi->dma) return true; /* alloc new page for storage */ if (!page) { page = netdev_alloc_page(netdev_ring(rx_ring)); if (unlikely(!page)) { rx_ring->rx_stats.alloc_rx_page_failed++; return false; } bi->page = page; } /* Map first half of page for use */ if (!bi->page_dma) { page_dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE / 2, DMA_FROM_DEVICE); if (dma_mapping_error(rx_ring->dev, page_dma)) { rx_ring->rx_stats.alloc_rx_page_failed++; return false; } bi->page_dma = page_dma; } /* Map secnd half of page, store xor result with first half */ page_dma = dma_map_page(rx_ring->dev, page, PAGE_SIZE / 2, PAGE_SIZE / 2, DMA_FROM_DEVICE); if (dma_mapping_error(rx_ring->dev, page_dma)) { rx_ring->rx_stats.alloc_rx_page_failed++; return false; } bi->dma = bi->page_dma ^ page_dma; bi->page_offset = 0; return true; } #ifndef __VMKLNX__ /** * ixgbe_alloc_rx_buffers_bb - Replace used receive buffers; bounce buffers * @rx_ring: ring to place buffers on * @cleaned_count: number of buffers to replace **/ static void ixgbe_alloc_rx_buffers_bb(struct ixgbe_ring *rx_ring, u16 cleaned_count) { union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *bi; u16 i = rx_ring->next_to_use; rx_desc = IXGBE_RX_DESC(rx_ring, i); bi = &rx_ring->rx_buffer_info[i]; i -= rx_ring->count; while (cleaned_count--) { if (!ixgbe_alloc_mapped_bb(rx_ring, bi)) break; rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); rx_desc++; bi++; i++; if (unlikely(!i)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); bi = rx_ring->rx_buffer_info; i -= rx_ring->count; } /* clear the hdr_addr for the next_to_use descriptor */ rx_desc->read.hdr_addr = 0; } i += rx_ring->count; if (rx_ring->next_to_use != i) ixgbe_release_rx_desc(rx_ring, i); } #endif void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) { /* nothing to do or no valid netdev defined */ if (!cleaned_count || !netdev_ring(rx_ring)) return; #ifndef __VMKLNX__ if (!rx_ring->rx_buf_len) ixgbe_alloc_rx_buffers_bb(rx_ring, cleaned_count); else #endif ixgbe_alloc_rx_buffers_ps(rx_ring, cleaned_count); } static inline u16 ixgbe_get_hlen(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc) { __le16 hdr_info = rx_desc->wb.lower.lo_dword.hs_rss.hdr_info; u16 hlen = le16_to_cpu(hdr_info) & IXGBE_RXDADV_HDRBUFLEN_MASK; /* if we are not packet split we just need to return upper length */ if (!ring_is_ps_enabled(rx_ring)) return le16_to_cpu(rx_desc->wb.upper.length); if (hdr_info & cpu_to_le16(IXGBE_RXDADV_SPH)) rx_ring->rx_stats.rx_hdr_split++; /* * HW will not DMA in data larger than the given buffer, even if it * parses the (NFS, of course) header to be larger. In that case, it * fills the header buffer and spills the rest into the page. */ if (hlen > (IXGBE_RX_HDR_SIZE << IXGBE_RXDADV_HDRBUFLEN_SHIFT)) hlen = IXGBE_RX_HDR_SIZE; else hlen >>= IXGBE_RXDADV_HDRBUFLEN_SHIFT; return hlen; } /** * ixgbe_merge_active_tail - merge active tail into lro skb * @tail: pointer to active tail in frag_list * * This function merges the length and data of an active tail into the * skb containing the frag_list. It resets the tail's pointer to the head, * but it leaves the heads pointer to tail intact. **/ static inline struct sk_buff *ixgbe_merge_active_tail(struct sk_buff *tail) { struct sk_buff *head = IXGBE_CB(tail)->head; if (!head) return tail; head->len += tail->len; head->data_len += tail->len; head->truesize += tail->len; IXGBE_CB(tail)->head = NULL; return head; } /** * ixgbe_add_active_tail - adds an active tail into the skb frag_list * @head: pointer to the start of the skb * @tail: pointer to active tail to add to frag_list * * This function adds an active tail to the end of the frag list. This tail * will still be receiving data so we cannot yet ad it's stats to the main * skb. That is done via ixgbe_merge_active_tail. **/ static inline void ixgbe_add_active_tail(struct sk_buff *head, struct sk_buff *tail) { struct sk_buff *old_tail = IXGBE_CB(head)->tail; if (old_tail) { ixgbe_merge_active_tail(old_tail); old_tail->next = tail; } else { skb_shinfo(head)->frag_list = tail; } IXGBE_CB(tail)->head = head; IXGBE_CB(head)->tail = tail; IXGBE_CB(head)->append_cnt++; } /** * ixgbe_close_active_frag_list - cleanup pointers on a frag_list skb * @head: pointer to head of an active frag list * * This function will clear the frag_tail_tracker pointer on an active * frag_list and returns true if the pointer was actually set **/ static inline bool ixgbe_close_active_frag_list(struct sk_buff *head) { struct sk_buff *tail = IXGBE_CB(head)->tail; if (!tail) return false; ixgbe_merge_active_tail(tail); IXGBE_CB(head)->tail = NULL; return true; } #ifdef __VMKLNX__ static void ixgbe_set_rsc_gso_size(struct net_device *netdev, struct sk_buff *skb) { u32 ethhdr_sz = eth_header_len((struct ethhdr *)skb->data); if (netdev->mtu < (skb->len - ethhdr_sz)) { struct iphdr *iph; struct tcphdr *tcph; iph = (struct iphdr *)(skb->data + ethhdr_sz); tcph = (struct tcphdr *)(skb->data + ethhdr_sz + (iph->ihl << 2)); skb_shinfo(skb)->gso_size = skb->len - (ethhdr_sz + (iph->ihl << 2) + (tcph->doff << 2)); skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; } } #endif #ifdef IXGBE_FCOE /** * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type * @adapter: address of board private structure * @rx_desc: advanced rx descriptor * * Returns : true if it is FCoE pkt */ static inline bool ixgbe_rx_is_fcoe(struct ixgbe_adapter *adapter, union ixgbe_adv_rx_desc *rx_desc) { __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; return (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) && ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE << IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); } #endif /* IXGBE_FCOE */ #ifdef HAVE_VLAN_RX_REGISTER /** * ixgbe_receive_skb - Send a completed packet up the stack * @q_vector: structure containing interrupt and ring information * @skb: packet to send up **/ static void ixgbe_receive_skb(struct ixgbe_q_vector *q_vector, struct sk_buff *skb) { struct ixgbe_adapter *adapter = q_vector->adapter; u16 vlan_tag = IXGBE_CB(skb)->vid; #ifdef NETIF_F_HW_VLAN_TX if (vlan_tag & VLAN_VID_MASK) { /* by placing vlgrp at start of structure we can alias it */ struct vlan_group **vlgrp = netdev_priv(skb->dev); #ifndef __VMKLNX__ if (!*vlgrp) #else if (!*vlgrp && !(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) #endif /* __VMKLNX__ */ dev_kfree_skb_any(skb); #ifdef CONFIG_IXGBE_NAPI else if (adapter->flags & IXGBE_FLAG_IN_NETPOLL) vlan_hwaccel_rx(skb, *vlgrp, vlan_tag); else vlan_gro_receive(&q_vector->napi, *vlgrp, vlan_tag, skb); #else else if (vlan_hwaccel_rx(skb, *vlgrp, vlan_tag) == NET_RX_DROP) adapter->rx_dropped_backlog++; #endif } else { #endif /* NETIF_F_HW_VLAN_TX */ #ifdef CONFIG_IXGBE_NAPI if (adapter->flags & IXGBE_FLAG_IN_NETPOLL) netif_rx(skb); else napi_gro_receive(&q_vector->napi, skb); #else if (netif_rx(skb) == NET_RX_DROP) adapter->rx_dropped_backlog++; #endif #ifdef NETIF_F_HW_VLAN_TX } #endif /* NETIF_F_HW_VLAN_TX */ } #endif /* HAVE_VLAN_RX_REGISTER */ /** * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum * @ring: structure containing ring specific data * @rx_desc: current Rx descriptor being processed * @skb: skb currently being received and modified **/ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb) { skb_checksum_none_assert(skb); /* Rx csum disabled */ #ifdef HAVE_NDO_SET_FEATURES if (!(netdev_ring(ring)->features & NETIF_F_RXCSUM)) #else if (!test_bit(__IXGBE_RX_CSUM_ENABLED, &ring->state)) #endif return; /* if IP and error */ if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { ring->rx_stats.csum_err++; return; } if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) return; if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; /* * 82599 errata, UDP frames with a 0 checksum can be marked as * checksum errors. */ if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) && test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) return; ring->rx_stats.csum_err++; return; } /* It must be a TCP or UDP packet with a valid checksum */ skb->ip_summed = CHECKSUM_UNNECESSARY; } static inline bool ixgbe_get_rsc_state(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc) { return ring_is_rsc_enabled(rx_ring) && (rx_desc->wb.lower.lo_dword.data & cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK)); } /** * ixgbe_cleanup_headers - Correct corrupted or empty headers * @skb: pointer to current skb being fixed * * Check for corrupted packet headers caused by senders on the local L2 * embedded NIC switch not setting up their Tx Descriptors right. These * should be very rare. * * Also address the case where we are pulling data in on pages only * and as such no data is present in the skb header. * * In addition if skb is not at least 60 bytes we need to pad it so that * it is large enough to qualify as a valid Ethernet frame. * * Returns true if an error was encountered and/or skb was freed. **/ static bool ixgbe_cleanup_headers(struct sk_buff *skb) { struct skb_frag_struct *frag; unsigned char *va; int pull_len = (skb->len <= 64) ? skb->len : ETH_HLEN; #ifndef CONFIG_IXGBE_NAPI /* we need to disable IRQs to use kmap_atomic in non-napi mode */ unsigned long flags; #endif /* * pskb_may_pull is not available in irq context, but we need * the header to contain the greater of either ETH_HLEN or 60 * bytes if the skb->len is less than 60 for skb_pad. */ if (likely(pull_len <= skb_headlen(skb))) return false; pull_len -= skb_headlen(skb); frag = &skb_shinfo(skb)->frags[0]; #ifndef CONFIG_IXGBE_NAPI local_irq_save(flags); #endif va = kmap_atomic(skb_frag_page(frag), KM_SKB_DATA_SOFTIRQ) + frag->page_offset; memcpy(skb_tail_pointer(skb), va, pull_len); kunmap_atomic(va, KM_SKB_DATA_SOFTIRQ); #ifndef CONFIG_IXGBE_NAPI local_irq_restore(flags); #endif /* update all of the pointers */ skb_frag_size_sub(frag, pull_len); frag->page_offset += pull_len; skb->data_len -= pull_len; skb->tail += pull_len; /* * if we sucked the frag empty then we should free it, * if there are other frags here something is screwed up in hardware */ if (skb_frag_size(frag) == 0) { BUG_ON(skb_shinfo(skb)->nr_frags != 1); skb_shinfo(skb)->nr_frags = 0; put_page(skb_frag_page(frag)); } /* if skb_pad returns an error the skb was freed */ if (skb->len < 60) { int pad_len = 60 - skb->len; if (skb_pad(skb, pad_len)) return true; __skb_put(skb, pad_len); } return false; } #ifndef __VMKLNX__ /** * ixgbe_clean_rx_irq_bb - Clean completed descriptors from Rx ring * @q_vector: structure containing interrupt and ring information * @rx_ring: rx descriptor ring to transact packets on * @budget: Total limit on number of packets to process * * This function provides a "bounce buffer" approach to Rx interrupt * processing. The advantage to this is that on systems that have * expensive overhead for IOMMU access this provides a means of avoiding * it by maintaining the mapping of the page to the syste. * * Returns true if all work is completed without reaching budget **/ static bool ixgbe_clean_rx_irq_bb(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *rx_ring, int budget) { union ixgbe_adv_rx_desc *rx_desc; unsigned int total_rx_bytes = 0, total_rx_packets = 0; const int current_node = numa_node_id(); #ifdef IXGBE_FCOE struct ixgbe_adapter *adapter = q_vector->adapter; int ddp_bytes = 0; #endif /* IXGBE_FCOE */ u16 ntc = rx_ring->next_to_clean; u16 cleaned_count = ixgbe_desc_unused(rx_ring); rx_desc = IXGBE_RX_DESC(rx_ring, ntc); while (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) { union ixgbe_adv_rx_desc *next_rxd; struct ixgbe_rx_buffer *rx_buffer_info; struct sk_buff *skb; bool pkt_is_rsc; rx_buffer_info = &rx_ring->rx_buffer_info[ntc]; skb = rx_buffer_info->skb; rx_buffer_info->skb = NULL; /* * This memory barrier is needed to keep us from reading * any other fields out of the rx_desc until we know the * RXD_STAT_DD bit is set */ rmb(); /* prefetch first cache line from page */ prefetch(page_address(rx_buffer_info->page) + rx_buffer_info->page_offset); /* allocate a skb to store the frags */ if (!skb) { skb = netdev_alloc_skb_ip_align(netdev_ring(rx_ring), IXGBE_RX_HDR_SIZE); if (!skb) { rx_ring->rx_stats.alloc_rx_buff_failed++; break; } /* * Delay unmapping of the first packet. It carries the * header information, HW may still access the header * after the writeback. Only unmap it when EOP is * reached */ IXGBE_CB(skb)->dma = rx_buffer_info->page_dma; /* initialize skb for ring */ skb_record_rx_queue(skb, ring_queue_index(rx_ring)); /* * we will be copying header into skb->data in * pskb_may_pull so it is in our interest to prefetch * it now to avoid a possible cache miss */ prefetchw(skb->data); } skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer_info->page, rx_buffer_info->page_offset, le16_to_cpu(rx_desc->wb.upper.length)); /* * page_dma contains the xor combined mappings for the 2 * halves of the page. In order to transition between the * bottom and top halves we need to xor page_dma by the * value stored in dma, and page_offset by PAGE_SIZE / 2. */ if ((page_count(rx_buffer_info->page) == 1) && (page_to_nid(rx_buffer_info->page) == current_node)) { if (rx_buffer_info->page_dma != IXGBE_CB(skb)->dma) dma_sync_single_for_cpu(rx_ring->dev, rx_buffer_info->page_dma, PAGE_SIZE / 2, DMA_FROM_DEVICE); rx_buffer_info->page_dma ^= rx_buffer_info->dma; dma_sync_single_for_device(rx_ring->dev, rx_buffer_info->page_dma, PAGE_SIZE / 2, DMA_FROM_DEVICE); rx_buffer_info->page_offset ^= PAGE_SIZE / 2; get_page(rx_buffer_info->page); } else { if (rx_buffer_info->page_dma != IXGBE_CB(skb)->dma) dma_unmap_page(rx_ring->dev, rx_buffer_info->page_dma, PAGE_SIZE / 2, DMA_FROM_DEVICE); else IXGBE_CB(skb)->delay_unmap = true; rx_buffer_info->page_dma ^= rx_buffer_info->dma; rx_buffer_info->dma = 0; dma_unmap_page(rx_ring->dev, rx_buffer_info->page_dma, PAGE_SIZE / 2, DMA_FROM_DEVICE); rx_buffer_info->page_dma = 0; rx_buffer_info->page = NULL; } ntc++; if (ntc == rx_ring->count) ntc = 0; next_rxd = IXGBE_RX_DESC(rx_ring, ntc); prefetch(next_rxd); pkt_is_rsc = ixgbe_get_rsc_state(rx_ring, rx_desc); if (unlikely(!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) { struct ixgbe_rx_buffer *next_buffer; if (pkt_is_rsc) { u32 nextp = le32_to_cpu(rx_desc->wb.upper.status_error); nextp &= IXGBE_RXDADV_NEXTP_MASK; nextp >>= IXGBE_RXDADV_NEXTP_SHIFT; next_buffer = &rx_ring->rx_buffer_info[nextp]; } else { next_buffer = &rx_ring->rx_buffer_info[ntc]; } next_buffer->skb = skb; rx_ring->rx_stats.non_eop_descs++; goto next_desc; } /* cleanup delayed unmapping of first buffer */ if (IXGBE_CB(skb)->delay_unmap) { dma_unmap_page(rx_ring->dev, IXGBE_CB(skb)->dma, PAGE_SIZE / 2, DMA_FROM_DEVICE); } else { dma_sync_single_for_cpu(rx_ring->dev, IXGBE_CB(skb)->dma, PAGE_SIZE / 2, DMA_FROM_DEVICE); } IXGBE_CB(skb)->delay_unmap = false; IXGBE_CB(skb)->dma = 0; if (pkt_is_rsc) { rx_ring->rx_stats.rsc_count += skb_shinfo(skb)->nr_frags; rx_ring->rx_stats.rsc_flush++; } /* verify the header layout is correct */ if (ixgbe_cleanup_headers(skb)) goto next_desc; /* ERR_MASK will only have valid bits if EOP set */ if (unlikely(ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { dev_kfree_skb_any(skb); goto next_desc; } #ifdef NETIF_F_RXHASH ixgbe_rx_hash(rx_ring, rx_desc, skb); #endif /* NETIF_F_RXHASH */ ixgbe_rx_checksum(rx_ring, rx_desc, skb); if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); #ifdef HAVE_VLAN_RX_REGISTER IXGBE_CB(skb)->vid = vid; } else { IXGBE_CB(skb)->vid = 0; #else __vlan_hwaccel_put_tag(skb, vid); #endif } /* probably a little skewed due to removing CRC */ total_rx_bytes += skb->len; total_rx_packets++; skb->protocol = eth_type_trans(skb, netdev_ring(rx_ring)); #ifdef IXGBE_FCOE /* if ddp, not passing to ULD unless for FCP_RSP or error */ if (ixgbe_rx_is_fcoe(adapter, rx_desc)) { ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb); if (!ddp_bytes) { dev_kfree_skb_any(skb); #ifndef NETIF_F_GRO netdev_ring(rx_ring)->last_rx = jiffies; #endif goto next_desc; } } #endif /* IXGBE_FCOE */ #ifdef HAVE_VLAN_RX_REGISTER ixgbe_receive_skb(q_vector, skb); #else #ifdef CONFIG_IXGBE_NAPI napi_gro_receive(&q_vector->napi, skb); #else if (netif_rx(skb) == NET_RX_DROP) q_vector->adapter->rx_dropped_backlog++; #endif #endif #ifndef NETIF_F_GRO netdev_ring(rx_ring)->last_rx = jiffies; #endif budget--; next_desc: if (!budget) break; cleaned_count++; /* return some buffers to hardware, one at a time is too slow */ if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { ixgbe_alloc_rx_buffers_bb(rx_ring, cleaned_count); cleaned_count = 0; } /* use prefetched values */ rx_desc = IXGBE_RX_DESC(rx_ring, ntc); } #ifdef IXGBE_FCOE /* include DDPed FCoE data */ if (ddp_bytes > 0) { unsigned int mss; mss = netdev_ring(rx_ring)->mtu - sizeof(struct fcoe_hdr) - sizeof(struct fc_frame_header) - sizeof(struct fcoe_crc_eof); if (mss > 512) mss &= ~511; total_rx_bytes += ddp_bytes; total_rx_packets += DIV_ROUND_UP(ddp_bytes, mss); } #endif /* IXGBE_FCOE */ rx_ring->next_to_clean = ntc; rx_ring->stats.packets += total_rx_packets; rx_ring->stats.bytes += total_rx_bytes; q_vector->rx.total_packets += total_rx_packets; q_vector->rx.total_bytes += total_rx_bytes; if (cleaned_count) ixgbe_alloc_rx_buffers_bb(rx_ring, cleaned_count); return !!budget; } #endif /* __VMKLNX__ */ /** * ixgbe_clean_rx_irq_ps - Clean completed descriptors from Rx ring * @q_vector: structure containing interrupt and ring information * @rx_ring: rx descriptor ring to transact packets on * @budget: Total limit on number of packets to process * * This function provides a "packet split" approach to Rx interrupt * handling. This version will perform better on systems with a low cost * dma mapping API. * * Returns true if all work is completed without reaching budget **/ static bool ixgbe_clean_rx_irq_ps(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *rx_ring, int budget) { union ixgbe_adv_rx_desc *rx_desc; unsigned int total_rx_bytes = 0, total_rx_packets = 0; const int current_node = numa_node_id(); #ifdef IXGBE_FCOE struct ixgbe_adapter *adapter = q_vector->adapter; int ddp_bytes = 0; #endif /* IXGBE_FCOE */ u32 len = 0; u16 i = rx_ring->next_to_clean; u16 cleaned_count = ixgbe_desc_unused(rx_ring); u16 hdr_info; rx_desc = IXGBE_RX_DESC(rx_ring, i); while (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) { union ixgbe_adv_rx_desc *next_rxd; struct ixgbe_rx_buffer *rx_buffer_info; struct sk_buff *skb; bool pkt_is_rsc; rx_buffer_info = &rx_ring->rx_buffer_info[i]; skb = rx_buffer_info->skb; rx_buffer_info->skb = NULL; /* * This memory barrier is needed to keep us from reading * any other fields out of the rx_desc until we know the * RXD_STAT_DD bit is set */ rmb(); pkt_is_rsc = ixgbe_get_rsc_state(rx_ring, rx_desc); prefetch(skb->data); len = ixgbe_get_hlen(rx_ring, rx_desc); #ifdef __VMKLNX__ /* small packet padding for queue-to-queue loopback */ if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_LB) && len < ETH_ZLEN) { memset(skb->data+len, 0, (ETH_ZLEN-len)); len = ETH_ZLEN; } #endif /* pull the header of the skb in if no data is already present */ if (!skb_is_nonlinear(skb)) { __skb_put(skb, len); /* * When HWRSC is enabled, delay unmapping of the first * packet. It carries the header information, HW may * still access the header after the writeback. Only * unmap it when EOP is reached */ if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP) && pkt_is_rsc && !IXGBE_CB(skb)->head) { IXGBE_CB(skb)->delay_unmap = true; IXGBE_CB(skb)->dma = rx_buffer_info->dma; } else { skb = ixgbe_merge_active_tail(skb); dma_unmap_single(rx_ring->dev, rx_buffer_info->dma, rx_ring->rx_buf_len, DMA_FROM_DEVICE); } rx_buffer_info->dma = 0; } if (ring_is_ps_enabled(rx_ring) && rx_desc->wb.upper.length) { skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer_info->page, rx_buffer_info->page_offset, le16_to_cpu(rx_desc->wb.upper.length)); #ifndef __VMKLNX__ if ((page_count(rx_buffer_info->page) == 1) && (page_to_nid(rx_buffer_info->page) == current_node)) get_page(rx_buffer_info->page); else #endif rx_buffer_info->page = NULL; dma_unmap_page(rx_ring->dev, rx_buffer_info->page_dma, #ifndef __VMKLNX__ PAGE_SIZE / 2, #else PAGE_SIZE, #endif DMA_FROM_DEVICE); rx_buffer_info->page_dma = 0; } i++; if (i == rx_ring->count) i = 0; next_rxd = IXGBE_RX_DESC(rx_ring, i); prefetch(next_rxd); if (unlikely(!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) { struct ixgbe_rx_buffer *next_buffer; if (pkt_is_rsc) { u32 nextp = le32_to_cpu(rx_desc->wb.upper.status_error); nextp &= IXGBE_RXDADV_NEXTP_MASK; nextp >>= IXGBE_RXDADV_NEXTP_SHIFT; next_buffer = &rx_ring->rx_buffer_info[nextp]; } else { next_buffer = &rx_ring->rx_buffer_info[i]; } if (ring_is_ps_enabled(rx_ring)) { rx_buffer_info->skb = next_buffer->skb; rx_buffer_info->dma = next_buffer->dma; next_buffer->skb = skb; next_buffer->dma = 0; } else { ixgbe_add_active_tail(skb, next_buffer->skb); IXGBE_CB(next_buffer->skb)->head = skb; } rx_ring->rx_stats.non_eop_descs++; goto next_desc; } if (ixgbe_close_active_frag_list(skb) && !pkt_is_rsc) { /* if we got here without RSC the packet is invalid */ dev_kfree_skb_any(skb); goto next_desc; } if (pkt_is_rsc) { if (IXGBE_CB(skb)->delay_unmap) { dma_unmap_single(rx_ring->dev, IXGBE_CB(skb)->dma, rx_ring->rx_buf_len, DMA_FROM_DEVICE); IXGBE_CB(skb)->dma = 0; IXGBE_CB(skb)->delay_unmap = false; } #ifdef __VMKLNX__ ixgbe_set_rsc_gso_size(netdev_ring(rx_ring), skb); #endif /* __VMKLNX__ */ if (ring_is_ps_enabled(rx_ring)) rx_ring->rx_stats.rsc_count += skb_shinfo(skb)->nr_frags; else rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt + 1; rx_ring->rx_stats.rsc_flush++; IXGBE_CB(skb)->append_cnt = 0; } /* verify the header layout is correct */ if (ixgbe_cleanup_headers(skb)) goto next_desc; /* ERR_MASK will only have valid bits if EOP set */ if (unlikely(ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { dev_kfree_skb_any(skb); goto next_desc; } #ifdef NETIF_F_RXHASH ixgbe_rx_hash(rx_ring, rx_desc, skb); #endif /* NETIF_F_RXHASH */ ixgbe_rx_checksum(rx_ring, rx_desc, skb); if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); #ifdef HAVE_VLAN_RX_REGISTER IXGBE_CB(skb)->vid = vid; } else { IXGBE_CB(skb)->vid = 0; #else __vlan_hwaccel_put_tag(skb, vid); #endif } /* probably a little skewed due to removing CRC */ total_rx_bytes += skb->len; total_rx_packets++; skb->protocol = eth_type_trans(skb, netdev_ring(rx_ring)); #ifdef IXGBE_FCOE /* if ddp, not passing to ULD unless for FCP_RSP or error */ if (ixgbe_rx_is_fcoe(adapter, rx_desc)) { ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb); if (!ddp_bytes) { dev_kfree_skb_any(skb); #ifndef NETIF_F_GRO netdev_ring(rx_ring)->last_rx = jiffies; #endif goto next_desc; } } #endif /* IXGBE_FCOE */ #ifdef __VMKLNX__ if ((rx_desc->wb.lower.lo_dword.hs_rss.pkt_info & IXGBE_RXDADV_RSSTYPE_MASK) != 0) { u8 q_id = ring_queue_index(rx_ring); if (q_id >= (adapter->num_rx_pools - 1)) { /* * Although hardware gives us the rss hash for all pkts * let us just set it in the skb for pkts that * came through the RSS queues for now. * * This way the higher layers can easily figure out the * pkt was from an RSS queue. We can set the rss hash * for all packets once we identify an extensible way * to let the higher layers know which of the queues * are rss queues and which ones are not. */ rss_skb_put_info(skb, rx_desc->wb.lower.hi_dword.rss, (vmklnx_rss_type)(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info & IXGBE_RXDADV_RSSTYPE_MASK)); } } #endif #ifdef HAVE_VLAN_RX_REGISTER ixgbe_receive_skb(q_vector, skb); #else #ifdef CONFIG_IXGBE_NAPI napi_gro_receive(&q_vector->napi, skb); #else if (netif_rx(skb) == NET_RX_DROP) q_vector->adapter->rx_dropped_backlog++; #endif #endif #ifndef NETIF_F_GRO netdev_ring(rx_ring)->last_rx = jiffies; #endif budget--; next_desc: if (!budget) break; cleaned_count++; /* return some buffers to hardware, one at a time is too slow */ if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { ixgbe_alloc_rx_buffers_ps(rx_ring, cleaned_count); cleaned_count = 0; } /* use prefetched values */ rx_desc = IXGBE_RX_DESC(rx_ring, i); } #ifdef IXGBE_FCOE /* include DDPed FCoE data */ if (ddp_bytes > 0) { unsigned int mss; mss = netdev_ring(rx_ring)->mtu - sizeof(struct fcoe_hdr) - sizeof(struct fc_frame_header) - sizeof(struct fcoe_crc_eof); if (mss > 512) mss &= ~511; total_rx_bytes += ddp_bytes; total_rx_packets += DIV_ROUND_UP(ddp_bytes, mss); } #endif /* IXGBE_FCOE */ rx_ring->next_to_clean = i; rx_ring->stats.packets += total_rx_packets; rx_ring->stats.bytes += total_rx_bytes; q_vector->rx.total_packets += total_rx_packets; q_vector->rx.total_bytes += total_rx_bytes; if (cleaned_count) ixgbe_alloc_rx_buffers_ps(rx_ring, cleaned_count); return !!budget; } #ifndef CONFIG_IXGBE_NAPI static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *rx_ring) { unsigned int budget = q_vector->rx.work_limit; #else static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *rx_ring, int budget) { #endif #ifndef __VMKLNX__ if (!rx_ring->rx_buf_len) return ixgbe_clean_rx_irq_bb(q_vector, rx_ring, budget); #endif return ixgbe_clean_rx_irq_ps(q_vector, rx_ring, budget); } /** * ixgbe_configure_msix - Configure MSI-X hardware * @adapter: board private structure * * ixgbe_configure_msix sets up the hardware to properly generate MSI-X * interrupts. **/ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) { struct ixgbe_q_vector *q_vector; int q_vectors, v_idx; u32 mask; q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; /* Populate MSIX to EITR Select */ if (adapter->num_vfs > 32) { u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1; IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel); } /* * Populate the IVAR table and set the ITR values to the * corresponding register. */ for (v_idx = 0; v_idx < q_vectors; v_idx++) { struct ixgbe_ring *ring; q_vector = adapter->q_vector[v_idx]; for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next) ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx); for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next) ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx); if (q_vector->tx.ring && !q_vector->rx.ring) { /* tx only vector */ if (adapter->tx_itr_setting == 1) q_vector->itr = IXGBE_10K_ITR; else q_vector->itr = adapter->tx_itr_setting; } else { /* rx or rx/tx vector */ if (adapter->rx_itr_setting == 1) q_vector->itr = IXGBE_20K_ITR; else q_vector->itr = adapter->rx_itr_setting; } ixgbe_write_eitr(q_vector); } switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX, v_idx); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: ixgbe_set_ivar(adapter, -1, 1, v_idx); break; default: break; } IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950); #ifdef IXGBE_TCP_TIMER ixgbe_set_ivar(adapter, -1, 0, ++v_idx); #endif /* IXGBE_TCP_TIMER */ /* set up to autoclear timer, and the vectors */ mask = IXGBE_EIMS_ENABLE_MASK; mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_MAILBOX | IXGBE_EIMS_LSC); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask); } enum latency_range { lowest_latency = 0, low_latency = 1, bulk_latency = 2, latency_invalid = 255 }; /** * ixgbe_update_itr - update the dynamic ITR value based on statistics * @q_vector: structure containing interrupt and ring information * @ring_container: structure containing ring performance data * * Stores a new ITR value based on packets and byte * counts during the last interrupt. The advantage of per interrupt * computation is faster updates and more accurate ITR for the current * traffic pattern. Constants in this function were computed * based on theoretical maximum wire speed and thresholds were set based * on testing data as well as attempting to minimize response time * while increasing bulk throughput. * this functionality is controlled by the InterruptThrottleRate module * parameter (see ixgbe_param.c) **/ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, struct ixgbe_ring_container *ring_container) { int bytes = ring_container->total_bytes; int packets = ring_container->total_packets; u32 timepassed_us; u64 bytes_perint; u8 itr_setting = ring_container->itr; if (packets == 0) return; /* simple throttlerate management * 0-10MB/s lowest (100000 ints/s) * 10-20MB/s low (20000 ints/s) * 20-1249MB/s bulk (8000 ints/s) */ /* what was last interrupt timeslice? */ timepassed_us = q_vector->itr >> 2; bytes_perint = bytes / timepassed_us; /* bytes/usec */ switch (itr_setting) { case lowest_latency: if (bytes_perint > 10) { itr_setting = low_latency; } break; case low_latency: if (bytes_perint > 20) { itr_setting = bulk_latency; } else if (bytes_perint <= 10) { itr_setting = lowest_latency; } break; case bulk_latency: if (bytes_perint <= 20) { itr_setting = low_latency; } break; } /* clear work counters since we have the values we need */ ring_container->total_bytes = 0; ring_container->total_packets = 0; /* write updated itr to ring container */ ring_container->itr = itr_setting; } /** * ixgbe_write_eitr - write EITR register in hardware specific way * @q_vector: structure containing interrupt and ring information * * This function is made to be called by ethtool and by the driver * when it needs to update EITR registers at runtime. Hardware * specific quirks/differences are taken care of here. */ void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector) { struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_hw *hw = &adapter->hw; int v_idx = q_vector->v_idx; u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: /* must write high and low 16 bits to reset counter */ itr_reg |= (itr_reg << 16); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: /* * set the WDIS bit to not clear the timer bits and cause an * immediate assertion of the interrupt */ itr_reg |= IXGBE_EITR_CNT_WDIS; break; default: break; } IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg); } static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector) { u32 new_itr = q_vector->itr; u8 current_itr; ixgbe_update_itr(q_vector, &q_vector->tx); ixgbe_update_itr(q_vector, &q_vector->rx); current_itr = max(q_vector->rx.itr, q_vector->tx.itr); switch (current_itr) { /* counts and packets in update_itr are dependent on these numbers */ case lowest_latency: new_itr = IXGBE_100K_ITR; break; case low_latency: new_itr = IXGBE_20K_ITR; break; case bulk_latency: new_itr = IXGBE_8K_ITR; break; default: break; } if (new_itr != q_vector->itr) { /* do an exponential smoothing */ new_itr = (10 * new_itr * q_vector->itr) / ((9 * new_itr) + q_vector->itr); /* save the algorithm value here */ q_vector->itr = new_itr; ixgbe_write_eitr(q_vector); } } /** * ixgbe_check_overtemp_subtask - check for over temperature * @adapter: pointer to adapter **/ static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 eicr = adapter->interrupt_event; if (test_bit(__IXGBE_DOWN, &adapter->state)) return; if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) && !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT)) return; adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT; switch (hw->device_id) { case IXGBE_DEV_ID_82599_T3_LOM: /* * Since the warning interrupt is for both ports * we don't have to check if: * - This interrupt wasn't for our port. * - We may have missed the interrupt so always have to * check if we got a LSC */ if (!(eicr & IXGBE_EICR_GPI_SDP0) && !(eicr & IXGBE_EICR_LSC)) return; if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) { u32 autoneg; bool link_up = false; hw->mac.ops.check_link(hw, &autoneg, &link_up, false); if (link_up) return; } /* Check if this is not due to overtemp */ if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP) return; break; default: if (!(eicr & IXGBE_EICR_GPI_SDP0)) return; break; } e_crit(drv, "Network adapter has been stopped because it has over heated. " "Restart the computer. If the problem persists, " "power off the system and replace the adapter\n"); adapter->interrupt_event = 0; } static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr) { struct ixgbe_hw *hw = &adapter->hw; if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && (eicr & IXGBE_EICR_GPI_SDP1)) { e_crit(probe, "Fan has stopped, replace the adapter\n"); /* write to clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); } } static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr) { if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)) return; switch (adapter->hw.mac.type) { case ixgbe_mac_82599EB: /* * Need to check link state so complete overtemp check * on service task */ if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) && (!test_bit(__IXGBE_DOWN, &adapter->state))) { adapter->interrupt_event = eicr; adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT; ixgbe_service_event_schedule(adapter); return; } return; case ixgbe_mac_X540: if (!(eicr & IXGBE_EICR_TS)) return; break; default: return; } e_crit(drv, "Network adapter has been stopped because it has over heated. " "Restart the computer. If the problem persists, " "power off the system and replace the adapter\n"); } static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr) { struct ixgbe_hw *hw = &adapter->hw; if (eicr & IXGBE_EICR_GPI_SDP2) { /* Clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); if (!test_bit(__IXGBE_DOWN, &adapter->state)) { adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; ixgbe_service_event_schedule(adapter); } } if (eicr & IXGBE_EICR_GPI_SDP1) { /* Clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); if (!test_bit(__IXGBE_DOWN, &adapter->state)) { adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; ixgbe_service_event_schedule(adapter); } } } static void ixgbe_check_lsc(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; adapter->lsc_int++; adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; adapter->link_check_timeout = jiffies; if (!test_bit(__IXGBE_DOWN, &adapter->state)) { IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); IXGBE_WRITE_FLUSH(hw); ixgbe_service_event_schedule(adapter); } } #ifdef IXGBE_TCP_TIMER static irqreturn_t ixgbe_msix_pba(int irq, void *data) { struct ixgbe_adapter *adapter = data; int i; u32 pba = readl(adapter->msix_addr + IXGBE_MSIXPBA); for (i = 0; i < MAX_MSIX_COUNT; i++) { if (pba & (1 << i)) adapter->msix_handlers[i](irq, data, regs); else adapter->pba_zero[i]++; } adapter->msix_pba++; return IRQ_HANDLED; } static irqreturn_t ixgbe_msix_tcp_timer(int irq, void *data) { struct ixgbe_adapter *adapter = data; adapter->msix_tcp_timer++; return IRQ_HANDLED; } #endif /* IXGBE_TCP_TIMER */ void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter, u64 qmask) { u32 mask; struct ixgbe_hw *hw = &adapter->hw; switch (hw->mac.type) { case ixgbe_mac_82598EB: mask = (IXGBE_EIMS_RTX_QUEUE & qmask); IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: mask = (qmask & 0xFFFFFFFF); if (mask) IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); mask = (qmask >> 32); if (mask) IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); break; default: break; } /* skip the flush */ } void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter, u64 qmask) { u32 mask; struct ixgbe_hw *hw = &adapter->hw; switch (hw->mac.type) { case ixgbe_mac_82598EB: mask = (IXGBE_EIMS_RTX_QUEUE & qmask); IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: mask = (qmask & 0xFFFFFFFF); if (mask) IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); mask = (qmask >> 32); if (mask) IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); break; default: break; } /* skip the flush */ } /** * ixgbe_irq_enable - Enable default interrupt generation settings * @adapter: board private structure **/ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, bool flush) { u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); /* don't reenable LSC while waiting for link */ if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE) mask &= ~IXGBE_EIMS_LSC; if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) switch (adapter->hw.mac.type) { case ixgbe_mac_82599EB: mask |= IXGBE_EIMS_GPI_SDP0; break; case ixgbe_mac_X540: mask |= IXGBE_EIMS_TS; break; default: break; } if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) mask |= IXGBE_EIMS_GPI_SDP1; switch (adapter->hw.mac.type) { case ixgbe_mac_82599EB: mask |= IXGBE_EIMS_GPI_SDP1; mask |= IXGBE_EIMS_GPI_SDP2; case ixgbe_mac_X540: mask |= IXGBE_EIMS_ECC; mask |= IXGBE_EIMS_MAILBOX; break; default: break; } if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) && !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) mask |= IXGBE_EIMS_FLOW_DIR; IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask); if (queues) ixgbe_irq_enable_queues(adapter, ~0); if (flush) IXGBE_WRITE_FLUSH(&adapter->hw); } static irqreturn_t ixgbe_msix_other(int irq, void *data) { struct ixgbe_adapter *adapter = data; struct ixgbe_hw *hw = &adapter->hw; u32 eicr; /* * Workaround for Silicon errata #26 on 82598. Use clear-by-write * instead of clear-by-read. Reading with EICS will return the * interrupt causes without clearing, which later be done * with the write to EICR. */ eicr = IXGBE_READ_REG(hw, IXGBE_EICS); IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); if (eicr & IXGBE_EICR_LSC) ixgbe_check_lsc(adapter); if (eicr & IXGBE_EICR_MAILBOX) ixgbe_msg_task(adapter); switch (hw->mac.type) { case ixgbe_mac_82599EB: case ixgbe_mac_X540: if (eicr & IXGBE_EICR_ECC) { e_info(link, "Received unrecoverable ECC Err, please " "reboot\n"); IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); } #ifdef HAVE_TX_MQ /* Handle Flow Director Full threshold interrupt */ if (eicr & IXGBE_EICR_FLOW_DIR) { int reinit_count = 0; int i; for (i = 0; i < adapter->num_tx_queues; i++) { struct ixgbe_ring *ring = adapter->tx_ring[i]; if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state)) reinit_count++; } if (reinit_count) { /* no more flow director interrupts until after init */ IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR); adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT; ixgbe_service_event_schedule(adapter); } } #endif ixgbe_check_sfp_event(adapter, eicr); ixgbe_check_overtemp_event(adapter, eicr); break; default: break; } ixgbe_check_fan_failure(adapter, eicr); /* re-enable the original interrupt state, no lsc, no queues */ if (!test_bit(__IXGBE_DOWN, &adapter->state)) ixgbe_irq_enable(adapter, false, false); return IRQ_HANDLED; } #ifndef CONFIG_IXGBE_NAPI static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data) { struct ixgbe_q_vector *q_vector = data; struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_ring *ring; bool clean_complete = true; if (!q_vector->tx.ring && !q_vector->rx.ring) return IRQ_HANDLED; #ifdef __VMKLNX__ /* EIAM not used in ESX */ ixgbe_irq_disable_queues(q_vector->adapter, ((u64)1 << q_vector->v_idx)); #endif if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) ixgbe_update_dca(q_vector); for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next) clean_complete &= ixgbe_clean_tx_irq(q_vector, ring); for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next) clean_complete &= ixgbe_clean_rx_irq(q_vector, ring); if (adapter->rx_itr_setting == 1) ixgbe_set_itr(q_vector); if (!test_bit(__IXGBE_DOWN, &adapter->state)) { u64 eics = ((u64)1 << q_vector->v_idx); ixgbe_irq_enable_queues(adapter, eics); if (!clean_complete) ixgbe_irq_rearm_queues(adapter, eics); } return IRQ_HANDLED; } #else /* CONFIG_IXGBE_NAPI */ static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data) { struct ixgbe_q_vector *q_vector = data; #ifdef __VMKLNX__ /* EIAM not used in ESX */ ixgbe_irq_disable_queues(q_vector->adapter, ((u64)1 << q_vector->v_idx)); #else /* EIAM disabled interrupts (on this vector) for us */ #endif if (q_vector->rx.ring || q_vector->tx.ring) napi_schedule(&q_vector->napi); return IRQ_HANDLED; } /** * ixgbe_poll - NAPI polling RX/TX cleanup routine * @napi: napi struct with our devices info in it * @budget: amount of work driver is allowed to do this pass, in packets * * This function will clean all queues associated with a q_vector. **/ static int ixgbe_poll(struct napi_struct *napi, int budget) { struct ixgbe_q_vector *q_vector = container_of(napi, struct ixgbe_q_vector, napi); struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_ring *ring; int per_ring_budget; bool clean_complete = true; if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) ixgbe_update_dca(q_vector); for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next) clean_complete &= ixgbe_clean_tx_irq(q_vector, ring); /* attempt to distribute budget to each queue fairly, but don't allow * the budget to go below 1 because we'll exit polling */ if (q_vector->rx.count > 1) per_ring_budget = max(budget/q_vector->rx.count, 1); else per_ring_budget = budget; for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next) clean_complete &= ixgbe_clean_rx_irq(q_vector, ring, per_ring_budget); #ifndef HAVE_NETDEV_NAPI_LIST if (!netif_running(adapter->netdev)) clean_complete = true; #endif /* If all work not completed, return budget and keep polling */ if (!clean_complete) return budget; /* all work done, exit the polling mode */ napi_complete(napi); if (adapter->rx_itr_setting == 1) ixgbe_set_itr(q_vector); if (!test_bit(__IXGBE_DOWN, &adapter->state)) ixgbe_irq_enable_queues(adapter, ((u64)1 << q_vector->v_idx)); return 0; } #endif /* CONFIG_IXGBE_NAPI */ static inline void map_vector_to_rxq(struct ixgbe_adapter *a, int v_idx, int r_idx) { struct ixgbe_q_vector *q_vector = a->q_vector[v_idx]; struct ixgbe_ring *rx_ring = a->rx_ring[r_idx]; rx_ring->q_vector = q_vector; rx_ring->next = q_vector->rx.ring; q_vector->rx.ring = rx_ring; q_vector->rx.count++; q_vector->rx.work_limit = a->rx_work_limit; #ifdef __VMKLNX__ #ifdef CONFIG_IXGBE_NAPI /* update the net device associated with the vector */ if (ring_type_is_cna(rx_ring)) { q_vector->napi.dev = a->cnadev; netif_set_poll_cna(&q_vector->napi); } #endif /* CONFIG_IXGBE_NAPI */ #endif } static inline void map_vector_to_txq(struct ixgbe_adapter *a, int v_idx, int t_idx) { struct ixgbe_q_vector *q_vector = a->q_vector[v_idx]; struct ixgbe_ring *tx_ring = a->tx_ring[t_idx]; tx_ring->q_vector = q_vector; tx_ring->next = q_vector->tx.ring; q_vector->tx.ring = tx_ring; q_vector->tx.count++; q_vector->tx.work_limit = a->tx_work_limit; } /** * ixgbe_map_rings_to_vectors - Maps descriptor rings to vectors * @adapter: board private structure to initialize * * This function maps descriptor rings to the queue-specific vectors * we were allotted through the MSI-X enabling code. Ideally, we'd have * one vector per ring/queue, but on a constrained vector budget, we * group the rings as "efficiently" as possible. You would add new * mapping configurations in here. **/ static void ixgbe_map_rings_to_vectors(struct ixgbe_adapter *adapter) { int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; int rxr_remaining = adapter->num_rx_queues, rxr_idx = 0; int txr_remaining = adapter->num_tx_queues, txr_idx = 0; int v_start = 0; /* only one q_vector if MSI-X is disabled. */ if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) q_vectors = 1; if ((q_vectors < (rxr_remaining + txr_remaining)) && (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED)) { int fcoe_i = adapter->ring_feature[RING_F_FCOE].indices; int fcoe_rxq, fcoe_txq; int fcoe_vector; int q_split; int i; if (q_vectors > MIN_MSIX_COUNT) { q_split = fcoe_i; q_vectors -= (fcoe_i * 2); } else { q_split = MIN_MSIX_Q_FCOE_VECTORS / 2; q_vectors -= MIN_MSIX_Q_FCOE_VECTORS; } /* remove FCOE vectors from regular Rx/Tx queue counts */ fcoe_rxq = rxr_remaining = adapter->num_rx_queues - fcoe_i; fcoe_txq = txr_remaining = adapter->num_tx_queues - fcoe_i; /* FCOE vectors start at the end of the Rx/Tx q_vectors */ fcoe_vector = q_vectors; /* map fcoe queues to vectors at end */ for (i = 0; i < q_split && fcoe_i; i++, fcoe_vector++) { int fqpv = DIV_ROUND_UP(fcoe_i, q_split - i); for (; fqpv; fqpv--, fcoe_rxq++, fcoe_txq++, fcoe_i--) { map_vector_to_rxq(adapter, fcoe_vector, fcoe_rxq); map_vector_to_txq(adapter, fcoe_vector + q_split, fcoe_txq); } } } /* * If we don't have enough vectors for a 1-to-1 mapping, we'll have to * group them so there are multiple queues per vector. * * Re-adjusting *qpv takes care of the remainder. */ for (; v_start < q_vectors && rxr_remaining; v_start++) { int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_start); for (; rqpv; rqpv--, rxr_idx++, rxr_remaining--) map_vector_to_rxq(adapter, v_start, rxr_idx); } /* * If there are not enough q_vectors for each ring to have it's own * vector then we must pair up Rx/Tx on a each vector */ if ((v_start + txr_remaining) > q_vectors) v_start = 0; for (; v_start < q_vectors && txr_remaining; v_start++) { int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_start); for (; tqpv; tqpv--, txr_idx++, txr_remaining--) map_vector_to_txq(adapter, v_start, txr_idx); } } /** * ixgbe_request_msix_irqs - Initialize MSI-X interrupts * @adapter: board private structure * * ixgbe_request_msix_irqs allocates MSI-X vectors and requests * interrupts from the kernel. **/ static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) { struct net_device *netdev = adapter->netdev; int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; int vector, err; int ri = 0, ti = 0; for (vector = 0; vector < q_vectors; vector++) { struct ixgbe_q_vector *q_vector = adapter->q_vector[vector]; struct msix_entry *entry = &adapter->msix_entries[vector]; if (q_vector->tx.ring && q_vector->rx.ring) { snprintf(q_vector->name, sizeof(q_vector->name) - 1, "%s-%s-%d", netdev->name, "TxRx", ri++); ti++; } else if (q_vector->rx.ring) { snprintf(q_vector->name, sizeof(q_vector->name) - 1, "%s-%s-%d", netdev->name, "rx", ri++); } else if (q_vector->tx.ring) { snprintf(q_vector->name, sizeof(q_vector->name) - 1, "%s-%s-%d", netdev->name, "tx", ti++); } else { /* skip this unused q_vector */ continue; } err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0, q_vector->name, q_vector); if (err) { e_err(probe, "request_irq failed for MSIX interrupt " "Error: %d\n", err); goto free_queue_irqs; } #ifdef HAVE_IRQ_AFFINITY_HINT /* If Flow Director is enabled, set interrupt affinity */ if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { /* assign the mask for this irq */ irq_set_affinity_hint(entry->vector, q_vector->affinity_mask); } #endif /* HAVE_IRQ_AFFINITY_HINT */ } err = request_irq(adapter->msix_entries[vector].vector, ixgbe_msix_other, 0, netdev->name, adapter); if (err) { e_err(probe, "request_irq for msix_other failed: %d\n", err); goto free_queue_irqs; } #ifdef IXGBE_TCP_TIMER vector++; sprintf(adapter->tcp_timer_name, "%s:timer", netdev->name); err = request_irq(adapter->msix_entries[vector].vector, &ixgbe_msix_tcp_timer, 0, adapter->tcp_timer_name, adapter); if (err) { e_err(probe, "request_irq for msix_tcp_timer failed: %d\n", err); /* Free "Other" interrupt */ free_irq(adapter->msix_entries[--vector].vector, netdev); goto free_queue_irqs; } #endif return 0; free_queue_irqs: while (vector) { vector--; #ifdef HAVE_IRQ_AFFINITY_HINT irq_set_affinity_hint(adapter->msix_entries[vector].vector, NULL); #endif free_irq(adapter->msix_entries[vector].vector, adapter->q_vector[vector]); } /* * should not clear this flag as we need it to identify * if we have malloced multiple vectors while freeing q vectors. * * adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED */ return err; } /** * ixgbe_intr - legacy mode Interrupt Handler * @irq: interrupt number * @data: pointer to a network interface device structure **/ static irqreturn_t ixgbe_intr(int irq, void *data) { struct ixgbe_adapter *adapter = data; struct ixgbe_hw *hw = &adapter->hw; struct ixgbe_q_vector *q_vector = adapter->q_vector[0]; u32 eicr; /* * Workaround of Silicon errata #26 on 82598. Mask the interrupt * before the read of EICR. */ IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read * therefore no explict interrupt disable is necessary */ eicr = IXGBE_READ_REG(hw, IXGBE_EICR); if (!eicr) { /* * shared interrupt alert! * make sure interrupts are enabled because the read will * have disabled interrupts due to EIAM * finish the workaround of silicon errata #26 on 82598. Unmask * the interrupt that we masked before the EICR read. */ if (!test_bit(__IXGBE_DOWN, &adapter->state)) ixgbe_irq_enable(adapter, true, true); return IRQ_NONE; /* Not our interrupt */ } if (eicr & IXGBE_EICR_LSC) ixgbe_check_lsc(adapter); switch (hw->mac.type) { case ixgbe_mac_82599EB: case ixgbe_mac_X540: if (eicr & IXGBE_EICR_ECC) e_info(link, "Received unrecoverable ECC Err, please " "reboot\n"); ixgbe_check_sfp_event(adapter, eicr); ixgbe_check_overtemp_event(adapter, eicr); break; default: break; } ixgbe_check_fan_failure(adapter, eicr); #ifdef CONFIG_IXGBE_NAPI /* would disable interrupts here but EIAM disabled it */ napi_schedule(&q_vector->napi); /* * re-enable link(maybe) and non-queue interrupts, no flush. * ixgbe_poll will re-enable the queue interrupts */ if (!test_bit(__IXGBE_DOWN, &adapter->state)) ixgbe_irq_enable(adapter, false, false); #else ixgbe_clean_tx_irq(q_vector, adapter->tx_ring[0]); ixgbe_clean_rx_irq(q_vector, adapter->rx_ring[0]); /* dynamically adjust throttle */ if (adapter->rx_itr_setting == 1) ixgbe_set_itr(q_vector); /* * Workaround of Silicon errata #26 on 82598. Unmask * the interrupt that we masked before the EICR read * no flush of the re-enable is necessary here */ if (!test_bit(__IXGBE_DOWN, &adapter->state)) ixgbe_irq_enable(adapter, true, false); #endif return IRQ_HANDLED; } static inline void ixgbe_reset_q_vectors(struct ixgbe_adapter *adapter) { int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; int i; /* legacy and MSI only use one vector */ if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) q_vectors = 1; for (i = 0; i < adapter->num_rx_queues; i++) { adapter->rx_ring[i]->q_vector = NULL; adapter->rx_ring[i]->next = NULL; } for (i = 0; i < adapter->num_tx_queues; i++) { adapter->tx_ring[i]->q_vector = NULL; adapter->tx_ring[i]->next = NULL; } for (i = 0; i < q_vectors; i++) { struct ixgbe_q_vector *q_vector = adapter->q_vector[i]; memset(&q_vector->rx, 0, sizeof(struct ixgbe_ring_container)); memset(&q_vector->tx, 0, sizeof(struct ixgbe_ring_container)); if (adapter->rx_itr_setting == 1) q_vector->itr = IXGBE_20K_ITR; else q_vector->itr = adapter->rx_itr_setting; } } /** * ixgbe_request_irq - initialize interrupts * @adapter: board private structure * * Attempts to configure interrupts using the best available * capabilities of the hardware and kernel. **/ static int ixgbe_request_irq(struct ixgbe_adapter *adapter) { struct net_device *netdev = adapter->netdev; int err; #ifdef HAVE_DEVICE_NUMA_NODE e_info(tx_err, "numa_node before request_irq %d\n", dev_to_node(&adapter->pdev->dev)); #endif /* HAVE_DEVICE_NUMA_NODE */ /* map all of the rings to the q_vectors */ ixgbe_map_rings_to_vectors(adapter); if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) err = ixgbe_request_msix_irqs(adapter); else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) err = request_irq(adapter->pdev->irq, &ixgbe_intr, 0, netdev->name, adapter); else err = request_irq(adapter->pdev->irq, &ixgbe_intr, IRQF_SHARED, netdev->name, adapter); if (err) { e_err(probe, "request_irq failed, Error %d\n", err); /* place q_vectors and rings back into a known good state */ ixgbe_reset_q_vectors(adapter); } return err; } static void ixgbe_free_irq(struct ixgbe_adapter *adapter) { if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { int i, q_vectors; q_vectors = adapter->num_msix_vectors; i = q_vectors - 1; #ifdef IXGBE_TCP_TIMER free_irq(adapter->msix_entries[i].vector, adapter); i--; #endif free_irq(adapter->msix_entries[i].vector, adapter); i--; for (; i >= 0; i--) { /* free only the irqs that were actually requested */ if (!adapter->q_vector[i]->rx.ring && !adapter->q_vector[i]->tx.ring) continue; #ifdef HAVE_IRQ_AFFINITY_HINT /* clear the affinity_mask in the IRQ descriptor */ irq_set_affinity_hint(adapter->msix_entries[i].vector, NULL); #endif free_irq(adapter->msix_entries[i].vector, adapter->q_vector[i]); } } else { free_irq(adapter->pdev->irq, adapter); } /* clear q_vector state information */ ixgbe_reset_q_vectors(adapter); } /** * ixgbe_irq_disable - Mask off interrupt generation on the NIC * @adapter: board private structure **/ static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter) { switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); break; default: break; } IXGBE_WRITE_FLUSH(&adapter->hw); if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { int i; for (i = 0; i < adapter->num_msix_vectors; i++) synchronize_irq(adapter->msix_entries[i].vector); } else { synchronize_irq(adapter->pdev->irq); } } /** * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts * **/ static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter) { struct ixgbe_q_vector *q_vector = adapter->q_vector[0]; /* rx/tx vector */ if (adapter->rx_itr_setting == 1) q_vector->itr = IXGBE_20K_ITR; else q_vector->itr = adapter->rx_itr_setting; ixgbe_write_eitr(q_vector); ixgbe_set_ivar(adapter, 0, 0, 0); ixgbe_set_ivar(adapter, 1, 0, 0); e_info(hw, "Legacy interrupt IVAR setup done\n"); } /** * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset * @adapter: board private structure * @ring: structure containing ring specific data * * Configure the Tx descriptor ring after a reset. **/ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u64 tdba = ring->dma; int wait_loop = 10; u32 txdctl = IXGBE_TXDCTL_ENABLE; u8 reg_idx = ring->reg_idx; /* disable queue to avoid issues while updating state */ IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0); IXGBE_WRITE_FLUSH(hw); IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx), (tdba & DMA_BIT_MASK(32))); IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32)); IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx), ring->count * sizeof(union ixgbe_adv_tx_desc)); IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0); IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0); ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx); /* * set WTHRESH to encourage burst writeback, it should not be set * higher than 1 when ITR is 0 as it could cause false TX hangs * * In order to avoid issues WTHRESH + PTHRESH should always be equal * to or less than the number of on chip descriptors, which is * currently 40. */ if (!adapter->tx_itr_setting || !adapter->rx_itr_setting) txdctl |= (1 << 16); /* WTHRESH = 1 */ else txdctl |= (8 << 16); /* WTHRESH = 8 */ /* * Setting PTHRESH to 32 both improves performance * and avoids a TX hang with DFP enabled */ txdctl |= (1 << 8) | /* HTHRESH = 1 */ 32; /* PTHRESH = 32 */ /* reinitialize flowdirector state */ if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) && adapter->atr_sample_rate) { ring->atr_sample_rate = adapter->atr_sample_rate; ring->atr_count = 0; set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state); } else { ring->atr_sample_rate = 0; } clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state); /* enable queue */ IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl); /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* poll to verify queue is enabled */ do { msleep(1); txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx)); } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); if (!wait_loop) e_err(drv, "Could not enable Tx Queue %d\n", reg_idx); } static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 rttdcs; u32 mask; u32 reg; u8 tcs = netdev_get_num_tc(adapter->netdev); if (hw->mac.type == ixgbe_mac_82598EB) return; /* disable the arbiter while setting MTQC */ rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); rttdcs |= IXGBE_RTTDCS_ARBDIS; IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); /* set transmit pool layout */ mask = IXGBE_FLAG_SRIOV_ENABLED | IXGBE_FLAG_VMDQ_ENABLED; mask |= IXGBE_FLAG_DCB_ENABLED; switch (adapter->flags & mask) { case IXGBE_FLAG_VMDQ_ENABLED: case IXGBE_FLAG_SRIOV_ENABLED: case (IXGBE_FLAG_SRIOV_ENABLED | IXGBE_FLAG_VMDQ_ENABLED): #ifdef __VMKLNX__ /* In RSS mode, we should work in 4 queues per pool mode */ if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) { IXGBE_WRITE_REG(hw, IXGBE_MTQC, (IXGBE_MTQC_VT_ENA | IXGBE_MTQC_32VF)); break; } #endif IXGBE_WRITE_REG(hw, IXGBE_MTQC, (IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF)); break; case (IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_DCB_ENABLED): case (IXGBE_FLAG_SRIOV_ENABLED | IXGBE_FLAG_DCB_ENABLED): case (IXGBE_FLAG_SRIOV_ENABLED | IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_DCB_ENABLED): IXGBE_WRITE_REG(hw, IXGBE_MTQC, (IXGBE_MTQC_RT_ENA | IXGBE_MTQC_VT_ENA | IXGBE_MTQC_4TC_4TQ)); break; case IXGBE_FLAG_DCB_ENABLED: if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) { IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_RT_ENA | IXGBE_MTQC_VT_ENA | IXGBE_MTQC_4TC_4TQ); break; } if (!tcs) reg = IXGBE_MTQC_64Q_1PB; else if (tcs <= 4) reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ; else reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ; IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg); /* Enable Security TX Buffer IFG for multiple pb */ if (tcs) { reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); reg |= IXGBE_SECTX_DCB; IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg); } break; default: IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); break; } /* re-enable the arbiter */ rttdcs &= ~IXGBE_RTTDCS_ARBDIS; IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); } /** * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset * @adapter: board private structure * * Configure the Tx unit of the MAC after a reset. **/ static void ixgbe_configure_tx(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 dmatxctl; u32 i; #ifdef CONFIG_NETDEVICES_MULTIQUEUE #ifndef __VMKLNX__ if (adapter->num_tx_queues > 1) adapter->netdev->features |= NETIF_F_MULTI_QUEUE; else adapter->netdev->features &= ~NETIF_F_MULTI_QUEUE; #endif /* __VMKLNX__ */ #endif ixgbe_setup_mtqc(adapter); if (hw->mac.type != ixgbe_mac_82598EB) { /* DMATXCTL.EN must be before Tx queues are enabled */ dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); dmatxctl |= IXGBE_DMATXCTL_TE; IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); } /* Setup the HW Tx Head and Tail descriptor pointers */ for (i = 0; i < adapter->num_tx_queues; i++) ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]); } #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring) { struct ixgbe_hw *hw = &adapter->hw; u32 srrctl; u8 reg_idx = rx_ring->reg_idx; switch (hw->mac.type) { case ixgbe_mac_82598EB: { struct ixgbe_ring_feature *feature = adapter->ring_feature; #ifdef IXGBE_VMDQ /* program one srrctl register per VMDq index */ if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) { unsigned long mask; long shift, len; mask = (unsigned long) feature[RING_F_VMDQ].mask; len = sizeof(feature[RING_F_VMDQ].mask) * 8; shift = find_first_bit(&mask, len); reg_idx = (reg_idx & mask) >> shift; } else { /* * if VMDq is not active we must program one srrctl * register per RSS queue since we have enabled * RDRXCTL.MVMEN */ const int mask = feature[RING_F_RSS].mask; reg_idx = reg_idx & mask; } #else /* IXGBE_VMDQ */ const int mask = feature[RING_F_RSS].mask; reg_idx = reg_idx & mask; #endif /* IXGBE_VMDQ */ } break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: default: break; } srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx)); srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; srrctl &= ~IXGBE_SRRCTL_DROP_EN; /* * We should set the drop enable bit if: * SR-IOV is enabled * or * Flow Control is disabled and number of RX queues > 1 * * This allows us to avoid head of line blocking for security * and performance reasons. */ if (adapter->num_vfs || (adapter->num_rx_queues > 1 && (hw->fc.disable_fc_autoneg || hw->fc.requested_mode == ixgbe_fc_none|| hw->fc.requested_mode == ixgbe_fc_rx_pause))) srrctl |= IXGBE_SRRCTL_DROP_EN; if (ring_is_ps_enabled(rx_ring)) { srrctl |= (IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) & IXGBE_SRRCTL_BSIZEHDR_MASK; #ifndef __VMKLNX__ #if (PAGE_SIZE / 2) > IXGBE_MAX_RXBUFFER srrctl |= IXGBE_MAX_RXBUFFER >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; #else srrctl |= (PAGE_SIZE / 2) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; #endif #else #if (PAGE_SIZE) > IXGBE_MAX_RXBUFFER srrctl |= IXGBE_MAX_RXBUFFER >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; #else srrctl |= (PAGE_SIZE) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; #endif #endif /* __VMKLNX__ */ /* rx_buf_len 0 indicates paged Rx for 82599 */ if (rx_ring->rx_buf_len) srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; else srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; } else { srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; } IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); } static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D, 0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE, 0x6A3E67EA, 0x14364D17, 0x3BED200D}; u32 mrqc = 0, reta = 0; u32 rxcsum; int i, j; int maxq = adapter->ring_feature[RING_F_RSS].indices; int mask; u8 tcs = netdev_get_num_tc(adapter->netdev); #ifdef HAVE_MQPRIO if (tcs) maxq = min(maxq, adapter->num_tx_queues / tcs); #endif /* Fill out hash function seeds */ for (i = 0; i < 10; i++) IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]); /* Fill out redirection table */ for (i = 0, j = 0; i < 128; i++, j++) { if (j == maxq) j = 0; /* reta = 4-byte sliding window of * 0x00..(indices-1)(indices-1)00..etc. */ reta = (reta << 8) | (j * 0x11); if ((i & 3) == 3) IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); } /* Disable indicating checksum in descriptor, enables RSS hash */ rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); rxcsum |= IXGBE_RXCSUM_PCSD; IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); mask = adapter->flags & (IXGBE_FLAG_RSS_ENABLED | IXGBE_FLAG_DCB_ENABLED #ifdef IXGBE_VMDQ | IXGBE_FLAG_VMDQ_ENABLED #endif | IXGBE_FLAG_SRIOV_ENABLED ); switch (mask) { case (IXGBE_FLAG_RSS_ENABLED): mrqc = IXGBE_MRQC_RSSEN; break; case (IXGBE_FLAG_SRIOV_ENABLED): mrqc = IXGBE_MRQC_VMDQEN; break; #ifdef IXGBE_VMDQ case (IXGBE_FLAG_VMDQ_ENABLED): case (IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_SRIOV_ENABLED): mrqc = IXGBE_MRQC_VMDQEN; break; case (IXGBE_FLAG_RSS_ENABLED | IXGBE_FLAG_VMDQ_ENABLED): case (IXGBE_FLAG_RSS_ENABLED | IXGBE_FLAG_SRIOV_ENABLED): case (IXGBE_FLAG_RSS_ENABLED | IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_SRIOV_ENABLED): if (adapter->ring_feature[RING_F_RSS].indices == 4) mrqc = IXGBE_MRQC_VMDQRSS32EN; else if (adapter->ring_feature[RING_F_RSS].indices == 2) mrqc = IXGBE_MRQC_VMDQRSS64EN; else mrqc = IXGBE_MRQC_VMDQEN; break; #endif /* IXGBE_VMDQ */ #ifdef IXGBE_VMDQ case (IXGBE_FLAG_DCB_ENABLED | IXGBE_FLAG_VMDQ_ENABLED): case (IXGBE_FLAG_DCB_ENABLED | IXGBE_FLAG_SRIOV_ENABLED): case (IXGBE_FLAG_DCB_ENABLED | IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_SRIOV_ENABLED): if (tcs <= 4) mrqc = IXGBE_MRQC_VMDQRT4TCEN; /* 4 TCs */ else mrqc = IXGBE_MRQC_VMDQRT8TCEN; /* 8 TCs */ break; #endif /* IXGBE_VMDQ */ case (IXGBE_FLAG_RSS_ENABLED | IXGBE_FLAG_DCB_ENABLED): if (!tcs) mrqc = IXGBE_MRQC_RSSEN; else if (tcs <= 4) mrqc = IXGBE_MRQC_RTRSS4TCEN; else mrqc = IXGBE_MRQC_RTRSS8TCEN; break; case (IXGBE_FLAG_DCB_ENABLED): mrqc = IXGBE_MRQC_RT8TCEN; break; default: break; } /* Perform hash on these packet types */ mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP | IXGBE_MRQC_RSS_FIELD_IPV6 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP; if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP) mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP) mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); } /** * ixgbe_clear_rscctl - disable RSC for the indicated ring * @adapter: address of board private structure * @ring: structure containing ring specific data **/ void ixgbe_clear_rscctl(struct ixgbe_adapter *adapter, struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u32 rscctrl; u8 reg_idx = ring->reg_idx; rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx)); rscctrl &= ~IXGBE_RSCCTL_RSCEN; IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl); clear_ring_rsc_enabled(ring); } /** * ixgbe_configure_rscctl - enable RSC for the indicated ring * @adapter: address of board private structure * @ring: structure containing ring specific data **/ void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u32 rscctrl; int rx_buf_len; u8 reg_idx = ring->reg_idx; if (!ring_is_rsc_enabled(ring)) return; rx_buf_len = ring->rx_buf_len; rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx)); rscctrl |= IXGBE_RSCCTL_RSCEN; /* * we must limit the number of descriptors so that the * total size of max desc * buf_len is not greater * than 65535 */ if (ring_is_ps_enabled(ring)) { #if (MAX_SKB_FRAGS > 16) rscctrl |= IXGBE_RSCCTL_MAXDESC_16; #elif (MAX_SKB_FRAGS > 8) rscctrl |= IXGBE_RSCCTL_MAXDESC_8; #elif (MAX_SKB_FRAGS > 4) rscctrl |= IXGBE_RSCCTL_MAXDESC_4; #else rscctrl |= IXGBE_RSCCTL_MAXDESC_1; #endif } else { if (rx_buf_len < IXGBE_RXBUFFER_4K) #ifndef __VMKLNX__ rscctrl |= IXGBE_RSCCTL_MAXDESC_16; #else /* * In ESX, restricting max aggregated * to 24K gives better perf compared 48K */ rscctrl |= IXGBE_RSCCTL_MAXDESC_8; #endif else if (rx_buf_len < IXGBE_RXBUFFER_8K) rscctrl |= IXGBE_RSCCTL_MAXDESC_8; else rscctrl |= IXGBE_RSCCTL_MAXDESC_4; } IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl); } /** * ixgbe_set_uta - Set unicast filter table address * @adapter: board private structure * * The unicast table address is a register array of 32-bit registers. * The table is meant to be used in a way similar to how the MTA is used * however due to certain limitations in the hardware it is necessary to * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous * enable bit to allow vlan tag stripping when promiscuous mode is enabled **/ static void ixgbe_set_uta(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int i; /* The UTA table only exists on 82599 hardware and newer */ if (hw->mac.type < ixgbe_mac_82599EB) return; /* we only need to do this if VMDq is enabled */ if (!(adapter->flags & (IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_SRIOV_ENABLED))) return; for (i = 0; i < 128; i++) IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0); } static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBE_MAX_RX_DESC_POLL; u32 rxdctl; u8 reg_idx = ring->reg_idx; /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; do { msleep(1); rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); if (!wait_loop) { e_err(drv, "RXDCTL.ENABLE on Rx queue %d " "not set within the polling period\n", reg_idx); } } void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter, struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBE_MAX_RX_DESC_POLL; u32 rxdctl; u8 reg_idx = ring->reg_idx; rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); rxdctl &= ~IXGBE_RXDCTL_ENABLE; /* write value back with RXDCTL.ENABLE bit cleared */ IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); if (hw->mac.type == ixgbe_mac_82598EB && !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* the hardware may take up to 100us to really disable the rx queue */ do { udelay(10); rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); if (!wait_loop) { e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within " "the polling period\n", reg_idx); } } void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u64 rdba = ring->dma; u32 rxdctl; u8 reg_idx = ring->reg_idx; /* disable queue to avoid issues while updating state */ rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); ixgbe_disable_rx_queue(adapter, ring); IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32))); IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32)); IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx), ring->count * sizeof(union ixgbe_adv_rx_desc)); IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0); IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0); ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx); ixgbe_configure_srrctl(adapter, ring); #ifndef __VMKLNX__ /* In ESX RSCCTL configuration is done by on demand. */ ixgbe_configure_rscctl(adapter, ring); #endif if (hw->mac.type == ixgbe_mac_82598EB) { /* * enable cache line friendly hardware writes: * PTHRESH=32 descriptors (half the internal cache), * this also removes ugly rx_no_buffer_count increment * HTHRESH=4 descriptors (to minimize latency on fetch) * WTHRESH=8 burst writeback up to two cache lines */ rxdctl &= ~0x3FFFFF; rxdctl |= 0x080420; } /* If operating in IOV mode set RLPML for X540 */ if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && hw->mac.type == ixgbe_mac_X540) { rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK; rxdctl |= ((ring->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN); } /* enable receive descriptor ring */ rxdctl |= IXGBE_RXDCTL_ENABLE; IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); ixgbe_rx_desc_queue_enable(adapter, ring); ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring)); } static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int p; /* PSRTYPE must be initialized in non 82598 adapters */ u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; if (hw->mac.type == ixgbe_mac_82598EB) return; for (p = 0; p < adapter->num_rx_pools; p++) IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(p)), psrtype); } static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 gcr_ext; #ifdef IXGBE_VMDQ u32 vt_reg; u32 vt_reg_bits; u32 pool; u32 vmdctl; if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED || adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) return; switch (hw->mac.type) { case ixgbe_mac_82598EB: vt_reg = IXGBE_VMD_CTL; vt_reg_bits = IXGBE_VMD_CTL_VMDQ_EN; vmdctl = IXGBE_READ_REG(hw, vt_reg); IXGBE_WRITE_REG(hw, vt_reg, vmdctl | vt_reg_bits); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: vt_reg = IXGBE_VT_CTL; vt_reg_bits = IXGBE_VMD_CTL_VMDQ_EN; if (adapter->num_vfs) { vt_reg_bits &= ~IXGBE_VT_CTL_POOL_MASK; vt_reg_bits |= (adapter->num_vfs << IXGBE_VT_CTL_POOL_SHIFT); vt_reg_bits |= IXGBE_VT_CTL_REPLEN; } vmdctl = IXGBE_READ_REG(hw, vt_reg); vmdctl &= ~IXGBE_VT_CTL_POOL_MASK; IXGBE_WRITE_REG(hw, vt_reg, vmdctl | vt_reg_bits); for (pool = 1; pool < adapter->num_rx_pools; pool++) { u32 vmolr; int vmdq_pool = VMDQ_P(pool); /* * accept untagged packets until a vlan tag * is specifically set for the VMDQ queue/pool */ vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(vmdq_pool)); vmolr |= IXGBE_VMOLR_AUPE; IXGBE_WRITE_REG(hw, IXGBE_VMOLR(vmdq_pool), vmolr); } IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), 0xFFFFFFFF); IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), 0xFFFFFFFF); IXGBE_WRITE_REG(hw, IXGBE_VFTE(0), 0xFFFFFFFF); IXGBE_WRITE_REG(hw, IXGBE_VFTE(1), 0xFFFFFFFF); break; default: break; } #endif /* IXGBE_VMDQ */ if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) return; /* * Set up VF register offsets for selected VT Mode, * i.e. 32 or 64 VFs for SR-IOV */ gcr_ext = IXGBE_READ_REG(hw, IXGBE_GCR_EXT); gcr_ext |= IXGBE_GCR_EXT_MSIX_EN; if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) gcr_ext |= IXGBE_GCR_EXT_VT_MODE_32; else if (adapter->flags & IXGBE_FLAG_RSS_ENABLED && adapter->ring_feature[RING_F_RSS].indices == 4) gcr_ext |= IXGBE_GCR_EXT_VT_MODE_32; else gcr_ext |= IXGBE_GCR_EXT_VT_MODE_64; IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext); /* enable Tx loopback for VF/PF communication */ if (adapter->flags & IXGBE_FLAG_SRIOV_L2LOOPBACK_ENABLE) IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN); else IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, 0); #ifdef __VMKLNX__ /* Only enable anti-spoofing on the VF pools. The shared code has * turned on anti-spoofing on all the pools except default, but we * only want it enabled for VF pools */ if (adapter->num_vfs != 0) { int i = 0; u32 pfvfspoof = 0; adapter->antispoofing_enabled = true; /* Only enable anti-spoofing on VF pools */ for (i=0; i < adapter->num_vfs ; i++) { pfvfspoof = IXGBE_READ_REG(hw, IXGBE_PFVFSPOOF(i >> 3)); pfvfspoof |= (1 << (i % 8)); IXGBE_WRITE_REG(hw, IXGBE_PFVFSPOOF(i >> 3), pfvfspoof); } } #endif /* __VMKLNX__ */ } static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; struct net_device *netdev = adapter->netdev; int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; int rx_buf_len; struct ixgbe_ring *rx_ring; int i; u32 mhadd, hlreg0; /* Decide whether to use packet split mode or not */ if (netdev->mtu > ETH_DATA_LEN) { if (adapter->flags & IXGBE_FLAG_RX_PS_CAPABLE) adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED; else adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; } else { if (adapter->flags & IXGBE_FLAG_RX_1BUF_CAPABLE) adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; else adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED; #ifdef __VMKLNX__ #endif /* __VMKLNX__ */ } #ifdef IXGBE_FCOE /* adjust max frame to be able to do baby jumbo for FCoE */ if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif /* IXGBE_FCOE */ mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { mhadd &= ~IXGBE_MHADD_MFS_MASK; mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); } if (!adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) /* MHADD will allow an extra 4 bytes past for vlan tagged frames */ max_frame += VLAN_HLEN; /* Set the RX buffer length according to the mode */ if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { #ifndef __VMKLNX__ /* set rx_buf_len to 0 in order to handle paged Rx for 82599 */ if (adapter->flags & IXGBE_FLAG_RX_BB_CAPABLE) rx_buf_len = 0; else #endif rx_buf_len = IXGBE_RX_HDR_SIZE; } else { if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) && (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)) rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE; #ifndef __VMKLNX__ /* * Make best use of allocation by using all but 1K of a * power of 2 allocation that will be used for skb->head. */ else if (max_frame <= IXGBE_RXBUFFER_3K) rx_buf_len = IXGBE_RXBUFFER_3K; else if (max_frame <= IXGBE_RXBUFFER_7K) rx_buf_len = IXGBE_RXBUFFER_7K; else if (max_frame <= IXGBE_RXBUFFER_15K) rx_buf_len = IXGBE_RXBUFFER_15K; else rx_buf_len = IXGBE_MAX_RXBUFFER; #else else rx_buf_len = ALIGN(max_frame, 1024); #endif /* __VMKLNX__ */ } hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */ hlreg0 |= IXGBE_HLREG0_JUMBOEN; IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); /* * Setup the HW Rx Head and Tail Descriptor Pointers and * the Base and Length of the Rx Descriptor Ring */ for (i = 0; i < adapter->num_rx_queues; i++) { rx_ring = adapter->rx_ring[i]; rx_ring->rx_buf_len = rx_buf_len; if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) set_ring_ps_enabled(rx_ring); else clear_ring_ps_enabled(rx_ring); #ifndef __VMKLNX__ /* RSC enablement is done on demand in ESX. */ if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) { set_ring_rsc_enabled(rx_ring); } else { clear_ring_rsc_enabled(rx_ring); } #endif /* __VMKLNX__ */ #ifdef IXGBE_FCOE if (netdev->features & NETIF_F_FCOE_MTU) { struct ixgbe_ring_feature *f; f = &adapter->ring_feature[RING_F_FCOE]; if ((i >= f->mask) && (i < f->mask + f->indices)) { clear_ring_ps_enabled(rx_ring); if (rx_buf_len < IXGBE_FCOE_JUMBO_FRAME_SIZE) rx_ring->rx_buf_len = IXGBE_FCOE_JUMBO_FRAME_SIZE; } } #endif /* IXGBE_FCOE */ } } static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); switch (hw->mac.type) { case ixgbe_mac_82598EB: /* * For VMDq support of different descriptor types or * buffer sizes through the use of multiple SRRCTL * registers, RDRXCTL.MVMEN must be set to 1 * * also, the manual doesn't mention it clearly but DCA hints * will only use queue 0's tags unless this bit is set. Side * effects of setting this bit are only that SRRCTL must be * fully programmed [0..15] */ rdrxctl |= IXGBE_RDRXCTL_MVMEN; break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: /* Disable RSC for ACK packets */ IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; /* hardware requires some bits to be set by default */ rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX); rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; break; default: /* We should do nothing since we don't know this hardware */ return; } IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); } /** * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset * @adapter: board private structure * * Configure the Rx unit of the MAC after a reset. **/ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int i; u32 rxctrl; /* disable receives while setting up the descriptors */ rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); ixgbe_setup_psrtype(adapter); ixgbe_setup_rdrxctl(adapter); /* Program registers for the distribution of queues */ ixgbe_setup_mrqc(adapter); ixgbe_set_uta(adapter); /* set_rx_buffer_len must be called before ring initialization */ ixgbe_set_rx_buffer_len(adapter); /* * Setup the HW Rx Head and Tail Descriptor Pointers and * the Base and Length of the Rx Descriptor Ring */ for (i = 0; i < adapter->num_rx_queues; i++) ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]); /* disable drop enable for 82598 parts */ if (hw->mac.type == ixgbe_mac_82598EB) rxctrl |= IXGBE_RXCTRL_DMBYPS; /* enable all receives */ rxctrl |= IXGBE_RXCTRL_RXEN; ixgbe_enable_rx_dma(hw, rxctrl); } #ifdef NETIF_F_HW_VLAN_TX static void ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; int pool_ndx = adapter->num_vfs; #ifdef __VMKLNX__ s32 regindex; u32 bitindex; u32 vfta; u32 targetbit; if (adapter->num_vfs) { switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: if (hw->mac.ops.set_vfta) hw->mac.ops.set_vfta(hw, vid, pool_ndx, true); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: regindex = (vid >> 5) & 0x7F; bitindex = vid & 0x1F; targetbit = (1 << bitindex); vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(regindex)); vfta |= targetbit; IXGBE_WRITE_REG(hw, IXGBE_VFTA(regindex), vfta); break; default: break; } } else if (hw->mac.ops.set_vfta) { #else /* add VID to filter table */ if (hw->mac.ops.set_vfta) { #endif #ifndef HAVE_VLAN_RX_REGISTER if (vid < VLAN_N_VID) set_bit(vid, adapter->active_vlans); #endif hw->mac.ops.set_vfta(hw, vid, pool_ndx, true); if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) { int i; switch (adapter->hw.mac.type) { case ixgbe_mac_82599EB: case ixgbe_mac_X540: /* enable vlan id for all pools */ for (i = 1; i < adapter->num_rx_pools; i++) hw->mac.ops.set_vfta(hw, vid, VMDQ_P(i), true); break; default: break; } } } #ifndef __VMKLNX__ #ifndef HAVE_NETDEV_VLAN_FEATURES /* * Copy feature flags from netdev to the vlan netdev for this vid. * This allows things like TSO to bubble down to our vlan device. * Some vlans, such as VLAN 0 for DCB will not have a v_netdev so * we will not have a netdev that needs updating. */ if (adapter->vlgrp) { struct vlan_group *vlgrp = adapter->vlgrp; struct net_device *v_netdev = vlan_group_get_device(vlgrp, vid); if (v_netdev) { v_netdev->features |= netdev->features; vlan_group_set_device(vlgrp, vid, v_netdev); } } #endif /* HAVE_NETDEV_VLAN_FEATURES */ #endif } static void ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; int pool_ndx = adapter->num_vfs; #ifdef __VMKLNX__ s32 regindex; u32 bitindex; u32 vfta; u32 targetbit; #endif /* User is not allowed to remove vlan ID 0 */ if (!vid) return; #ifdef HAVE_VLAN_RX_REGISTER #ifndef __VMKLNX__ if (!test_bit(__IXGBE_DOWN, &adapter->state)) ixgbe_irq_disable(adapter); vlan_group_set_device(adapter->vlgrp, vid, NULL); if (!test_bit(__IXGBE_DOWN, &adapter->state)) ixgbe_irq_enable(adapter, true, true); #endif #endif /* HAVE_VLAN_RX_REGISTER */ /* remove VID from filter table */ #ifdef __VMKLNX__ if (adapter->num_vfs) { switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: if (hw->mac.ops.set_vfta) hw->mac.ops.set_vfta(hw, vid, pool_ndx, false); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: regindex = (vid >> 5) & 0x7F; bitindex = vid & 0x1F; targetbit = (1 << bitindex); vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(regindex)); vfta &= ~targetbit; IXGBE_WRITE_REG(hw, IXGBE_VFTA(regindex), vfta); break; default: break; } } else if (hw->mac.ops.set_vfta) { #else if (hw->mac.ops.set_vfta) { #endif hw->mac.ops.set_vfta(hw, vid, pool_ndx, false); if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) { int i; switch (adapter->hw.mac.type) { case ixgbe_mac_82599EB: case ixgbe_mac_X540: /* remove vlan id from all pools */ for (i = 1; i < adapter->num_rx_pools; i++) hw->mac.ops.set_vfta(hw, vid, VMDQ_P(i), false); break; default: break; } } } #ifndef HAVE_VLAN_RX_REGISTER clear_bit(vid, adapter->active_vlans); #endif } /** * ixgbe_vlan_stripping_disable - helper to disable vlan tag stripping * @adapter: driver data */ void ixgbe_vlan_stripping_disable(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 vlnctrl; int i; /* leave vlan tag stripping enabled for DCB */ if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) return; switch (hw->mac.type) { case ixgbe_mac_82598EB: vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); vlnctrl &= ~IXGBE_VLNCTRL_VME; IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: for (i = 0; i < adapter->num_rx_queues; i++) { u8 reg_idx = adapter->rx_ring[i]->reg_idx; vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); vlnctrl &= ~IXGBE_RXDCTL_VME; IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), vlnctrl); } break; default: break; } } /** * ixgbe_vlan_stripping_enable - helper to enable vlan tag stripping * @adapter: driver data */ void ixgbe_vlan_stripping_enable(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 vlnctrl; int i; switch (hw->mac.type) { case ixgbe_mac_82598EB: vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); vlnctrl |= IXGBE_VLNCTRL_VME; IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: for (i = 0; i < adapter->num_rx_queues; i++) { u8 reg_idx = adapter->rx_ring[i]->reg_idx; vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); vlnctrl |= IXGBE_RXDCTL_VME; IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), vlnctrl); } break; default: break; } } #ifdef HAVE_VLAN_RX_REGISTER static void ixgbe_vlan_mode(struct net_device *netdev, struct vlan_group *grp) #else void ixgbe_vlan_mode(struct net_device *netdev, u32 features) #endif { struct ixgbe_adapter *adapter = netdev_priv(netdev); #ifdef HAVE_VLAN_RX_REGISTER if (!test_bit(__IXGBE_DOWN, &adapter->state)) ixgbe_irq_disable(adapter); adapter->vlgrp = grp; if (!test_bit(__IXGBE_DOWN, &adapter->state)) ixgbe_irq_enable(adapter, true, true); #endif #ifdef HAVE_VLAN_RX_REGISTER bool enable = (grp || (adapter->flags & IXGBE_FLAG_DCB_ENABLED)); #else bool enable = !!(features & NETIF_F_HW_VLAN_RX); #endif if (enable) /* enable VLAN tag insert/strip */ ixgbe_vlan_stripping_enable(adapter); else /* disable VLAN tag insert/strip */ ixgbe_vlan_stripping_disable(adapter); } static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter) { #ifdef HAVE_VLAN_RX_REGISTER ixgbe_vlan_mode(adapter->netdev, adapter->vlgrp); /* * add vlan ID 0 and enable vlan tag stripping so we * always accept priority-tagged traffic */ ixgbe_vlan_rx_add_vid(adapter->netdev, 0); if (adapter->vlgrp) { u16 vid; for (vid = 0; vid < VLAN_N_VID; vid++) { if (!vlan_group_get_device(adapter->vlgrp, vid)) continue; ixgbe_vlan_rx_add_vid(adapter->netdev, vid); } } #else struct net_device *netdev = adapter->netdev; u16 vid; ixgbe_vlan_mode(netdev, netdev->features); for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) ixgbe_vlan_rx_add_vid(netdev, vid); #endif } #endif static u8 *ixgbe_addr_list_itr(struct ixgbe_hw *hw, u8 **mc_addr_ptr, u32 *vmdq) { #ifdef NETDEV_HW_ADDR_T_MULTICAST struct netdev_hw_addr *mc_ptr; #else struct dev_mc_list *mc_ptr; #endif struct ixgbe_adapter *adapter = hw->back; u8 *addr = *mc_addr_ptr; *vmdq = adapter->num_vfs; #ifdef NETDEV_HW_ADDR_T_MULTICAST mc_ptr = container_of(addr, struct netdev_hw_addr, addr[0]); if (mc_ptr->list.next) { struct netdev_hw_addr *ha; ha = list_entry(mc_ptr->list.next, struct netdev_hw_addr, list); *mc_addr_ptr = ha->addr; } #else mc_ptr = container_of(addr, struct dev_mc_list, dmi_addr[0]); if (mc_ptr->next) *mc_addr_ptr = mc_ptr->next->dmi_addr; #endif else *mc_addr_ptr = NULL; return addr; } /** * ixgbe_write_mc_addr_list - write multicast addresses to MTA * @netdev: network interface device structure * * Writes multicast address list to the MTA hash table. * Returns: -ENOMEM on failure * 0 on no addresses written * X on writing X addresses to MTA **/ int ixgbe_write_mc_addr_list(struct net_device *netdev) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; #ifdef NETDEV_HW_ADDR_T_MULTICAST struct netdev_hw_addr *ha; #endif u8 *addr_list = NULL; int addr_count = 0; if (!hw->mac.ops.update_mc_addr_list) return -ENOMEM; if (!netif_running(netdev)) return 0; hw->mac.ops.update_mc_addr_list(hw, NULL, 0, ixgbe_addr_list_itr, true); if (!netdev_mc_empty(netdev)) { #ifdef NETDEV_HW_ADDR_T_MULTICAST ha = list_first_entry(&netdev->mc.list, struct netdev_hw_addr, list); addr_list = ha->addr; #else addr_list = netdev->mc_list->dmi_addr; #endif addr_count = netdev_mc_count(netdev); hw->mac.ops.update_mc_addr_list(hw, addr_list, addr_count, ixgbe_addr_list_itr, false); } ixgbe_restore_vf_multicasts(adapter); return addr_count; } void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int i; for (i = 0; i < hw->mac.num_rar_entries; i++) { if (adapter->mac_table[i].state & IXGBE_MAC_STATE_IN_USE) { hw->mac.ops.set_rar(hw, i, adapter->mac_table[i].addr, adapter->mac_table[i].queue, IXGBE_RAH_AV); } else { hw->mac.ops.clear_rar(hw, i); } } } void ixgbe_sync_mac_table(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int i; for (i = 0; i < hw->mac.num_rar_entries; i++) { if (adapter->mac_table[i].state & IXGBE_MAC_STATE_MODIFIED) { if (adapter->mac_table[i].state & IXGBE_MAC_STATE_IN_USE) { hw->mac.ops.set_rar(hw, i, adapter->mac_table[i].addr, adapter->mac_table[i].queue, IXGBE_RAH_AV); } else { hw->mac.ops.clear_rar(hw, i); } adapter->mac_table[i].state &= ~(IXGBE_MAC_STATE_MODIFIED); } } } int ixgbe_available_rars(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int i, count = 0; for (i = 0; i < hw->mac.num_rar_entries; i++) { if (adapter->mac_table[i].state == 0) count++; } return count; } int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter, u8 *addr, u16 queue) { struct ixgbe_hw *hw = &adapter->hw; int i; if (is_zero_ether_addr(addr)) { DPRINTK(PROBE, ERR, "MAC address is zero\n"); return -EINVAL; } for (i = 0; i < hw->mac.num_rar_entries; i++) { if (adapter->mac_table[i].state & IXGBE_MAC_STATE_IN_USE) { /* Dont allow programming of duplicate MAC * address on the same PF for different VFs(queues) */ if (!compare_ether_addr(addr, hw->mac.perm_addr)) return i; if (!compare_ether_addr (addr, adapter->mac_table[i].addr)) { if (adapter->mac_table[i].queue == queue) { return i; } else { DPRINTK(PROBE, ERR, "Duplicate MAC address not allowed\n"); return -EEXIST; } } else { continue; } } adapter->mac_table[i].state |= (IXGBE_MAC_STATE_MODIFIED | IXGBE_MAC_STATE_IN_USE); memcpy(adapter->mac_table[i].addr, addr, ETH_ALEN); adapter->mac_table[i].queue = queue; ixgbe_sync_mac_table(adapter); return i; } return -ENOMEM; } #ifdef __VMKLNX__ void ixgbe_flush_sw_mac_table(struct ixgbe_adapter *adapter) { int i; struct ixgbe_hw *hw = &adapter->hw; for (i = 0; i < hw->mac.num_rar_entries; i++) { adapter->mac_table[i].state |= IXGBE_MAC_STATE_MODIFIED; adapter->mac_table[i].state &= ~IXGBE_MAC_STATE_IN_USE; memset(adapter->mac_table[i].addr, 0, ETH_ALEN); adapter->mac_table[i].queue = 0; } ixgbe_sync_mac_table(adapter); } void ixgbe_del_mac_filter_by_index(struct ixgbe_adapter *adapter, int index) { struct ixgbe_hw *hw = &adapter->hw; adapter->mac_table[index].state |= IXGBE_MAC_STATE_MODIFIED; adapter->mac_table[index].state &= ~IXGBE_MAC_STATE_IN_USE; memset(adapter->mac_table[index].addr, 0, ETH_ALEN); adapter->mac_table[index].queue = 0; ixgbe_sync_mac_table(adapter); } #endif int ixgbe_del_mac_filter(struct ixgbe_adapter *adapter, u8* addr, u16 queue) { /* search table for addr, if found, set to 0 and sync */ int i; struct ixgbe_hw *hw = &adapter->hw; if (is_zero_ether_addr(addr)) return -EINVAL; for (i = 0; i < hw->mac.num_rar_entries; i++) { if (!compare_ether_addr(addr, hw->mac.perm_addr)) return 0; if (!compare_ether_addr(addr, adapter->mac_table[i].addr) && adapter->mac_table[i].queue == queue) { adapter->mac_table[i].state |= IXGBE_MAC_STATE_MODIFIED; adapter->mac_table[i].state &= ~IXGBE_MAC_STATE_IN_USE; memset(adapter->mac_table[i].addr, 0, ETH_ALEN); adapter->mac_table[i].queue = 0; ixgbe_sync_mac_table(adapter); return 0; } } return -ENOMEM; } #ifdef HAVE_SET_RX_MODE /** * ixgbe_write_uc_addr_list - write unicast addresses to RAR table * @netdev: network interface device structure * * Writes unicast address list to the RAR table. * Returns: -ENOMEM on failure/insufficient address space * 0 on no addresses written * X on writing X addresses to the RAR table **/ int ixgbe_write_uc_addr_list(struct ixgbe_adapter *adapter, struct net_device *netdev, int vfn) { int count = 0; /* return ENOMEM indicating insufficient memory for addresses */ if (netdev_uc_count(netdev) > ixgbe_available_rars(adapter)) return -ENOMEM; if (!netdev_uc_empty(netdev)) { #ifdef NETDEV_HW_ADDR_T_UNICAST struct netdev_hw_addr *ha; #else struct dev_mc_list *ha; #endif netdev_for_each_uc_addr(ha, netdev) { #ifdef NETDEV_HW_ADDR_T_UNICAST ixgbe_del_mac_filter(adapter, ha->addr, vfn); ixgbe_add_mac_filter(adapter, ha->addr, vfn); #else ixgbe_del_mac_filter(adapter, ha->da_addr, vfn); ixgbe_add_mac_filter(adapter, ha->da_addr, vfn); #endif count++; } } return count; } #endif /** * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set * @netdev: network interface device structure * * The set_rx_method entry point is called whenever the unicast/multicast * address list or the network interface flags are updated. This routine is * responsible for configuring the hardware for proper unicast, multicast and * promiscuous mode. **/ void ixgbe_set_rx_mode(struct net_device *netdev) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE; u32 vlnctrl; int count; /* Check for Promiscuous and All Multicast modes */ fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); /* set all bits that we expect to always be set */ fctrl |= IXGBE_FCTRL_BAM; fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */ fctrl |= IXGBE_FCTRL_PMCF; /* clear the bits we are changing the status of */ fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN); /* IFF_PROMISC mode is not set in SR-IOV mode, so set here */ if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { netdev->flags |= IFF_PROMISC; /* enable hardware vlan filtering */ vlnctrl |= IXGBE_VLNCTRL_VFE; } if (netdev->flags & IFF_PROMISC) { hw->addr_ctrl.user_set_promisc = true; fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); vmolr |= IXGBE_VMOLR_MPE; } else { if (netdev->flags & IFF_ALLMULTI) { fctrl |= IXGBE_FCTRL_MPE; vmolr |= IXGBE_VMOLR_MPE; } else { /* * Write addresses to the MTA, if the attempt fails * then we should just turn on promiscous mode so * that we can at least receive multicast traffic */ count = ixgbe_write_mc_addr_list(netdev); if (count < 0) { fctrl |= IXGBE_FCTRL_MPE; vmolr |= IXGBE_VMOLR_MPE; } else if (count) { vmolr |= IXGBE_VMOLR_ROMPE; } } #ifdef NETIF_F_HW_VLAN_TX /* enable hardware vlan filtering */ vlnctrl |= IXGBE_VLNCTRL_VFE; #endif hw->addr_ctrl.user_set_promisc = false; #ifdef HAVE_SET_RX_MODE /* * Write addresses to available RAR registers, if there is not * sufficient space to store all the addresses then enable * unicast promiscous mode */ count = ixgbe_write_uc_addr_list(adapter, netdev, adapter->num_vfs); if (count < 0) { fctrl |= IXGBE_FCTRL_UPE; vmolr |= IXGBE_VMOLR_ROPE; } #endif } if (hw->mac.type != ixgbe_mac_82598EB) { vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(adapter->num_vfs)) & ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE | IXGBE_VMOLR_ROPE); IXGBE_WRITE_REG(hw, IXGBE_VMOLR(adapter->num_vfs), vmolr); } IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); } static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter) { #ifdef CONFIG_IXGBE_NAPI int q_idx; struct ixgbe_q_vector *q_vector; int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; /* legacy and MSI only use one vector */ if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) q_vectors = 1; for (q_idx = 0; q_idx < q_vectors; q_idx++) { q_vector = adapter->q_vector[q_idx]; napi_enable(&q_vector->napi); } #endif /* CONFIG_IXGBE_NAPI */ } static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter) { #ifdef CONFIG_IXGBE_NAPI int q_idx; struct ixgbe_q_vector *q_vector; int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; /* legacy and MSI only use one vector */ if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) q_vectors = 1; for (q_idx = 0; q_idx < q_vectors; q_idx++) { q_vector = adapter->q_vector[q_idx]; napi_disable(&q_vector->napi); } #endif } s32 ixgbe_dcb_hw_ets(struct ixgbe_hw *hw, struct ieee_ets *ets, int max_frame) { __u16 refill[IEEE_8021QAZ_MAX_TCS], max[IEEE_8021QAZ_MAX_TCS]; __u8 prio_type[IEEE_8021QAZ_MAX_TCS]; int i; /* naively give each TC a bwg to map onto CEE hardware */ __u8 bwg_id[IEEE_8021QAZ_MAX_TCS] = {0, 1, 2, 3, 4, 5, 6, 7}; /* Map TSA onto CEE prio type */ for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { switch (ets->tc_tsa[i]) { case IEEE_8021QAZ_TSA_STRICT: prio_type[i] = 2; break; case IEEE_8021QAZ_TSA_ETS: prio_type[i] = 0; break; default: /* Hardware only supports priority strict or * ETS transmission selection algorithms if * we receive some other value from dcbnl * throw an error */ return -EINVAL; } } ixgbe_dcb_calculate_tc_credits(ets->tc_tx_bw, refill, max, max_frame); return ixgbe_dcb_hw_config(hw, refill, max, bwg_id, prio_type, ets->prio_tc); } /* * ixgbe_configure_dcb - Configure DCB hardware * @adapter: ixgbe adapter struct * * This is called by the driver on open to configure the DCB hardware. * This is also called by the gennetlink interface when reconfiguring * the DCB state. */ static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN; if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) { if (hw->mac.type == ixgbe_mac_82598EB) netif_set_gso_max_size(adapter->netdev, 65536); return; } if (hw->mac.type == ixgbe_mac_82598EB) netif_set_gso_max_size(adapter->netdev, 32768); #ifdef IXGBE_FCOE if (adapter->netdev->features & NETIF_F_FCOE_MTU) max_frame = max_t(int, max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); #endif /* IXGBE_FCOE */ adapter->dcb_cfg.num_tcs.pg_tcs = 4; adapter->dcb_cfg.vt_mode = true; if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE) { if (adapter->ixgbe_ieee_ets) ixgbe_dcb_hw_ets(&adapter->hw, adapter->ixgbe_ieee_ets, max_frame); if (adapter->ixgbe_ieee_pfc && adapter->ixgbe_ieee_ets) { struct ieee_pfc *pfc = adapter->ixgbe_ieee_pfc; u8 *tc = adapter->ixgbe_ieee_ets->prio_tc; ixgbe_dcb_config_pfc(&adapter->hw, pfc->pfc_en, tc); } } else { ixgbe_dcb_calculate_tc_credits_cee(hw, &adapter->dcb_cfg, max_frame, IXGBE_DCB_TX_CONFIG); ixgbe_dcb_calculate_tc_credits_cee(hw, &adapter->dcb_cfg, max_frame, IXGBE_DCB_RX_CONFIG); ixgbe_dcb_hw_config_cee(hw, &adapter->dcb_cfg); } } #ifndef IXGBE_NO_LLI static void ixgbe_configure_lli_82599(struct ixgbe_adapter *adapter) { u16 port; if (adapter->lli_etype) { IXGBE_WRITE_REG(&adapter->hw, IXGBE_L34T_IMIR(0), (IXGBE_IMIR_LLI_EN_82599 | IXGBE_IMIR_SIZE_BP_82599 | IXGBE_IMIR_CTRL_BP_82599)); IXGBE_WRITE_REG(&adapter->hw, IXGBE_ETQS(0), IXGBE_ETQS_LLI); IXGBE_WRITE_REG(&adapter->hw, IXGBE_ETQF(0), (adapter->lli_etype | IXGBE_ETQF_FILTER_EN)); } if (adapter->lli_port) { port = swab16(adapter->lli_port); IXGBE_WRITE_REG(&adapter->hw, IXGBE_L34T_IMIR(0), (IXGBE_IMIR_LLI_EN_82599 | IXGBE_IMIR_SIZE_BP_82599 | IXGBE_IMIR_CTRL_BP_82599)); IXGBE_WRITE_REG(&adapter->hw, IXGBE_FTQF(0), (IXGBE_FTQF_POOL_MASK_EN | (IXGBE_FTQF_PRIORITY_MASK << IXGBE_FTQF_PRIORITY_SHIFT) | (IXGBE_FTQF_DEST_PORT_MASK << IXGBE_FTQF_5TUPLE_MASK_SHIFT))); IXGBE_WRITE_REG(&adapter->hw, IXGBE_SDPQF(0), (port << 16)); } if (adapter->flags & IXGBE_FLAG_LLI_PUSH) { switch (adapter->hw.mac.type) { case ixgbe_mac_82599EB: IXGBE_WRITE_REG(&adapter->hw, IXGBE_L34T_IMIR(0), (IXGBE_IMIR_LLI_EN_82599 | IXGBE_IMIR_SIZE_BP_82599 | IXGBE_IMIR_CTRL_PSH_82599 | IXGBE_IMIR_CTRL_SYN_82599 | IXGBE_IMIR_CTRL_URG_82599 | IXGBE_IMIR_CTRL_ACK_82599 | IXGBE_IMIR_CTRL_RST_82599 | IXGBE_IMIR_CTRL_FIN_82599)); IXGBE_WRITE_REG(&adapter->hw, IXGBE_LLITHRESH, 0xfc000000); break; case ixgbe_mac_X540: IXGBE_WRITE_REG(&adapter->hw, IXGBE_L34T_IMIR(0), (IXGBE_IMIR_LLI_EN_82599 | IXGBE_IMIR_SIZE_BP_82599 | IXGBE_IMIR_CTRL_PSH_82599)); break; default: break; } IXGBE_WRITE_REG(&adapter->hw, IXGBE_FTQF(0), (IXGBE_FTQF_POOL_MASK_EN | (IXGBE_FTQF_PRIORITY_MASK << IXGBE_FTQF_PRIORITY_SHIFT) | (IXGBE_FTQF_5TUPLE_MASK_MASK << IXGBE_FTQF_5TUPLE_MASK_SHIFT))); IXGBE_WRITE_REG(&adapter->hw, IXGBE_SYNQF, 0x80000100); } if (adapter->lli_size) { IXGBE_WRITE_REG(&adapter->hw, IXGBE_L34T_IMIR(0), (IXGBE_IMIR_LLI_EN_82599 | IXGBE_IMIR_CTRL_BP_82599)); IXGBE_WRITE_REG(&adapter->hw, IXGBE_LLITHRESH, adapter->lli_size); IXGBE_WRITE_REG(&adapter->hw, IXGBE_FTQF(0), (IXGBE_FTQF_POOL_MASK_EN | (IXGBE_FTQF_PRIORITY_MASK << IXGBE_FTQF_PRIORITY_SHIFT) | (IXGBE_FTQF_5TUPLE_MASK_MASK << IXGBE_FTQF_5TUPLE_MASK_SHIFT))); } if (adapter->lli_vlan_pri) { IXGBE_WRITE_REG(&adapter->hw, IXGBE_IMIRVP, (IXGBE_IMIRVP_PRIORITY_EN | adapter->lli_vlan_pri)); } } static void ixgbe_configure_lli(struct ixgbe_adapter *adapter) { u16 port; /* lli should only be enabled with MSI-X and MSI */ if (!(adapter->flags & IXGBE_FLAG_MSI_ENABLED) && !(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) return; if (adapter->hw.mac.type != ixgbe_mac_82598EB) { ixgbe_configure_lli_82599(adapter); return; } if (adapter->lli_port) { /* use filter 0 for port */ port = swab16(adapter->lli_port); IXGBE_WRITE_REG(&adapter->hw, IXGBE_IMIR(0), (port | IXGBE_IMIR_PORT_IM_EN)); IXGBE_WRITE_REG(&adapter->hw, IXGBE_IMIREXT(0), (IXGBE_IMIREXT_SIZE_BP | IXGBE_IMIREXT_CTRL_BP)); } if (adapter->flags & IXGBE_FLAG_LLI_PUSH) { /* use filter 1 for push flag */ IXGBE_WRITE_REG(&adapter->hw, IXGBE_IMIR(1), (IXGBE_IMIR_PORT_BP | IXGBE_IMIR_PORT_IM_EN)); IXGBE_WRITE_REG(&adapter->hw, IXGBE_IMIREXT(1), (IXGBE_IMIREXT_SIZE_BP | IXGBE_IMIREXT_CTRL_PSH)); } if (adapter->lli_size) { /* use filter 2 for size */ IXGBE_WRITE_REG(&adapter->hw, IXGBE_IMIR(2), (IXGBE_IMIR_PORT_BP | IXGBE_IMIR_PORT_IM_EN)); IXGBE_WRITE_REG(&adapter->hw, IXGBE_IMIREXT(2), (adapter->lli_size | IXGBE_IMIREXT_CTRL_BP)); } } #endif /* IXGBE_NO_LLI */ /* Additional bittime to account for IXGBE framing */ #define IXGBE_ETH_FRAMING 20 /* * ixgbe_hpbthresh - calculate high water mark for flow control * * @adapter: board private structure to calculate for * @pb - packet buffer to calculate */ static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) { struct ixgbe_hw *hw = &adapter->hw; struct net_device *dev = adapter->netdev; int link, tc, kb, marker; u32 dv_id, rx_pba; /* Calculate max LAN frame size */ tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING; #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ if (dev->features & NETIF_F_FCOE_MTU) { int fcoe_pb = 0; fcoe_pb = netdev_get_prio_tc_map(dev, adapter->fcoe.up); if (fcoe_pb == pb && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; } #endif /* Calculate delay value for device */ switch (hw->mac.type) { case ixgbe_mac_X540: dv_id = IXGBE_DV_X540(link, tc); break; default: dv_id = IXGBE_DV(link, tc); break; } /* Loopback switch introduces additional latency */ if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) dv_id += IXGBE_B2BT(tc); /* Delay value is calculated in bit times convert to KB */ kb = IXGBE_BT2KB(dv_id); rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10; marker = rx_pba - kb; /* It is possible that the packet buffer is not large enough * to provide required headroom. In this case throw an error * to user and a do the best we can. */ if (marker < 0) { e_warn(drv, "Packet Buffer(%i) can not provide enough" "headroom to suppport flow control." "Decrease MTU or number of traffic classes\n", pb); marker = tc + 1; } return marker; } /* * ixgbe_lpbthresh - calculate low water mark for for flow control * * @adapter: board private structure to calculate for * @pb - packet buffer to calculate */ static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; struct net_device *dev = adapter->netdev; int tc; u32 dv_id; /* Calculate max LAN frame size */ tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN; /* Calculate delay value for device */ switch (hw->mac.type) { case ixgbe_mac_X540: dv_id = IXGBE_LOW_DV_X540(tc); break; default: dv_id = IXGBE_LOW_DV(tc); break; } /* Delay value is calculated in bit times convert to KB */ return IXGBE_BT2KB(dv_id); } /* * ixgbe_pbthresh_setup - calculate and setup high low water marks */ static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int num_tc = netdev_get_num_tc(adapter->netdev); int i; if (!num_tc) num_tc = 1; hw->fc.low_water = ixgbe_lpbthresh(adapter); for (i = 0; i < num_tc; i++) { hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i); /* Low water marks must not be larger than high water marks */ if (hw->fc.low_water > hw->fc.high_water[i]) hw->fc.low_water = 0; } } static void ixgbe_configure_pb(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int hdrm; u8 tc = netdev_get_num_tc(adapter->netdev); if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE || adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) hdrm = 32 << adapter->fdir_pballoc; else hdrm = 0; hw->mac.ops.setup_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL); ixgbe_pbthresh_setup(adapter); } static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; struct hlist_node *node, *node2; struct ixgbe_fdir_filter *filter; spin_lock(&adapter->fdir_perfect_lock); if (!hlist_empty(&adapter->fdir_filter_list)) ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask); hlist_for_each_entry_safe(filter, node, node2, &adapter->fdir_filter_list, fdir_node) { ixgbe_fdir_write_perfect_filter_82599(hw, &filter->filter, filter->sw_idx, (filter->action == IXGBE_FDIR_DROP_QUEUE) ? IXGBE_FDIR_DROP_QUEUE : adapter->rx_ring[filter->action]->reg_idx); } spin_unlock(&adapter->fdir_perfect_lock); } static void ixgbe_configure(struct ixgbe_adapter *adapter) { ixgbe_configure_pb(adapter); ixgbe_configure_dcb(adapter); ixgbe_set_rx_mode(adapter->netdev); #ifdef NETIF_F_HW_VLAN_TX ixgbe_restore_vlan(adapter); #endif #ifdef IXGBE_FCOE if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) ixgbe_configure_fcoe(adapter); #endif /* IXGBE_FCOE */ if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { ixgbe_init_fdir_signature_82599(&adapter->hw, adapter->fdir_pballoc); } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) { ixgbe_init_fdir_perfect_82599(&adapter->hw, adapter->fdir_pballoc); ixgbe_fdir_filter_restore(adapter); } ixgbe_configure_virtualization(adapter); ixgbe_configure_tx(adapter); ixgbe_configure_rx(adapter); } static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw) { switch (hw->phy.type) { case ixgbe_phy_sfp_avago: case ixgbe_phy_sfp_ftl: case ixgbe_phy_sfp_intel: case ixgbe_phy_sfp_unknown: case ixgbe_phy_sfp_passive_tyco: case ixgbe_phy_sfp_passive_unknown: case ixgbe_phy_sfp_active_unknown: case ixgbe_phy_sfp_ftl_active: return true; case ixgbe_phy_nl: if (hw->mac.type == ixgbe_mac_82598EB) return true; default: return false; } } /** * ixgbe_sfp_link_config - set up SFP+ link * @adapter: pointer to private adapter struct **/ static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter) { /* * We are assuming the worst case scenerio here, and that * is that an SFP was inserted/removed after the reset * but before SFP detection was enabled. As such the best * solution is to just start searching as soon as we start */ if (adapter->hw.mac.type == ixgbe_mac_82598EB) adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP; adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; } /** * ixgbe_non_sfp_link_config - set up non-SFP+ link * @hw: pointer to private hardware struct * * Returns 0 on success, negative on failure **/ static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw) { u32 autoneg; bool negotiation, link_up = false; u32 ret = IXGBE_ERR_LINK_SETUP; if (hw->mac.ops.check_link) ret = hw->mac.ops.check_link(hw, &autoneg, &link_up, false); if (ret) goto link_cfg_out; autoneg = hw->phy.autoneg_advertised; if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) ret = hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation); if (ret) goto link_cfg_out; if (hw->mac.ops.setup_link) ret = hw->mac.ops.setup_link(hw, autoneg, negotiation, link_up); link_cfg_out: return ret; } /** * ixgbe_clear_vf_stats_counters - Clear out VF stats after reset * @adapter: board private structure * * On a reset we need to clear out the VF stats or accounting gets * messed up because they're not clear on read. **/ void ixgbe_clear_vf_stats_counters(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int i; for(i = 0; i < adapter->num_vfs; i++) { adapter->vfinfo[i].last_vfstats.gprc = IXGBE_READ_REG(hw, IXGBE_PVFGPRC(i)); adapter->vfinfo[i].saved_rst_vfstats.gprc += adapter->vfinfo[i].vfstats.gprc; adapter->vfinfo[i].vfstats.gprc = 0; adapter->vfinfo[i].last_vfstats.gptc = IXGBE_READ_REG(hw, IXGBE_PVFGPTC(i)); adapter->vfinfo[i].saved_rst_vfstats.gptc += adapter->vfinfo[i].vfstats.gptc; adapter->vfinfo[i].vfstats.gptc = 0; adapter->vfinfo[i].last_vfstats.gorc = IXGBE_READ_REG(hw, IXGBE_PVFGORC_LSB(i)); adapter->vfinfo[i].saved_rst_vfstats.gorc += adapter->vfinfo[i].vfstats.gorc; adapter->vfinfo[i].vfstats.gorc = 0; adapter->vfinfo[i].last_vfstats.gotc = IXGBE_READ_REG(hw, IXGBE_PVFGOTC_LSB(i)); adapter->vfinfo[i].saved_rst_vfstats.gotc += adapter->vfinfo[i].vfstats.gotc; adapter->vfinfo[i].vfstats.gotc = 0; adapter->vfinfo[i].last_vfstats.mprc = IXGBE_READ_REG(hw, IXGBE_PVFMPRC(i)); adapter->vfinfo[i].saved_rst_vfstats.mprc += adapter->vfinfo[i].vfstats.mprc; adapter->vfinfo[i].vfstats.mprc = 0; } } static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 gpie = 0; if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD; #ifdef CONFIG_IXGBE_NAPI #ifndef __VMKLNX__ gpie |= IXGBE_GPIE_EIAME; /* * use EIAM to auto-mask when MSI-X interrupt is asserted * this saves a register write for every interrupt */ switch (hw->mac.type) { case ixgbe_mac_82598EB: IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: default: IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); break; } } else { /* legacy interrupts, use EIAM to auto-mask when reading EICR, * specifically only auto mask tx and rx interrupts */ IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); #endif /* __VMKLNX__ */ #endif } /* XXX: to interrupt immediately for EICS writes, enable this */ /* gpie |= IXGBE_GPIE_EIMEN; */ if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { gpie &= ~IXGBE_GPIE_VTMODE_MASK; if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) gpie |= IXGBE_GPIE_VTMODE_32; else if (adapter->flags & IXGBE_FLAG_RSS_ENABLED && adapter->ring_feature[RING_F_RSS].indices == 4) gpie |= IXGBE_GPIE_VTMODE_32; else gpie |= IXGBE_GPIE_VTMODE_64; } /* Enable Thermal over heat sensor interrupt */ if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) switch (adapter->hw.mac.type) { case ixgbe_mac_82599EB: gpie |= IXGBE_SDP0_GPIEN; break; default: break; } /* Enable fan failure interrupt */ if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) gpie |= IXGBE_SDP1_GPIEN; if (hw->mac.type == ixgbe_mac_82599EB) { gpie |= IXGBE_SDP1_GPIEN; gpie |= IXGBE_SDP2_GPIEN; } IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); #ifdef IXGBE_TCP_TIMER if ((adapter->flags & IXGBE_FLAG_MSIX_ENABLED) || (adapter->flags & IXGBE_FLAG_MSI_ENABLED)) { u32 tcp_timer = IXGBE_READ_REG(hw, IXGBE_TCPTIMER); tcp_timer |= IXGBE_TCPTIMER_DURATION_MASK; tcp_timer |= (IXGBE_TCPTIMER_KS | IXGBE_TCPTIMER_COUNT_ENABLE | IXGBE_TCPTIMER_LOOP); IXGBE_WRITE_REG(hw, IXGBE_TCPTIMER, tcp_timer); tcp_timer = IXGBE_READ_REG(hw, IXGBE_TCPTIMER); } #endif } static void ixgbe_up_complete(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int err; u32 ctrl_ext; ixgbe_get_hw_control(adapter); ixgbe_setup_gpie(adapter); if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) ixgbe_configure_msix(adapter); else ixgbe_configure_msi_and_legacy(adapter); /* enable the optics */ if ((hw->phy.multispeed_fiber) || ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) && (hw->mac.type == ixgbe_mac_82599EB))) ixgbe_enable_tx_laser(hw); clear_bit(__IXGBE_DOWN, &adapter->state); ixgbe_napi_enable_all(adapter); #ifndef IXGBE_NO_LLI ixgbe_configure_lli(adapter); #endif if (ixgbe_is_sfp(hw)) { ixgbe_sfp_link_config(adapter); } else { err = ixgbe_non_sfp_link_config(hw); if (err) e_err(probe, "link_config FAILED %d\n", err); } /* clear any pending interrupts, may auto mask */ IXGBE_READ_REG(hw, IXGBE_EICR); ixgbe_irq_enable(adapter, true, true); /* * If this adapter has a fan, check to see if we had a failure * before we enabled the interrupt. */ if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) { u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); if (esdp & IXGBE_ESDP_SDP1) e_crit(drv, "Fan has stopped, replace the adapter\n"); } #ifndef __VMKLNX__ /* enable transmits */ netif_tx_start_all_queues(adapter->netdev); #endif /* __VMKLNX__ */ /* bring the link up in the watchdog, this could race with our first * link up interrupt but shouldn't be a problem */ adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; adapter->link_check_timeout = jiffies; mod_timer(&adapter->service_timer, jiffies); ixgbe_clear_vf_stats_counters(adapter); /* Set PF Reset Done bit so PF/VF Mail Ops can work */ ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); } void ixgbe_reinit_locked(struct ixgbe_adapter *adapter) { WARN_ON(in_interrupt()); /* put off any impending NetWatchDogTimeout */ adapter->netdev->trans_start = jiffies; while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state)) usleep_range(1000, 2000); ixgbe_down(adapter); /* * If SR-IOV enabled then wait a bit before bringing the adapter * back up to give the VFs time to respond to the reset. The * two second wait is based upon the watchdog timer cycle in * the VF driver. */ if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) msleep(2000); ixgbe_up(adapter); clear_bit(__IXGBE_RESETTING, &adapter->state); } void ixgbe_up(struct ixgbe_adapter *adapter) { /* hardware has been reset, we need to reload some things */ ixgbe_configure(adapter); ixgbe_up_complete(adapter); #ifdef __VMKLNX__ /* Invalidate netqueue state as filters were lost on reinit */ vmknetddi_queueops_invalidate_state(adapter->netdev); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) vmknetddi_queueops_invalidate_state(adapter->cnadev); #endif } void ixgbe_reset(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; struct net_device *netdev = adapter->netdev; int err; /* lock SFP init bit to prevent race conditions with the watchdog */ while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) usleep_range(1000, 2000); /* clear all SFP and link config related flags while holding SFP_INIT */ adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP | IXGBE_FLAG2_SFP_NEEDS_RESET); adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; err = hw->mac.ops.init_hw(hw); switch (err) { case 0: case IXGBE_ERR_SFP_NOT_PRESENT: case IXGBE_ERR_SFP_NOT_SUPPORTED: break; case IXGBE_ERR_MASTER_REQUESTS_PENDING: e_dev_err("master disable timed out\n"); break; case IXGBE_ERR_EEPROM_VERSION: /* We are running on a pre-production device, log a warning */ e_dev_warn("This device is a pre-production adapter/LOM. " "Please be aware there may be issues associated " "with your hardware. If you are experiencing " "problems please contact your Intel or hardware " "representative who provided you with this " "hardware.\n"); break; default: e_dev_err("Hardware Error: %d\n", err); } clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); #ifdef __VMKLNX__ ixgbe_flush_sw_mac_table(adapter); memcpy(&adapter->mac_table[0].addr, hw->mac.perm_addr, netdev->addr_len); adapter->mac_table[0].queue = adapter->num_vfs; adapter->mac_table[0].state = (IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE); hw->mac.ops.set_rar(hw, 0, adapter->mac_table[0].addr, adapter->mac_table[0].queue, IXGBE_RAH_AV); #else ixgbe_full_sync_mac_table(adapter); #endif } /** * ixgbe_clean_rx_ring - Free Rx Buffers per Queue * @rx_ring: ring to free buffers from **/ void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) { struct device *dev = rx_ring->dev; unsigned long size; u16 i; /* ring already cleared, nothing to do */ if (!rx_ring->rx_buffer_info) return; /* Free all the Rx ring sk_buffs */ for (i = 0; i < rx_ring->count; i++) { struct ixgbe_rx_buffer *rx_buffer_info; rx_buffer_info = &rx_ring->rx_buffer_info[i]; if (rx_buffer_info->dma) { if (rx_ring->rx_buf_len == 0) dma_unmap_page(dev, rx_buffer_info->dma ^ rx_buffer_info->page_dma, PAGE_SIZE / 2, DMA_FROM_DEVICE); else dma_unmap_single(dev, rx_buffer_info->dma, rx_ring->rx_buf_len, DMA_FROM_DEVICE); rx_buffer_info->dma = 0; } if (rx_buffer_info->skb) { struct sk_buff *skb = rx_buffer_info->skb; rx_buffer_info->skb = NULL; /* We need to clean up RSC frag lists */ skb = ixgbe_merge_active_tail(skb); ixgbe_close_active_frag_list(skb); if (IXGBE_CB(skb)->delay_unmap) { if (rx_ring->rx_buf_len == 0) dma_unmap_page(dev, IXGBE_CB(skb)->dma, PAGE_SIZE / 2, DMA_FROM_DEVICE); else dma_unmap_single(dev, IXGBE_CB(skb)->dma, rx_ring->rx_buf_len, DMA_FROM_DEVICE); IXGBE_CB(skb)->dma = 0; IXGBE_CB(skb)->delay_unmap = false; } dev_kfree_skb(skb); } if (!rx_buffer_info->page) continue; if (rx_buffer_info->page_dma) { dma_unmap_page(dev, rx_buffer_info->page_dma, #ifndef __VMKLNX__ PAGE_SIZE / 2, #else PAGE_SIZE, #endif /* __VMKLNX__ */ DMA_FROM_DEVICE); rx_buffer_info->page_dma = 0; } put_page(rx_buffer_info->page); rx_buffer_info->page = NULL; rx_buffer_info->page_offset = 0; } size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; memset(rx_ring->rx_buffer_info, 0, size); /* Zero out the descriptor ring */ memset(rx_ring->desc, 0, rx_ring->size); rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; } /** * ixgbe_clean_tx_ring - Free Tx Buffers * @tx_ring: ring to be cleaned **/ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) { struct ixgbe_tx_buffer *tx_buffer_info; unsigned long size; u16 i; /* ring already cleared, nothing to do */ if (!tx_ring->tx_buffer_info) return; /* Free all the Tx ring sk_buffs */ for (i = 0; i < tx_ring->count; i++) { tx_buffer_info = &tx_ring->tx_buffer_info[i]; ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); } size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; memset(tx_ring->tx_buffer_info, 0, size); /* Zero out the descriptor ring */ memset(tx_ring->desc, 0, tx_ring->size); tx_ring->next_to_use = 0; tx_ring->next_to_clean = 0; } /** * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues * @adapter: board private structure **/ static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter) { int i; for (i = 0; i < adapter->num_rx_queues; i++) ixgbe_clean_rx_ring(adapter->rx_ring[i]); } /** * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues * @adapter: board private structure **/ static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter) { int i; for (i = 0; i < adapter->num_tx_queues; i++) ixgbe_clean_tx_ring(adapter->tx_ring[i]); } static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter) { struct hlist_node *node, *node2; struct ixgbe_fdir_filter *filter; spin_lock(&adapter->fdir_perfect_lock); hlist_for_each_entry_safe(filter, node, node2, &adapter->fdir_filter_list, fdir_node) { hlist_del(&filter->fdir_node); kfree(filter); } adapter->fdir_filter_count = 0; spin_unlock(&adapter->fdir_perfect_lock); } void ixgbe_down(struct ixgbe_adapter *adapter) { struct net_device *netdev = adapter->netdev; struct ixgbe_hw *hw = &adapter->hw; u32 rxctrl; int i; /* signal that we are down to the interrupt handler */ set_bit(__IXGBE_DOWN, &adapter->state); /* disable receives */ rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); /* disable all enabled rx queues */ for (i = 0; i < adapter->num_rx_queues; i++) /* this call also flushes the previous write */ ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]); usleep_range(10000, 20000); netif_tx_stop_all_queues(netdev); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) netif_tx_stop_all_queues(adapter->cnadev); /* call carrier off first to avoid false dev_watchdog timeouts */ netif_carrier_off(netdev); netif_tx_disable(netdev); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) { netif_carrier_off(adapter->cnadev); netif_tx_disable(adapter->cnadev); } ixgbe_irq_disable(adapter); ixgbe_napi_disable_all(adapter); adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT | IXGBE_FLAG2_RESET_REQUESTED); adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; del_timer_sync(&adapter->service_timer); if (adapter->num_vfs) { /* Clear EITR Select mapping */ IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0); /* Mark all the VFs as inactive */ for (i = 0 ; i < adapter->num_vfs; i++) adapter->vfinfo[i].clear_to_send = 0; /* ping all the active vfs to let them know we are going down */ ixgbe_ping_all_vfs(adapter); /* Disable all VFTE/VFRE TX/RX */ ixgbe_disable_tx_rx(adapter); } /* disable transmits in the hardware now that interrupts are off */ for (i = 0; i < adapter->num_tx_queues; i++) { u8 reg_idx = adapter->tx_ring[i]->reg_idx; IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); } /* Disable the Tx DMA engine on 82599 and X540 */ switch (hw->mac.type) { case ixgbe_mac_82599EB: case ixgbe_mac_X540: IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) & ~IXGBE_DMATXCTL_TE)); break; default: break; } #ifdef __VMKLNX__ ixgbe_reset(adapter); #else #ifdef HAVE_PCI_ERS if (!pci_channel_offline(adapter->pdev)) #endif ixgbe_reset(adapter); #endif /* power down the optics */ if ((hw->phy.multispeed_fiber) || ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) && (hw->mac.type == ixgbe_mac_82599EB))) ixgbe_disable_tx_laser(hw); ixgbe_clean_all_tx_rings(adapter); ixgbe_clean_all_rx_rings(adapter); /* since we reset the hardware DCA settings were cleared */ ixgbe_setup_dca(adapter); } /* Artificial max queue cap per traffic class in DCB mode */ #define DCB_QUEUE_CAP 8 /** * ixgbe_set_dcb_queues: Allocate queues for a DCB-enabled device * @adapter: board private structure to initialize * * When DCB (Data Center Bridging) is enabled, allocate queues for * each traffic class. If multiqueue isn't availabe, then abort DCB * initialization. * **/ static inline bool ixgbe_set_dcb_queues(struct ixgbe_adapter *adapter) { int tcs; #ifdef HAVE_MQPRIO int per_tc_q, q, i, offset = 0; struct net_device *dev = adapter->netdev; /* Map queue offset and counts onto allocated tx queues */ tcs = netdev_get_num_tc(dev); if (!tcs) return false; per_tc_q = min(dev->num_tx_queues / tcs, (unsigned int)DCB_QUEUE_CAP); q = min((int)num_online_cpus(), per_tc_q); for (i = 0; i < tcs; i++) { netdev_set_tc_queue(dev, i, q, offset); offset += q; } adapter->num_tx_queues = q * tcs; adapter->num_rx_queues = q * tcs; #ifdef IXGBE_FCOE /* FCoE enabled queues require special configuration indexed * by feature specific indices and mask. Here we map FCoE * indices onto the DCB queue pairs allowing FCoE to own * configuration later. */ if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) { int tc; struct ixgbe_ring_feature *fcoe = &adapter->ring_feature[RING_F_FCOE]; struct netdev_tc_txq *tc_to_txq; tc = netdev_get_prio_tc_map(dev, adapter->fcoe.up); tc_to_txq = ixgbe_get_netdev_tc_txq(dev, tc); fcoe->indices = tc_to_txq->count; fcoe->mask = tc_to_txq->offset; } #endif /* IXGBE_FCOE */ #else if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) return false; /* Enable one Queue per traffic class */ tcs = adapter->tc; if (!tcs) return false; adapter->num_rx_queues = tcs; adapter->num_tx_queues = tcs; #ifdef IXGBE_FCOE if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) { struct ixgbe_ring_feature *fcoe = &adapter->ring_feature[RING_F_FCOE]; fcoe->mask = tcs; fcoe->indices = min((int)num_online_cpus(), fcoe->indices); adapter->num_rx_queues += fcoe->indices; adapter->num_tx_queues += fcoe->indices; } #endif /* IXGBE_FCOE */ #endif /* HAVE_MQ */ return true; } #ifdef IXGBE_VMDQ /** * ixgbe_set_vmdq_queues: Allocate queues for VMDq devices * @adapter: board private structure to initialize * * When VMDq (Virtual Machine Devices queue) is enabled, allocate queues * and VM pools where appropriate. If RSS is available, then also try and * enable RSS and map accordingly. * **/ static inline bool ixgbe_set_vmdq_queues(struct ixgbe_adapter *adapter) { int vmdq_i = adapter->ring_feature[RING_F_VMDQ].indices; int vmdq_m = 0; int rss_i = adapter->ring_feature[RING_F_RSS].indices; int fcoe_i = adapter->ring_feature[RING_F_FCOE].indices; unsigned long i; int rss_shift; bool ret = false; switch (adapter->flags & (IXGBE_FLAG_RSS_ENABLED | IXGBE_FLAG_DCB_ENABLED | IXGBE_FLAG_VMDQ_ENABLED)) { case (IXGBE_FLAG_RSS_ENABLED | IXGBE_FLAG_VMDQ_ENABLED): switch (adapter->hw.mac.type) { case ixgbe_mac_82599EB: case ixgbe_mac_X540: vmdq_i = min(IXGBE_MAX_VMDQ_INDICES, vmdq_i); if (vmdq_i > 32) rss_i = 2; else rss_i = 4; i = rss_i; rss_shift = find_first_bit(&i, sizeof(i) * 8); vmdq_m = ((IXGBE_MAX_VMDQ_INDICES - 1) << rss_shift) & (MAX_RX_QUEUES - 1); break; default: break; } #ifndef __VMKLNX__ adapter->num_rx_queues = vmdq_i * rss_i; adapter->num_tx_queues = min(MAX_TX_QUEUES, vmdq_i * rss_i); #endif ret = true; break; case (IXGBE_FLAG_DCB_ENABLED): if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) { /* Enable VMDq flag so device will be set in VM mode */ adapter->flags |= IXGBE_FLAG_VMDQ_ENABLED; vmdq_i = 1; } /* FALLTHROUGH */ case (IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_DCB_ENABLED): if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) vmdq_i += fcoe_i; vmdq_m = (IXGBE_MAX_VMDQ_INDICES - 1) << 1; adapter->num_rx_queues = vmdq_i; adapter->num_tx_queues = vmdq_i; ret = true; break; case (IXGBE_FLAG_VMDQ_ENABLED): switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: vmdq_m = (IXGBE_MAX_VMDQ_INDICES - 1); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: vmdq_m = (IXGBE_MAX_VMDQ_INDICES - 1) << 1; break; default: break; } adapter->num_rx_queues = vmdq_i; adapter->num_tx_queues = vmdq_i; ret = true; break; case (IXGBE_FLAG_RSS_ENABLED): /* RSS enabled, no VMDQ. possibly SRIOV enabled */ adapter->num_rx_queues = min(MAX_RX_QUEUES, 1 + rss_i); /* default queue + RSS */ adapter->num_tx_queues = min(MAX_TX_QUEUES, 2); /* default + 1 for the RSS pool */ adapter->num_rx_pools = 2; /* default + RSS */ adapter->num_rx_queues_per_pool = 1; ret = true; goto vmdq_queues_out; default: ret = false; goto vmdq_queues_out; } /* * See VMware PR 295512: For Jumbo Frame, we limit number of netqueues * to 4 to support multiple ports with limited netPktHeap */ if ((adapter->netdev->mtu > ETH_DATA_LEN) && (adapter->num_rx_queues > 4)) { adapter->num_rx_queues = min(4, adapter->num_rx_queues); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) adapter->num_rx_queues += fcoe_i; vmdq_i = adapter->num_rx_queues; /* * Do the same for tx queues as well. */ adapter->num_tx_queues = min(4, adapter->num_tx_queues); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) adapter->num_tx_queues += fcoe_i; } if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) { adapter->num_rx_queues = min(MAX_RX_QUEUES, vmdq_i + rss_i); adapter->num_tx_queues = min(MAX_TX_QUEUES, vmdq_i + 1); /* +1 for the RSS pool */ vmdq_i++; /* Extra pool for RSS queues */ } if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) { adapter->num_rx_pools = vmdq_i; adapter->num_rx_queues_per_pool = adapter->num_rx_queues / vmdq_i; if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) { /* we use 1 queue per pool. This cannot be calculated * by regular division */ adapter->num_rx_queues_per_pool = 1; } } else { adapter->num_rx_pools = adapter->num_rx_queues; adapter->num_rx_queues_per_pool = 1; } /* save the mask for later use */ adapter->ring_feature[RING_F_VMDQ].mask = vmdq_m; /* * Make sure we've got the queues to give to CNA without * giving away the default queue. */ if ((adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) && (adapter->num_rx_queues > fcoe_i)) { adapter->ring_feature[RING_F_FCOE].mask = adapter->num_rx_queues - fcoe_i; } else { if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) DPRINTK(PROBE, INFO, "Disabling CNA: " "not enough VMDq queues available\n"); adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; adapter->flags2 &= ~IXGBE_FLAG2_CNA_ENABLED; adapter->ring_feature[RING_F_FCOE].indices = 0; adapter->ring_feature[RING_F_FCOE].mask = adapter->num_rx_queues; } vmdq_queues_out: return ret; } #endif /* IXGBE_VMDQ */ /** * ixgbe_set_rss_queues: Allocate queues for RSS * @adapter: board private structure to initialize * * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU. * **/ static inline bool ixgbe_set_rss_queues(struct ixgbe_adapter *adapter) { bool ret = false; struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_RSS]; if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) { f->mask = 0xF; adapter->num_rx_queues = f->indices; #ifdef HAVE_TX_MQ adapter->num_tx_queues = f->indices; #endif ret = true; } return ret; } /** * ixgbe_set_fdir_queues: Allocate queues for Flow Director * @adapter: board private structure to initialize * * Flow Director is an advanced Rx filter, attempting to get Rx flows back * to the original CPU that initiated the Tx session. This runs in addition * to RSS, so if a packet doesn't match an FDIR filter, we can still spread the * Rx load across CPUs using RSS. * **/ static inline bool ixgbe_set_fdir_queues(struct ixgbe_adapter *adapter) { bool ret = false; struct ixgbe_ring_feature *f_fdir = &adapter->ring_feature[RING_F_FDIR]; f_fdir->indices = min((int)num_online_cpus(), f_fdir->indices); f_fdir->mask = 0; /* * Use RSS in addition to Flow Director to ensure the best * distribution of flows across cores, even when an FDIR flow * isn't matched. */ if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) && (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) { #ifdef HAVE_TX_MQ adapter->num_tx_queues = f_fdir->indices; #endif adapter->num_rx_queues = f_fdir->indices; ret = true; } else { adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; } return ret; } #ifdef IXGBE_FCOE /** * ixgbe_set_fcoe_queues: Allocate queues for Fiber Channel over Ethernet (FCoE) * @adapter: board private structure to initialize * * FCoE RX FCRETA can use up to 8 rx queues for up to 8 different exchanges. * The ring feature mask is not used as a mask for FCoE, as it can take any 8 * rx queues out of the max number of rx queues, instead, it is used as the * index of the first rx queue. * **/ static inline bool ixgbe_set_fcoe_queues(struct ixgbe_adapter *adapter) { bool ret = false; struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE]; f->indices = min((int)num_online_cpus(), f->indices); if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) { adapter->num_rx_queues = 1; adapter->num_tx_queues = 1; if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) { ixgbe_set_dcb_queues(adapter); } if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) { if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) ixgbe_set_fdir_queues(adapter); else ixgbe_set_rss_queues(adapter); } /* adding FCoE queues */ f->mask = adapter->num_rx_queues; adapter->num_rx_queues += f->indices; adapter->num_tx_queues += f->indices; ret = true; } return ret; } #endif /* IXGBE_FCOE */ /** * ixgbe_set_sriov_queues: Allocate queues for IOV use * @adapter: board private structure to initialize * * IOV doesn't actually use anything, so just NAK the * request for now and let the other queue routines * figure out what to do. */ static inline bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter) { return false; } /* * ixgbe_set_num_queues: Allocate queues for device, feature dependant * @adapter: board private structure to initialize * * This is the top level queue allocation routine. The order here is very * important, starting with the "most" number of features turned on at once, * and ending with the smallest set of features. This way large combinations * can be allocated if they're turned on, and smaller combinations are the * fallthrough conditions. * **/ static void ixgbe_set_num_queues(struct ixgbe_adapter *adapter) { /* Start with base case */ adapter->num_rx_queues = 1; adapter->num_tx_queues = 1; adapter->num_rx_pools = adapter->num_rx_queues; adapter->num_rx_queues_per_pool = 1; if (ixgbe_set_sriov_queues(adapter)) return; #ifdef IXGBE_VMDQ if (ixgbe_set_vmdq_queues(adapter)) return; #endif /* IXGBE_VMDQ */ if (ixgbe_set_dcb_queues(adapter)) return; #ifdef IXGBE_FCOE if (ixgbe_set_fcoe_queues(adapter)) return; #endif /* IXGBE_FCOE */ if (ixgbe_set_fdir_queues(adapter)) return; if (ixgbe_set_rss_queues(adapter)) return; } static void ixgbe_acquire_msix_vectors(struct ixgbe_adapter *adapter, int vectors) { int err, vector_threshold; /* * We'll want at least 2 (vector_threshold): * 1) TxQ[0] + RxQ[0] handler * 2) Other (Link Status Change, etc.) */ #ifdef __VMKLNX__ if(!(adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED)) { /* * MIN_MSIX_COUNT includes 2 vectors * for FCoE, but we won't need them * if CNA is disabled */ vector_threshold = MIN_MSIX_COUNT - MIN_MSIX_Q_FCOE_VECTORS; } else #endif /* __VMKLNX__ */ vector_threshold = MIN_MSIX_COUNT; /* * The more we get, the more we will assign to Tx/Rx Cleanup * for the separate queues...where Rx Cleanup >= Tx Cleanup. * Right now, we simply care about how many we'll get; we'll * set them up later while requesting irq's. */ while (vectors >= vector_threshold) { err = pci_enable_msix(adapter->pdev, adapter->msix_entries, vectors); if (!err) /* Success in acquiring all requested vectors. */ break; else if (err < 0) vectors = 0; /* Nasty failure, quit now */ else vectors = 0; } if (vectors < vector_threshold) { /* Can't allocate enough MSI-X interrupts? Oh well. * This just means we'll go with either a single MSI * vector or fall back to legacy interrupts. */ e_warn(hw, "Unable to allocate MSI-X interrupts\n"); adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED; kfree(adapter->msix_entries); adapter->msix_entries = NULL; } else { adapter->flags |= IXGBE_FLAG_MSIX_ENABLED; /* Woot! */ /* * Adjust for only the vectors we'll use, which is minimum * of max_msix_q_vectors + NON_Q_VECTORS, or the number of * vectors we were allocated. */ adapter->num_msix_vectors = min(vectors, adapter->max_msix_q_vectors + NON_Q_VECTORS); } } /** * ixgbe_cache_ring_rss - Descriptor ring to register mapping for RSS * @adapter: board private structure to initialize * * Cache the descriptor ring offsets for RSS to the assigned rings. * **/ static inline bool ixgbe_cache_ring_rss(struct ixgbe_adapter *adapter) { int i; if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED)) return false; for (i = 0; i < adapter->num_rx_queues; i++) adapter->rx_ring[i]->reg_idx = i; for (i = 0; i < adapter->num_tx_queues; i++) adapter->tx_ring[i]->reg_idx = i; return true; } /* ixgbe_get_first_reg_idx - Return first register index associated with ring */ static void ixgbe_get_first_reg_idx(struct ixgbe_adapter *adapter, u8 tc, unsigned int *tx, unsigned int *rx) { struct ixgbe_hw *hw = &adapter->hw; struct net_device *dev = adapter->netdev; u8 num_tcs = netdev_get_num_tc(dev); *tx = 0; *rx = 0; switch (hw->mac.type) { case ixgbe_mac_82598EB: *tx = tc << 2; *rx = tc << 3; break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: if (num_tcs > 4) { if (tc < 3) { *tx = tc << 5; *rx = tc << 4; } else if (tc < 5) { *tx = ((tc + 2) << 4); *rx = tc << 4; } else if (tc < num_tcs) { *tx = ((tc + 8) << 3); *rx = tc << 4; } } else { *rx = tc << 5; switch (tc) { case 0: *tx = 0; break; case 1: *tx = 64; break; case 2: *tx = 96; break; case 3: *tx = 112; break; default: break; } } break; default: break; } } /** * ixgbe_cache_ring_dcb - Descriptor ring to register mapping for DCB * @adapter: board private structure to initialize * * Cache the descriptor ring offsets for DCB to the assigned rings. * **/ static inline bool ixgbe_cache_ring_dcb(struct ixgbe_adapter *adapter) { int i, j, k; struct net_device *dev = adapter->netdev; u8 num_tcs = netdev_get_num_tc(dev); if (!num_tcs) return false; for (i = 0, k = 0; i < num_tcs; i++) { unsigned int tx_s, rx_s; u16 count; #ifdef HAVE_MQPRIO struct netdev_tc_txq *tc_to_txq = ixgbe_get_netdev_tc_txq(dev, i); count = tc_to_txq->count; #else count = 1; #endif ixgbe_get_first_reg_idx(adapter, i, &tx_s, &rx_s); for (j = 0; j < count; j++, k++) { adapter->tx_ring[k]->reg_idx = tx_s + j; adapter->rx_ring[k]->reg_idx = rx_s + j; adapter->tx_ring[k]->dcb_tc = i; adapter->rx_ring[k]->dcb_tc = i; } } return true; } #ifdef IXGBE_VMDQ /** * ixgbe_cache_ring_vmdq - Descriptor ring to register mapping for VMDq * @adapter: board private structure to initialize * * Cache the descriptor ring offsets for VMDq to the assigned rings. It * will also try to cache the proper offsets if RSS/DCB/CNA/SRIOV are all * enabled along with VMDq. * **/ static inline bool ixgbe_cache_ring_vmdq(struct ixgbe_adapter *adapter) { int i; bool ret = false; #ifdef IXGBE_FCOE struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE]; #endif /* IXGBE_FCOE */ u8 tc = netdev_get_num_tc(adapter->netdev); switch (adapter->flags & (IXGBE_FLAG_RSS_ENABLED | IXGBE_FLAG_DCB_ENABLED | IXGBE_FLAG_VMDQ_ENABLED)) { case IXGBE_FLAG_RSS_ENABLED: case (IXGBE_FLAG_RSS_ENABLED | IXGBE_FLAG_VMDQ_ENABLED): switch (adapter->hw.mac.type) { case ixgbe_mac_82599EB: case ixgbe_mac_X540: /* Configure non RSS queues */ for (i = 0; i < adapter->num_rx_pools - 1; i++) adapter->rx_ring[i]->reg_idx = VMDQ_P(i) * IXGBE_ESX_HW_QUEUES_PER_POOL; /* Configure RSS queues - all queues in the same pool */ for (i = 0; i < IXGBE_ESX_RSS_QUEUES ; i++) adapter->rx_ring[adapter->num_rx_pools-1 + i]-> reg_idx = (VMDQ_P(adapter->num_rx_pools - 1) * IXGBE_ESX_HW_QUEUES_PER_POOL) + i; /* Configure TX queues */ for (i = 0; i < adapter->num_tx_queues; i++) adapter->tx_ring[i]->reg_idx = VMDQ_P(i) * IXGBE_ESX_HW_QUEUES_PER_POOL; ret = true; break; default: break; } break; case (IXGBE_FLAG_DCB_ENABLED): if (!(adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED)) break; case (IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_DCB_ENABLED): switch (adapter->hw.mac.type) { case ixgbe_mac_82599EB: case ixgbe_mac_X540: for (i = 0; i < adapter->num_rx_queues; i++) { adapter->rx_ring[i]->reg_idx = VMDQ_P(i) * tc; #ifdef IXGBE_FCOE if (i >= adapter->num_rx_queues - f->indices) adapter->rx_ring[i]->reg_idx += (adapter->fcoe.tc); #endif /* IXGBE_FCOE */ } for (i = 0; i < adapter->num_tx_queues; i++) { adapter->tx_ring[i]->reg_idx = VMDQ_P(i) * tc; #ifdef IXGBE_FCOE if (i >= adapter->num_tx_queues - f->indices) adapter->tx_ring[i]->reg_idx += adapter->fcoe.tc; #endif /* IXGBE_FCOE */ } ret = true; break; default: break; } break; case (IXGBE_FLAG_VMDQ_ENABLED): switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: for (i = 0; i < adapter->num_rx_queues; i++) adapter->rx_ring[i]->reg_idx = i; for (i = 0; i < adapter->num_tx_queues; i++) adapter->tx_ring[i]->reg_idx = i; ret = true; break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: /* even without rss, there are 2 queues per * pool, the odd numbered ones are unused. */ for (i = 0; i < adapter->num_rx_queues; i++) adapter->rx_ring[i]->reg_idx = VMDQ_P(i) * 2; for (i = 0; i < adapter->num_tx_queues; i++) adapter->tx_ring[i]->reg_idx = VMDQ_P(i) * 2; ret = true; break; default: break; } break; } return ret; } #endif /* IXGBE_VMDQ */ /** * ixgbe_cache_ring_fdir - Descriptor ring to register mapping for Flow Director * @adapter: board private structure to initialize * * Cache the descriptor ring offsets for Flow Director to the assigned rings. * **/ static inline bool ixgbe_cache_ring_fdir(struct ixgbe_adapter *adapter) { int i; bool ret = false; if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) && (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) { for (i = 0; i < adapter->num_rx_queues; i++) adapter->rx_ring[i]->reg_idx = i; for (i = 0; i < adapter->num_tx_queues; i++) adapter->tx_ring[i]->reg_idx = i; ret = true; } return ret; } #ifdef IXGBE_FCOE /** * ixgbe_cache_ring_fcoe - Descriptor ring to register mapping for the FCoE * @adapter: board private structure to initialize * * Cache the descriptor ring offsets for FCoE mode to the assigned rings. * */ static inline bool ixgbe_cache_ring_fcoe(struct ixgbe_adapter *adapter) { struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE]; int i; unsigned int fcoe_rx_i = 0, fcoe_tx_i = 0; if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) return false; if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) { struct net_device *dev = adapter->netdev; int tc = netdev_get_prio_tc_map(dev, adapter->fcoe.up); ixgbe_cache_ring_dcb(adapter); /* find out queues in TC for FCoE */ ixgbe_get_first_reg_idx(adapter, tc, &fcoe_tx_i, &fcoe_rx_i); /* * In 82599, the number of Tx queues for each traffic * class for both 8-TC and 4-TC modes are: * TCs : TC0 TC1 TC2 TC3 TC4 TC5 TC6 TC7 * 8 TCs: 32 32 16 16 8 8 8 8 * 4 TCs: 64 64 32 32 * We have max 8 queues for FCoE, where 8 the is * FCoE redirection table size. If TC for FCoE is * less than or equal to TC3, we have enough queues * to add max of 8 queues for FCoE, so we start FCoE * tx queue from the next one, i.e., reg_idx + 1. * If TC for FCoE is above TC3, implying 8 TC mode, * and we need 8 for FCoE, we have to take all queues * in that traffic class for FCoE. */ if ((f->indices == IXGBE_FCRETA_SIZE) && (tc > 3)) fcoe_tx_i--; } else if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) { if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) ixgbe_cache_ring_fdir(adapter); else ixgbe_cache_ring_rss(adapter); fcoe_rx_i = f->mask; fcoe_tx_i = f->mask; } for (i = 0; i < f->indices; i++, fcoe_rx_i++, fcoe_tx_i++) { adapter->rx_ring[f->mask + i]->reg_idx = fcoe_rx_i; adapter->tx_ring[f->mask + i]->reg_idx = fcoe_tx_i; } return true; } #endif /* IXGBE_FCOE */ /** * ixgbe_cache_ring_sriov - Descriptor ring to register mapping for sriov * @adapter: board private structure to initialize * * SR-IOV doesn't use any descriptor rings but changes the default if * no other mapping is used. * */ static inline bool ixgbe_cache_ring_sriov(struct ixgbe_adapter *adapter) { adapter->rx_ring[0]->reg_idx = adapter->num_vfs * 2; adapter->tx_ring[0]->reg_idx = adapter->num_vfs * 2; return false; } /** * ixgbe_cache_ring_register - Descriptor ring to register mapping * @adapter: board private structure to initialize * * Once we know the feature-set enabled for the device, we'll cache * the register offset the descriptor ring is assigned to. * * Note, the order the various feature calls is important. It must start with * the "most" features enabled at the same time, then trickle down to the * least amount of features turned on at once. **/ static void ixgbe_cache_ring_register(struct ixgbe_adapter *adapter) { /* start with default case */ adapter->rx_ring[0]->reg_idx = 0; adapter->tx_ring[0]->reg_idx = 0; #ifdef IXGBE_VMDQ if (ixgbe_cache_ring_vmdq(adapter)) return; #endif /* IXGBE_VMDQ */ if (ixgbe_cache_ring_sriov(adapter)) return; #ifdef IXGBE_FCOE if (ixgbe_cache_ring_fcoe(adapter)) return; #endif /* IXGBE_FCOE */ if (ixgbe_cache_ring_dcb(adapter)) return; if (ixgbe_cache_ring_fdir(adapter)) return; if (ixgbe_cache_ring_rss(adapter)) return; } #ifdef __VMKLNX__ int ixgbe_calculate_rx_ring_size(struct ixgbe_adapter *adapter) { u32 rx_count = IXGBE_DEFAULT_RXD; if (adapter->netdev->mtu > ETH_DATA_LEN) { rx_count = min(IXGBE_MAX_RXD / adapter->num_rx_queues, IXGBE_JUMBO_FRAME_DEFAULT_RXD); } else { rx_count = min(IXGBE_MAX_RXD / adapter->num_rx_queues, rx_count); } rx_count = max(rx_count, (u32)IXGBE_MIN_RXD); rx_count = ALIGN(rx_count, IXGBE_REQ_RX_DESCRIPTOR_MULTIPLE); return rx_count; } #endif /* __VMKLNX__ */ /** * ixgbe_alloc_queues - Allocate memory for all rings * @adapter: board private structure to initialize * * We allocate one ring per queue at run-time since we don't know the * number of queues at compile-time. The polling_netdev array is * intended for Multiqueue, but should work fine with a single queue. **/ static int ixgbe_alloc_queues(struct ixgbe_adapter *adapter) { int i; int rx_count; #ifdef HAVE_DEVICE_NUMA_NODE int orig_node = adapter->node; WARN_ON(orig_node != -1 && !node_online(orig_node)); #endif /* HAVE_DEVICE_NUMA_NODE */ for (i = 0; i < adapter->num_tx_queues; i++) { struct ixgbe_ring *ring = adapter->tx_ring[i]; #ifdef HAVE_DEVICE_NUMA_NODE if (orig_node == -1) { int cur_node = next_online_node(adapter->node); if (cur_node == MAX_NUMNODES) cur_node = first_online_node; adapter->node = cur_node; } #endif /* HAVE_DEVICE_NUMA_NODE */ ring = kzalloc_node(sizeof(struct ixgbe_ring), GFP_KERNEL, adapter->node); if (!ring) ring = kzalloc(sizeof(struct ixgbe_ring), GFP_KERNEL); if (!ring) goto err_tx_ring_allocation; ring->count = adapter->tx_ring_count; ring->queue_index = i; ring->dev = pci_dev_to_dev(adapter->pdev); ring->netdev = adapter->netdev; ring->numa_node = adapter->node; adapter->tx_ring[i] = ring; } #ifdef HAVE_DEVICE_NUMA_NODE /* Restore the adapter's original node */ adapter->node = orig_node; #endif /* HAVE_DEVICE_NUMA_NODE */ #ifdef __VMKLNX__ rx_count = ixgbe_calculate_rx_ring_size(adapter); adapter->rx_ring_count = rx_count; DPRINTK(PROBE, ERR, "using rx_count = %d \n", rx_count); #else rx_count = adapter->rx_ring_count; #endif for (i = 0; i < adapter->num_rx_queues; i++) { struct ixgbe_ring *ring = adapter->rx_ring[i]; #ifdef HAVE_DEVICE_NUMA_NODE if (orig_node == -1) { int cur_node = next_online_node(adapter->node); if (cur_node == MAX_NUMNODES) cur_node = first_online_node; adapter->node = cur_node; } #endif /* HAVE_DEVICE_NUMA_NODE */ ring = kzalloc_node(sizeof(struct ixgbe_ring), GFP_KERNEL, adapter->node); if (!ring) ring = kzalloc(sizeof(struct ixgbe_ring), GFP_KERNEL); if (!ring) goto err_rx_ring_allocation; ring->count = rx_count; ring->queue_index = i; ring->dev = pci_dev_to_dev(adapter->pdev); ring->netdev = adapter->netdev; ring->numa_node = adapter->node; #ifndef HAVE_NDO_SET_FEATURES /* enable rx csum by default */ set_bit(__IXGBE_RX_CSUM_ENABLED, &ring->state); #endif /* * 82599 errata, UDP frames with a 0 checksum can be marked as * checksum errors. */ if (adapter->hw.mac.type == ixgbe_mac_82599EB) set_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state); adapter->rx_ring[i] = ring; } #ifdef HAVE_DEVICE_NUMA_NODE /* Restore the adapter's original node */ adapter->node = orig_node; #endif /* HAVE_DEVICE_NUMA_NODE */ ixgbe_cache_ring_register(adapter); return 0; err_rx_ring_allocation: for (i = 0; i < adapter->num_tx_queues; i++) kfree(adapter->tx_ring[i]); err_tx_ring_allocation: return -ENOMEM; } /** * ixgbe_set_interrupt_capability - set MSI-X or MSI if supported * @adapter: board private structure to initialize * * Attempt to configure the interrupts using the best available * capabilities of the hardware and the kernel. **/ static int ixgbe_set_interrupt_capability(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int err = 0; int vector, v_budget; if (!(adapter->flags & IXGBE_FLAG_MSIX_CAPABLE)) goto try_msi; /* * It's easy to be greedy for MSI-X vectors, but it really * doesn't do us much good if we have a lot more vectors * than CPU's. So let's be conservative and only ask for * (roughly) the same number of vectors as there are CPU's. * the default is to use pairs of vectors */ v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); #ifndef __VMKLNX__ /* * To avoid netq QueuePair confusions */ v_budget = min(v_budget, (int)num_online_cpus()); #endif v_budget += NON_Q_VECTORS; #ifdef __VMKLNX__ if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) v_budget += adapter->ring_feature[RING_F_FCOE].indices; #endif /* * At the same time, hardware can only support a maximum of * hw.mac->max_msix_vectors vectors. With features * such as RSS and VMDq, we can easily surpass the number of Rx and Tx * descriptor queues supported by our device. Thus, we cap it off in * those rare cases where the cpu count also exceeds our vector limit. */ v_budget = min(v_budget, (int)hw->mac.max_msix_vectors); /* A failure in MSI-X entry allocation isn't fatal, but it does * mean we disable MSI-X capabilities of the adapter. */ adapter->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), GFP_KERNEL); if (adapter->msix_entries) { for (vector = 0; vector < v_budget; vector++) adapter->msix_entries[vector].entry = vector; ixgbe_acquire_msix_vectors(adapter, v_budget); if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) goto out; } adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED; adapter->flags &= ~IXGBE_FLAG_DCB_CAPABLE; if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { e_err(probe, "Flow Director is not supported while multiple " "queues are disabled. Disabling Flow Director\n"); } adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; adapter->atr_sample_rate = 0; #ifdef IXGBE_VMDQ adapter->flags &= ~IXGBE_FLAG_VMDQ_ENABLED; #endif if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) ixgbe_cna_disable(adapter); if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { e_err(probe, "MSI-X interrupt not available - disabling " "SR-IOV\n"); ixgbe_disable_sriov(adapter); } adapter->flags &= ~IXGBE_FLAG_RSS_ENABLED; ixgbe_set_num_queues(adapter); try_msi: if (!(adapter->flags & IXGBE_FLAG_MSI_CAPABLE)) goto out; err = pci_enable_msi(adapter->pdev); if (!err) { adapter->flags |= IXGBE_FLAG_MSI_ENABLED; } else { e_warn(hw, "Unable to allocate MSI interrupt, " "falling back to legacy. Error: %d\n", err); /* reset err */ err = 0; } out: /* Notify the stack of the (possibly) reduced Tx Queue count. */ #ifdef __VMKLNX__ netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); #else netif_set_real_num_tx_queues(adapter->netdev, adapter->num_rx_pools > 1 ? 1 : adapter->num_tx_queues); #endif return err; } /** * ixgbe_alloc_q_vectors - Allocate memory for interrupt vectors * @adapter: board private structure to initialize * * We allocate one q_vector per queue interrupt. If allocation fails we * return -ENOMEM. **/ static int ixgbe_alloc_q_vectors(struct ixgbe_adapter *adapter) { int v_idx, num_q_vectors; struct ixgbe_q_vector *q_vector; int rx_vectors; if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; rx_vectors = adapter->num_rx_queues; } else { num_q_vectors = 1; rx_vectors = 1; } for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { q_vector = kzalloc_node(sizeof(struct ixgbe_q_vector), GFP_KERNEL, adapter->node); if (!q_vector) q_vector = kzalloc(sizeof(struct ixgbe_q_vector), GFP_KERNEL); if (!q_vector) goto err_out; q_vector->adapter = adapter; q_vector->v_idx = v_idx; #ifdef HAVE_IRQ_AFFINITY_HINT /* Allocate the affinity_hint cpumask, configure the mask */ if (!alloc_cpumask_var(&q_vector->affinity_mask, GFP_KERNEL)) goto err_out; cpumask_set_cpu(v_idx, q_vector->affinity_mask); #endif #ifdef CONFIG_IXGBE_NAPI netif_napi_add(adapter->netdev, &q_vector->napi, ixgbe_poll, 64); #endif /* CONFIG_IXGBE_NAPI */ adapter->q_vector[v_idx] = q_vector; } return 0; err_out: while (v_idx) { v_idx--; q_vector = adapter->q_vector[v_idx]; #ifdef CONFIG_IXGBE_NAPI netif_napi_del(&q_vector->napi); #endif #ifdef HAVE_IRQ_AFFINITY_HINT free_cpumask_var(q_vector->affinity_mask); #endif kfree(q_vector); adapter->q_vector[v_idx] = NULL; } return -ENOMEM; } /** * ixgbe_free_q_vectors - Free memory allocated for interrupt vectors * @adapter: board private structure to initialize * * This function frees the memory allocated to the q_vectors. In addition if * NAPI is enabled it will delete any references to the NAPI struct prior * to freeing the q_vector. **/ static void ixgbe_free_q_vectors(struct ixgbe_adapter *adapter) { int v_idx, num_q_vectors; if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; else num_q_vectors = 1; for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { struct ixgbe_q_vector *q_vector = adapter->q_vector[v_idx]; adapter->q_vector[v_idx] = NULL; #ifdef CONFIG_IXGBE_NAPI netif_napi_del(&q_vector->napi); #endif #ifdef HAVE_IRQ_AFFINITY_HINT free_cpumask_var(q_vector->affinity_mask); #endif kfree(q_vector); } } static void ixgbe_reset_interrupt_capability(struct ixgbe_adapter *adapter) { if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED; pci_disable_msix(adapter->pdev); kfree(adapter->msix_entries); adapter->msix_entries = NULL; } else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) { adapter->flags &= ~IXGBE_FLAG_MSI_ENABLED; pci_disable_msi(adapter->pdev); } } /** * ixgbe_init_interrupt_scheme - Determine proper interrupt scheme * @adapter: board private structure to initialize * * We determine which interrupt scheme to use based on... * - Kernel support (MSI, MSI-X) * - which can be user-defined (via MODULE_PARAM) * - Hardware queue count (num_*_queues) * - defined by miscellaneous hardware support/features (RSS, etc.) **/ int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter) { int err; /* Number of supported queues */ ixgbe_set_num_queues(adapter); err = ixgbe_set_interrupt_capability(adapter); if (err) { e_err(probe, "Unable to setup interrupt capabilities\n"); goto err_set_interrupt; } err = ixgbe_alloc_q_vectors(adapter); if (err) { e_err(probe, "Unable to allocate memory for queue vectors\n"); goto err_alloc_q_vectors; } err = ixgbe_alloc_queues(adapter); if (err) { e_err(probe, "Unable to allocate memory for queues\n"); goto err_alloc_queues; } set_bit(__IXGBE_DOWN, &adapter->state); return 0; err_alloc_queues: ixgbe_free_q_vectors(adapter); err_alloc_q_vectors: ixgbe_reset_interrupt_capability(adapter); err_set_interrupt: return err; } /** * ixgbe_clear_interrupt_scheme - Clear the current interrupt scheme settings * @adapter: board private structure to clear interrupt scheme on * * We go through and clear interrupt specific resources and reset the structure * to pre-load conditions **/ void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter) { int i; for (i = 0; i < adapter->num_tx_queues; i++) { kfree(adapter->tx_ring[i]); adapter->tx_ring[i] = NULL; } for (i = 0; i < adapter->num_rx_queues; i++) { kfree(adapter->rx_ring[i]); adapter->rx_ring[i] = NULL; } adapter->num_tx_queues = 0; adapter->num_rx_queues = 0; ixgbe_free_q_vectors(adapter); ixgbe_reset_interrupt_capability(adapter); /* reinit the dev->napi_list */ INIT_LIST_HEAD(&adapter->netdev->napi_list); } /** * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter) * @adapter: board private structure to initialize * * ixgbe_sw_init initializes the Adapter private data structure. * Fields are initialized based on PCI device information and * OS network device settings (MTU size). **/ static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; struct pci_dev *pdev = adapter->pdev; int err; /* PCI config space info */ hw->vendor_id = pdev->vendor; hw->device_id = pdev->device; pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); hw->subsystem_vendor_id = pdev->subsystem_vendor; hw->subsystem_device_id = pdev->subsystem_device; err = ixgbe_init_shared_code(hw); if (err) { e_err(probe, "init_shared_code failed: %d\n", err); goto out; } adapter->mac_table = kzalloc(sizeof(struct ixgbe_mac_addr) * hw->mac.num_rar_entries, GFP_ATOMIC); if (!adapter->mac_table) { err = IXGBE_ERR_OUT_OF_MEM; e_err(probe, "mac_table allocation failed: %d\n", err); goto out; } /* Set capability flags */ switch (hw->mac.type) { case ixgbe_mac_82598EB: adapter->flags |= IXGBE_FLAG_MSI_CAPABLE | IXGBE_FLAG_MSIX_CAPABLE | IXGBE_FLAG_MQ_CAPABLE | IXGBE_FLAG_RSS_CAPABLE; adapter->flags |= IXGBE_FLAG_DCB_CAPABLE; #ifdef IXGBE_VMDQ adapter->flags |= IXGBE_FLAG_VMDQ_CAPABLE; #endif #if defined(CONFIG_DCA) || defined(CONFIG_DCA_MODULE) adapter->flags |= IXGBE_FLAG_DCA_CAPABLE; #endif adapter->flags &= ~IXGBE_FLAG_SRIOV_CAPABLE; adapter->flags2 &= ~IXGBE_FLAG2_RSC_CAPABLE; if (hw->device_id == IXGBE_DEV_ID_82598AT) adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; adapter->max_msix_q_vectors = IXGBE_MAX_MSIX_Q_VECTORS_82598; break; case ixgbe_mac_X540: adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; case ixgbe_mac_82599EB: adapter->flags |= IXGBE_FLAG_MSI_CAPABLE | IXGBE_FLAG_MSIX_CAPABLE | IXGBE_FLAG_MQ_CAPABLE | IXGBE_FLAG_RSS_CAPABLE; adapter->flags |= IXGBE_FLAG_DCB_CAPABLE; #ifdef IXGBE_VMDQ adapter->flags |= IXGBE_FLAG_VMDQ_CAPABLE; #endif #if defined(CONFIG_DCA) || defined(CONFIG_DCA_MODULE) adapter->flags |= IXGBE_FLAG_DCA_CAPABLE; #endif adapter->flags |= IXGBE_FLAG_SRIOV_CAPABLE; adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE; #ifdef IXGBE_FCOE adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE; adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; adapter->ring_feature[RING_F_FCOE].indices = 0; #ifdef CONFIG_DCB /* Default traffic class to use for FCoE */ adapter->fcoe.tc = IXGBE_FCOE_DEFTC; adapter->fcoe.up = IXGBE_FCOE_DEFTC; adapter->fcoe.up_set = IXGBE_FCOE_DEFTC; #endif #endif if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; #ifndef IXGBE_NO_SMART_SPEED hw->phy.smart_speed = ixgbe_smart_speed_on; #else hw->phy.smart_speed = ixgbe_smart_speed_off; #endif adapter->max_msix_q_vectors = IXGBE_MAX_MSIX_Q_VECTORS_82599; default: break; } /* n-tuple support exists, always init our spinlock */ spin_lock_init(&adapter->fdir_perfect_lock); #ifdef __VMKLNX__ /* Disable explicitly the packet split for ESX */ adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; adapter->flags &= ~IXGBE_FLAG_RX_PS_CAPABLE; #endif if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE) { int j; struct ixgbe_dcb_tc_config *tc; int dcb_i = IXGBE_DCB_MAX_TRAFFIC_CLASS; /* ESX CNA intitializes to DCB and four traffic classes */ adapter->tc = 4; dcb_i = 4; adapter->dcb_cfg.num_tcs.pg_tcs = dcb_i; adapter->dcb_cfg.num_tcs.pfc_tcs = dcb_i; for (j = 0; j < dcb_i; j++) { tc = &adapter->dcb_cfg.tc_config[j]; tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = 0; tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 100 / dcb_i; tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = 0; tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 100 / dcb_i; tc->pfc = ixgbe_dcb_pfc_disabled; if (j == 0) { /* total of all TCs bandwidth needs to be 100 */ tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent += 100 % dcb_i; tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent += 100 % dcb_i; } } /* Initialize default user to priority mapping, UPx->TC0 */ tc = &adapter->dcb_cfg.tc_config[0]; tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF; tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF; adapter->dcb_cfg.bw_percentage[IXGBE_DCB_TX_CONFIG][0] = 100; adapter->dcb_cfg.bw_percentage[IXGBE_DCB_RX_CONFIG][0] = 100; adapter->dcb_cfg.rx_pba_cfg = ixgbe_dcb_pba_equal; adapter->dcb_cfg.pfc_mode_enable = false; adapter->dcb_cfg.round_robin_enable = false; adapter->dcb_set_bitmap = 0x00; #ifdef CONFIG_DCB adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE; #endif /* CONFIG_DCB */ if (hw->mac.type == ixgbe_mac_X540) { adapter->dcb_cfg.num_tcs.pg_tcs = 4; adapter->dcb_cfg.num_tcs.pfc_tcs = 4; } } if (hw->mac.type == ixgbe_mac_82599EB || hw->mac.type == ixgbe_mac_X540) hw->mbx.ops.init_params(hw); /* default flow control settings */ hw->fc.requested_mode = ixgbe_fc_full; hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */ adapter->last_lfc_mode = hw->fc.current_mode; ixgbe_pbthresh_setup(adapter); hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE; hw->fc.send_xon = true; hw->fc.disable_fc_autoneg = false; /* set default ring sizes */ adapter->tx_ring_count = IXGBE_DEFAULT_TXD; adapter->rx_ring_count = IXGBE_DEFAULT_RXD; /* set default work limits */ adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK; adapter->rx_work_limit = IXGBE_DEFAULT_RX_WORK; set_bit(__IXGBE_DOWN, &adapter->state); out: return err; } /** * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors) * @tx_ring: tx descriptor ring (for a specific queue) to setup * * Return 0 on success, negative on failure **/ int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring) { struct device *dev = tx_ring->dev; int orig_node = dev_to_node(dev); int size; size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node); if (!tx_ring->tx_buffer_info) tx_ring->tx_buffer_info = vzalloc(size); if (!tx_ring->tx_buffer_info) goto err; /* round up to nearest 4K */ tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); tx_ring->size = ALIGN(tx_ring->size, 4096); set_dev_node(dev, tx_ring->numa_node); tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, &tx_ring->dma, GFP_KERNEL); set_dev_node(dev, orig_node); if (!tx_ring->desc) tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, &tx_ring->dma, GFP_KERNEL); if (!tx_ring->desc) goto err; tx_ring->next_to_use = 0; tx_ring->next_to_clean = 0; return 0; err: vfree(tx_ring->tx_buffer_info); tx_ring->tx_buffer_info = NULL; dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n"); return -ENOMEM; } /** * ixgbe_setup_all_tx_resources - allocate all queues Tx resources * @adapter: board private structure * * If this function returns with an error, then it's possible one or * more of the rings is populated (while the rest are not). It is the * callers duty to clean those orphaned rings. * * Return 0 on success, negative on failure **/ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) { int i, err = 0; int net_tx_queues = adapter->num_tx_queues - adapter->ring_feature[RING_F_FCOE].indices; for (i = 0; i < adapter->num_tx_queues; i++) { if (i < net_tx_queues) { adapter->tx_ring[i]->netdev = adapter->netdev; clear_ring_type_cna(adapter->tx_ring[i]); } else { adapter->tx_ring[i]->netdev = adapter->cnadev; set_ring_type_cna(adapter->tx_ring[i]); } #ifdef HAVE_DEVICE_NUMA_NODE e_info(tx_err, "tx[%02d] bd: %d - assigning node %d\n", i, adapter->bd_number, adapter->tx_ring[i]->numa_node); #endif /* HAVE_DEVICE_NUMA_NODE */ err = ixgbe_setup_tx_resources(adapter->tx_ring[i]); if (!err) continue; e_err(probe, "Allocation for Tx Queue %u failed\n", i); break; } #ifdef __VMKLNX__ adapter->n_tx_queues_allocated = 0; #endif return err; } /** * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) * @rx_ring: rx descriptor ring (for a specific queue) to setup * * Returns 0 on success, negative on failure **/ int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring) { struct device *dev = rx_ring->dev; int orig_node = dev_to_node(dev); int size; size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node); if (!rx_ring->rx_buffer_info) rx_ring->rx_buffer_info = vzalloc(size); if (!rx_ring->rx_buffer_info) goto err; /* Round up to nearest 4K */ rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); set_dev_node(dev, rx_ring->numa_node); rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, &rx_ring->dma, GFP_KERNEL); set_dev_node(dev, orig_node); if (!rx_ring->desc) rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, &rx_ring->dma, GFP_KERNEL); if (!rx_ring->desc) goto err; rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; #ifdef __VMKLNX__ rx_ring->active = 0; clear_ring_allocated(rx_ring); clear_bit(__IXGBE_RING_CLEAN_BUSY, &rx_ring->state); #endif return 0; err: vfree(rx_ring->rx_buffer_info); rx_ring->rx_buffer_info = NULL; dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); return -ENOMEM; } /** * ixgbe_setup_all_rx_resources - allocate all queues Rx resources * @adapter: board private structure * * If this function returns with an error, then it's possible one or * more of the rings is populated (while the rest are not). It is the * callers duty to clean those orphaned rings. * * Return 0 on success, negative on failure **/ static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter) { int i, err = 0; int net_rx_queues = adapter->num_rx_queues - adapter->ring_feature[RING_F_FCOE].indices; for (i = 0; i < adapter->num_rx_queues; i++) { if (i < net_rx_queues) { adapter->rx_ring[i]->netdev = adapter->netdev; clear_ring_type_cna(adapter->rx_ring[i]); } else { adapter->rx_ring[i]->netdev = adapter->cnadev; set_ring_type_cna(adapter->rx_ring[i]); } #ifdef HAVE_DEVICE_NUMA_NODE e_info(rx_err, "rx[%02d] bd: %d - assigning node %d\n", i, adapter->bd_number, adapter->rx_ring[i]->numa_node); #endif /* HAVE_DEVICE_NUMA_NODE */ err = ixgbe_setup_rx_resources(adapter->rx_ring[i]); if (!err) { #ifdef __VMKLNX__ /* * ixgbe_setup_rx_resources sets the ring to not * allocated for VMware. Set default queue to * allocated so filter calls will not fail. * Other queues will be allocated when used for VMQ */ if (i==0) set_ring_allocated(adapter->rx_ring[0]); #endif continue; } e_err(probe, "Allocation for Rx Queue %u failed\n", i); break; } #ifdef __VMKLNX__ adapter->n_rx_queues_allocated = 0; #endif return err; } /** * ixgbe_free_tx_resources - Free Tx Resources per Queue * @tx_ring: Tx descriptor ring for a specific queue * * Free all transmit software resources **/ void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring) { ixgbe_clean_tx_ring(tx_ring); vfree(tx_ring->tx_buffer_info); tx_ring->tx_buffer_info = NULL; /* if not set, then don't free */ if (!tx_ring->desc) return; dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, tx_ring->dma); tx_ring->desc = NULL; } /** * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues * @adapter: board private structure * * Free all transmit software resources **/ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) { int i; for (i = 0; i < adapter->num_tx_queues; i++) if (adapter->tx_ring[i]->desc) ixgbe_free_tx_resources(adapter->tx_ring[i]); #ifdef __VMKLNX__ adapter->n_tx_queues_allocated = 0; #endif } /** * ixgbe_free_rx_resources - Free Rx Resources * @rx_ring: ring to clean the resources from * * Free all receive software resources **/ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { ixgbe_clean_rx_ring(rx_ring); vfree(rx_ring->rx_buffer_info); rx_ring->rx_buffer_info = NULL; /* if not set, then don't free */ if (!rx_ring->desc) return; dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; } /** * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues * @adapter: board private structure * * Free all receive software resources **/ static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) { int i; for (i = 0; i < adapter->num_rx_queues; i++) if (adapter->rx_ring[i]->desc) ixgbe_free_rx_resources(adapter->rx_ring[i]); #ifdef __VMKLNX__ adapter->n_rx_queues_allocated = 0; #endif } /** * ixgbe_change_mtu - Change the Maximum Transfer Unit * @netdev: network interface device structure * @new_mtu: new value for maximum frame size * * Returns 0 on success, negative on failure **/ static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; #ifdef __VMKLNX__ int old_mtu = netdev->mtu; #endif #ifndef __VMKLNX__ /* MTU < 68 is an error and causes problems on some kernels */ if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED && hw->mac.type != ixgbe_mac_X540) { if ((new_mtu < 68) || (max_frame > MAXIMUM_ETHERNET_VLAN_SIZE)) return -EINVAL; } else { if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE)) return -EINVAL; } #else if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE)) return -EINVAL; #endif /* __VMKLNX__ */ e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); DPRINTK(PROBE, ERR, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); /* must set new MTU before calling down or up */ netdev->mtu = new_mtu; if (netif_running(netdev)) { #ifdef __VMKLNX__ if ((old_mtu <= ETH_DATA_LEN && new_mtu > ETH_DATA_LEN) || (old_mtu > ETH_DATA_LEN && new_mtu <= ETH_DATA_LEN)) ixgbe_reinit_locked_change_queues(adapter); else ixgbe_reinit_locked(adapter); #else ixgbe_reinit_locked(adapter); #endif /* __VMKLNX__ */ } return 0; } /** * ixgbe_open - Called when a network interface is made active * @netdev: network interface device structure * * Returns 0 on success, negative value on failure * * The open entry point is called when a network interface is made * active by the system (IFF_UP). At this point all resources needed * for transmit and receive operations are allocated, the interrupt * handler is registered with the OS, the watchdog timer is started, * and the stack is notified that the interface is ready. **/ static int ixgbe_open(struct net_device *netdev) { struct ixgbe_adapter *adapter = netdev_priv(netdev); int err; struct ixgbe_hw *hw = &adapter->hw; u32 rxctrl; /* disallow open during test */ if (test_bit(__IXGBE_TESTING, &adapter->state)) return -EBUSY; netif_carrier_off(netdev); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) netif_carrier_off(adapter->cnadev); /* allocate transmit descriptors */ err = ixgbe_setup_all_tx_resources(adapter); if (err) goto err_setup_tx; /* allocate receive descriptors */ err = ixgbe_setup_all_rx_resources(adapter); if (err) goto err_setup_rx; ixgbe_configure(adapter); err = ixgbe_request_irq(adapter); if (err) { e_err(probe, "ixgbe_request_irq failed for %s, errno %d\n", adapter->netdev->name, err); goto err_req_irq; } ixgbe_up_complete(adapter); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) dev_open(adapter->cnadev); #ifdef __VMKLNX__ /* Invalidate netqueue state as filters have been lost after reinit */ vmknetddi_queueops_invalidate_state(adapter->netdev); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) vmknetddi_queueops_invalidate_state(adapter->cnadev); #endif return 0; err_req_irq: /* signal that we are down to the interrupt handler */ set_bit(__IXGBE_DOWN, &adapter->state); /* * disable RX, here we don't disable TX * since there is no pkt TXed out here, * this is because the ixgbe_open has not * yet finished and the ixgbe_down is set. */ rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); #ifdef IXGBE_FCOE if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) ixgbe_cleanup_fcoe(adapter); #endif err_setup_rx: ixgbe_free_all_rx_resources(adapter); err_setup_tx: ixgbe_free_all_tx_resources(adapter); ixgbe_reset(adapter); return err; } /** * ixgbe_close - Disables a network interface * @netdev: network interface device structure * * Returns 0, this is not allowed to fail * * The close entry point is called when an interface is de-activated * by the OS. The hardware is still under the drivers control, but * needs to be disabled. A global MAC reset is issued to stop the * hardware, and all transmit and receive resources are freed. **/ static int ixgbe_close(struct net_device *netdev) { struct ixgbe_adapter *adapter = netdev_priv(netdev); ixgbe_down(adapter); ixgbe_free_irq(adapter); ixgbe_fdir_filter_exit(adapter); ixgbe_free_all_tx_resources(adapter); ixgbe_free_all_rx_resources(adapter); ixgbe_release_hw_control(adapter); return 0; } #ifdef __VMKLNX__ /** * PR295512: When changing MTU, reset the NIC and resize number of queues * according to MTU size. This is needed when netPktHeap has limited * low memory space. **/ int ixgbe_reinit_locked_change_queues(struct ixgbe_adapter *adapter) { WARN_ON(in_interrupt()); /* put off any impending NetWatchDogTimeout */ adapter->netdev->trans_start = jiffies; while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state)) usleep_range(1000, 2000); /* start out the same as a close(), but don't release the hw */ ixgbe_down(adapter); ixgbe_free_irq(adapter); ixgbe_free_all_rx_resources(adapter); ixgbe_free_all_tx_resources(adapter); ixgbe_clear_interrupt_scheme(adapter); /* Resize number of queues and rings */ ixgbe_init_interrupt_scheme(adapter); /* start back up */ ixgbe_open(adapter->netdev); clear_bit(__IXGBE_RESETTING, &adapter->state); #ifdef __VMKLNX__ /* Invalidate netqueue state as filters have been lost after reinit */ vmknetddi_queueops_invalidate_state(adapter->netdev); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) vmknetddi_queueops_invalidate_state(adapter->cnadev); #endif return 0; } #endif /* __VMKLNX__ */ #ifdef CONFIG_PM static int ixgbe_resume(struct pci_dev *pdev) { struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); struct net_device *netdev = adapter->netdev; u32 err; pci_set_power_state(pdev, PCI_D0); #ifndef __VMKLNX__ pci_restore_state(pdev); /* * pci_restore_state clears dev->state_saved so call * pci_save_state to restore it. */ pci_save_state(pdev); #endif err = pci_enable_device_mem(pdev); if (err) { e_dev_err("Cannot enable PCI device from suspend\n"); return err; } pci_set_master(pdev); pci_wake_from_d3(pdev, false); err = ixgbe_init_interrupt_scheme(adapter); if (err) { e_dev_err("Cannot initialize interrupts for device\n"); return err; } ixgbe_reset(adapter); IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); if (netif_running(netdev)) { err = ixgbe_open(netdev); if (err) return err; } netif_device_attach(netdev); return 0; } #endif /* CONFIG_PM */ /* * __ixgbe_shutdown is not used when power management * is disabled on older kernels (<2.6.12). causes a compile * warning/error, because it is defined and not used. */ #if defined(CONFIG_PM) || !defined(USE_REBOOT_NOTIFIER) static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake) { struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); struct net_device *netdev = adapter->netdev; struct ixgbe_hw *hw = &adapter->hw; u32 ctrl, fctrl; u32 wufc = adapter->wol; #ifdef CONFIG_PM int retval = 0; #endif netif_device_detach(netdev); if (netif_running(netdev)) { rtnl_lock(); ixgbe_down(adapter); ixgbe_free_irq(adapter); ixgbe_free_all_tx_resources(adapter); ixgbe_free_all_rx_resources(adapter); rtnl_unlock(); } ixgbe_clear_interrupt_scheme(adapter); #ifdef CONFIG_PM #ifndef __VMKLNX__ retval = pci_save_state(pdev); if (retval) return retval; #endif #endif if (wufc) { ixgbe_set_rx_mode(netdev); /* turn on all-multi mode if wake on multicast is enabled */ if (wufc & IXGBE_WUFC_MC) { fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); fctrl |= IXGBE_FCTRL_MPE; IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); } ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL); ctrl |= IXGBE_CTRL_GIO_DIS; IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl); IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc); } else { IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); } switch (hw->mac.type) { case ixgbe_mac_82598EB: pci_wake_from_d3(pdev, false); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: pci_wake_from_d3(pdev, !!wufc); break; default: break; } *enable_wake = !!wufc; ixgbe_release_hw_control(adapter); pci_disable_device(pdev); return 0; } #endif /* defined(CONFIG_PM) || !defined(USE_REBOOT_NOTIFIER) */ #ifdef CONFIG_PM static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state) { int retval; bool wake; retval = __ixgbe_shutdown(pdev, &wake); if (retval) return retval; if (wake) { pci_prepare_to_sleep(pdev); } else { pci_wake_from_d3(pdev, false); pci_set_power_state(pdev, PCI_D3hot); } return 0; } #endif /* CONFIG_PM */ #ifndef USE_REBOOT_NOTIFIER static void ixgbe_shutdown(struct pci_dev *pdev) { bool wake; __ixgbe_shutdown(pdev, &wake); #ifndef __VMKLNX__ if (system_state == SYSTEM_POWER_OFF) { pci_wake_from_d3(pdev, wake); pci_set_power_state(pdev, PCI_D3hot); } #endif } #endif /** * ixgbe_get_stats - Get System Network Statistics * @netdev: network interface device structure * * Returns the address of the device statistics structure. * The statistics are actually updated from the timer callback. **/ static struct net_device_stats *ixgbe_get_stats(struct net_device *netdev) { #ifdef HAVE_NETDEV_STATS_IN_NETDEV /* only return the current stats */ return &netdev->stats; #else struct ixgbe_adapter *adapter = netdev_priv(netdev); /* only return the current stats */ return &adapter->net_stats; #endif /* HAVE_NETDEV_STATS_IN_NETDEV */ } /** * ixgbe_update_stats - Update the board statistics counters. * @adapter: board private structure **/ void ixgbe_update_stats(struct ixgbe_adapter *adapter) { struct net_device_stats *net_stats = ixgbe_get_stats(adapter->netdev); struct ixgbe_hw *hw = &adapter->hw; struct ixgbe_hw_stats *hwstats = &adapter->stats; u64 total_mpc = 0; u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot; u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0; u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0; u64 bytes = 0, packets = 0, hw_csum_rx_error = 0; u64 rx_hdr_split = 0; #ifdef IXGBE_FCOE struct ixgbe_fcoe *fcoe = &adapter->fcoe; unsigned int cpu; u64 fcoe_noddp_counts_sum = 0, fcoe_noddp_ext_buff_counts_sum = 0; #endif /* IXGBE_FCOE */ if (test_bit(__IXGBE_DOWN, &adapter->state) || test_bit(__IXGBE_RESETTING, &adapter->state)) return; if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) { u64 rsc_count = 0; u64 rsc_flush = 0; for (i = 0; i < adapter->num_rx_queues; i++) { rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count; rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush; } adapter->rsc_total_count = rsc_count; adapter->rsc_total_flush = rsc_flush; } for (i = 0; i < adapter->num_rx_queues; i++) { struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; non_eop_descs += rx_ring->rx_stats.non_eop_descs; alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed; alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed; hw_csum_rx_error += rx_ring->rx_stats.csum_err; rx_hdr_split += rx_ring->rx_stats.rx_hdr_split; bytes += rx_ring->stats.bytes; packets += rx_ring->stats.packets; } adapter->non_eop_descs = non_eop_descs; adapter->alloc_rx_page_failed = alloc_rx_page_failed; adapter->alloc_rx_buff_failed = alloc_rx_buff_failed; adapter->hw_csum_rx_error = hw_csum_rx_error; adapter->rx_hdr_split = rx_hdr_split; net_stats->rx_bytes = bytes; net_stats->rx_packets = packets; bytes = 0; packets = 0; /* gather some stats to the adapter struct that are per queue */ for (i = 0; i < adapter->num_tx_queues; i++) { struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; restart_queue += tx_ring->tx_stats.restart_queue; tx_busy += tx_ring->tx_stats.tx_busy; bytes += tx_ring->stats.bytes; packets += tx_ring->stats.packets; } adapter->restart_queue = restart_queue; adapter->tx_busy = tx_busy; net_stats->tx_bytes = bytes; net_stats->tx_packets = packets; for (i = 0; i < 16; i++) { hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); } hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); for (i = 0; i < 8; i++) { /* for packet buffers not used, the register should read 0 */ mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i)); missed_rx += mpc; hwstats->mpc[i] += mpc; total_mpc += hwstats->mpc[i]; if (hw->mac.type == ixgbe_mac_82598EB) hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); switch (hw->mac.type) { case ixgbe_mac_82598EB: hwstats->pxonrxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: hwstats->pxonrxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i)); break; default: break; } hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); } hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); /* work around hardware counting issue */ hwstats->gprc -= missed_rx; ixgbe_update_xoff_received(adapter); /* 82598 hardware only has a 32 bit counter in the high register */ switch (hw->mac.type) { case ixgbe_mac_82598EB: hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH); break; case ixgbe_mac_X540: /* OS2BMC stats are X540 only*/ hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC); hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC); hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC); hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC); case ixgbe_mac_82599EB: for (i = 0; i < 16; i++) adapter->hw_rx_no_dma_resources += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */ hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */ hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL); IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */ hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); #ifdef HAVE_TX_MQ hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH); hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS); #endif /* HAVE_TX_MQ */ #ifdef IXGBE_FCOE hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); hwstats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST); hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); /* Add up per cpu counters for total ddp aloc fail */ if (fcoe && fcoe->pcpu_noddp && fcoe->pcpu_noddp_ext_buff) { for_each_possible_cpu(cpu) { #ifndef __VMKLNX__ fcoe_noddp_counts_sum += *per_cpu_ptr(fcoe->pcpu_noddp, cpu); fcoe_noddp_ext_buff_counts_sum += *per_cpu_ptr(fcoe-> pcpu_noddp_ext_buff, cpu); #else fcoe_noddp_counts_sum += *ESX_PER_CPU_PTR(fcoe->pcpu_noddp, cpu, u64); fcoe_noddp_ext_buff_counts_sum += *ESX_PER_CPU_PTR(fcoe-> pcpu_noddp_ext_buff, cpu,u64); #endif } } hwstats->fcoe_noddp = fcoe_noddp_counts_sum; hwstats->fcoe_noddp_ext_buff = fcoe_noddp_ext_buff_counts_sum; #endif /* IXGBE_FCOE */ break; default: break; } bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); hwstats->bprc += bprc; hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); if (hw->mac.type == ixgbe_mac_82598EB) hwstats->mprc -= bprc; hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC); hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); hwstats->lxontxc += lxon; lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); hwstats->lxofftxc += lxoff; hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); /* * 82598 errata - tx of flow control packets is included in tx counters */ xon_off_tot = lxon + lxoff; hwstats->gptc -= xon_off_tot; hwstats->mptc -= xon_off_tot; hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN)); hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC); hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC); hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC); hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR); hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); hwstats->ptc64 -= xon_off_tot; hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); /* Fill out the OS statistics structure */ net_stats->multicast = hwstats->mprc; /* Rx Errors */ net_stats->rx_errors = hwstats->crcerrs + hwstats->rlec; net_stats->rx_dropped = 0; net_stats->rx_length_errors = hwstats->rlec; net_stats->rx_crc_errors = hwstats->crcerrs; net_stats->rx_missed_errors = total_mpc; /* * VF Stats Collection - skip while resetting because these * are not clear on read and otherwise you'll sometimes get * crazy values. */ if (!test_bit(__IXGBE_RESETTING, &adapter->state)) { for(i = 0; i < adapter->num_vfs; i++) { UPDATE_VF_COUNTER_32bit(IXGBE_PVFGPRC(i), \ adapter->vfinfo[i].last_vfstats.gprc, \ adapter->vfinfo[i].vfstats.gprc); UPDATE_VF_COUNTER_32bit(IXGBE_PVFGPTC(i), \ adapter->vfinfo[i].last_vfstats.gptc, \ adapter->vfinfo[i].vfstats.gptc); UPDATE_VF_COUNTER_36bit(IXGBE_PVFGORC_LSB(i), \ IXGBE_PVFGORC_MSB(i), \ adapter->vfinfo[i].last_vfstats.gorc, \ adapter->vfinfo[i].vfstats.gorc); UPDATE_VF_COUNTER_36bit(IXGBE_PVFGOTC_LSB(i), \ IXGBE_PVFGOTC_MSB(i), \ adapter->vfinfo[i].last_vfstats.gotc, \ adapter->vfinfo[i].vfstats.gotc); UPDATE_VF_COUNTER_32bit(IXGBE_PVFMPRC(i), \ adapter->vfinfo[i].last_vfstats.mprc, \ adapter->vfinfo[i].vfstats.mprc); } } } #ifdef HAVE_TX_MQ /** * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table * @adapter - pointer to the device adapter structure **/ static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int i; if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) return; adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT; /* if interface is down do nothing */ if (test_bit(__IXGBE_DOWN, &adapter->state)) return; /* do nothing if we are not using signature filters */ if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) return; adapter->fdir_overflow++; if (ixgbe_reinit_fdir_tables_82599(hw) == 0) { for (i = 0; i < adapter->num_tx_queues; i++) set_bit(__IXGBE_TX_FDIR_INIT_DONE, &(adapter->tx_ring[i]->state)); /* re-enable flow director interrupts */ IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); } else { e_err(probe, "failed to finish FDIR re-initialization, " "ignored adding FDIR ATR filters\n"); } } #endif /* HAVE_TX_MQ */ /** * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts * @adapter - pointer to the device adapter structure * * This function serves two purposes. First it strobes the interrupt lines * in order to make certain interrupts are occuring. Secondly it sets the * bits needed to check for TX hangs. As a result we should immediately * determine if a hang has occured. */ static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u64 eics = 0; int i; /* If we're down or resetting, just bail */ if (test_bit(__IXGBE_DOWN, &adapter->state) || test_bit(__IXGBE_RESETTING, &adapter->state)) return; /* Force detection of hung controller */ if (netif_carrier_ok(adapter->netdev)) { for (i = 0; i < adapter->num_tx_queues; i++) set_check_for_tx_hang(adapter->tx_ring[i]); } if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) { /* * for legacy and MSI interrupts don't set any bits * that are enabled for EIAM, because this operation * would set *both* EIMS and EICS for any bit in EIAM */ IXGBE_WRITE_REG(hw, IXGBE_EICS, (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER)); } else { /* get one bit for every active tx/rx interrupt vector */ for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { struct ixgbe_q_vector *qv = adapter->q_vector[i]; if (qv->rx.ring || qv->tx.ring) eics |= ((u64)1 << i); } } /* Cause software interrupt to ensure rings are cleaned */ ixgbe_irq_rearm_queues(adapter, eics); } /** * ixgbe_watchdog_update_link - update the link status * @adapter - pointer to the device adapter structure * @link_speed - pointer to a u32 to store the link_speed **/ static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 link_speed = adapter->link_speed; bool link_up = adapter->link_up; int i; if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) return; if (hw->mac.ops.check_link) { hw->mac.ops.check_link(hw, &link_speed, &link_up, false); } else { /* always assume link is up, if no check link function */ link_speed = IXGBE_LINK_SPEED_10GB_FULL; link_up = true; } if (link_up) { if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) { for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) hw->mac.ops.fc_enable(hw, i); } else { hw->mac.ops.fc_enable(hw, 0); } } if (link_up || time_after(jiffies, (adapter->link_check_timeout + IXGBE_TRY_LINK_TIMEOUT))) { adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC); IXGBE_WRITE_FLUSH(hw); } adapter->link_up = link_up; adapter->link_speed = link_speed; } /** * ixgbe_watchdog_link_is_up - update netif_carrier status and * print link up message * @adapter - pointer to the device adapter structure **/ static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter) { struct net_device *netdev = adapter->netdev; struct ixgbe_hw *hw = &adapter->hw; u32 link_speed = adapter->link_speed; bool flow_rx, flow_tx; /* only continue if link was previously down */ if (netif_carrier_ok(netdev)) return; adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP; switch (hw->mac.type) { case ixgbe_mac_82598EB: { u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL); u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS); flow_rx = !!(frctl & IXGBE_FCTRL_RFCE); flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X); } break; case ixgbe_mac_X540: case ixgbe_mac_82599EB: { u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN); u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG); flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE); flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X); } break; default: flow_tx = false; flow_rx = false; break; } e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", (link_speed == IXGBE_LINK_SPEED_10GB_FULL ? "10 Gbps" : (link_speed == IXGBE_LINK_SPEED_1GB_FULL ? "1 Gbps" : (link_speed == IXGBE_LINK_SPEED_100_FULL ? "100 Mbps" : "unknown speed"))), ((flow_rx && flow_tx) ? "RX/TX" : (flow_rx ? "RX" : (flow_tx ? "TX" : "None")))); netif_carrier_on(netdev); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) { DPRINTK(LINK, ERR, "NIC CNA Link is Up\n"); netif_carrier_on(adapter->cnadev); } netif_tx_wake_all_queues(netdev); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) netif_tx_wake_all_queues(adapter->cnadev); } /** * ixgbe_watchdog_link_is_down - update netif_carrier status and * print link down message * @adapter - pointer to the adapter structure **/ static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter* adapter) { struct net_device *netdev = adapter->netdev; struct ixgbe_hw *hw = &adapter->hw; adapter->link_up = false; adapter->link_speed = 0; /* only continue if link was up previously */ if (!netif_carrier_ok(netdev)) return; /* poll for SFP+ cable when link is down */ if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB) adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP; e_info(drv, "NIC Link is Down\n"); netif_carrier_off(netdev); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) { DPRINTK(LINK, ERR, "NIC CNA Link is Down\n"); netif_carrier_off(adapter->cnadev); } netif_tx_stop_all_queues(netdev); if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) netif_tx_stop_all_queues(adapter->cnadev); } /** * ixgbe_watchdog_flush_tx - flush queues on link down * @adapter - pointer to the device adapter structure **/ static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter) { int i; int some_tx_pending = 0; if (!netif_carrier_ok(adapter->netdev)) { for (i = 0; i < adapter->num_tx_queues; i++) { struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; if (tx_ring->next_to_use != tx_ring->next_to_clean) { some_tx_pending = 1; break; } } if (some_tx_pending) { /* We've lost link, so the controller stops DMA, * but we've got queued Tx work that's never going * to get done, so reset controller to flush Tx. * (Do the reset outside of interrupt context). */ adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED; } } } static void ixgbe_spoof_check(struct ixgbe_adapter *adapter) { u32 ssvpc; /* Do not perform spoof check for 82598 */ if (adapter->hw.mac.type == ixgbe_mac_82598EB) return; ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC); /* * ssvpc register is cleared on read, if zero then no * spoofed packets in the last interval. */ if (!ssvpc) return; e_warn(drv, "%d Spoofed packets detected\n", ssvpc); } /** * ixgbe_watchdog_subtask - check and bring link up * @adapter - pointer to the device adapter structure **/ static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter) { /* if interface is down do nothing */ if (test_bit(__IXGBE_DOWN, &adapter->state) || test_bit(__IXGBE_RESETTING, &adapter->state)) return; ixgbe_watchdog_update_link(adapter); if (adapter->link_up) ixgbe_watchdog_link_is_up(adapter); else ixgbe_watchdog_link_is_down(adapter); ixgbe_spoof_check(adapter); ixgbe_update_stats(adapter); ixgbe_watchdog_flush_tx(adapter); } /** * ixgbe_sfp_detection_subtask - poll for SFP+ cable * @adapter - the ixgbe adapter structure **/ static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; s32 err; /* not searching for SFP so there is nothing to do here */ if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) && !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) return; /* someone else is in init, wait until next service event */ if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) return; err = hw->phy.ops.identify_sfp(hw); if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) goto sfp_out; if (err == IXGBE_ERR_SFP_NOT_PRESENT) { /* If no cable is present, then we need to reset * the next time we find a good cable. */ adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; } /* exit on error */ if (err) goto sfp_out; /* exit if reset not needed */ if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) goto sfp_out; adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET; /* * A module may be identified correctly, but the EEPROM may not have * support for that module. setup_sfp() will fail in that case, so * we should not allow that module to load. */ if (hw->mac.type == ixgbe_mac_82598EB) err = hw->phy.ops.reset(hw); else err = hw->mac.ops.setup_sfp(hw); if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) goto sfp_out; adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type); sfp_out: clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) && adapter->netdev_registered) { e_dev_err("failed to initialize because an unsupported " "SFP+ module type was detected.\n"); e_dev_err("Reload the driver after installing a " "supported module.\n"); #ifndef __VMKLNX__ unregister_netdev(adapter->netdev); adapter->netdev_registered = false; #else /* * The original approach unregister_netdev() will release * the dev->sharedData which is used by Uplink module, * then a PSOD will occer, details is described in PR1137688. * Here we close the device using dev_close() and clear * IFF_DEV_IS_OPEN flag to let up layer know the change. */ rtnl_lock(); if (adapter->netdev->gflags & IFF_DEV_IS_OPEN) { e_info(probe, "closing %s due to unsupported SFP+\n", adapter->netdev->name); dev_close(adapter->netdev); adapter->netdev->gflags &= ~IFF_DEV_IS_OPEN; } rtnl_unlock(); #endif } } /** * ixgbe_sfp_link_config_subtask - set up link SFP after module install * @adapter - the ixgbe adapter structure **/ static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 autoneg; bool negotiation; if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG)) return; /* someone else is in init, wait until next service event */ if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) return; adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; autoneg = hw->phy.autoneg_advertised; if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation); if (hw->mac.ops.setup_link) hw->mac.ops.setup_link(hw, autoneg, negotiation, true); adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; adapter->link_check_timeout = jiffies; clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); } static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter) { int vf; struct ixgbe_hw *hw = &adapter->hw; struct net_device *netdev = adapter->netdev; u32 gpc; u32 ciaa, ciad; gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC); if (gpc) /* If incrementing then no need for the check below */ return; /* * Check to see if a bad DMA write target from an errant or * malicious VF has caused a PCIe error. If so then we can * issue a VFLR to the offending VF(s) and then resume without * requesting a full slot reset. */ for (vf = 0; vf < adapter->num_vfs; vf++) { ciaa = (vf << 16) | 0x80000000; /* 32 bit read so align, we really want status at offset 6 */ ciaa |= PCI_COMMAND; IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); ciad = IXGBE_READ_REG(hw, IXGBE_CIAD_82599); ciaa &= 0x7FFFFFFF; /* disable debug mode asap after reading data */ IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); /* Get the upper 16 bits which will be the PCI status reg */ ciad >>= 16; if (ciad & PCI_STATUS_REC_MASTER_ABORT) { netdev_err(netdev, "VF %d Hung DMA\n", vf); /* Issue VFLR */ ciaa = (vf << 16) | 0x80000000; ciaa |= 0xA8; IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); ciad = 0x00008000; /* VFLR */ IXGBE_WRITE_REG(hw, IXGBE_CIAD_82599, ciad); ciaa &= 0x7FFFFFFF; IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); } } } /** * ixgbe_service_timer - Timer Call-back * @data: pointer to adapter cast into an unsigned long **/ static void ixgbe_service_timer(unsigned long data) { struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data; unsigned long next_event_offset; bool ready = false; /* * don't bother with SR-IOV VF DMA hang check if there are * no VFs or the link is down */ if (!adapter->num_vfs || (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) { ready = true; goto normal_timer_service; } /* If we have VFs allocated then we must check for DMA hangs */ ixgbe_check_for_bad_vf(adapter); next_event_offset = HZ / 50; adapter->timer_event_accumulator++; if (adapter->timer_event_accumulator >= 100) { ready = true; adapter->timer_event_accumulator = 0; } goto schedule_event; normal_timer_service: /* poll faster when waiting for link */ if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE) next_event_offset = HZ / 10; else next_event_offset = HZ * 2; schedule_event: /* Reset the timer */ mod_timer(&adapter->service_timer, next_event_offset + jiffies); if (ready) ixgbe_service_event_schedule(adapter); } static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter) { if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED)) return; adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED; /* If we're already down or resetting, just bail */ if (test_bit(__IXGBE_DOWN, &adapter->state) || test_bit(__IXGBE_RESETTING, &adapter->state)) return; adapter->tx_timeout_count++; netdev_err(adapter->netdev, "Reset adapter\n"); ixgbe_reinit_locked(adapter); } /** * ixgbe_service_task - manages and runs subtasks * @work: pointer to work_struct containing our data **/ static void ixgbe_service_task(struct work_struct *work) { struct ixgbe_adapter *adapter = container_of(work, struct ixgbe_adapter, service_task); ixgbe_reset_subtask(adapter); ixgbe_sfp_detection_subtask(adapter); ixgbe_sfp_link_config_subtask(adapter); ixgbe_check_overtemp_subtask(adapter); ixgbe_watchdog_subtask(adapter); #ifdef HAVE_TX_MQ ixgbe_fdir_reinit_subtask(adapter); #endif ixgbe_check_hang_subtask(adapter); ixgbe_service_event_complete(adapter); } void ixgbe_tx_ctxtdesc(struct ixgbe_ring *tx_ring, u32 vlan_macip_lens, u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx) { struct ixgbe_adv_tx_context_desc *context_desc; u16 i = tx_ring->next_to_use; context_desc = IXGBE_TX_CTXTDESC(tx_ring, i); i++; tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; /* set bits to identify this as an advanced context descriptor */ type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); context_desc->seqnum_seed = cpu_to_le32(fcoe_sof_eof); context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); } static int ixgbe_tso(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first, u8 *hdr_len) { #ifdef NETIF_F_TSO struct sk_buff *skb = first->skb; u32 vlan_macip_lens, type_tucmd; u32 mss_l4len_idx, l4len; if (!skb_is_gso(skb)) #endif /* NETIF_F_TSO */ return 0; #ifdef NETIF_F_TSO if (skb_header_cloned(skb)) { int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); if (err) return err; } /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; if (first->protocol == __constant_htons(ETH_P_IP)) { struct iphdr *iph = ip_hdr(skb); iph->tot_len = 0; iph->check = 0; tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, IPPROTO_TCP, 0); type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; first->tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_IPV4; #ifdef NETIF_F_TSO6 } else if (skb_is_gso_v6(skb)) { ipv6_hdr(skb)->payload_len = 0; tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); first->tx_flags |= IXGBE_TX_FLAGS_TSO; #endif } /* compute header lengths */ l4len = tcp_hdrlen(skb); *hdr_len = skb_transport_offset(skb) + l4len; /* update gso size and bytecount with header size */ first->gso_segs = skb_shinfo(skb)->gso_segs; first->bytecount += (first->gso_segs - 1) * *hdr_len; /* mss_l4len_id: use 1 as index for TSO */ mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT; /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ vlan_macip_lens = skb_network_header_len(skb); vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd, mss_l4len_idx); return 1; #endif } static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first) { struct sk_buff *skb = first->skb; u32 vlan_macip_lens = 0; u32 mss_l4len_idx = 0; u32 type_tucmd = 0; if (skb->ip_summed != CHECKSUM_PARTIAL) { if (!(first->tx_flags & IXGBE_TX_FLAGS_HW_VLAN) && !(first->tx_flags & IXGBE_TX_FLAGS_TXSW)) return; } else { u8 l4_hdr = 0; switch (first->protocol) { case __constant_htons(ETH_P_IP): vlan_macip_lens |= skb_network_header_len(skb); type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; l4_hdr = ip_hdr(skb)->protocol; break; #ifdef NETIF_F_IPV6_CSUM case __constant_htons(ETH_P_IPV6): vlan_macip_lens |= skb_network_header_len(skb); l4_hdr = ipv6_hdr(skb)->nexthdr; break; #endif default: #ifndef __VMKLNX__ if (unlikely(net_ratelimit())) { dev_warn(tx_ring->dev, "partial checksum but proto=%x!\n", first->protocol); } #endif break; } switch (l4_hdr) { case IPPROTO_TCP: type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; mss_l4len_idx = tcp_hdrlen(skb) << IXGBE_ADVTXD_L4LEN_SHIFT; break; #ifdef HAVE_SCTP case IPPROTO_SCTP: type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; mss_l4len_idx = sizeof(struct sctphdr) << IXGBE_ADVTXD_L4LEN_SHIFT; break; #endif case IPPROTO_UDP: mss_l4len_idx = sizeof(struct udphdr) << IXGBE_ADVTXD_L4LEN_SHIFT; break; default: #ifndef __VMKLNX__ if (unlikely(net_ratelimit())) { dev_warn(tx_ring->dev, "partial checksum but l4 proto=%x!\n", l4_hdr); } #endif break; } /* update TX checksum flag */ first->tx_flags |= IXGBE_TX_FLAGS_CSUM; } /* vlan_macip_lens: MACLEN, VLAN tag */ vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd, mss_l4len_idx); } static __le32 ixgbe_tx_cmd_type(u32 tx_flags) { /* set type for advanced descriptor with frame checksum insertion */ __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); /* set HW vlan bit if vlan is present */ if (tx_flags & IXGBE_TX_FLAGS_HW_VLAN) cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); if (tx_flags & IXGBE_TX_FLAGS_TSTAMP) cmd_type |= cpu_to_le32(IXGBE_ADVTXD_MAC_TSTAMP); /* set segmentation enable bits for TSO/FSO */ #ifdef IXGBE_FCOE if ((tx_flags & IXGBE_TX_FLAGS_TSO) || (tx_flags & IXGBE_TX_FLAGS_FSO)) #else if (tx_flags & IXGBE_TX_FLAGS_TSO) #endif cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); return cmd_type; } static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, u32 tx_flags, unsigned int paylen) { u32 olinfo_status = paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; /* enable L4 checksum for TSO and TX checksum offload */ if (tx_flags & IXGBE_TX_FLAGS_CSUM) { olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM; } else if (tx_flags & IXGBE_TX_FLAGS_TSO) { olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM | (1 << IXGBE_ADVTXD_IDX_SHIFT); /* enble IPv4 checksum for TSO */ if (tx_flags & IXGBE_TX_FLAGS_IPV4) olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM; #ifdef IXGBE_FCOE /* use index 1 context for FCOE/FSO */ } else if (tx_flags & IXGBE_TX_FLAGS_FCOE) { olinfo_status |= IXGBE_ADVTXD_CC | (1 << IXGBE_ADVTXD_IDX_SHIFT); #endif } /* * Check Context must be set if Tx switch is enabled, which it * always is for case where virtual functions are running */ if (tx_flags & IXGBE_TX_FLAGS_TXSW) olinfo_status |= IXGBE_ADVTXD_CC; tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); } #define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \ IXGBE_TXD_CMD_RS) static void ixgbe_tx_map(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first, const u8 hdr_len) { struct sk_buff *skb = first->skb; struct ixgbe_tx_buffer *tx_buffer_info; union ixgbe_adv_tx_desc *tx_desc; dma_addr_t dma; __le32 cmd_type; #ifdef MAX_SKB_FRAGS struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; unsigned int data_len = skb->data_len; #endif unsigned int size = skb_headlen(skb); unsigned int paylen = skb->len - hdr_len; u32 tx_flags = first->tx_flags; u16 i = tx_ring->next_to_use; tx_desc = IXGBE_TX_DESC(tx_ring, i); ixgbe_tx_olinfo_status(tx_desc, tx_flags, paylen); cmd_type = ixgbe_tx_cmd_type(tx_flags); #ifdef IXGBE_FCOE if (tx_flags & IXGBE_TX_FLAGS_FCOE) { if (data_len < sizeof(struct fcoe_crc_eof)) { size -= sizeof(struct fcoe_crc_eof) - data_len; data_len = 0; } else { data_len -= sizeof(struct fcoe_crc_eof); } } #endif dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); if (dma_mapping_error(tx_ring->dev, dma)) goto dma_error; /* record length, and DMA address */ first->length = size; first->dma = dma; tx_desc->read.buffer_addr = cpu_to_le64(dma); #ifdef MAX_SKB_FRAGS for (;;) { #endif while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); i++; tx_desc++; if (i == tx_ring->count) { tx_desc = IXGBE_TX_DESC(tx_ring, 0); i = 0; } dma += IXGBE_MAX_DATA_PER_TXD; size -= IXGBE_MAX_DATA_PER_TXD; tx_desc->read.buffer_addr = cpu_to_le64(dma); tx_desc->read.olinfo_status = 0; } #ifdef MAX_SKB_FRAGS if (likely(!data_len)) break; tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); i++; tx_desc++; if (i == tx_ring->count) { tx_desc = IXGBE_TX_DESC(tx_ring, 0); i = 0; } #ifdef IXGBE_FCOE size = min_t(unsigned int, data_len, skb_frag_size(frag)); #else size = skb_frag_size(frag); #endif data_len -= size; dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, DMA_TO_DEVICE); if (dma_mapping_error(tx_ring->dev, dma)) goto dma_error; tx_buffer_info = &tx_ring->tx_buffer_info[i]; tx_buffer_info->length = size; tx_buffer_info->dma = dma; tx_desc->read.buffer_addr = cpu_to_le64(dma); tx_desc->read.olinfo_status = 0; frag++; } #endif /* MAX_SKB_FRAGS */ /* write last descriptor with RS and EOP bits */ cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); tx_desc->read.cmd_type_len = cmd_type; /* set the timestamp */ first->time_stamp = jiffies; /* * Force memory writes to complete before letting h/w know there * are new descriptors to fetch. (Only applicable for weak-ordered * memory model archs, such as IA-64). * * We also need this memory barrier to make certain all of the * status bits have been updated before next_to_watch is written. */ wmb(); /* set next_to_watch value indicating a packet is present */ first->next_to_watch = tx_desc; i++; if (i == tx_ring->count) i = 0; tx_ring->next_to_use = i; /* notify HW of packet */ writel(i, tx_ring->tail); return; dma_error: dev_err(tx_ring->dev, "TX DMA map failed\n"); /* clear dma mappings for failed tx_buffer_info map */ for (;;) { tx_buffer_info = &tx_ring->tx_buffer_info[i]; ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); if (tx_buffer_info == first) break; if (i == 0) i = tx_ring->count; i--; } tx_ring->next_to_use = i; } static void ixgbe_atr(struct ixgbe_ring *ring, struct ixgbe_tx_buffer *first) { struct ixgbe_q_vector *q_vector = ring->q_vector; union ixgbe_atr_hash_dword input = { .dword = 0 }; union ixgbe_atr_hash_dword common = { .dword = 0 }; union { unsigned char *network; struct iphdr *ipv4; struct ipv6hdr *ipv6; } hdr; struct tcphdr *th; __be16 vlan_id; /* if ring doesn't have a interrupt vector, cannot perform ATR */ if (!q_vector) return; /* do nothing if sampling is disabled */ if (!ring->atr_sample_rate) return; ring->atr_count++; /* snag network header to get L4 type and address */ hdr.network = skb_network_header(first->skb); /* Currently only IPv4/IPv6 with TCP is supported */ if ((first->protocol != __constant_htons(ETH_P_IPV6) || hdr.ipv6->nexthdr != IPPROTO_TCP) && (first->protocol != __constant_htons(ETH_P_IP) || hdr.ipv4->protocol != IPPROTO_TCP)) return; th = tcp_hdr(first->skb); /* skip this packet since it is invalid or the socket is closing */ if (!th || th->fin) return; /* sample on all syn packets or once every atr sample count */ if (!th->syn && (ring->atr_count < ring->atr_sample_rate)) return; /* reset sample count */ ring->atr_count = 0; vlan_id = htons(first->tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT); /* * src and dst are inverted, think how the receiver sees them * * The input is broken into two sections, a non-compressed section * containing vm_pool, vlan_id, and flow_type. The rest of the data * is XORed together and stored in the compressed dword. */ input.formatted.vlan_id = vlan_id; /* * since src port and flex bytes occupy the same word XOR them together * and write the value to source port portion of compressed dword */ if (first->tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN)) common.port.src ^= th->dest ^ __constant_htons(ETH_P_8021Q); else common.port.src ^= th->dest ^ first->protocol; common.port.dst ^= th->source; if (first->protocol == __constant_htons(ETH_P_IP)) { input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4; common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr; } else { input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6; common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^ hdr.ipv6->saddr.s6_addr32[1] ^ hdr.ipv6->saddr.s6_addr32[2] ^ hdr.ipv6->saddr.s6_addr32[3] ^ hdr.ipv6->daddr.s6_addr32[0] ^ hdr.ipv6->daddr.s6_addr32[1] ^ hdr.ipv6->daddr.s6_addr32[2] ^ hdr.ipv6->daddr.s6_addr32[3]; } /* This assumes the Rx queue and Tx queue are bound to the same CPU */ ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw, input, common, ring->queue_index); } static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) { netif_stop_subqueue(netdev_ring(tx_ring), ring_queue_index(tx_ring)); /* Herbert's original patch had: * smp_mb__after_netif_stop_queue(); * but since that doesn't exist yet, just open code it. */ smp_mb(); /* We need to check again in a case another CPU has just * made room available. */ if (likely(ixgbe_desc_unused(tx_ring) < size)) return -EBUSY; /* A reprieve! - use start_queue because it doesn't call schedule */ netif_start_subqueue(netdev_ring(tx_ring), ring_queue_index(tx_ring)); ++tx_ring->tx_stats.restart_queue; return 0; } static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) { if (likely(ixgbe_desc_unused(tx_ring) >= size)) return 0; return __ixgbe_maybe_stop_tx(tx_ring, size); } netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring) { struct ixgbe_tx_buffer *first; int tso; u32 tx_flags = 0; #ifdef MAX_SKB_FRAGS #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD unsigned short f; #endif #endif u16 count = TXD_USE_COUNT(skb_headlen(skb)); __be16 protocol = skb->protocol; u8 hdr_len = 0; /* * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, * + 1 desc for skb_head_len/IXGBE_MAX_DATA_PER_TXD, * + 2 desc gap to keep tail from touching head, * + 1 desc for context descriptor, * otherwise try next time */ #ifdef MAX_SKB_FRAGS #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); #else count += skb_shinfo(skb)->nr_frags; #endif #endif if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) { tx_ring->tx_stats.tx_busy++; return NETDEV_TX_BUSY; } /* record the location of the first descriptor for this packet */ first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; first->skb = skb; first->bytecount = skb->len; first->gso_segs = 1; /* if we have a HW VLAN tag being added default to the HW one */ if (vlan_tx_tag_present(skb)) { tx_flags |= vlan_tx_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT; tx_flags |= IXGBE_TX_FLAGS_HW_VLAN; /* else if it is a SW VLAN check the next protocol and store the tag */ } else if (protocol == __constant_htons(ETH_P_8021Q)) { struct vlan_hdr *vhdr, _vhdr; vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); if (!vhdr) goto out_drop; protocol = vhdr->h_vlan_encapsulated_proto; tx_flags |= ntohs(vhdr->h_vlan_TCI) << IXGBE_TX_FLAGS_VLAN_SHIFT; tx_flags |= IXGBE_TX_FLAGS_SW_VLAN; } /* * Use the l2switch_enable flag - would be false if the DMA * Tx switch had been disabled. */ if (adapter->flags & IXGBE_FLAG_SRIOV_L2SWITCH_ENABLE) tx_flags |= IXGBE_TX_FLAGS_TXSW; #ifndef __VMKLNX__ #ifdef HAVE_TX_MQ if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || (skb->priority != TC_PRIO_CONTROL))) { tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK; #ifdef IXGBE_FCOE /* for FCoE with DCB, we force the priority to what * was specified by the switch */ if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && ((protocol == __constant_htons(ETH_P_FCOE)) || (protocol == __constant_htons(ETH_P_FIP)))) tx_flags |= adapter->fcoe.up << IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT; else #endif /* IXGBE_FCOE */ tx_flags |= skb->priority << IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT; if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) { struct vlan_ethhdr *vhdr; if (skb_header_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) goto out_drop; vhdr = (struct vlan_ethhdr *)skb->data; vhdr->h_vlan_TCI = htons(tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT); } else { tx_flags |= IXGBE_TX_FLAGS_HW_VLAN; } } #endif /* HAVE_TX_MQ */ #endif /* __VMKLNX__ */ /* record initial flags and protocol */ first->tx_flags = tx_flags; first->protocol = protocol; #ifdef IXGBE_FCOE /* setup tx offload for FCoE */ if ((protocol == __constant_htons(ETH_P_FCOE)) && (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) { tso = ixgbe_fso(tx_ring, first, &hdr_len); if (tso < 0) goto out_drop; goto xmit_fcoe; } #endif /* IXGBE_FCOE */ tso = ixgbe_tso(tx_ring, first, &hdr_len); if (tso < 0) goto out_drop; else if (!tso) ixgbe_tx_csum(tx_ring, first); /* add the ATR filter if ATR is on */ if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state)) ixgbe_atr(tx_ring, first); #ifdef IXGBE_FCOE xmit_fcoe: #endif /* IXGBE_FCOE */ ixgbe_tx_map(tx_ring, first, hdr_len); #ifndef HAVE_TRANS_START_IN_QUEUE netdev_ring(tx_ring)->trans_start = jiffies; #endif ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED); return NETDEV_TX_OK; out_drop: dev_kfree_skb_any(first->skb); first->skb = NULL; return NETDEV_TX_OK; } static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, struct net_device *netdev) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_ring *tx_ring; #ifdef HAVE_TX_MQ unsigned int r_idx = skb->queue_mapping; #endif if (skb->len <= 0) { dev_kfree_skb_any(skb); return NETDEV_TX_OK; } /* * The minimum packet size for olinfo paylen is 17 so pad the skb * in order to meet this minimum size requirement. */ if (skb->len < 17) { if (skb_padto(skb, 17)) return NETDEV_TX_OK; skb->len = 17; } #ifdef HAVE_TX_MQ if (r_idx >= adapter->num_tx_queues) r_idx = r_idx % adapter->num_tx_queues; tx_ring = adapter->tx_ring[r_idx]; #else tx_ring = adapter->tx_ring[0]; #endif #ifdef IXGBE_FCOE if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && ((skb->protocol == __constant_htons(ETH_P_FCOE)) || (skb->protocol == __constant_htons(ETH_P_FIP))) && (netdev->features & NETIF_F_CNA)) { int r_idx; struct ixgbe_ring_feature *f; f = &adapter->ring_feature[RING_F_FCOE]; r_idx = smp_processor_id(); r_idx &= (f->indices - 1); r_idx += adapter->num_tx_queues - f->indices; tx_ring = adapter->tx_ring[r_idx]; } #endif /* IXGBE_FCOE */ return ixgbe_xmit_frame_ring(skb, adapter, tx_ring); } /** * ixgbe_set_mac - Change the Ethernet Address of the NIC * @netdev: network interface device structure * @p: pointer to an address structure * * Returns 0 on success, negative on failure **/ static int ixgbe_set_mac(struct net_device *netdev, void *p) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; struct sockaddr *addr = p; int ret; if (!is_valid_ether_addr(addr->sa_data)) return -EADDRNOTAVAIL; ixgbe_del_mac_filter(adapter, hw->mac.addr, adapter->num_vfs); memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); /* set the correct pool for the new PF MAC address in entry 0 */ ret = ixgbe_add_mac_filter(adapter, hw->mac.addr, adapter->num_vfs); return (ret >= 0 ? 0 : ret); } #if defined(HAVE_NETDEV_STORAGE_ADDRESS) && defined(NETDEV_HW_ADDR_T_SAN) /** * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding * netdev->dev_addr_list * @netdev: network interface device structure * * Returns non-zero on failure **/ static int ixgbe_add_sanmac_netdev(struct net_device *dev) { int err = 0; struct ixgbe_adapter *adapter = netdev_priv(dev); struct ixgbe_mac_info *mac = &adapter->hw.mac; if (is_valid_ether_addr(mac->san_addr)) { rtnl_lock(); err = dev_addr_add(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN); rtnl_unlock(); } return err; } /** * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding * netdev->dev_addr_list * @netdev: network interface device structure * * Returns non-zero on failure **/ static int ixgbe_del_sanmac_netdev(struct net_device *dev) { int err = 0; struct ixgbe_adapter *adapter = netdev_priv(dev); struct ixgbe_mac_info *mac = &adapter->hw.mac; if (is_valid_ether_addr(mac->san_addr)) { rtnl_lock(); err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN); rtnl_unlock(); } return err; } #endif /* (HAVE_NETDEV_STORAGE_ADDRESS) && defined(NETDEV_HW_ADDR_T_SAN) */ /** * ixgbe_ioctl - * @netdev: * @ifreq: * @cmd: **/ static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) { switch (cmd) { #ifdef ETHTOOL_OPS_COMPAT case SIOCETHTOOL: return ethtool_ioctl(ifr); #endif default: return -EOPNOTSUPP; } } #ifdef CONFIG_NET_POLL_CONTROLLER /* * Polling 'interrupt' - used by things like netconsole to send skbs * without having to re-enable interrupts. It's not called while * the interrupt routine is executing. */ static void ixgbe_netpoll(struct net_device *netdev) { struct ixgbe_adapter *adapter = netdev_priv(netdev); int i; /* if interface is down do nothing */ if (test_bit(__IXGBE_DOWN, &adapter->state)) return; #ifndef CONFIG_IXGBE_NAPI ixgbe_irq_disable(adapter); #endif adapter->flags |= IXGBE_FLAG_IN_NETPOLL; if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { int num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; for (i = 0; i < num_q_vectors; i++) { struct ixgbe_q_vector *q_vector = adapter->q_vector[i]; ixgbe_msix_clean_rings(0, q_vector); } } else { ixgbe_intr(0, adapter); } adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL; #ifndef CONFIG_IXGBE_NAPI ixgbe_irq_enable(adapter, true, true); #endif } #endif #ifdef HAVE_NETDEV_SELECT_QUEUE static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb) { struct ixgbe_adapter *adapter = netdev_priv(dev); int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : smp_processor_id(); #ifdef IXGBE_FCOE if ((skb->protocol == __constant_htons(ETH_P_FCOE)) || (skb->protocol == __constant_htons(ETH_P_FIP))) { if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) { txq &= (adapter->ring_feature[RING_F_FCOE].indices - 1); txq += adapter->ring_feature[RING_F_FCOE].mask; return txq; } else if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) { txq = adapter->fcoe.up; return txq; } } #endif /* IXGBE_FCOE */ if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { while (unlikely(txq >= dev->real_num_tx_queues)) txq -= dev->real_num_tx_queues; return txq; } #ifndef HAVE_MQPRIO if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) { if (skb->priority == TC_PRIO_CONTROL) txq = adapter->tc - 1; else txq = (skb->vlan_tci & IXGBE_TX_FLAGS_VLAN_PRIO_MASK) >> 13; return txq; } #endif return skb_tx_hash(dev, skb); } #endif /* HAVE_NETDEV_SELECT_QUEUE */ /* ixgbe_validate_rtr - verify 802.1Qp to rx packet buffer mapping is valid. * #adapter: pointer to ixgbe_adapter * @tc: number of traffic classes currently enabled * * Configure a valid 802.1Qp to rx packet buffer mapping ie confirm * 802.1Q priority maps to a packet buffer that exists. */ static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc) { struct ixgbe_hw *hw = &adapter->hw; u32 reg, rsave; int i; /* 82598 have a static priority to TC mapping that can not * be changed so no validation is needed. */ if (hw->mac.type == ixgbe_mac_82598EB) return; reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC); rsave = reg; for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT); /* If up2tc is out of bounds default to zero */ if (up2tc > tc) reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT); } if (reg != rsave) IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg); return; } /* ixgbe_setup_tc - routine to configure net_device for multiple traffic * classes. * * @netdev: net device to configure * @tc: number of traffic classes to enable */ int ixgbe_setup_tc(struct net_device *dev, u8 tc) { struct ixgbe_adapter *adapter = netdev_priv(dev); struct ixgbe_hw *hw = &adapter->hw; /* If DCB is anabled do not remove traffic classes, multiple * traffic classes are required to implement DCB */ if (!tc && (adapter->flags & IXGBE_FLAG_DCB_ENABLED)) return 0; /* Hardware supports up to 8 traffic classes */ if (tc > adapter->dcb_cfg.num_tcs.pg_tcs || (hw->mac.type == ixgbe_mac_82598EB && tc < IXGBE_DCB_MAX_TRAFFIC_CLASS)) return -EINVAL; /* Hardware has to reinitialize queues and interrupts to * match packet buffer alignment. Unfortunantly, the * hardware is not flexible enough to do this dynamically. */ if (netif_running(dev)) ixgbe_close(dev); ixgbe_clear_interrupt_scheme(adapter); #ifdef HAVE_MQPRIO if (tc) netdev_set_num_tc(dev, tc); else netdev_reset_tc(dev); #else adapter->tc = tc; #endif ixgbe_init_interrupt_scheme(adapter); ixgbe_validate_rtr(adapter, tc); if (netif_running(dev)) ixgbe_open(dev); return 0; } void ixgbe_do_reset(struct net_device *netdev) { struct ixgbe_adapter *adapter = netdev_priv(netdev); if (netif_running(netdev)) ixgbe_reinit_locked(adapter); else ixgbe_reset(adapter); } #ifdef HAVE_NDO_SET_FEATURES static u32 ixgbe_fix_features(struct net_device *netdev, u32 features) { struct ixgbe_adapter *adapter = netdev_priv(netdev); #ifdef CONFIG_DCB if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) features |= NETIF_F_HW_VLAN_RX; #endif /* return error if RXHASH is being enabled when RSS is not supported */ if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED)) features &= ~NETIF_F_RXHASH; /* If Rx checksum is disabled, then RSC/LRO should also be disabled */ if (!(features & NETIF_F_RXCSUM)) features &= ~NETIF_F_LRO; /* Turn off LRO if not RSC capable or invalid ITR settings */ if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)) features &= ~NETIF_F_LRO; return features; } static int ixgbe_set_features(struct net_device *netdev, u32 features) { struct ixgbe_adapter *adapter = netdev_priv(netdev); u32 changed = netdev->features ^ features; bool need_reset = false; /* Make sure RSC matches LRO, reset if change */ if (!(features & NETIF_F_LRO)) { if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) need_reset = true; adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED; } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) && !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) { if (adapter->rx_itr_setting == 1 || adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; need_reset = true; } else if (changed & NETIF_F_LRO) { e_info(probe, "rx-usecs set too low, " "disabling RSC\n"); } } /* * Check if Flow Director n-tuple support was enabled or disabled. If * the state changed, we need to reset. */ if (!(features & NETIF_F_NTUPLE)) { if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) { /* turn off Flow Director, set ATR and reset */ if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) && !(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; need_reset = true; } adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE; } else if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) { /* turn off ATR, enable perfect filters and reset */ adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE; need_reset = true; } if (need_reset) ixgbe_do_reset(netdev); return 0; } #endif /* HAVE_NDO_SET_FEATURES */ #ifdef HAVE_NET_DEVICE_OPS static const struct net_device_ops ixgbe_netdev_ops = { .ndo_open = &ixgbe_open, .ndo_stop = &ixgbe_close, .ndo_start_xmit = &ixgbe_xmit_frame, .ndo_get_stats = &ixgbe_get_stats, .ndo_set_rx_mode = &ixgbe_set_rx_mode, .ndo_validate_addr = eth_validate_addr, .ndo_set_mac_address = &ixgbe_set_mac, .ndo_change_mtu = &ixgbe_change_mtu, .ndo_do_ioctl = &ixgbe_ioctl, .ndo_tx_timeout = &ixgbe_tx_timeout, #ifdef NETIF_F_HW_VLAN_TX .ndo_vlan_rx_add_vid = &ixgbe_vlan_rx_add_vid, .ndo_vlan_rx_kill_vid = &ixgbe_vlan_rx_kill_vid, #endif #ifdef HAVE_SETUP_TC .ndo_setup_tc = &ixgbe_setup_tc, #endif #ifdef HAVE_IPLINK_VF_CONFIG .ndo_set_vf_mac = ixgbe_ndo_set_vf_mac, .ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan, .ndo_set_vf_tx_rate = ixgbe_ndo_set_vf_bw, .ndo_get_vf_config = ixgbe_ndo_get_vf_config, #endif #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = &ixgbe_netpoll, #endif .ndo_select_queue = &ixgbe_select_queue, #ifdef IXGBE_FCOE .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get, #ifdef HAVE_NETDEV_OPS_FCOE_DDP_TARGET .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target, #endif .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put, #ifdef HAVE_NETDEV_OPS_FCOE_ENABLE .ndo_fcoe_enable = ixgbe_fcoe_enable, .ndo_fcoe_disable = ixgbe_fcoe_disable, #endif #ifdef HAVE_NETDEV_OPS_FCOE_GETWWN .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn, #endif #endif /* IXGBE_FCOE */ #ifdef HAVE_NDO_SET_FEATURES .ndo_set_features = ixgbe_set_features, .ndo_fix_features = ixgbe_fix_features, #endif /* HAVE_NDO_SET_FEATURES */ #ifdef HAVE_VLAN_RX_REGISTER .ndo_vlan_rx_register = &ixgbe_vlan_mode, #endif }; #endif /* HAVE_NET_DEVICE_OPS */ void ixgbe_assign_netdev_ops(struct net_device *dev) { #ifdef HAVE_NET_DEVICE_OPS dev->netdev_ops = &ixgbe_netdev_ops; #else /* HAVE_NET_DEVICE_OPS */ dev->open = &ixgbe_open; dev->stop = &ixgbe_close; dev->hard_start_xmit = &ixgbe_xmit_frame; dev->get_stats = &ixgbe_get_stats; #ifdef HAVE_SET_RX_MODE dev->set_rx_mode = &ixgbe_set_rx_mode; #endif dev->set_multicast_list = &ixgbe_set_rx_mode; dev->set_mac_address = &ixgbe_set_mac; dev->change_mtu = &ixgbe_change_mtu; dev->do_ioctl = &ixgbe_ioctl; #ifdef HAVE_TX_TIMEOUT dev->tx_timeout = &ixgbe_tx_timeout; #endif #ifdef NETIF_F_HW_VLAN_TX dev->vlan_rx_register = &ixgbe_vlan_mode; dev->vlan_rx_add_vid = &ixgbe_vlan_rx_add_vid; dev->vlan_rx_kill_vid = &ixgbe_vlan_rx_kill_vid; #endif #ifdef CONFIG_NET_POLL_CONTROLLER dev->poll_controller = &ixgbe_netpoll; #endif #ifdef HAVE_NETDEV_SELECT_QUEUE dev->select_queue = &ixgbe_select_queue; #endif /* HAVE_NETDEV_SELECT_QUEUE */ #endif /* HAVE_NET_DEVICE_OPS */ ixgbe_set_ethtool_ops(dev); dev->watchdog_timeo = 5 * HZ; } #if defined(CONFIG_PCI_IOV) && defined(HAVE_IPLINK_VF_CONFIG) && defined(__VMKLNX__) static VMK_ReturnStatus ixgbe_passthru_ops(struct net_device *netdev, vmk_NetPTOP op, void *pargs) { struct ixgbe_adapter *adapter = netdev_priv(netdev); VMK_ReturnStatus ret; switch (op) { case VMK_NETPTOP_VF_SET_MAC: { vmk_NetPTOPVFSetMacArgs *args = pargs; if (is_zero_ether_addr(args->mac)) { /* Remove the VF mac address */ ixgbe_del_mac_filter(adapter, adapter->vfinfo[args->vf].vf_mac_addresses, args->vf); memset(adapter->vfinfo[args->vf].vf_mac_addresses, 0, ETH_ALEN); ret = VMK_OK; } else { DPRINTK(PROBE, ERR, "Setting ndo MAC addr" " %02x:%02x:%02x:%02x:%02x:%02x" " for vf %u\n", args->mac[0], args->mac[1], args->mac[2], args->mac[3], args->mac[4], args->mac[5], args->vf); if (ixgbe_ndo_set_vf_mac(netdev, args->vf, args->mac) < 0) ret = VMK_FAILURE; else ret = VMK_OK; } break; } case VMK_NETPTOP_VF_SET_DEFAULT_VLAN: { vmk_NetPTOPVFSetDefaultVlanArgs *args = pargs; if (args->enable) { DPRINTK(PROBE, ERR, "Setting ndo port VLAN %d on VF %d\n", (int)args->vid, (u32)args->vf); ret = ixgbe_ndo_set_vf_vlan(netdev, args->vf, args->vid, args->prio) ? VMK_FAILURE : VMK_OK; } else { DPRINTK(PROBE, ERR, "Removing ndo port VLAN from VF %d\n", (u32)args->vf); ret = ixgbe_ndo_set_vf_vlan(netdev, args->vf, 0, 0) ? VMK_FAILURE : VMK_OK; } break; } case VMK_NETPTOP_VF_ADD_VLAN_RANGE: { vmk_NetPTOPVFVlanRangeArgs *args = pargs; DPRINTK(PROBE, ERR, "Passthru OP to set guest VLAN %d-%d" " received for VF %d\n", (int)args->first, (int)args->last, (u32)args->vf); ret = ixgbe_set_vf_vlan(adapter, 1, args->first, args->vf); break; } case VMK_NETPTOP_VF_DEL_VLAN_RANGE: { vmk_NetPTOPVFVlanRangeArgs *args = pargs; DPRINTK(PROBE, ERR, "Passthru OP to remove guest VLAN %d" " received for VF %d\n", (int)args->first, (u32)args->vf); ret = ixgbe_set_vf_vlan(adapter, 0, args->first, args->vf); break; } case VMK_NETPTOP_VF_GET_QUEUE_STATS: { vmk_NetPTOPVFGetQueueStatsArgs *args = pargs; ret = ixgbe_vf_get_stats(netdev, args->vf, args->numTxQueues, args->numRxQueues, args->tqStats, args->rqStats); break; } case VMK_NETPTOP_VF_SET_MTU: { vmk_NetPTOPVFSetMtuArgs *args = pargs; ret = ixgbe_set_vf_lpe(adapter, args->vf, args->mtu); break; } default: printk("Unhandled OP %d\n", op); ret = VMK_FAILURE; break; } return ret; } VMK_ReturnStatus ixgbe_passthru_config(struct ixgbe_adapter *adapter, u32 vfIdx, int change, void *data) { vmk_NetVFCfgInfo cfginfo; VMK_ReturnStatus ret; struct vf_vlan *guest_vlan = 0; u32 *new_mtu = 0; u8 guest_vid[512]; s16 array_index; u8 bit_index; u8 bit; int i; switch(change) { case VMK_CFG_MAC_CHANGED: printk("ixgbe_passthru_config: MAC Address changed"); if (ETH_ALEN != sizeof(cfginfo.macAddr)) { printk("Invalid MAC address.\n"); return VMK_FAILURE; } cfginfo.cfgChanged = VMK_CFG_MAC_CHANGED; memcpy(cfginfo.macAddr, ((u8*)data), ETH_ALEN); DPRINTK(PROBE, ERR, "Guest OS requesting MAC addr %02x:%02x:%02x:%02x:%02x:%02x" " for VF %d\n", cfginfo.macAddr[0], cfginfo.macAddr[1], cfginfo.macAddr[2],cfginfo.macAddr[3], cfginfo.macAddr[4], cfginfo.macAddr[5], vfIdx); break; case VMK_CFG_GUEST_VLAN_ADD: case VMK_CFG_GUEST_VLAN_REMOVE: guest_vlan = (struct vf_vlan*)data; DPRINTK(PROBE, ERR, "Guest OS requesting %s of vlan %d\n", guest_vlan->add ? "addition" : "deletion", guest_vlan->vid); memset(guest_vid, 0, 512 * sizeof(u8)); array_index = (guest_vlan->vid >> 3) & 0x1FF; bit_index = guest_vlan->vid & 0x7; bit = (1 << bit_index); guest_vid[array_index] |= bit; DPRINTK(PROBE, ERR, "vid: %d, Bit index: %d, array index: %d, bit: %x\n", guest_vlan->vid, bit_index, array_index, bit); if (guest_vlan->add == TRUE) cfginfo.cfgChanged = VMK_CFG_GUEST_VLAN_ADD; else cfginfo.cfgChanged = VMK_CFG_GUEST_VLAN_REMOVE; memcpy(cfginfo.vlan.guestVlans, guest_vid, sizeof(cfginfo.vlan.guestVlans)); DPRINTK(PROBE, ERR, "Passing bit array to kernel\n"); break; case VMK_CFG_MTU_CHANGED: new_mtu = (u32*)data; cfginfo.cfgChanged = VMK_CFG_MTU_CHANGED; DPRINTK(PROBE, ERR, "Guest OS requesting MTU change to %d\n", *new_mtu); memcpy(&cfginfo.mtu, new_mtu, sizeof(cfginfo.mtu)); break; default: printk("Invalid VF configuration change request.\n"); return VMK_FAILURE; } return vmklnx_configure_net_vf(adapter->pdev, (void *)&cfginfo, vfIdx); } #endif static void __devinit ixgbe_probe_vf(struct ixgbe_adapter *adapter) { int err; int num_vf_macvlans, i; struct vf_macvlans *mv_list; /* The 82599 supports up to 64 VFs per physical function * but this implementation limits allocation to 63 so that * basic networking resources are still available to the * physical function */ adapter->num_vfs = (adapter->num_vfs > 63) ? 63 : adapter->num_vfs; err = pci_enable_sriov(adapter->pdev, adapter->num_vfs); if (err) { e_err(probe, "Failed to enable PCI sriov: %d\n", err); goto err_novfs; } num_vf_macvlans = adapter->hw.mac.num_rar_entries - (IXGBE_MAX_PF_MACVLANS + 1 + adapter->num_vfs); adapter->mv_list = mv_list = kcalloc(num_vf_macvlans, sizeof(struct vf_macvlans), GFP_KERNEL); if (mv_list) { /* Initialize list of VF macvlans */ INIT_LIST_HEAD(&adapter->vf_mvs.l); for (i = 0; i < num_vf_macvlans; i++) { mv_list->vf = -1; mv_list->free = true; list_add(&mv_list->l, &adapter->vf_mvs.l); mv_list++; } } /* If call to enable VFs succeeded then allocate memory * for per VF control structures. */ adapter->vfinfo = kcalloc(adapter->num_vfs, sizeof(struct vf_data_storage), GFP_KERNEL); if (adapter->vfinfo) { adapter->flags |= IXGBE_FLAG_SRIOV_L2SWITCH_ENABLE | IXGBE_FLAG_SRIOV_REPLICATION_ENABLE; #if defined(CONFIG_PCI_IOV) && defined(HAVE_IPLINK_VF_CONFIG) && defined(__VMKLNX__) /* Register control callback */ VMK_REGISTER_PT_OPS(adapter->netdev, ixgbe_passthru_ops); printk("Registered Passthru Ops\n"); #endif return; } /* Oh oh */ e_err(probe, "Unable to allocate memory for VF Data Storage - " "SRIOV disabled\n"); ixgbe_disable_sriov(adapter); err_novfs: adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED; adapter->num_vfs = 0; } /** * ixgbe_probe - Device Initialization Routine * @pdev: PCI device information struct * @ent: entry in ixgbe_pci_tbl * * Returns 0 on success, negative on failure * * ixgbe_probe initializes an adapter identified by a pci_dev structure. * The OS initialization, configuring of the adapter private structure, * and a hardware reset occur. **/ static int __devinit ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct net_device *netdev; struct ixgbe_adapter *adapter = NULL; struct ixgbe_hw *hw = NULL; static int cards_found; int i, err, pci_using_dac; char *info_string, *i_s_var; #ifdef HAVE_TX_MQ unsigned int indices; #endif u8 part_str[IXGBE_PBANUM_LENGTH]; enum ixgbe_mac_type mac_type = ixgbe_mac_unknown; #ifdef IXGBE_FCOE u16 device_caps; #endif u16 wol_cap; err = pci_enable_device_mem(pdev); if (err) return err; if (!dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64)) && !dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64))) { pci_using_dac = 1; } else { err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32)); if (err) { err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32)); if (err) { dev_err(pci_dev_to_dev(pdev), "No usable DMA " "configuration, aborting\n"); goto err_dma; } } pci_using_dac = 0; } err = pci_request_selected_regions(pdev, pci_select_bars(pdev, IORESOURCE_MEM), ixgbe_driver_name); if (err) { dev_err(pci_dev_to_dev(pdev), "pci_request_selected_regions failed 0x%x\n", err); goto err_pci_reg; } #ifndef __VMKLNX__ /* * The mac_type is needed before we have the adapter is set up * so rather than maintain two devID -> MAC tables we dummy up * an ixgbe_hw stuct and use ixgbe_set_mac_type. */ hw = vmalloc(sizeof(struct ixgbe_hw)); if (!hw) { printk(KERN_INFO "Unable to allocate memory for early mac " "check\n"); } else { hw->vendor_id = pdev->vendor; hw->device_id = pdev->device; ixgbe_set_mac_type(hw); mac_type = hw->mac.type; vfree(hw); } /* * Workaround of Silicon errata on 82598. Disable LOs in the PCI switch * port to which the 82598 is connected to prevent duplicate * completions caused by LOs. We need the mac type so that we only * do this on 82598 devices, ixgbe_set_mac_type does this for us if * we set it's device ID. */ if (mac_type == ixgbe_mac_82598EB) pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S); #endif pci_enable_pcie_error_reporting(pdev); pci_set_master(pdev); #ifdef HAVE_TX_MQ #ifndef __VMKLNX__ indices = num_possible_cpus(); #ifdef CONFIG_DCB #ifdef HAVE_MQPRIO indices *= IXGBE_DCB_MAX_TRAFFIC_CLASS; #else indices = max_t(unsigned int, indices, IXGBE_MAX_DCB_INDICES); #endif /* HAVE_MQPRIO */ #endif /* CONFIG_DCB */ if (mac_type == ixgbe_mac_unknown) indices = max_t(unsigned int, IXGBE_MAX_RSS_INDICES, IXGBE_MAX_FDIR_INDICES); else if (mac_type == ixgbe_mac_82598EB) indices = min_t(unsigned int, indices, IXGBE_MAX_RSS_INDICES); else indices = min_t(unsigned int, indices, IXGBE_MAX_FDIR_INDICES); #ifdef IXGBE_FCOE indices += min_t(unsigned int, num_possible_cpus(), IXGBE_MAX_FCOE_INDICES); #endif #else indices = IXGBE_MAX_VMDQ_INDICES; #endif netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices); #else netdev = alloc_etherdev(sizeof(struct ixgbe_adapter)); #endif if (!netdev) { err = -ENOMEM; goto err_alloc_etherdev; } SET_NETDEV_DEV(netdev, &pdev->dev); adapter = netdev_priv(netdev); pci_set_drvdata(pdev, adapter); adapter->netdev = netdev; adapter->pdev = pdev; hw = &adapter->hw; hw->back = adapter; adapter->msg_enable = (1 << DEFAULT_DEBUG_LEVEL_SHIFT) - 1; #ifdef HAVE_DEVICE_NUMA_NODE e_info(tx_err, "my (original) node was: %d\n", dev_to_node(&pdev->dev)); #endif /* HAVE_DEVICE_NUMA_NODE */ #ifdef HAVE_PCI_ERS #ifndef __VMKLNX__ /* * call save state here in standalone driver because it relies on * adapter struct to exist, and needs to call netdev_priv */ pci_save_state(pdev); #endif #endif hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pci_resource_len(pdev, 0)); if (!hw->hw_addr) { err = -EIO; goto err_ioremap; } ixgbe_assign_netdev_ops(netdev); strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1); adapter->bd_number = cards_found; #ifdef IXGBE_TCP_TIMER adapter->msix_addr = ioremap(pci_resource_start(pdev, 3), pci_resource_len(pdev, 3)); if (!adapter->msix_addr) { err = -EIO; printk("Error in ioremap of BAR3\n"); goto err_map_msix; } #endif /* setup the private structure */ err = ixgbe_sw_init(adapter); if (err) goto err_sw_init; /* Make it possible the adapter to be woken up via WOL */ switch(adapter->hw.mac.type) { case ixgbe_mac_82599EB: case ixgbe_mac_X540: IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); break; default: break; } /* * If we have a fan, this is as early we know, warn if we * have had a failure. */ if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) { u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); if (esdp & IXGBE_ESDP_SDP1) e_crit(probe, "Fan has stopped, replace the adapter\n"); } /* reset_hw fills in the perm_addr as well */ hw->phy.reset_if_overtemp = true; err = hw->mac.ops.reset_hw(hw); hw->phy.reset_if_overtemp = false; if (err == IXGBE_ERR_SFP_NOT_PRESENT && hw->mac.type == ixgbe_mac_82598EB) { err = 0; } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { e_dev_err("failed to load because an unsupported SFP+ " "module type was detected.\n"); e_dev_err("Reload the driver after installing a supported " "module.\n"); goto err_sw_init; } else if (err) { e_dev_err("HW Init failed: %d\n", err); goto err_sw_init; } /* * check_options must be called before setup_link to set up * hw->fc completely */ ixgbe_check_options(adapter); e_info(tx_err, "my (preferred) node is: %d\n", adapter->node); if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) ixgbe_probe_vf(adapter); if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) { hw->fc.requested_mode = ixgbe_fc_pfc; hw->fc.current_mode = ixgbe_fc_pfc; /* init for ethtool output */ } #ifdef MAX_SKB_FRAGS netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM; #ifdef NETIF_F_IPV6_CSUM netdev->features |= NETIF_F_IPV6_CSUM; #endif #ifdef NETIF_F_HW_VLAN_TX netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX; #endif #ifdef NETIF_F_TSO netdev->features |= NETIF_F_TSO; #endif /* NETIF_F_TSO */ #ifdef NETIF_F_TSO6 netdev->features |= NETIF_F_TSO6; #endif /* NETIF_F_TSO6 */ #ifdef NETIF_F_RXHASH netdev->features |= NETIF_F_RXHASH; #endif /* NETIF_F_RXHASH */ #ifdef __VMKLNX__ #ifdef NETIF_F_OFFLOAD_8OFFSET netdev->features |= NETIF_F_OFFLOAD_8OFFSET; #endif // NETIF_F_OFFLOAD_8OFFSET /* * We want load balancer to call apply_filter for mac addresses applied * to default queue to have successful VF to emulated communication */ if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) netdev->features |= NETIF_F_DEFQ_L2_FLTR; #endif // __VMKLNX__ #ifdef HAVE_NDO_SET_FEATURES netdev->features |= NETIF_F_RXCSUM; /* copy netdev features into list of user selectable features */ netdev->hw_features |= netdev->features; /* give us the option of enabling RSC/LRO later */ if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) netdev->hw_features |= NETIF_F_LRO; #else #ifdef NETIF_F_GRO /* this is only needed on kernels prior to 2.6.39 */ netdev->features |= NETIF_F_GRO; #endif /* NETIF_F_GRO */ #endif #ifndef __VMKLNX__ /* set this bit last since it cannot be part of hw_features */ netdev->features |= NETIF_F_HW_VLAN_FILTER; #endif switch(adapter->hw.mac.type) { case ixgbe_mac_82599EB: case ixgbe_mac_X540: netdev->features |= NETIF_F_SCTP_CSUM; #ifdef HAVE_NDO_SET_FEATURES netdev->hw_features |= NETIF_F_SCTP_CSUM | NETIF_F_NTUPLE; #endif break; default: break; } #ifdef HAVE_NETDEV_VLAN_FEATURES netdev->vlan_features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_TSO | NETIF_F_TSO6; #endif /* HAVE_NETDEV_VLAN_FEATURES */ /* * If perfect filters were enabled in check_options(), enable them * on the netdevice too. */ if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) netdev->features |= NETIF_F_NTUPLE; #ifdef IXGBE_VMDQ #endif /* IXGBE_VMDQ */ if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) adapter->flags &= ~IXGBE_FLAG_RSS_ENABLED; if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) { adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; /* clear n-tuple support in the netdev unconditionally */ netdev->features &= ~NETIF_F_NTUPLE; } #ifdef NETIF_F_RXHASH if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED)) netdev->features &= ~NETIF_F_RXHASH; #endif /* NETIF_F_RXHASH */ if (netdev->features & NETIF_F_LRO) { if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) && ((adapter->rx_itr_setting == 1) || (adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR))) { adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; } else if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) { e_info(probe, "InterruptThrottleRate set too high, " "disabling RSC\n"); } } #ifdef CONFIG_DCB netdev->dcbnl_ops = &dcbnl_ops; #endif #ifdef IXGBE_FCOE #ifdef NETIF_F_FSO if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) { ixgbe_get_device_caps(hw, &device_caps); if ((ixgbe_validate_mac_addr(adapter->hw.mac.san_addr) == IXGBE_ERR_INVALID_MAC_ADDR) || !(adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) || (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)) { adapter->flags2 &= ~IXGBE_FLAG2_CNA_ENABLED; adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE; e_info(probe, "FCoE offload feature is not available. " "Disabling FCoE offload feature\n"); } #ifndef HAVE_NETDEV_OPS_FCOE_ENABLE else { adapter->flags |= IXGBE_FLAG_FCOE_ENABLED; netdev->features |= NETIF_F_FSO | NETIF_F_FCOE_CRC | NETIF_F_FCOE_MTU; netdev->fcoe_ddp_xid = IXGBE_FCOE_DDP_MAX - 1; } #endif /* HAVE_NETDEV_OPS_FCOE_ENABLE */ #ifdef HAVE_NETDEV_VLAN_FEATURES netdev->vlan_features |= NETIF_F_FSO | NETIF_F_FCOE_CRC | NETIF_F_FCOE_MTU; #endif /* HAVE_NETDEV_VLAN_FEATURES */ } #endif /* NETIF_F_FSO */ #endif /* IXGBE_FCOE */ if (pci_using_dac) { netdev->features |= NETIF_F_HIGHDMA; #ifdef HAVE_NETDEV_VLAN_FEATURES netdev->vlan_features |= NETIF_F_HIGHDMA; #endif /* HAVE_NETDEV_VLAN_FEATURES */ } #endif /* MAX_SKB_FRAGS */ #ifdef __VMKLNX__ //Enable LATENCY NETQ adapter->flags2 |= IXGBE_FLAG2_LATENCY_ENABLED; #endif /* __VMKLNX__ */ /* make sure the EEPROM is good */ if (hw->eeprom.ops.validate_checksum && (hw->eeprom.ops.validate_checksum(hw, NULL) < 0)) { e_dev_err("The EEPROM Checksum Is Not Valid\n"); err = -EIO; goto err_sw_init; } memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len); #ifdef ETHTOOL_GPERMADDR memcpy(netdev->perm_addr, hw->mac.perm_addr, netdev->addr_len); if (ixgbe_validate_mac_addr(netdev->perm_addr)) { e_dev_err("invalid MAC address\n"); err = -EIO; goto err_sw_init; } #else if (ixgbe_validate_mac_addr(netdev->dev_addr)) { e_dev_err("invalid MAC address\n"); err = -EIO; goto err_sw_init; } #endif memcpy(&adapter->mac_table[0].addr, hw->mac.perm_addr, netdev->addr_len); adapter->mac_table[0].queue = adapter->num_vfs; adapter->mac_table[0].state = (IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE); hw->mac.ops.set_rar(hw, 0, adapter->mac_table[0].addr, adapter->mac_table[0].queue, IXGBE_RAH_AV); setup_timer(&adapter->service_timer, &ixgbe_service_timer, (unsigned long) adapter); INIT_WORK(&adapter->service_task, ixgbe_service_task); clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state); err = ixgbe_init_interrupt_scheme(adapter); if (err) goto err_sw_init; /* WOL not supported for all but the following */ adapter->wol = 0; switch (pdev->device) { case IXGBE_DEV_ID_82599_SFP: /* Only this subdevice supports WOL */ if (pdev->subsystem_device == IXGBE_SUBDEV_ID_82599_SFP) adapter->wol = IXGBE_WUFC_MAG; break; case IXGBE_DEV_ID_82599_COMBO_BACKPLANE: /* All except this subdevice support WOL */ if (pdev->subsystem_device != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ) adapter->wol = IXGBE_WUFC_MAG; break; case IXGBE_DEV_ID_82599_KX4: adapter->wol = IXGBE_WUFC_MAG; break; case IXGBE_DEV_ID_X540T: /* Check eeprom to see if it is enabled */ ixgbe_read_eeprom(hw, 0x2c, &adapter->eeprom_cap); wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK; if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) || ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && (hw->bus.func == 0))) adapter->wol = IXGBE_WUFC_MAG; break; } device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol); /* save off EEPROM version number */ ixgbe_read_eeprom(hw, 0x2e, &adapter->eeprom_verh); ixgbe_read_eeprom(hw, 0x2d, &adapter->eeprom_verl); /* reset the hardware with the new settings */ err = hw->mac.ops.start_hw(hw); if (err == IXGBE_ERR_EEPROM_VERSION) { /* We are running on a pre-production device, log a warning */ e_dev_warn("This device is a pre-production adapter/LOM. " "Please be aware there may be issues associated " "with your hardware. If you are experiencing " "problems please contact your Intel or hardware " "representative who provided you with this " "hardware.\n"); } /* pick up the PCI bus settings for reporting later */ if (hw->mac.ops.get_bus_info) hw->mac.ops.get_bus_info(hw); #ifndef __VMKLNX__ strcpy(netdev->name, "eth%d"); #endif err = register_netdev(netdev); if (err) goto err_register; if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) { err = ixgbe_cna_enable(adapter); if (err) e_dev_err("ixgbe_cna_enable failed " "return value %d\n", err); } adapter->netdev_registered = true; /* power down the optics */ if ((hw->phy.multispeed_fiber) || ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) && (hw->mac.type == ixgbe_mac_82599EB))) ixgbe_disable_tx_laser(hw); /* carrier off reporting is important to ethtool even BEFORE open */ netif_carrier_off(netdev); /* keep stopping all the transmit queues for older kernels */ netif_tx_stop_all_queues(netdev); #if defined(__VMKLNX__) && defined(__VMKNETDDI_QUEUEOPS__) if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) { DPRINTK(PROBE, INFO, "Registering for VMware NetQueue Ops\n"); VMKNETDDI_REGISTER_QUEUEOPS(netdev, ixgbe_netqueue_ops); } #endif if (adapter->flags & IXGBE_FLAG_DCA_CAPABLE) { err = dca_add_requester(&pdev->dev); switch (err) { case 0: adapter->flags |= IXGBE_FLAG_DCA_ENABLED; ixgbe_setup_dca(adapter); break; /* -19 is returned from the kernel when no provider is found */ case -19: e_info(rx_err, "No DCA provider found. Please " "start ioatdma for DCA functionality.\n"); break; default: e_info(probe, "DCA registration failed: %d\n", err); break; } } /* print all messages at the end so that we use our eth%d name */ /* print bus type/speed/width info */ e_dev_info("(PCI Express:%s:%s) ", (hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" : hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" : "Unknown"), (hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" : hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" : hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" : "Unknown")); /* print the MAC address */ for (i = 0; i < 6; i++) printk("%2.2x%c", netdev->dev_addr[i], i == 5 ? '\n' : ':'); /* First try to read PBA as a string */ err = ixgbe_read_pba_string(hw, part_str, IXGBE_PBANUM_LENGTH); if (err) strncpy(part_str, "Unknown", IXGBE_PBANUM_LENGTH); if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present) e_info(probe, "MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n", hw->mac.type, hw->phy.type, hw->phy.sfp_type, part_str); else e_info(probe, "MAC: %d, PHY: %d, PBA No: %s\n", hw->mac.type, hw->phy.type, part_str); if (((hw->bus.speed == ixgbe_bus_speed_2500) && (hw->bus.width <= ixgbe_bus_width_pcie_x4)) || (hw->bus.width <= ixgbe_bus_width_pcie_x2)) { e_dev_warn("PCI-Express bandwidth available for this card is " "not sufficient for optimal performance.\n"); e_dev_warn("For optimal performance a x8 PCI-Express slot " "is required.\n"); } #define INFO_STRING_LEN 255 info_string = kzalloc(INFO_STRING_LEN, GFP_KERNEL); if (!info_string) { e_err(probe, "allocation for info string failed\n"); goto no_info_string; } i_s_var = info_string; i_s_var += sprintf(info_string, "Enabled Features: "); i_s_var += sprintf(i_s_var, "RxQ: %d TxQ: %d ", adapter->num_rx_queues, adapter->num_tx_queues); #ifdef IXGBE_FCOE if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) i_s_var += sprintf(i_s_var, "FCoE "); #endif if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) i_s_var += sprintf(i_s_var, "FdirHash "); if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) i_s_var += sprintf(i_s_var, "FdirPerfect "); if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) i_s_var += sprintf(i_s_var, "DCB "); if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) i_s_var += sprintf(i_s_var, "RSS "); if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) i_s_var += sprintf(i_s_var, "DCA "); if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) i_s_var += sprintf(i_s_var, "RSC "); BUG_ON(i_s_var > (info_string + INFO_STRING_LEN)); /* end features printing */ e_info(probe, "%s\n", info_string); kfree(info_string); no_info_string: if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { DPRINTK(PROBE, INFO, "IOV is enabled with %d VFs\n", adapter->num_vfs); for (i = 0; i < adapter->num_vfs; i++) ixgbe_vf_configuration(pdev, (i | 0x10000000)); } /* firmware requires blank driver version */ ixgbe_set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF, 0xFF); #if defined(HAVE_NETDEV_STORAGE_ADDRESS) && defined(NETDEV_HW_ADDR_T_SAN) /* add san mac addr to netdev */ ixgbe_add_sanmac_netdev(netdev); #endif /* (HAVE_NETDEV_STORAGE_ADDRESS) && (NETDEV_HW_ADDR_T_SAN) */ e_info(probe, "Intel(R) 10 Gigabit Network Connection\n"); cards_found++; return 0; err_register: ixgbe_clear_interrupt_scheme(adapter); ixgbe_release_hw_control(adapter); err_sw_init: if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) ixgbe_disable_sriov(adapter); adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP; kfree(adapter->mac_table); #ifdef IXGBE_TCP_TIMER iounmap(adapter->msix_addr); err_map_msix: #endif iounmap(hw->hw_addr); err_ioremap: free_netdev(netdev); err_alloc_etherdev: pci_release_selected_regions(pdev, pci_select_bars(pdev, IORESOURCE_MEM)); err_pci_reg: err_dma: pci_disable_device(pdev); return err; } /** * ixgbe_remove - Device Removal Routine * @pdev: PCI device information struct * * ixgbe_remove is called by the PCI subsystem to alert the driver * that it should release a PCI device. The could be caused by a * Hot-Plug event, or because the driver is going to be removed from * memory. **/ static void __devexit ixgbe_remove(struct pci_dev *pdev) { struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); struct net_device *netdev = adapter->netdev; if (netdev->reg_state == NETREG_UNINITIALIZED || netdev->reg_state == NETREG_RELEASED || netdev->reg_state == 0xffffffff) { dev_err(pci_dev_to_dev(pdev), "ixgbe_remove: netdev uninitialized" ", reg_state=0x%x, state=0x%x\n", netdev->reg_state, netdev->state); return; } set_bit(__IXGBE_DOWN, &adapter->state); cancel_work_sync(&adapter->service_task); if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED; dca_remove_requester(&pdev->dev); IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1); } #ifdef IXGBE_FCOE if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) ixgbe_cleanup_fcoe(adapter); #endif /* IXGBE_FCOE */ #if defined(HAVE_NETDEV_STORAGE_ADDRESS) && defined(NETDEV_HW_ADDR_T_SAN) /* remove the added san mac */ ixgbe_del_sanmac_netdev(netdev); #endif /* (HAVE_NETDEV_STORAGE_ADDRESS) && (NETDEV_HW_ADDR_T_SAN) */ if (adapter->flags2 & IXGBE_FLAG2_CNA_ENABLED) ixgbe_cna_disable(adapter); #ifdef __VMKLNX__ unregister_netdev(netdev); adapter->netdev_registered = false; #else if (adapter->netdev_registered) { unregister_netdev(netdev); adapter->netdev_registered = false; } #endif if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) ixgbe_disable_sriov(adapter); ixgbe_clear_interrupt_scheme(adapter); ixgbe_release_hw_control(adapter); #ifdef IXGBE_TCP_TIMER iounmap(adapter->msix_addr); #endif iounmap(adapter->hw.hw_addr); pci_release_selected_regions(pdev, pci_select_bars(pdev, IORESOURCE_MEM)); kfree(adapter->mac_table); free_netdev(netdev); pci_disable_pcie_error_reporting(pdev); pci_disable_device(pdev); } u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg) { u16 value; struct ixgbe_adapter *adapter = hw->back; pci_read_config_word(adapter->pdev, reg, &value); return value; } void ixgbe_write_pci_cfg_word(struct ixgbe_hw *hw, u32 reg, u16 value) { struct ixgbe_adapter *adapter = hw->back; pci_write_config_word(adapter->pdev, reg, value); } #ifdef HAVE_PCI_ERS /** * ixgbe_io_error_detected - called when PCI error is detected * @pdev: Pointer to PCI device * @state: The current pci connection state * * This function is called after a PCI bus error affecting * this device has been detected. */ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state) { struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); struct net_device *netdev = adapter->netdev; netif_device_detach(netdev); if (state == pci_channel_io_perm_failure) return PCI_ERS_RESULT_DISCONNECT; if (netif_running(netdev)) ixgbe_down(adapter); pci_disable_device(pdev); /* Request a slot reset. */ return PCI_ERS_RESULT_NEED_RESET; } /** * ixgbe_io_slot_reset - called after the pci bus has been reset. * @pdev: Pointer to PCI device * * Restart the card from scratch, as if from a cold-boot. */ static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev) { struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); pci_ers_result_t result; if (pci_enable_device_mem(pdev)) { e_err(probe, "Cannot re-enable PCI device after reset.\n"); result = PCI_ERS_RESULT_DISCONNECT; } else { pci_set_master(pdev); #ifndef __VMKLNX__ pci_restore_state(pdev); /* * After second error pci->state_saved is false, this * resets it so EEH doesn't break. */ pci_save_state(pdev); #endif pci_wake_from_d3(pdev, false); adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED; ixgbe_service_event_schedule(adapter); IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); result = PCI_ERS_RESULT_RECOVERED; } pci_cleanup_aer_uncorrect_error_status(pdev); return result; } /** * ixgbe_io_resume - called when traffic can start flowing again. * @pdev: Pointer to PCI device * * This callback is called when the error recovery driver tells us that * its OK to resume normal operation. */ static void ixgbe_io_resume(struct pci_dev *pdev) { struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); struct net_device *netdev = adapter->netdev; if (netif_running(netdev)) ixgbe_up(adapter); netif_device_attach(netdev); } static struct pci_error_handlers ixgbe_err_handler = { .error_detected = ixgbe_io_error_detected, .slot_reset = ixgbe_io_slot_reset, .resume = ixgbe_io_resume, }; #endif static struct pci_driver ixgbe_driver = { .name = ixgbe_driver_name, .id_table = ixgbe_pci_tbl, .probe = ixgbe_probe, .remove = __devexit_p(ixgbe_remove), #ifdef CONFIG_PM .suspend = ixgbe_suspend, .resume = ixgbe_resume, #endif #ifndef USE_REBOOT_NOTIFIER .shutdown = ixgbe_shutdown, #endif #ifdef HAVE_PCI_ERS .err_handler = &ixgbe_err_handler #endif }; bool ixgbe_is_ixgbe(struct pci_dev *pcidev) { #ifndef __VMKLNX__ if (pci_dev_driver(pcidev) != &ixgbe_driver) return false; else #endif return true; } /** * ixgbe_init_module - Driver Registration Routine * * ixgbe_init_module is the first routine called when the driver is * loaded. All it does is register with the PCI subsystem. **/ static int __init ixgbe_init_module(void) { int ret; pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version); pr_info("%s\n", ixgbe_copyright); #ifdef __VMKLNX__ if (!vmk_set_module_version("%s", ixgbe_driver_version)) { printk(KERN_ERR "Incompatible Module Version\n"); return -ENODEV; } #endif #if defined(CONFIG_DCA) || defined(CONFIG_DCA_MODULE) dca_register_notify(&dca_notifier); #endif ret = pci_register_driver(&ixgbe_driver); return ret; } module_init(ixgbe_init_module); /** * ixgbe_exit_module - Driver Exit Cleanup Routine * * ixgbe_exit_module is called just before the driver is removed * from memory. **/ static void __exit ixgbe_exit_module(void) { #if defined(CONFIG_DCA) || defined(CONFIG_DCA_MODULE) dca_unregister_notify(&dca_notifier); #endif pci_unregister_driver(&ixgbe_driver); } #if defined(CONFIG_DCA) || defined(CONFIG_DCA_MODULE) static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event, void *p) { int ret_val; ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event, __ixgbe_notify_dca); return ret_val ? NOTIFY_BAD : NOTIFY_DONE; } #endif module_exit(ixgbe_exit_module); /* ixgbe_main.c */