whiterose

linux unikernel
Log | Files | Refs | README | LICENSE | git clone https://git.ne02ptzero.me/git/whiterose

i40e_main.c (419598B)


      1 // SPDX-License-Identifier: GPL-2.0
      2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
      3 
      4 #include <linux/etherdevice.h>
      5 #include <linux/of_net.h>
      6 #include <linux/pci.h>
      7 #include <linux/bpf.h>
      8 
      9 /* Local includes */
     10 #include "i40e.h"
     11 #include "i40e_diag.h"
     12 #include "i40e_xsk.h"
     13 #include <net/udp_tunnel.h>
     14 #include <net/xdp_sock.h>
     15 /* All i40e tracepoints are defined by the include below, which
     16  * must be included exactly once across the whole kernel with
     17  * CREATE_TRACE_POINTS defined
     18  */
     19 #define CREATE_TRACE_POINTS
     20 #include "i40e_trace.h"
     21 
     22 const char i40e_driver_name[] = "i40e";
     23 static const char i40e_driver_string[] =
     24 			"Intel(R) Ethernet Connection XL710 Network Driver";
     25 
     26 #define DRV_KERN "-k"
     27 
     28 #define DRV_VERSION_MAJOR 2
     29 #define DRV_VERSION_MINOR 8
     30 #define DRV_VERSION_BUILD 10
     31 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
     32 	     __stringify(DRV_VERSION_MINOR) "." \
     33 	     __stringify(DRV_VERSION_BUILD)    DRV_KERN
     34 const char i40e_driver_version_str[] = DRV_VERSION;
     35 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
     36 
     37 /* a bit of forward declarations */
     38 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
     39 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
     40 static int i40e_add_vsi(struct i40e_vsi *vsi);
     41 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
     42 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
     43 static int i40e_setup_misc_vector(struct i40e_pf *pf);
     44 static void i40e_determine_queue_usage(struct i40e_pf *pf);
     45 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
     46 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
     47 static int i40e_reset(struct i40e_pf *pf);
     48 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
     49 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
     50 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
     51 static int i40e_get_capabilities(struct i40e_pf *pf,
     52 				 enum i40e_admin_queue_opc list_type);
     53 
     54 
     55 /* i40e_pci_tbl - PCI Device ID Table
     56  *
     57  * Last entry must be all 0s
     58  *
     59  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
     60  *   Class, Class Mask, private data (not used) }
     61  */
     62 static const struct pci_device_id i40e_pci_tbl[] = {
     63 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
     64 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
     65 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
     66 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
     67 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
     68 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
     69 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
     70 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
     71 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
     72 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
     73 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
     74 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
     75 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
     76 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
     77 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
     78 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
     79 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
     80 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
     81 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
     82 	/* required last entry */
     83 	{0, }
     84 };
     85 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
     86 
     87 #define I40E_MAX_VF_COUNT 128
     88 static int debug = -1;
     89 module_param(debug, uint, 0);
     90 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
     91 
     92 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
     93 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
     94 MODULE_LICENSE("GPL v2");
     95 MODULE_VERSION(DRV_VERSION);
     96 
     97 static struct workqueue_struct *i40e_wq;
     98 
     99 /**
    100  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
    101  * @hw:   pointer to the HW structure
    102  * @mem:  ptr to mem struct to fill out
    103  * @size: size of memory requested
    104  * @alignment: what to align the allocation to
    105  **/
    106 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
    107 			    u64 size, u32 alignment)
    108 {
    109 	struct i40e_pf *pf = (struct i40e_pf *)hw->back;
    110 
    111 	mem->size = ALIGN(size, alignment);
    112 	mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
    113 				     GFP_KERNEL);
    114 	if (!mem->va)
    115 		return -ENOMEM;
    116 
    117 	return 0;
    118 }
    119 
    120 /**
    121  * i40e_free_dma_mem_d - OS specific memory free for shared code
    122  * @hw:   pointer to the HW structure
    123  * @mem:  ptr to mem struct to free
    124  **/
    125 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
    126 {
    127 	struct i40e_pf *pf = (struct i40e_pf *)hw->back;
    128 
    129 	dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
    130 	mem->va = NULL;
    131 	mem->pa = 0;
    132 	mem->size = 0;
    133 
    134 	return 0;
    135 }
    136 
    137 /**
    138  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
    139  * @hw:   pointer to the HW structure
    140  * @mem:  ptr to mem struct to fill out
    141  * @size: size of memory requested
    142  **/
    143 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
    144 			     u32 size)
    145 {
    146 	mem->size = size;
    147 	mem->va = kzalloc(size, GFP_KERNEL);
    148 
    149 	if (!mem->va)
    150 		return -ENOMEM;
    151 
    152 	return 0;
    153 }
    154 
    155 /**
    156  * i40e_free_virt_mem_d - OS specific memory free for shared code
    157  * @hw:   pointer to the HW structure
    158  * @mem:  ptr to mem struct to free
    159  **/
    160 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
    161 {
    162 	/* it's ok to kfree a NULL pointer */
    163 	kfree(mem->va);
    164 	mem->va = NULL;
    165 	mem->size = 0;
    166 
    167 	return 0;
    168 }
    169 
    170 /**
    171  * i40e_get_lump - find a lump of free generic resource
    172  * @pf: board private structure
    173  * @pile: the pile of resource to search
    174  * @needed: the number of items needed
    175  * @id: an owner id to stick on the items assigned
    176  *
    177  * Returns the base item index of the lump, or negative for error
    178  *
    179  * The search_hint trick and lack of advanced fit-finding only work
    180  * because we're highly likely to have all the same size lump requests.
    181  * Linear search time and any fragmentation should be minimal.
    182  **/
    183 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
    184 			 u16 needed, u16 id)
    185 {
    186 	int ret = -ENOMEM;
    187 	int i, j;
    188 
    189 	if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
    190 		dev_info(&pf->pdev->dev,
    191 			 "param err: pile=%s needed=%d id=0x%04x\n",
    192 			 pile ? "<valid>" : "<null>", needed, id);
    193 		return -EINVAL;
    194 	}
    195 
    196 	/* start the linear search with an imperfect hint */
    197 	i = pile->search_hint;
    198 	while (i < pile->num_entries) {
    199 		/* skip already allocated entries */
    200 		if (pile->list[i] & I40E_PILE_VALID_BIT) {
    201 			i++;
    202 			continue;
    203 		}
    204 
    205 		/* do we have enough in this lump? */
    206 		for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
    207 			if (pile->list[i+j] & I40E_PILE_VALID_BIT)
    208 				break;
    209 		}
    210 
    211 		if (j == needed) {
    212 			/* there was enough, so assign it to the requestor */
    213 			for (j = 0; j < needed; j++)
    214 				pile->list[i+j] = id | I40E_PILE_VALID_BIT;
    215 			ret = i;
    216 			pile->search_hint = i + j;
    217 			break;
    218 		}
    219 
    220 		/* not enough, so skip over it and continue looking */
    221 		i += j;
    222 	}
    223 
    224 	return ret;
    225 }
    226 
    227 /**
    228  * i40e_put_lump - return a lump of generic resource
    229  * @pile: the pile of resource to search
    230  * @index: the base item index
    231  * @id: the owner id of the items assigned
    232  *
    233  * Returns the count of items in the lump
    234  **/
    235 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
    236 {
    237 	int valid_id = (id | I40E_PILE_VALID_BIT);
    238 	int count = 0;
    239 	int i;
    240 
    241 	if (!pile || index >= pile->num_entries)
    242 		return -EINVAL;
    243 
    244 	for (i = index;
    245 	     i < pile->num_entries && pile->list[i] == valid_id;
    246 	     i++) {
    247 		pile->list[i] = 0;
    248 		count++;
    249 	}
    250 
    251 	if (count && index < pile->search_hint)
    252 		pile->search_hint = index;
    253 
    254 	return count;
    255 }
    256 
    257 /**
    258  * i40e_find_vsi_from_id - searches for the vsi with the given id
    259  * @pf: the pf structure to search for the vsi
    260  * @id: id of the vsi it is searching for
    261  **/
    262 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
    263 {
    264 	int i;
    265 
    266 	for (i = 0; i < pf->num_alloc_vsi; i++)
    267 		if (pf->vsi[i] && (pf->vsi[i]->id == id))
    268 			return pf->vsi[i];
    269 
    270 	return NULL;
    271 }
    272 
    273 /**
    274  * i40e_service_event_schedule - Schedule the service task to wake up
    275  * @pf: board private structure
    276  *
    277  * If not already scheduled, this puts the task into the work queue
    278  **/
    279 void i40e_service_event_schedule(struct i40e_pf *pf)
    280 {
    281 	if (!test_bit(__I40E_DOWN, pf->state) &&
    282 	    !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
    283 		queue_work(i40e_wq, &pf->service_task);
    284 }
    285 
    286 /**
    287  * i40e_tx_timeout - Respond to a Tx Hang
    288  * @netdev: network interface device structure
    289  *
    290  * If any port has noticed a Tx timeout, it is likely that the whole
    291  * device is munged, not just the one netdev port, so go for the full
    292  * reset.
    293  **/
    294 static void i40e_tx_timeout(struct net_device *netdev)
    295 {
    296 	struct i40e_netdev_priv *np = netdev_priv(netdev);
    297 	struct i40e_vsi *vsi = np->vsi;
    298 	struct i40e_pf *pf = vsi->back;
    299 	struct i40e_ring *tx_ring = NULL;
    300 	unsigned int i, hung_queue = 0;
    301 	u32 head, val;
    302 
    303 	pf->tx_timeout_count++;
    304 
    305 	/* find the stopped queue the same way the stack does */
    306 	for (i = 0; i < netdev->num_tx_queues; i++) {
    307 		struct netdev_queue *q;
    308 		unsigned long trans_start;
    309 
    310 		q = netdev_get_tx_queue(netdev, i);
    311 		trans_start = q->trans_start;
    312 		if (netif_xmit_stopped(q) &&
    313 		    time_after(jiffies,
    314 			       (trans_start + netdev->watchdog_timeo))) {
    315 			hung_queue = i;
    316 			break;
    317 		}
    318 	}
    319 
    320 	if (i == netdev->num_tx_queues) {
    321 		netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
    322 	} else {
    323 		/* now that we have an index, find the tx_ring struct */
    324 		for (i = 0; i < vsi->num_queue_pairs; i++) {
    325 			if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
    326 				if (hung_queue ==
    327 				    vsi->tx_rings[i]->queue_index) {
    328 					tx_ring = vsi->tx_rings[i];
    329 					break;
    330 				}
    331 			}
    332 		}
    333 	}
    334 
    335 	if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
    336 		pf->tx_timeout_recovery_level = 1;  /* reset after some time */
    337 	else if (time_before(jiffies,
    338 		      (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
    339 		return;   /* don't do any new action before the next timeout */
    340 
    341 	/* don't kick off another recovery if one is already pending */
    342 	if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
    343 		return;
    344 
    345 	if (tx_ring) {
    346 		head = i40e_get_head(tx_ring);
    347 		/* Read interrupt register */
    348 		if (pf->flags & I40E_FLAG_MSIX_ENABLED)
    349 			val = rd32(&pf->hw,
    350 			     I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
    351 						tx_ring->vsi->base_vector - 1));
    352 		else
    353 			val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
    354 
    355 		netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
    356 			    vsi->seid, hung_queue, tx_ring->next_to_clean,
    357 			    head, tx_ring->next_to_use,
    358 			    readl(tx_ring->tail), val);
    359 	}
    360 
    361 	pf->tx_timeout_last_recovery = jiffies;
    362 	netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
    363 		    pf->tx_timeout_recovery_level, hung_queue);
    364 
    365 	switch (pf->tx_timeout_recovery_level) {
    366 	case 1:
    367 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
    368 		break;
    369 	case 2:
    370 		set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
    371 		break;
    372 	case 3:
    373 		set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
    374 		break;
    375 	default:
    376 		netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
    377 		break;
    378 	}
    379 
    380 	i40e_service_event_schedule(pf);
    381 	pf->tx_timeout_recovery_level++;
    382 }
    383 
    384 /**
    385  * i40e_get_vsi_stats_struct - Get System Network Statistics
    386  * @vsi: the VSI we care about
    387  *
    388  * Returns the address of the device statistics structure.
    389  * The statistics are actually updated from the service task.
    390  **/
    391 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
    392 {
    393 	return &vsi->net_stats;
    394 }
    395 
    396 /**
    397  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
    398  * @ring: Tx ring to get statistics from
    399  * @stats: statistics entry to be updated
    400  **/
    401 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
    402 					    struct rtnl_link_stats64 *stats)
    403 {
    404 	u64 bytes, packets;
    405 	unsigned int start;
    406 
    407 	do {
    408 		start = u64_stats_fetch_begin_irq(&ring->syncp);
    409 		packets = ring->stats.packets;
    410 		bytes   = ring->stats.bytes;
    411 	} while (u64_stats_fetch_retry_irq(&ring->syncp, start));
    412 
    413 	stats->tx_packets += packets;
    414 	stats->tx_bytes   += bytes;
    415 }
    416 
    417 /**
    418  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
    419  * @netdev: network interface device structure
    420  * @stats: data structure to store statistics
    421  *
    422  * Returns the address of the device statistics structure.
    423  * The statistics are actually updated from the service task.
    424  **/
    425 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
    426 				  struct rtnl_link_stats64 *stats)
    427 {
    428 	struct i40e_netdev_priv *np = netdev_priv(netdev);
    429 	struct i40e_vsi *vsi = np->vsi;
    430 	struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
    431 	struct i40e_ring *ring;
    432 	int i;
    433 
    434 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
    435 		return;
    436 
    437 	if (!vsi->tx_rings)
    438 		return;
    439 
    440 	rcu_read_lock();
    441 	for (i = 0; i < vsi->num_queue_pairs; i++) {
    442 		u64 bytes, packets;
    443 		unsigned int start;
    444 
    445 		ring = READ_ONCE(vsi->tx_rings[i]);
    446 		if (!ring)
    447 			continue;
    448 		i40e_get_netdev_stats_struct_tx(ring, stats);
    449 
    450 		if (i40e_enabled_xdp_vsi(vsi)) {
    451 			ring++;
    452 			i40e_get_netdev_stats_struct_tx(ring, stats);
    453 		}
    454 
    455 		ring++;
    456 		do {
    457 			start   = u64_stats_fetch_begin_irq(&ring->syncp);
    458 			packets = ring->stats.packets;
    459 			bytes   = ring->stats.bytes;
    460 		} while (u64_stats_fetch_retry_irq(&ring->syncp, start));
    461 
    462 		stats->rx_packets += packets;
    463 		stats->rx_bytes   += bytes;
    464 
    465 	}
    466 	rcu_read_unlock();
    467 
    468 	/* following stats updated by i40e_watchdog_subtask() */
    469 	stats->multicast	= vsi_stats->multicast;
    470 	stats->tx_errors	= vsi_stats->tx_errors;
    471 	stats->tx_dropped	= vsi_stats->tx_dropped;
    472 	stats->rx_errors	= vsi_stats->rx_errors;
    473 	stats->rx_dropped	= vsi_stats->rx_dropped;
    474 	stats->rx_crc_errors	= vsi_stats->rx_crc_errors;
    475 	stats->rx_length_errors	= vsi_stats->rx_length_errors;
    476 }
    477 
    478 /**
    479  * i40e_vsi_reset_stats - Resets all stats of the given vsi
    480  * @vsi: the VSI to have its stats reset
    481  **/
    482 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
    483 {
    484 	struct rtnl_link_stats64 *ns;
    485 	int i;
    486 
    487 	if (!vsi)
    488 		return;
    489 
    490 	ns = i40e_get_vsi_stats_struct(vsi);
    491 	memset(ns, 0, sizeof(*ns));
    492 	memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
    493 	memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
    494 	memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
    495 	if (vsi->rx_rings && vsi->rx_rings[0]) {
    496 		for (i = 0; i < vsi->num_queue_pairs; i++) {
    497 			memset(&vsi->rx_rings[i]->stats, 0,
    498 			       sizeof(vsi->rx_rings[i]->stats));
    499 			memset(&vsi->rx_rings[i]->rx_stats, 0,
    500 			       sizeof(vsi->rx_rings[i]->rx_stats));
    501 			memset(&vsi->tx_rings[i]->stats, 0,
    502 			       sizeof(vsi->tx_rings[i]->stats));
    503 			memset(&vsi->tx_rings[i]->tx_stats, 0,
    504 			       sizeof(vsi->tx_rings[i]->tx_stats));
    505 		}
    506 	}
    507 	vsi->stat_offsets_loaded = false;
    508 }
    509 
    510 /**
    511  * i40e_pf_reset_stats - Reset all of the stats for the given PF
    512  * @pf: the PF to be reset
    513  **/
    514 void i40e_pf_reset_stats(struct i40e_pf *pf)
    515 {
    516 	int i;
    517 
    518 	memset(&pf->stats, 0, sizeof(pf->stats));
    519 	memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
    520 	pf->stat_offsets_loaded = false;
    521 
    522 	for (i = 0; i < I40E_MAX_VEB; i++) {
    523 		if (pf->veb[i]) {
    524 			memset(&pf->veb[i]->stats, 0,
    525 			       sizeof(pf->veb[i]->stats));
    526 			memset(&pf->veb[i]->stats_offsets, 0,
    527 			       sizeof(pf->veb[i]->stats_offsets));
    528 			pf->veb[i]->stat_offsets_loaded = false;
    529 		}
    530 	}
    531 	pf->hw_csum_rx_error = 0;
    532 }
    533 
    534 /**
    535  * i40e_stat_update48 - read and update a 48 bit stat from the chip
    536  * @hw: ptr to the hardware info
    537  * @hireg: the high 32 bit reg to read
    538  * @loreg: the low 32 bit reg to read
    539  * @offset_loaded: has the initial offset been loaded yet
    540  * @offset: ptr to current offset value
    541  * @stat: ptr to the stat
    542  *
    543  * Since the device stats are not reset at PFReset, they likely will not
    544  * be zeroed when the driver starts.  We'll save the first values read
    545  * and use them as offsets to be subtracted from the raw values in order
    546  * to report stats that count from zero.  In the process, we also manage
    547  * the potential roll-over.
    548  **/
    549 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
    550 			       bool offset_loaded, u64 *offset, u64 *stat)
    551 {
    552 	u64 new_data;
    553 
    554 	if (hw->device_id == I40E_DEV_ID_QEMU) {
    555 		new_data = rd32(hw, loreg);
    556 		new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
    557 	} else {
    558 		new_data = rd64(hw, loreg);
    559 	}
    560 	if (!offset_loaded)
    561 		*offset = new_data;
    562 	if (likely(new_data >= *offset))
    563 		*stat = new_data - *offset;
    564 	else
    565 		*stat = (new_data + BIT_ULL(48)) - *offset;
    566 	*stat &= 0xFFFFFFFFFFFFULL;
    567 }
    568 
    569 /**
    570  * i40e_stat_update32 - read and update a 32 bit stat from the chip
    571  * @hw: ptr to the hardware info
    572  * @reg: the hw reg to read
    573  * @offset_loaded: has the initial offset been loaded yet
    574  * @offset: ptr to current offset value
    575  * @stat: ptr to the stat
    576  **/
    577 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
    578 			       bool offset_loaded, u64 *offset, u64 *stat)
    579 {
    580 	u32 new_data;
    581 
    582 	new_data = rd32(hw, reg);
    583 	if (!offset_loaded)
    584 		*offset = new_data;
    585 	if (likely(new_data >= *offset))
    586 		*stat = (u32)(new_data - *offset);
    587 	else
    588 		*stat = (u32)((new_data + BIT_ULL(32)) - *offset);
    589 }
    590 
    591 /**
    592  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
    593  * @hw: ptr to the hardware info
    594  * @reg: the hw reg to read and clear
    595  * @stat: ptr to the stat
    596  **/
    597 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
    598 {
    599 	u32 new_data = rd32(hw, reg);
    600 
    601 	wr32(hw, reg, 1); /* must write a nonzero value to clear register */
    602 	*stat += new_data;
    603 }
    604 
    605 /**
    606  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
    607  * @vsi: the VSI to be updated
    608  **/
    609 void i40e_update_eth_stats(struct i40e_vsi *vsi)
    610 {
    611 	int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
    612 	struct i40e_pf *pf = vsi->back;
    613 	struct i40e_hw *hw = &pf->hw;
    614 	struct i40e_eth_stats *oes;
    615 	struct i40e_eth_stats *es;     /* device's eth stats */
    616 
    617 	es = &vsi->eth_stats;
    618 	oes = &vsi->eth_stats_offsets;
    619 
    620 	/* Gather up the stats that the hw collects */
    621 	i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
    622 			   vsi->stat_offsets_loaded,
    623 			   &oes->tx_errors, &es->tx_errors);
    624 	i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
    625 			   vsi->stat_offsets_loaded,
    626 			   &oes->rx_discards, &es->rx_discards);
    627 	i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
    628 			   vsi->stat_offsets_loaded,
    629 			   &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
    630 	i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
    631 			   vsi->stat_offsets_loaded,
    632 			   &oes->tx_errors, &es->tx_errors);
    633 
    634 	i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
    635 			   I40E_GLV_GORCL(stat_idx),
    636 			   vsi->stat_offsets_loaded,
    637 			   &oes->rx_bytes, &es->rx_bytes);
    638 	i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
    639 			   I40E_GLV_UPRCL(stat_idx),
    640 			   vsi->stat_offsets_loaded,
    641 			   &oes->rx_unicast, &es->rx_unicast);
    642 	i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
    643 			   I40E_GLV_MPRCL(stat_idx),
    644 			   vsi->stat_offsets_loaded,
    645 			   &oes->rx_multicast, &es->rx_multicast);
    646 	i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
    647 			   I40E_GLV_BPRCL(stat_idx),
    648 			   vsi->stat_offsets_loaded,
    649 			   &oes->rx_broadcast, &es->rx_broadcast);
    650 
    651 	i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
    652 			   I40E_GLV_GOTCL(stat_idx),
    653 			   vsi->stat_offsets_loaded,
    654 			   &oes->tx_bytes, &es->tx_bytes);
    655 	i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
    656 			   I40E_GLV_UPTCL(stat_idx),
    657 			   vsi->stat_offsets_loaded,
    658 			   &oes->tx_unicast, &es->tx_unicast);
    659 	i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
    660 			   I40E_GLV_MPTCL(stat_idx),
    661 			   vsi->stat_offsets_loaded,
    662 			   &oes->tx_multicast, &es->tx_multicast);
    663 	i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
    664 			   I40E_GLV_BPTCL(stat_idx),
    665 			   vsi->stat_offsets_loaded,
    666 			   &oes->tx_broadcast, &es->tx_broadcast);
    667 	vsi->stat_offsets_loaded = true;
    668 }
    669 
    670 /**
    671  * i40e_update_veb_stats - Update Switch component statistics
    672  * @veb: the VEB being updated
    673  **/
    674 static void i40e_update_veb_stats(struct i40e_veb *veb)
    675 {
    676 	struct i40e_pf *pf = veb->pf;
    677 	struct i40e_hw *hw = &pf->hw;
    678 	struct i40e_eth_stats *oes;
    679 	struct i40e_eth_stats *es;     /* device's eth stats */
    680 	struct i40e_veb_tc_stats *veb_oes;
    681 	struct i40e_veb_tc_stats *veb_es;
    682 	int i, idx = 0;
    683 
    684 	idx = veb->stats_idx;
    685 	es = &veb->stats;
    686 	oes = &veb->stats_offsets;
    687 	veb_es = &veb->tc_stats;
    688 	veb_oes = &veb->tc_stats_offsets;
    689 
    690 	/* Gather up the stats that the hw collects */
    691 	i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
    692 			   veb->stat_offsets_loaded,
    693 			   &oes->tx_discards, &es->tx_discards);
    694 	if (hw->revision_id > 0)
    695 		i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
    696 				   veb->stat_offsets_loaded,
    697 				   &oes->rx_unknown_protocol,
    698 				   &es->rx_unknown_protocol);
    699 	i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
    700 			   veb->stat_offsets_loaded,
    701 			   &oes->rx_bytes, &es->rx_bytes);
    702 	i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
    703 			   veb->stat_offsets_loaded,
    704 			   &oes->rx_unicast, &es->rx_unicast);
    705 	i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
    706 			   veb->stat_offsets_loaded,
    707 			   &oes->rx_multicast, &es->rx_multicast);
    708 	i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
    709 			   veb->stat_offsets_loaded,
    710 			   &oes->rx_broadcast, &es->rx_broadcast);
    711 
    712 	i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
    713 			   veb->stat_offsets_loaded,
    714 			   &oes->tx_bytes, &es->tx_bytes);
    715 	i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
    716 			   veb->stat_offsets_loaded,
    717 			   &oes->tx_unicast, &es->tx_unicast);
    718 	i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
    719 			   veb->stat_offsets_loaded,
    720 			   &oes->tx_multicast, &es->tx_multicast);
    721 	i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
    722 			   veb->stat_offsets_loaded,
    723 			   &oes->tx_broadcast, &es->tx_broadcast);
    724 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
    725 		i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
    726 				   I40E_GLVEBTC_RPCL(i, idx),
    727 				   veb->stat_offsets_loaded,
    728 				   &veb_oes->tc_rx_packets[i],
    729 				   &veb_es->tc_rx_packets[i]);
    730 		i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
    731 				   I40E_GLVEBTC_RBCL(i, idx),
    732 				   veb->stat_offsets_loaded,
    733 				   &veb_oes->tc_rx_bytes[i],
    734 				   &veb_es->tc_rx_bytes[i]);
    735 		i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
    736 				   I40E_GLVEBTC_TPCL(i, idx),
    737 				   veb->stat_offsets_loaded,
    738 				   &veb_oes->tc_tx_packets[i],
    739 				   &veb_es->tc_tx_packets[i]);
    740 		i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
    741 				   I40E_GLVEBTC_TBCL(i, idx),
    742 				   veb->stat_offsets_loaded,
    743 				   &veb_oes->tc_tx_bytes[i],
    744 				   &veb_es->tc_tx_bytes[i]);
    745 	}
    746 	veb->stat_offsets_loaded = true;
    747 }
    748 
    749 /**
    750  * i40e_update_vsi_stats - Update the vsi statistics counters.
    751  * @vsi: the VSI to be updated
    752  *
    753  * There are a few instances where we store the same stat in a
    754  * couple of different structs.  This is partly because we have
    755  * the netdev stats that need to be filled out, which is slightly
    756  * different from the "eth_stats" defined by the chip and used in
    757  * VF communications.  We sort it out here.
    758  **/
    759 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
    760 {
    761 	struct i40e_pf *pf = vsi->back;
    762 	struct rtnl_link_stats64 *ons;
    763 	struct rtnl_link_stats64 *ns;   /* netdev stats */
    764 	struct i40e_eth_stats *oes;
    765 	struct i40e_eth_stats *es;     /* device's eth stats */
    766 	u32 tx_restart, tx_busy;
    767 	struct i40e_ring *p;
    768 	u32 rx_page, rx_buf;
    769 	u64 bytes, packets;
    770 	unsigned int start;
    771 	u64 tx_linearize;
    772 	u64 tx_force_wb;
    773 	u64 rx_p, rx_b;
    774 	u64 tx_p, tx_b;
    775 	u16 q;
    776 
    777 	if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
    778 	    test_bit(__I40E_CONFIG_BUSY, pf->state))
    779 		return;
    780 
    781 	ns = i40e_get_vsi_stats_struct(vsi);
    782 	ons = &vsi->net_stats_offsets;
    783 	es = &vsi->eth_stats;
    784 	oes = &vsi->eth_stats_offsets;
    785 
    786 	/* Gather up the netdev and vsi stats that the driver collects
    787 	 * on the fly during packet processing
    788 	 */
    789 	rx_b = rx_p = 0;
    790 	tx_b = tx_p = 0;
    791 	tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
    792 	rx_page = 0;
    793 	rx_buf = 0;
    794 	rcu_read_lock();
    795 	for (q = 0; q < vsi->num_queue_pairs; q++) {
    796 		/* locate Tx ring */
    797 		p = READ_ONCE(vsi->tx_rings[q]);
    798 
    799 		do {
    800 			start = u64_stats_fetch_begin_irq(&p->syncp);
    801 			packets = p->stats.packets;
    802 			bytes = p->stats.bytes;
    803 		} while (u64_stats_fetch_retry_irq(&p->syncp, start));
    804 		tx_b += bytes;
    805 		tx_p += packets;
    806 		tx_restart += p->tx_stats.restart_queue;
    807 		tx_busy += p->tx_stats.tx_busy;
    808 		tx_linearize += p->tx_stats.tx_linearize;
    809 		tx_force_wb += p->tx_stats.tx_force_wb;
    810 
    811 		/* Rx queue is part of the same block as Tx queue */
    812 		p = &p[1];
    813 		do {
    814 			start = u64_stats_fetch_begin_irq(&p->syncp);
    815 			packets = p->stats.packets;
    816 			bytes = p->stats.bytes;
    817 		} while (u64_stats_fetch_retry_irq(&p->syncp, start));
    818 		rx_b += bytes;
    819 		rx_p += packets;
    820 		rx_buf += p->rx_stats.alloc_buff_failed;
    821 		rx_page += p->rx_stats.alloc_page_failed;
    822 	}
    823 	rcu_read_unlock();
    824 	vsi->tx_restart = tx_restart;
    825 	vsi->tx_busy = tx_busy;
    826 	vsi->tx_linearize = tx_linearize;
    827 	vsi->tx_force_wb = tx_force_wb;
    828 	vsi->rx_page_failed = rx_page;
    829 	vsi->rx_buf_failed = rx_buf;
    830 
    831 	ns->rx_packets = rx_p;
    832 	ns->rx_bytes = rx_b;
    833 	ns->tx_packets = tx_p;
    834 	ns->tx_bytes = tx_b;
    835 
    836 	/* update netdev stats from eth stats */
    837 	i40e_update_eth_stats(vsi);
    838 	ons->tx_errors = oes->tx_errors;
    839 	ns->tx_errors = es->tx_errors;
    840 	ons->multicast = oes->rx_multicast;
    841 	ns->multicast = es->rx_multicast;
    842 	ons->rx_dropped = oes->rx_discards;
    843 	ns->rx_dropped = es->rx_discards;
    844 	ons->tx_dropped = oes->tx_discards;
    845 	ns->tx_dropped = es->tx_discards;
    846 
    847 	/* pull in a couple PF stats if this is the main vsi */
    848 	if (vsi == pf->vsi[pf->lan_vsi]) {
    849 		ns->rx_crc_errors = pf->stats.crc_errors;
    850 		ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
    851 		ns->rx_length_errors = pf->stats.rx_length_errors;
    852 	}
    853 }
    854 
    855 /**
    856  * i40e_update_pf_stats - Update the PF statistics counters.
    857  * @pf: the PF to be updated
    858  **/
    859 static void i40e_update_pf_stats(struct i40e_pf *pf)
    860 {
    861 	struct i40e_hw_port_stats *osd = &pf->stats_offsets;
    862 	struct i40e_hw_port_stats *nsd = &pf->stats;
    863 	struct i40e_hw *hw = &pf->hw;
    864 	u32 val;
    865 	int i;
    866 
    867 	i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
    868 			   I40E_GLPRT_GORCL(hw->port),
    869 			   pf->stat_offsets_loaded,
    870 			   &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
    871 	i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
    872 			   I40E_GLPRT_GOTCL(hw->port),
    873 			   pf->stat_offsets_loaded,
    874 			   &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
    875 	i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
    876 			   pf->stat_offsets_loaded,
    877 			   &osd->eth.rx_discards,
    878 			   &nsd->eth.rx_discards);
    879 	i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
    880 			   I40E_GLPRT_UPRCL(hw->port),
    881 			   pf->stat_offsets_loaded,
    882 			   &osd->eth.rx_unicast,
    883 			   &nsd->eth.rx_unicast);
    884 	i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
    885 			   I40E_GLPRT_MPRCL(hw->port),
    886 			   pf->stat_offsets_loaded,
    887 			   &osd->eth.rx_multicast,
    888 			   &nsd->eth.rx_multicast);
    889 	i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
    890 			   I40E_GLPRT_BPRCL(hw->port),
    891 			   pf->stat_offsets_loaded,
    892 			   &osd->eth.rx_broadcast,
    893 			   &nsd->eth.rx_broadcast);
    894 	i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
    895 			   I40E_GLPRT_UPTCL(hw->port),
    896 			   pf->stat_offsets_loaded,
    897 			   &osd->eth.tx_unicast,
    898 			   &nsd->eth.tx_unicast);
    899 	i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
    900 			   I40E_GLPRT_MPTCL(hw->port),
    901 			   pf->stat_offsets_loaded,
    902 			   &osd->eth.tx_multicast,
    903 			   &nsd->eth.tx_multicast);
    904 	i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
    905 			   I40E_GLPRT_BPTCL(hw->port),
    906 			   pf->stat_offsets_loaded,
    907 			   &osd->eth.tx_broadcast,
    908 			   &nsd->eth.tx_broadcast);
    909 
    910 	i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
    911 			   pf->stat_offsets_loaded,
    912 			   &osd->tx_dropped_link_down,
    913 			   &nsd->tx_dropped_link_down);
    914 
    915 	i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
    916 			   pf->stat_offsets_loaded,
    917 			   &osd->crc_errors, &nsd->crc_errors);
    918 
    919 	i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
    920 			   pf->stat_offsets_loaded,
    921 			   &osd->illegal_bytes, &nsd->illegal_bytes);
    922 
    923 	i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
    924 			   pf->stat_offsets_loaded,
    925 			   &osd->mac_local_faults,
    926 			   &nsd->mac_local_faults);
    927 	i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
    928 			   pf->stat_offsets_loaded,
    929 			   &osd->mac_remote_faults,
    930 			   &nsd->mac_remote_faults);
    931 
    932 	i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
    933 			   pf->stat_offsets_loaded,
    934 			   &osd->rx_length_errors,
    935 			   &nsd->rx_length_errors);
    936 
    937 	i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
    938 			   pf->stat_offsets_loaded,
    939 			   &osd->link_xon_rx, &nsd->link_xon_rx);
    940 	i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
    941 			   pf->stat_offsets_loaded,
    942 			   &osd->link_xon_tx, &nsd->link_xon_tx);
    943 	i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
    944 			   pf->stat_offsets_loaded,
    945 			   &osd->link_xoff_rx, &nsd->link_xoff_rx);
    946 	i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
    947 			   pf->stat_offsets_loaded,
    948 			   &osd->link_xoff_tx, &nsd->link_xoff_tx);
    949 
    950 	for (i = 0; i < 8; i++) {
    951 		i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
    952 				   pf->stat_offsets_loaded,
    953 				   &osd->priority_xoff_rx[i],
    954 				   &nsd->priority_xoff_rx[i]);
    955 		i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
    956 				   pf->stat_offsets_loaded,
    957 				   &osd->priority_xon_rx[i],
    958 				   &nsd->priority_xon_rx[i]);
    959 		i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
    960 				   pf->stat_offsets_loaded,
    961 				   &osd->priority_xon_tx[i],
    962 				   &nsd->priority_xon_tx[i]);
    963 		i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
    964 				   pf->stat_offsets_loaded,
    965 				   &osd->priority_xoff_tx[i],
    966 				   &nsd->priority_xoff_tx[i]);
    967 		i40e_stat_update32(hw,
    968 				   I40E_GLPRT_RXON2OFFCNT(hw->port, i),
    969 				   pf->stat_offsets_loaded,
    970 				   &osd->priority_xon_2_xoff[i],
    971 				   &nsd->priority_xon_2_xoff[i]);
    972 	}
    973 
    974 	i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
    975 			   I40E_GLPRT_PRC64L(hw->port),
    976 			   pf->stat_offsets_loaded,
    977 			   &osd->rx_size_64, &nsd->rx_size_64);
    978 	i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
    979 			   I40E_GLPRT_PRC127L(hw->port),
    980 			   pf->stat_offsets_loaded,
    981 			   &osd->rx_size_127, &nsd->rx_size_127);
    982 	i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
    983 			   I40E_GLPRT_PRC255L(hw->port),
    984 			   pf->stat_offsets_loaded,
    985 			   &osd->rx_size_255, &nsd->rx_size_255);
    986 	i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
    987 			   I40E_GLPRT_PRC511L(hw->port),
    988 			   pf->stat_offsets_loaded,
    989 			   &osd->rx_size_511, &nsd->rx_size_511);
    990 	i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
    991 			   I40E_GLPRT_PRC1023L(hw->port),
    992 			   pf->stat_offsets_loaded,
    993 			   &osd->rx_size_1023, &nsd->rx_size_1023);
    994 	i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
    995 			   I40E_GLPRT_PRC1522L(hw->port),
    996 			   pf->stat_offsets_loaded,
    997 			   &osd->rx_size_1522, &nsd->rx_size_1522);
    998 	i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
    999 			   I40E_GLPRT_PRC9522L(hw->port),
   1000 			   pf->stat_offsets_loaded,
   1001 			   &osd->rx_size_big, &nsd->rx_size_big);
   1002 
   1003 	i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
   1004 			   I40E_GLPRT_PTC64L(hw->port),
   1005 			   pf->stat_offsets_loaded,
   1006 			   &osd->tx_size_64, &nsd->tx_size_64);
   1007 	i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
   1008 			   I40E_GLPRT_PTC127L(hw->port),
   1009 			   pf->stat_offsets_loaded,
   1010 			   &osd->tx_size_127, &nsd->tx_size_127);
   1011 	i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
   1012 			   I40E_GLPRT_PTC255L(hw->port),
   1013 			   pf->stat_offsets_loaded,
   1014 			   &osd->tx_size_255, &nsd->tx_size_255);
   1015 	i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
   1016 			   I40E_GLPRT_PTC511L(hw->port),
   1017 			   pf->stat_offsets_loaded,
   1018 			   &osd->tx_size_511, &nsd->tx_size_511);
   1019 	i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
   1020 			   I40E_GLPRT_PTC1023L(hw->port),
   1021 			   pf->stat_offsets_loaded,
   1022 			   &osd->tx_size_1023, &nsd->tx_size_1023);
   1023 	i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
   1024 			   I40E_GLPRT_PTC1522L(hw->port),
   1025 			   pf->stat_offsets_loaded,
   1026 			   &osd->tx_size_1522, &nsd->tx_size_1522);
   1027 	i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
   1028 			   I40E_GLPRT_PTC9522L(hw->port),
   1029 			   pf->stat_offsets_loaded,
   1030 			   &osd->tx_size_big, &nsd->tx_size_big);
   1031 
   1032 	i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
   1033 			   pf->stat_offsets_loaded,
   1034 			   &osd->rx_undersize, &nsd->rx_undersize);
   1035 	i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
   1036 			   pf->stat_offsets_loaded,
   1037 			   &osd->rx_fragments, &nsd->rx_fragments);
   1038 	i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
   1039 			   pf->stat_offsets_loaded,
   1040 			   &osd->rx_oversize, &nsd->rx_oversize);
   1041 	i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
   1042 			   pf->stat_offsets_loaded,
   1043 			   &osd->rx_jabber, &nsd->rx_jabber);
   1044 
   1045 	/* FDIR stats */
   1046 	i40e_stat_update_and_clear32(hw,
   1047 			I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
   1048 			&nsd->fd_atr_match);
   1049 	i40e_stat_update_and_clear32(hw,
   1050 			I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
   1051 			&nsd->fd_sb_match);
   1052 	i40e_stat_update_and_clear32(hw,
   1053 			I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
   1054 			&nsd->fd_atr_tunnel_match);
   1055 
   1056 	val = rd32(hw, I40E_PRTPM_EEE_STAT);
   1057 	nsd->tx_lpi_status =
   1058 		       (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
   1059 			I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
   1060 	nsd->rx_lpi_status =
   1061 		       (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
   1062 			I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
   1063 	i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
   1064 			   pf->stat_offsets_loaded,
   1065 			   &osd->tx_lpi_count, &nsd->tx_lpi_count);
   1066 	i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
   1067 			   pf->stat_offsets_loaded,
   1068 			   &osd->rx_lpi_count, &nsd->rx_lpi_count);
   1069 
   1070 	if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
   1071 	    !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
   1072 		nsd->fd_sb_status = true;
   1073 	else
   1074 		nsd->fd_sb_status = false;
   1075 
   1076 	if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
   1077 	    !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
   1078 		nsd->fd_atr_status = true;
   1079 	else
   1080 		nsd->fd_atr_status = false;
   1081 
   1082 	pf->stat_offsets_loaded = true;
   1083 }
   1084 
   1085 /**
   1086  * i40e_update_stats - Update the various statistics counters.
   1087  * @vsi: the VSI to be updated
   1088  *
   1089  * Update the various stats for this VSI and its related entities.
   1090  **/
   1091 void i40e_update_stats(struct i40e_vsi *vsi)
   1092 {
   1093 	struct i40e_pf *pf = vsi->back;
   1094 
   1095 	if (vsi == pf->vsi[pf->lan_vsi])
   1096 		i40e_update_pf_stats(pf);
   1097 
   1098 	i40e_update_vsi_stats(vsi);
   1099 }
   1100 
   1101 /**
   1102  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
   1103  * @vsi: the VSI to be searched
   1104  * @macaddr: the MAC address
   1105  * @vlan: the vlan
   1106  *
   1107  * Returns ptr to the filter object or NULL
   1108  **/
   1109 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
   1110 						const u8 *macaddr, s16 vlan)
   1111 {
   1112 	struct i40e_mac_filter *f;
   1113 	u64 key;
   1114 
   1115 	if (!vsi || !macaddr)
   1116 		return NULL;
   1117 
   1118 	key = i40e_addr_to_hkey(macaddr);
   1119 	hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
   1120 		if ((ether_addr_equal(macaddr, f->macaddr)) &&
   1121 		    (vlan == f->vlan))
   1122 			return f;
   1123 	}
   1124 	return NULL;
   1125 }
   1126 
   1127 /**
   1128  * i40e_find_mac - Find a mac addr in the macvlan filters list
   1129  * @vsi: the VSI to be searched
   1130  * @macaddr: the MAC address we are searching for
   1131  *
   1132  * Returns the first filter with the provided MAC address or NULL if
   1133  * MAC address was not found
   1134  **/
   1135 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
   1136 {
   1137 	struct i40e_mac_filter *f;
   1138 	u64 key;
   1139 
   1140 	if (!vsi || !macaddr)
   1141 		return NULL;
   1142 
   1143 	key = i40e_addr_to_hkey(macaddr);
   1144 	hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
   1145 		if ((ether_addr_equal(macaddr, f->macaddr)))
   1146 			return f;
   1147 	}
   1148 	return NULL;
   1149 }
   1150 
   1151 /**
   1152  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
   1153  * @vsi: the VSI to be searched
   1154  *
   1155  * Returns true if VSI is in vlan mode or false otherwise
   1156  **/
   1157 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
   1158 {
   1159 	/* If we have a PVID, always operate in VLAN mode */
   1160 	if (vsi->info.pvid)
   1161 		return true;
   1162 
   1163 	/* We need to operate in VLAN mode whenever we have any filters with
   1164 	 * a VLAN other than I40E_VLAN_ALL. We could check the table each
   1165 	 * time, incurring search cost repeatedly. However, we can notice two
   1166 	 * things:
   1167 	 *
   1168 	 * 1) the only place where we can gain a VLAN filter is in
   1169 	 *    i40e_add_filter.
   1170 	 *
   1171 	 * 2) the only place where filters are actually removed is in
   1172 	 *    i40e_sync_filters_subtask.
   1173 	 *
   1174 	 * Thus, we can simply use a boolean value, has_vlan_filters which we
   1175 	 * will set to true when we add a VLAN filter in i40e_add_filter. Then
   1176 	 * we have to perform the full search after deleting filters in
   1177 	 * i40e_sync_filters_subtask, but we already have to search
   1178 	 * filters here and can perform the check at the same time. This
   1179 	 * results in avoiding embedding a loop for VLAN mode inside another
   1180 	 * loop over all the filters, and should maintain correctness as noted
   1181 	 * above.
   1182 	 */
   1183 	return vsi->has_vlan_filter;
   1184 }
   1185 
   1186 /**
   1187  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
   1188  * @vsi: the VSI to configure
   1189  * @tmp_add_list: list of filters ready to be added
   1190  * @tmp_del_list: list of filters ready to be deleted
   1191  * @vlan_filters: the number of active VLAN filters
   1192  *
   1193  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
   1194  * behave as expected. If we have any active VLAN filters remaining or about
   1195  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
   1196  * so that they only match against untagged traffic. If we no longer have any
   1197  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
   1198  * so that they match against both tagged and untagged traffic. In this way,
   1199  * we ensure that we correctly receive the desired traffic. This ensures that
   1200  * when we have an active VLAN we will receive only untagged traffic and
   1201  * traffic matching active VLANs. If we have no active VLANs then we will
   1202  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
   1203  *
   1204  * Finally, in a similar fashion, this function also corrects filters when
   1205  * there is an active PVID assigned to this VSI.
   1206  *
   1207  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
   1208  *
   1209  * This function is only expected to be called from within
   1210  * i40e_sync_vsi_filters.
   1211  *
   1212  * NOTE: This function expects to be called while under the
   1213  * mac_filter_hash_lock
   1214  */
   1215 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
   1216 					 struct hlist_head *tmp_add_list,
   1217 					 struct hlist_head *tmp_del_list,
   1218 					 int vlan_filters)
   1219 {
   1220 	s16 pvid = le16_to_cpu(vsi->info.pvid);
   1221 	struct i40e_mac_filter *f, *add_head;
   1222 	struct i40e_new_mac_filter *new;
   1223 	struct hlist_node *h;
   1224 	int bkt, new_vlan;
   1225 
   1226 	/* To determine if a particular filter needs to be replaced we
   1227 	 * have the three following conditions:
   1228 	 *
   1229 	 * a) if we have a PVID assigned, then all filters which are
   1230 	 *    not marked as VLAN=PVID must be replaced with filters that
   1231 	 *    are.
   1232 	 * b) otherwise, if we have any active VLANS, all filters
   1233 	 *    which are marked as VLAN=-1 must be replaced with
   1234 	 *    filters marked as VLAN=0
   1235 	 * c) finally, if we do not have any active VLANS, all filters
   1236 	 *    which are marked as VLAN=0 must be replaced with filters
   1237 	 *    marked as VLAN=-1
   1238 	 */
   1239 
   1240 	/* Update the filters about to be added in place */
   1241 	hlist_for_each_entry(new, tmp_add_list, hlist) {
   1242 		if (pvid && new->f->vlan != pvid)
   1243 			new->f->vlan = pvid;
   1244 		else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
   1245 			new->f->vlan = 0;
   1246 		else if (!vlan_filters && new->f->vlan == 0)
   1247 			new->f->vlan = I40E_VLAN_ANY;
   1248 	}
   1249 
   1250 	/* Update the remaining active filters */
   1251 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
   1252 		/* Combine the checks for whether a filter needs to be changed
   1253 		 * and then determine the new VLAN inside the if block, in
   1254 		 * order to avoid duplicating code for adding the new filter
   1255 		 * then deleting the old filter.
   1256 		 */
   1257 		if ((pvid && f->vlan != pvid) ||
   1258 		    (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
   1259 		    (!vlan_filters && f->vlan == 0)) {
   1260 			/* Determine the new vlan we will be adding */
   1261 			if (pvid)
   1262 				new_vlan = pvid;
   1263 			else if (vlan_filters)
   1264 				new_vlan = 0;
   1265 			else
   1266 				new_vlan = I40E_VLAN_ANY;
   1267 
   1268 			/* Create the new filter */
   1269 			add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
   1270 			if (!add_head)
   1271 				return -ENOMEM;
   1272 
   1273 			/* Create a temporary i40e_new_mac_filter */
   1274 			new = kzalloc(sizeof(*new), GFP_ATOMIC);
   1275 			if (!new)
   1276 				return -ENOMEM;
   1277 
   1278 			new->f = add_head;
   1279 			new->state = add_head->state;
   1280 
   1281 			/* Add the new filter to the tmp list */
   1282 			hlist_add_head(&new->hlist, tmp_add_list);
   1283 
   1284 			/* Put the original filter into the delete list */
   1285 			f->state = I40E_FILTER_REMOVE;
   1286 			hash_del(&f->hlist);
   1287 			hlist_add_head(&f->hlist, tmp_del_list);
   1288 		}
   1289 	}
   1290 
   1291 	vsi->has_vlan_filter = !!vlan_filters;
   1292 
   1293 	return 0;
   1294 }
   1295 
   1296 /**
   1297  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
   1298  * @vsi: the PF Main VSI - inappropriate for any other VSI
   1299  * @macaddr: the MAC address
   1300  *
   1301  * Remove whatever filter the firmware set up so the driver can manage
   1302  * its own filtering intelligently.
   1303  **/
   1304 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
   1305 {
   1306 	struct i40e_aqc_remove_macvlan_element_data element;
   1307 	struct i40e_pf *pf = vsi->back;
   1308 
   1309 	/* Only appropriate for the PF main VSI */
   1310 	if (vsi->type != I40E_VSI_MAIN)
   1311 		return;
   1312 
   1313 	memset(&element, 0, sizeof(element));
   1314 	ether_addr_copy(element.mac_addr, macaddr);
   1315 	element.vlan_tag = 0;
   1316 	/* Ignore error returns, some firmware does it this way... */
   1317 	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
   1318 	i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
   1319 
   1320 	memset(&element, 0, sizeof(element));
   1321 	ether_addr_copy(element.mac_addr, macaddr);
   1322 	element.vlan_tag = 0;
   1323 	/* ...and some firmware does it this way. */
   1324 	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
   1325 			I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
   1326 	i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
   1327 }
   1328 
   1329 /**
   1330  * i40e_add_filter - Add a mac/vlan filter to the VSI
   1331  * @vsi: the VSI to be searched
   1332  * @macaddr: the MAC address
   1333  * @vlan: the vlan
   1334  *
   1335  * Returns ptr to the filter object or NULL when no memory available.
   1336  *
   1337  * NOTE: This function is expected to be called with mac_filter_hash_lock
   1338  * being held.
   1339  **/
   1340 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
   1341 					const u8 *macaddr, s16 vlan)
   1342 {
   1343 	struct i40e_mac_filter *f;
   1344 	u64 key;
   1345 
   1346 	if (!vsi || !macaddr)
   1347 		return NULL;
   1348 
   1349 	f = i40e_find_filter(vsi, macaddr, vlan);
   1350 	if (!f) {
   1351 		f = kzalloc(sizeof(*f), GFP_ATOMIC);
   1352 		if (!f)
   1353 			return NULL;
   1354 
   1355 		/* Update the boolean indicating if we need to function in
   1356 		 * VLAN mode.
   1357 		 */
   1358 		if (vlan >= 0)
   1359 			vsi->has_vlan_filter = true;
   1360 
   1361 		ether_addr_copy(f->macaddr, macaddr);
   1362 		f->vlan = vlan;
   1363 		f->state = I40E_FILTER_NEW;
   1364 		INIT_HLIST_NODE(&f->hlist);
   1365 
   1366 		key = i40e_addr_to_hkey(macaddr);
   1367 		hash_add(vsi->mac_filter_hash, &f->hlist, key);
   1368 
   1369 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
   1370 		set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
   1371 	}
   1372 
   1373 	/* If we're asked to add a filter that has been marked for removal, it
   1374 	 * is safe to simply restore it to active state. __i40e_del_filter
   1375 	 * will have simply deleted any filters which were previously marked
   1376 	 * NEW or FAILED, so if it is currently marked REMOVE it must have
   1377 	 * previously been ACTIVE. Since we haven't yet run the sync filters
   1378 	 * task, just restore this filter to the ACTIVE state so that the
   1379 	 * sync task leaves it in place
   1380 	 */
   1381 	if (f->state == I40E_FILTER_REMOVE)
   1382 		f->state = I40E_FILTER_ACTIVE;
   1383 
   1384 	return f;
   1385 }
   1386 
   1387 /**
   1388  * __i40e_del_filter - Remove a specific filter from the VSI
   1389  * @vsi: VSI to remove from
   1390  * @f: the filter to remove from the list
   1391  *
   1392  * This function should be called instead of i40e_del_filter only if you know
   1393  * the exact filter you will remove already, such as via i40e_find_filter or
   1394  * i40e_find_mac.
   1395  *
   1396  * NOTE: This function is expected to be called with mac_filter_hash_lock
   1397  * being held.
   1398  * ANOTHER NOTE: This function MUST be called from within the context of
   1399  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
   1400  * instead of list_for_each_entry().
   1401  **/
   1402 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
   1403 {
   1404 	if (!f)
   1405 		return;
   1406 
   1407 	/* If the filter was never added to firmware then we can just delete it
   1408 	 * directly and we don't want to set the status to remove or else an
   1409 	 * admin queue command will unnecessarily fire.
   1410 	 */
   1411 	if ((f->state == I40E_FILTER_FAILED) ||
   1412 	    (f->state == I40E_FILTER_NEW)) {
   1413 		hash_del(&f->hlist);
   1414 		kfree(f);
   1415 	} else {
   1416 		f->state = I40E_FILTER_REMOVE;
   1417 	}
   1418 
   1419 	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
   1420 	set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
   1421 }
   1422 
   1423 /**
   1424  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
   1425  * @vsi: the VSI to be searched
   1426  * @macaddr: the MAC address
   1427  * @vlan: the VLAN
   1428  *
   1429  * NOTE: This function is expected to be called with mac_filter_hash_lock
   1430  * being held.
   1431  * ANOTHER NOTE: This function MUST be called from within the context of
   1432  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
   1433  * instead of list_for_each_entry().
   1434  **/
   1435 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
   1436 {
   1437 	struct i40e_mac_filter *f;
   1438 
   1439 	if (!vsi || !macaddr)
   1440 		return;
   1441 
   1442 	f = i40e_find_filter(vsi, macaddr, vlan);
   1443 	__i40e_del_filter(vsi, f);
   1444 }
   1445 
   1446 /**
   1447  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
   1448  * @vsi: the VSI to be searched
   1449  * @macaddr: the mac address to be filtered
   1450  *
   1451  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
   1452  * go through all the macvlan filters and add a macvlan filter for each
   1453  * unique vlan that already exists. If a PVID has been assigned, instead only
   1454  * add the macaddr to that VLAN.
   1455  *
   1456  * Returns last filter added on success, else NULL
   1457  **/
   1458 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
   1459 					    const u8 *macaddr)
   1460 {
   1461 	struct i40e_mac_filter *f, *add = NULL;
   1462 	struct hlist_node *h;
   1463 	int bkt;
   1464 
   1465 	if (vsi->info.pvid)
   1466 		return i40e_add_filter(vsi, macaddr,
   1467 				       le16_to_cpu(vsi->info.pvid));
   1468 
   1469 	if (!i40e_is_vsi_in_vlan(vsi))
   1470 		return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
   1471 
   1472 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
   1473 		if (f->state == I40E_FILTER_REMOVE)
   1474 			continue;
   1475 		add = i40e_add_filter(vsi, macaddr, f->vlan);
   1476 		if (!add)
   1477 			return NULL;
   1478 	}
   1479 
   1480 	return add;
   1481 }
   1482 
   1483 /**
   1484  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
   1485  * @vsi: the VSI to be searched
   1486  * @macaddr: the mac address to be removed
   1487  *
   1488  * Removes a given MAC address from a VSI regardless of what VLAN it has been
   1489  * associated with.
   1490  *
   1491  * Returns 0 for success, or error
   1492  **/
   1493 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
   1494 {
   1495 	struct i40e_mac_filter *f;
   1496 	struct hlist_node *h;
   1497 	bool found = false;
   1498 	int bkt;
   1499 
   1500 	lockdep_assert_held(&vsi->mac_filter_hash_lock);
   1501 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
   1502 		if (ether_addr_equal(macaddr, f->macaddr)) {
   1503 			__i40e_del_filter(vsi, f);
   1504 			found = true;
   1505 		}
   1506 	}
   1507 
   1508 	if (found)
   1509 		return 0;
   1510 	else
   1511 		return -ENOENT;
   1512 }
   1513 
   1514 /**
   1515  * i40e_set_mac - NDO callback to set mac address
   1516  * @netdev: network interface device structure
   1517  * @p: pointer to an address structure
   1518  *
   1519  * Returns 0 on success, negative on failure
   1520  **/
   1521 static int i40e_set_mac(struct net_device *netdev, void *p)
   1522 {
   1523 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   1524 	struct i40e_vsi *vsi = np->vsi;
   1525 	struct i40e_pf *pf = vsi->back;
   1526 	struct i40e_hw *hw = &pf->hw;
   1527 	struct sockaddr *addr = p;
   1528 
   1529 	if (!is_valid_ether_addr(addr->sa_data))
   1530 		return -EADDRNOTAVAIL;
   1531 
   1532 	if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
   1533 		netdev_info(netdev, "already using mac address %pM\n",
   1534 			    addr->sa_data);
   1535 		return 0;
   1536 	}
   1537 
   1538 	if (test_bit(__I40E_DOWN, pf->state) ||
   1539 	    test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
   1540 		return -EADDRNOTAVAIL;
   1541 
   1542 	if (ether_addr_equal(hw->mac.addr, addr->sa_data))
   1543 		netdev_info(netdev, "returning to hw mac address %pM\n",
   1544 			    hw->mac.addr);
   1545 	else
   1546 		netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
   1547 
   1548 	/* Copy the address first, so that we avoid a possible race with
   1549 	 * .set_rx_mode().
   1550 	 * - Remove old address from MAC filter
   1551 	 * - Copy new address
   1552 	 * - Add new address to MAC filter
   1553 	 */
   1554 	spin_lock_bh(&vsi->mac_filter_hash_lock);
   1555 	i40e_del_mac_filter(vsi, netdev->dev_addr);
   1556 	ether_addr_copy(netdev->dev_addr, addr->sa_data);
   1557 	i40e_add_mac_filter(vsi, netdev->dev_addr);
   1558 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
   1559 
   1560 	if (vsi->type == I40E_VSI_MAIN) {
   1561 		i40e_status ret;
   1562 
   1563 		ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
   1564 						addr->sa_data, NULL);
   1565 		if (ret)
   1566 			netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
   1567 				    i40e_stat_str(hw, ret),
   1568 				    i40e_aq_str(hw, hw->aq.asq_last_status));
   1569 	}
   1570 
   1571 	/* schedule our worker thread which will take care of
   1572 	 * applying the new filter changes
   1573 	 */
   1574 	i40e_service_event_schedule(pf);
   1575 	return 0;
   1576 }
   1577 
   1578 /**
   1579  * i40e_config_rss_aq - Prepare for RSS using AQ commands
   1580  * @vsi: vsi structure
   1581  * @seed: RSS hash seed
   1582  **/
   1583 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
   1584 			      u8 *lut, u16 lut_size)
   1585 {
   1586 	struct i40e_pf *pf = vsi->back;
   1587 	struct i40e_hw *hw = &pf->hw;
   1588 	int ret = 0;
   1589 
   1590 	if (seed) {
   1591 		struct i40e_aqc_get_set_rss_key_data *seed_dw =
   1592 			(struct i40e_aqc_get_set_rss_key_data *)seed;
   1593 		ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
   1594 		if (ret) {
   1595 			dev_info(&pf->pdev->dev,
   1596 				 "Cannot set RSS key, err %s aq_err %s\n",
   1597 				 i40e_stat_str(hw, ret),
   1598 				 i40e_aq_str(hw, hw->aq.asq_last_status));
   1599 			return ret;
   1600 		}
   1601 	}
   1602 	if (lut) {
   1603 		bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
   1604 
   1605 		ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
   1606 		if (ret) {
   1607 			dev_info(&pf->pdev->dev,
   1608 				 "Cannot set RSS lut, err %s aq_err %s\n",
   1609 				 i40e_stat_str(hw, ret),
   1610 				 i40e_aq_str(hw, hw->aq.asq_last_status));
   1611 			return ret;
   1612 		}
   1613 	}
   1614 	return ret;
   1615 }
   1616 
   1617 /**
   1618  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
   1619  * @vsi: VSI structure
   1620  **/
   1621 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
   1622 {
   1623 	struct i40e_pf *pf = vsi->back;
   1624 	u8 seed[I40E_HKEY_ARRAY_SIZE];
   1625 	u8 *lut;
   1626 	int ret;
   1627 
   1628 	if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
   1629 		return 0;
   1630 	if (!vsi->rss_size)
   1631 		vsi->rss_size = min_t(int, pf->alloc_rss_size,
   1632 				      vsi->num_queue_pairs);
   1633 	if (!vsi->rss_size)
   1634 		return -EINVAL;
   1635 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
   1636 	if (!lut)
   1637 		return -ENOMEM;
   1638 
   1639 	/* Use the user configured hash keys and lookup table if there is one,
   1640 	 * otherwise use default
   1641 	 */
   1642 	if (vsi->rss_lut_user)
   1643 		memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
   1644 	else
   1645 		i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
   1646 	if (vsi->rss_hkey_user)
   1647 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
   1648 	else
   1649 		netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
   1650 	ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
   1651 	kfree(lut);
   1652 	return ret;
   1653 }
   1654 
   1655 /**
   1656  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
   1657  * @vsi: the VSI being configured,
   1658  * @ctxt: VSI context structure
   1659  * @enabled_tc: number of traffic classes to enable
   1660  *
   1661  * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
   1662  **/
   1663 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
   1664 					   struct i40e_vsi_context *ctxt,
   1665 					   u8 enabled_tc)
   1666 {
   1667 	u16 qcount = 0, max_qcount, qmap, sections = 0;
   1668 	int i, override_q, pow, num_qps, ret;
   1669 	u8 netdev_tc = 0, offset = 0;
   1670 
   1671 	if (vsi->type != I40E_VSI_MAIN)
   1672 		return -EINVAL;
   1673 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
   1674 	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
   1675 	vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
   1676 	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
   1677 	num_qps = vsi->mqprio_qopt.qopt.count[0];
   1678 
   1679 	/* find the next higher power-of-2 of num queue pairs */
   1680 	pow = ilog2(num_qps);
   1681 	if (!is_power_of_2(num_qps))
   1682 		pow++;
   1683 	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
   1684 		(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
   1685 
   1686 	/* Setup queue offset/count for all TCs for given VSI */
   1687 	max_qcount = vsi->mqprio_qopt.qopt.count[0];
   1688 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   1689 		/* See if the given TC is enabled for the given VSI */
   1690 		if (vsi->tc_config.enabled_tc & BIT(i)) {
   1691 			offset = vsi->mqprio_qopt.qopt.offset[i];
   1692 			qcount = vsi->mqprio_qopt.qopt.count[i];
   1693 			if (qcount > max_qcount)
   1694 				max_qcount = qcount;
   1695 			vsi->tc_config.tc_info[i].qoffset = offset;
   1696 			vsi->tc_config.tc_info[i].qcount = qcount;
   1697 			vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
   1698 		} else {
   1699 			/* TC is not enabled so set the offset to
   1700 			 * default queue and allocate one queue
   1701 			 * for the given TC.
   1702 			 */
   1703 			vsi->tc_config.tc_info[i].qoffset = 0;
   1704 			vsi->tc_config.tc_info[i].qcount = 1;
   1705 			vsi->tc_config.tc_info[i].netdev_tc = 0;
   1706 		}
   1707 	}
   1708 
   1709 	/* Set actual Tx/Rx queue pairs */
   1710 	vsi->num_queue_pairs = offset + qcount;
   1711 
   1712 	/* Setup queue TC[0].qmap for given VSI context */
   1713 	ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
   1714 	ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
   1715 	ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
   1716 	ctxt->info.valid_sections |= cpu_to_le16(sections);
   1717 
   1718 	/* Reconfigure RSS for main VSI with max queue count */
   1719 	vsi->rss_size = max_qcount;
   1720 	ret = i40e_vsi_config_rss(vsi);
   1721 	if (ret) {
   1722 		dev_info(&vsi->back->pdev->dev,
   1723 			 "Failed to reconfig rss for num_queues (%u)\n",
   1724 			 max_qcount);
   1725 		return ret;
   1726 	}
   1727 	vsi->reconfig_rss = true;
   1728 	dev_dbg(&vsi->back->pdev->dev,
   1729 		"Reconfigured rss with num_queues (%u)\n", max_qcount);
   1730 
   1731 	/* Find queue count available for channel VSIs and starting offset
   1732 	 * for channel VSIs
   1733 	 */
   1734 	override_q = vsi->mqprio_qopt.qopt.count[0];
   1735 	if (override_q && override_q < vsi->num_queue_pairs) {
   1736 		vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
   1737 		vsi->next_base_queue = override_q;
   1738 	}
   1739 	return 0;
   1740 }
   1741 
   1742 /**
   1743  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
   1744  * @vsi: the VSI being setup
   1745  * @ctxt: VSI context structure
   1746  * @enabled_tc: Enabled TCs bitmap
   1747  * @is_add: True if called before Add VSI
   1748  *
   1749  * Setup VSI queue mapping for enabled traffic classes.
   1750  **/
   1751 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
   1752 				     struct i40e_vsi_context *ctxt,
   1753 				     u8 enabled_tc,
   1754 				     bool is_add)
   1755 {
   1756 	struct i40e_pf *pf = vsi->back;
   1757 	u16 sections = 0;
   1758 	u8 netdev_tc = 0;
   1759 	u16 numtc = 1;
   1760 	u16 qcount;
   1761 	u8 offset;
   1762 	u16 qmap;
   1763 	int i;
   1764 	u16 num_tc_qps = 0;
   1765 
   1766 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
   1767 	offset = 0;
   1768 
   1769 	/* Number of queues per enabled TC */
   1770 	num_tc_qps = vsi->alloc_queue_pairs;
   1771 	if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
   1772 		/* Find numtc from enabled TC bitmap */
   1773 		for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   1774 			if (enabled_tc & BIT(i)) /* TC is enabled */
   1775 				numtc++;
   1776 		}
   1777 		if (!numtc) {
   1778 			dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
   1779 			numtc = 1;
   1780 		}
   1781 		num_tc_qps = num_tc_qps / numtc;
   1782 		num_tc_qps = min_t(int, num_tc_qps,
   1783 				   i40e_pf_get_max_q_per_tc(pf));
   1784 	}
   1785 
   1786 	vsi->tc_config.numtc = numtc;
   1787 	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
   1788 
   1789 	/* Do not allow use more TC queue pairs than MSI-X vectors exist */
   1790 	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
   1791 		num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
   1792 
   1793 	/* Setup queue offset/count for all TCs for given VSI */
   1794 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   1795 		/* See if the given TC is enabled for the given VSI */
   1796 		if (vsi->tc_config.enabled_tc & BIT(i)) {
   1797 			/* TC is enabled */
   1798 			int pow, num_qps;
   1799 
   1800 			switch (vsi->type) {
   1801 			case I40E_VSI_MAIN:
   1802 				if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
   1803 				    I40E_FLAG_FD_ATR_ENABLED)) ||
   1804 				    vsi->tc_config.enabled_tc != 1) {
   1805 					qcount = min_t(int, pf->alloc_rss_size,
   1806 						       num_tc_qps);
   1807 					break;
   1808 				}
   1809 				/* fall through */
   1810 			case I40E_VSI_FDIR:
   1811 			case I40E_VSI_SRIOV:
   1812 			case I40E_VSI_VMDQ2:
   1813 			default:
   1814 				qcount = num_tc_qps;
   1815 				WARN_ON(i != 0);
   1816 				break;
   1817 			}
   1818 			vsi->tc_config.tc_info[i].qoffset = offset;
   1819 			vsi->tc_config.tc_info[i].qcount = qcount;
   1820 
   1821 			/* find the next higher power-of-2 of num queue pairs */
   1822 			num_qps = qcount;
   1823 			pow = 0;
   1824 			while (num_qps && (BIT_ULL(pow) < qcount)) {
   1825 				pow++;
   1826 				num_qps >>= 1;
   1827 			}
   1828 
   1829 			vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
   1830 			qmap =
   1831 			    (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
   1832 			    (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
   1833 
   1834 			offset += qcount;
   1835 		} else {
   1836 			/* TC is not enabled so set the offset to
   1837 			 * default queue and allocate one queue
   1838 			 * for the given TC.
   1839 			 */
   1840 			vsi->tc_config.tc_info[i].qoffset = 0;
   1841 			vsi->tc_config.tc_info[i].qcount = 1;
   1842 			vsi->tc_config.tc_info[i].netdev_tc = 0;
   1843 
   1844 			qmap = 0;
   1845 		}
   1846 		ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
   1847 	}
   1848 
   1849 	/* Set actual Tx/Rx queue pairs */
   1850 	vsi->num_queue_pairs = offset;
   1851 	if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
   1852 		if (vsi->req_queue_pairs > 0)
   1853 			vsi->num_queue_pairs = vsi->req_queue_pairs;
   1854 		else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
   1855 			vsi->num_queue_pairs = pf->num_lan_msix;
   1856 	}
   1857 
   1858 	/* Scheduler section valid can only be set for ADD VSI */
   1859 	if (is_add) {
   1860 		sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
   1861 
   1862 		ctxt->info.up_enable_bits = enabled_tc;
   1863 	}
   1864 	if (vsi->type == I40E_VSI_SRIOV) {
   1865 		ctxt->info.mapping_flags |=
   1866 				     cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
   1867 		for (i = 0; i < vsi->num_queue_pairs; i++)
   1868 			ctxt->info.queue_mapping[i] =
   1869 					       cpu_to_le16(vsi->base_queue + i);
   1870 	} else {
   1871 		ctxt->info.mapping_flags |=
   1872 					cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
   1873 		ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
   1874 	}
   1875 	ctxt->info.valid_sections |= cpu_to_le16(sections);
   1876 }
   1877 
   1878 /**
   1879  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
   1880  * @netdev: the netdevice
   1881  * @addr: address to add
   1882  *
   1883  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
   1884  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
   1885  */
   1886 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
   1887 {
   1888 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   1889 	struct i40e_vsi *vsi = np->vsi;
   1890 
   1891 	if (i40e_add_mac_filter(vsi, addr))
   1892 		return 0;
   1893 	else
   1894 		return -ENOMEM;
   1895 }
   1896 
   1897 /**
   1898  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
   1899  * @netdev: the netdevice
   1900  * @addr: address to add
   1901  *
   1902  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
   1903  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
   1904  */
   1905 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
   1906 {
   1907 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   1908 	struct i40e_vsi *vsi = np->vsi;
   1909 
   1910 	/* Under some circumstances, we might receive a request to delete
   1911 	 * our own device address from our uc list. Because we store the
   1912 	 * device address in the VSI's MAC/VLAN filter list, we need to ignore
   1913 	 * such requests and not delete our device address from this list.
   1914 	 */
   1915 	if (ether_addr_equal(addr, netdev->dev_addr))
   1916 		return 0;
   1917 
   1918 	i40e_del_mac_filter(vsi, addr);
   1919 
   1920 	return 0;
   1921 }
   1922 
   1923 /**
   1924  * i40e_set_rx_mode - NDO callback to set the netdev filters
   1925  * @netdev: network interface device structure
   1926  **/
   1927 static void i40e_set_rx_mode(struct net_device *netdev)
   1928 {
   1929 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   1930 	struct i40e_vsi *vsi = np->vsi;
   1931 
   1932 	spin_lock_bh(&vsi->mac_filter_hash_lock);
   1933 
   1934 	__dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
   1935 	__dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
   1936 
   1937 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
   1938 
   1939 	/* check for other flag changes */
   1940 	if (vsi->current_netdev_flags != vsi->netdev->flags) {
   1941 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
   1942 		set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
   1943 	}
   1944 }
   1945 
   1946 /**
   1947  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
   1948  * @vsi: Pointer to VSI struct
   1949  * @from: Pointer to list which contains MAC filter entries - changes to
   1950  *        those entries needs to be undone.
   1951  *
   1952  * MAC filter entries from this list were slated for deletion.
   1953  **/
   1954 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
   1955 					 struct hlist_head *from)
   1956 {
   1957 	struct i40e_mac_filter *f;
   1958 	struct hlist_node *h;
   1959 
   1960 	hlist_for_each_entry_safe(f, h, from, hlist) {
   1961 		u64 key = i40e_addr_to_hkey(f->macaddr);
   1962 
   1963 		/* Move the element back into MAC filter list*/
   1964 		hlist_del(&f->hlist);
   1965 		hash_add(vsi->mac_filter_hash, &f->hlist, key);
   1966 	}
   1967 }
   1968 
   1969 /**
   1970  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
   1971  * @vsi: Pointer to vsi struct
   1972  * @from: Pointer to list which contains MAC filter entries - changes to
   1973  *        those entries needs to be undone.
   1974  *
   1975  * MAC filter entries from this list were slated for addition.
   1976  **/
   1977 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
   1978 					 struct hlist_head *from)
   1979 {
   1980 	struct i40e_new_mac_filter *new;
   1981 	struct hlist_node *h;
   1982 
   1983 	hlist_for_each_entry_safe(new, h, from, hlist) {
   1984 		/* We can simply free the wrapper structure */
   1985 		hlist_del(&new->hlist);
   1986 		kfree(new);
   1987 	}
   1988 }
   1989 
   1990 /**
   1991  * i40e_next_entry - Get the next non-broadcast filter from a list
   1992  * @next: pointer to filter in list
   1993  *
   1994  * Returns the next non-broadcast filter in the list. Required so that we
   1995  * ignore broadcast filters within the list, since these are not handled via
   1996  * the normal firmware update path.
   1997  */
   1998 static
   1999 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
   2000 {
   2001 	hlist_for_each_entry_continue(next, hlist) {
   2002 		if (!is_broadcast_ether_addr(next->f->macaddr))
   2003 			return next;
   2004 	}
   2005 
   2006 	return NULL;
   2007 }
   2008 
   2009 /**
   2010  * i40e_update_filter_state - Update filter state based on return data
   2011  * from firmware
   2012  * @count: Number of filters added
   2013  * @add_list: return data from fw
   2014  * @add_head: pointer to first filter in current batch
   2015  *
   2016  * MAC filter entries from list were slated to be added to device. Returns
   2017  * number of successful filters. Note that 0 does NOT mean success!
   2018  **/
   2019 static int
   2020 i40e_update_filter_state(int count,
   2021 			 struct i40e_aqc_add_macvlan_element_data *add_list,
   2022 			 struct i40e_new_mac_filter *add_head)
   2023 {
   2024 	int retval = 0;
   2025 	int i;
   2026 
   2027 	for (i = 0; i < count; i++) {
   2028 		/* Always check status of each filter. We don't need to check
   2029 		 * the firmware return status because we pre-set the filter
   2030 		 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
   2031 		 * request to the adminq. Thus, if it no longer matches then
   2032 		 * we know the filter is active.
   2033 		 */
   2034 		if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
   2035 			add_head->state = I40E_FILTER_FAILED;
   2036 		} else {
   2037 			add_head->state = I40E_FILTER_ACTIVE;
   2038 			retval++;
   2039 		}
   2040 
   2041 		add_head = i40e_next_filter(add_head);
   2042 		if (!add_head)
   2043 			break;
   2044 	}
   2045 
   2046 	return retval;
   2047 }
   2048 
   2049 /**
   2050  * i40e_aqc_del_filters - Request firmware to delete a set of filters
   2051  * @vsi: ptr to the VSI
   2052  * @vsi_name: name to display in messages
   2053  * @list: the list of filters to send to firmware
   2054  * @num_del: the number of filters to delete
   2055  * @retval: Set to -EIO on failure to delete
   2056  *
   2057  * Send a request to firmware via AdminQ to delete a set of filters. Uses
   2058  * *retval instead of a return value so that success does not force ret_val to
   2059  * be set to 0. This ensures that a sequence of calls to this function
   2060  * preserve the previous value of *retval on successful delete.
   2061  */
   2062 static
   2063 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
   2064 			  struct i40e_aqc_remove_macvlan_element_data *list,
   2065 			  int num_del, int *retval)
   2066 {
   2067 	struct i40e_hw *hw = &vsi->back->hw;
   2068 	i40e_status aq_ret;
   2069 	int aq_err;
   2070 
   2071 	aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
   2072 	aq_err = hw->aq.asq_last_status;
   2073 
   2074 	/* Explicitly ignore and do not report when firmware returns ENOENT */
   2075 	if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
   2076 		*retval = -EIO;
   2077 		dev_info(&vsi->back->pdev->dev,
   2078 			 "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
   2079 			 vsi_name, i40e_stat_str(hw, aq_ret),
   2080 			 i40e_aq_str(hw, aq_err));
   2081 	}
   2082 }
   2083 
   2084 /**
   2085  * i40e_aqc_add_filters - Request firmware to add a set of filters
   2086  * @vsi: ptr to the VSI
   2087  * @vsi_name: name to display in messages
   2088  * @list: the list of filters to send to firmware
   2089  * @add_head: Position in the add hlist
   2090  * @num_add: the number of filters to add
   2091  *
   2092  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
   2093  * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
   2094  * space for more filters.
   2095  */
   2096 static
   2097 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
   2098 			  struct i40e_aqc_add_macvlan_element_data *list,
   2099 			  struct i40e_new_mac_filter *add_head,
   2100 			  int num_add)
   2101 {
   2102 	struct i40e_hw *hw = &vsi->back->hw;
   2103 	int aq_err, fcnt;
   2104 
   2105 	i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
   2106 	aq_err = hw->aq.asq_last_status;
   2107 	fcnt = i40e_update_filter_state(num_add, list, add_head);
   2108 
   2109 	if (fcnt != num_add) {
   2110 		set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
   2111 		dev_warn(&vsi->back->pdev->dev,
   2112 			 "Error %s adding RX filters on %s, promiscuous mode forced on\n",
   2113 			 i40e_aq_str(hw, aq_err),
   2114 			 vsi_name);
   2115 	}
   2116 }
   2117 
   2118 /**
   2119  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
   2120  * @vsi: pointer to the VSI
   2121  * @vsi_name: the VSI name
   2122  * @f: filter data
   2123  *
   2124  * This function sets or clears the promiscuous broadcast flags for VLAN
   2125  * filters in order to properly receive broadcast frames. Assumes that only
   2126  * broadcast filters are passed.
   2127  *
   2128  * Returns status indicating success or failure;
   2129  **/
   2130 static i40e_status
   2131 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
   2132 			  struct i40e_mac_filter *f)
   2133 {
   2134 	bool enable = f->state == I40E_FILTER_NEW;
   2135 	struct i40e_hw *hw = &vsi->back->hw;
   2136 	i40e_status aq_ret;
   2137 
   2138 	if (f->vlan == I40E_VLAN_ANY) {
   2139 		aq_ret = i40e_aq_set_vsi_broadcast(hw,
   2140 						   vsi->seid,
   2141 						   enable,
   2142 						   NULL);
   2143 	} else {
   2144 		aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
   2145 							    vsi->seid,
   2146 							    enable,
   2147 							    f->vlan,
   2148 							    NULL);
   2149 	}
   2150 
   2151 	if (aq_ret) {
   2152 		set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
   2153 		dev_warn(&vsi->back->pdev->dev,
   2154 			 "Error %s, forcing overflow promiscuous on %s\n",
   2155 			 i40e_aq_str(hw, hw->aq.asq_last_status),
   2156 			 vsi_name);
   2157 	}
   2158 
   2159 	return aq_ret;
   2160 }
   2161 
   2162 /**
   2163  * i40e_set_promiscuous - set promiscuous mode
   2164  * @pf: board private structure
   2165  * @promisc: promisc on or off
   2166  *
   2167  * There are different ways of setting promiscuous mode on a PF depending on
   2168  * what state/environment we're in.  This identifies and sets it appropriately.
   2169  * Returns 0 on success.
   2170  **/
   2171 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
   2172 {
   2173 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
   2174 	struct i40e_hw *hw = &pf->hw;
   2175 	i40e_status aq_ret;
   2176 
   2177 	if (vsi->type == I40E_VSI_MAIN &&
   2178 	    pf->lan_veb != I40E_NO_VEB &&
   2179 	    !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
   2180 		/* set defport ON for Main VSI instead of true promisc
   2181 		 * this way we will get all unicast/multicast and VLAN
   2182 		 * promisc behavior but will not get VF or VMDq traffic
   2183 		 * replicated on the Main VSI.
   2184 		 */
   2185 		if (promisc)
   2186 			aq_ret = i40e_aq_set_default_vsi(hw,
   2187 							 vsi->seid,
   2188 							 NULL);
   2189 		else
   2190 			aq_ret = i40e_aq_clear_default_vsi(hw,
   2191 							   vsi->seid,
   2192 							   NULL);
   2193 		if (aq_ret) {
   2194 			dev_info(&pf->pdev->dev,
   2195 				 "Set default VSI failed, err %s, aq_err %s\n",
   2196 				 i40e_stat_str(hw, aq_ret),
   2197 				 i40e_aq_str(hw, hw->aq.asq_last_status));
   2198 		}
   2199 	} else {
   2200 		aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
   2201 						  hw,
   2202 						  vsi->seid,
   2203 						  promisc, NULL,
   2204 						  true);
   2205 		if (aq_ret) {
   2206 			dev_info(&pf->pdev->dev,
   2207 				 "set unicast promisc failed, err %s, aq_err %s\n",
   2208 				 i40e_stat_str(hw, aq_ret),
   2209 				 i40e_aq_str(hw, hw->aq.asq_last_status));
   2210 		}
   2211 		aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
   2212 						  hw,
   2213 						  vsi->seid,
   2214 						  promisc, NULL);
   2215 		if (aq_ret) {
   2216 			dev_info(&pf->pdev->dev,
   2217 				 "set multicast promisc failed, err %s, aq_err %s\n",
   2218 				 i40e_stat_str(hw, aq_ret),
   2219 				 i40e_aq_str(hw, hw->aq.asq_last_status));
   2220 		}
   2221 	}
   2222 
   2223 	if (!aq_ret)
   2224 		pf->cur_promisc = promisc;
   2225 
   2226 	return aq_ret;
   2227 }
   2228 
   2229 /**
   2230  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
   2231  * @vsi: ptr to the VSI
   2232  *
   2233  * Push any outstanding VSI filter changes through the AdminQ.
   2234  *
   2235  * Returns 0 or error value
   2236  **/
   2237 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
   2238 {
   2239 	struct hlist_head tmp_add_list, tmp_del_list;
   2240 	struct i40e_mac_filter *f;
   2241 	struct i40e_new_mac_filter *new, *add_head = NULL;
   2242 	struct i40e_hw *hw = &vsi->back->hw;
   2243 	bool old_overflow, new_overflow;
   2244 	unsigned int failed_filters = 0;
   2245 	unsigned int vlan_filters = 0;
   2246 	char vsi_name[16] = "PF";
   2247 	int filter_list_len = 0;
   2248 	i40e_status aq_ret = 0;
   2249 	u32 changed_flags = 0;
   2250 	struct hlist_node *h;
   2251 	struct i40e_pf *pf;
   2252 	int num_add = 0;
   2253 	int num_del = 0;
   2254 	int retval = 0;
   2255 	u16 cmd_flags;
   2256 	int list_size;
   2257 	int bkt;
   2258 
   2259 	/* empty array typed pointers, kcalloc later */
   2260 	struct i40e_aqc_add_macvlan_element_data *add_list;
   2261 	struct i40e_aqc_remove_macvlan_element_data *del_list;
   2262 
   2263 	while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
   2264 		usleep_range(1000, 2000);
   2265 	pf = vsi->back;
   2266 
   2267 	old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
   2268 
   2269 	if (vsi->netdev) {
   2270 		changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
   2271 		vsi->current_netdev_flags = vsi->netdev->flags;
   2272 	}
   2273 
   2274 	INIT_HLIST_HEAD(&tmp_add_list);
   2275 	INIT_HLIST_HEAD(&tmp_del_list);
   2276 
   2277 	if (vsi->type == I40E_VSI_SRIOV)
   2278 		snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
   2279 	else if (vsi->type != I40E_VSI_MAIN)
   2280 		snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
   2281 
   2282 	if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
   2283 		vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
   2284 
   2285 		spin_lock_bh(&vsi->mac_filter_hash_lock);
   2286 		/* Create a list of filters to delete. */
   2287 		hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
   2288 			if (f->state == I40E_FILTER_REMOVE) {
   2289 				/* Move the element into temporary del_list */
   2290 				hash_del(&f->hlist);
   2291 				hlist_add_head(&f->hlist, &tmp_del_list);
   2292 
   2293 				/* Avoid counting removed filters */
   2294 				continue;
   2295 			}
   2296 			if (f->state == I40E_FILTER_NEW) {
   2297 				/* Create a temporary i40e_new_mac_filter */
   2298 				new = kzalloc(sizeof(*new), GFP_ATOMIC);
   2299 				if (!new)
   2300 					goto err_no_memory_locked;
   2301 
   2302 				/* Store pointer to the real filter */
   2303 				new->f = f;
   2304 				new->state = f->state;
   2305 
   2306 				/* Add it to the hash list */
   2307 				hlist_add_head(&new->hlist, &tmp_add_list);
   2308 			}
   2309 
   2310 			/* Count the number of active (current and new) VLAN
   2311 			 * filters we have now. Does not count filters which
   2312 			 * are marked for deletion.
   2313 			 */
   2314 			if (f->vlan > 0)
   2315 				vlan_filters++;
   2316 		}
   2317 
   2318 		retval = i40e_correct_mac_vlan_filters(vsi,
   2319 						       &tmp_add_list,
   2320 						       &tmp_del_list,
   2321 						       vlan_filters);
   2322 		if (retval)
   2323 			goto err_no_memory_locked;
   2324 
   2325 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
   2326 	}
   2327 
   2328 	/* Now process 'del_list' outside the lock */
   2329 	if (!hlist_empty(&tmp_del_list)) {
   2330 		filter_list_len = hw->aq.asq_buf_size /
   2331 			    sizeof(struct i40e_aqc_remove_macvlan_element_data);
   2332 		list_size = filter_list_len *
   2333 			    sizeof(struct i40e_aqc_remove_macvlan_element_data);
   2334 		del_list = kzalloc(list_size, GFP_ATOMIC);
   2335 		if (!del_list)
   2336 			goto err_no_memory;
   2337 
   2338 		hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
   2339 			cmd_flags = 0;
   2340 
   2341 			/* handle broadcast filters by updating the broadcast
   2342 			 * promiscuous flag and release filter list.
   2343 			 */
   2344 			if (is_broadcast_ether_addr(f->macaddr)) {
   2345 				i40e_aqc_broadcast_filter(vsi, vsi_name, f);
   2346 
   2347 				hlist_del(&f->hlist);
   2348 				kfree(f);
   2349 				continue;
   2350 			}
   2351 
   2352 			/* add to delete list */
   2353 			ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
   2354 			if (f->vlan == I40E_VLAN_ANY) {
   2355 				del_list[num_del].vlan_tag = 0;
   2356 				cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
   2357 			} else {
   2358 				del_list[num_del].vlan_tag =
   2359 					cpu_to_le16((u16)(f->vlan));
   2360 			}
   2361 
   2362 			cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
   2363 			del_list[num_del].flags = cmd_flags;
   2364 			num_del++;
   2365 
   2366 			/* flush a full buffer */
   2367 			if (num_del == filter_list_len) {
   2368 				i40e_aqc_del_filters(vsi, vsi_name, del_list,
   2369 						     num_del, &retval);
   2370 				memset(del_list, 0, list_size);
   2371 				num_del = 0;
   2372 			}
   2373 			/* Release memory for MAC filter entries which were
   2374 			 * synced up with HW.
   2375 			 */
   2376 			hlist_del(&f->hlist);
   2377 			kfree(f);
   2378 		}
   2379 
   2380 		if (num_del) {
   2381 			i40e_aqc_del_filters(vsi, vsi_name, del_list,
   2382 					     num_del, &retval);
   2383 		}
   2384 
   2385 		kfree(del_list);
   2386 		del_list = NULL;
   2387 	}
   2388 
   2389 	if (!hlist_empty(&tmp_add_list)) {
   2390 		/* Do all the adds now. */
   2391 		filter_list_len = hw->aq.asq_buf_size /
   2392 			       sizeof(struct i40e_aqc_add_macvlan_element_data);
   2393 		list_size = filter_list_len *
   2394 			       sizeof(struct i40e_aqc_add_macvlan_element_data);
   2395 		add_list = kzalloc(list_size, GFP_ATOMIC);
   2396 		if (!add_list)
   2397 			goto err_no_memory;
   2398 
   2399 		num_add = 0;
   2400 		hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
   2401 			/* handle broadcast filters by updating the broadcast
   2402 			 * promiscuous flag instead of adding a MAC filter.
   2403 			 */
   2404 			if (is_broadcast_ether_addr(new->f->macaddr)) {
   2405 				if (i40e_aqc_broadcast_filter(vsi, vsi_name,
   2406 							      new->f))
   2407 					new->state = I40E_FILTER_FAILED;
   2408 				else
   2409 					new->state = I40E_FILTER_ACTIVE;
   2410 				continue;
   2411 			}
   2412 
   2413 			/* add to add array */
   2414 			if (num_add == 0)
   2415 				add_head = new;
   2416 			cmd_flags = 0;
   2417 			ether_addr_copy(add_list[num_add].mac_addr,
   2418 					new->f->macaddr);
   2419 			if (new->f->vlan == I40E_VLAN_ANY) {
   2420 				add_list[num_add].vlan_tag = 0;
   2421 				cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
   2422 			} else {
   2423 				add_list[num_add].vlan_tag =
   2424 					cpu_to_le16((u16)(new->f->vlan));
   2425 			}
   2426 			add_list[num_add].queue_number = 0;
   2427 			/* set invalid match method for later detection */
   2428 			add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
   2429 			cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
   2430 			add_list[num_add].flags = cpu_to_le16(cmd_flags);
   2431 			num_add++;
   2432 
   2433 			/* flush a full buffer */
   2434 			if (num_add == filter_list_len) {
   2435 				i40e_aqc_add_filters(vsi, vsi_name, add_list,
   2436 						     add_head, num_add);
   2437 				memset(add_list, 0, list_size);
   2438 				num_add = 0;
   2439 			}
   2440 		}
   2441 		if (num_add) {
   2442 			i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
   2443 					     num_add);
   2444 		}
   2445 		/* Now move all of the filters from the temp add list back to
   2446 		 * the VSI's list.
   2447 		 */
   2448 		spin_lock_bh(&vsi->mac_filter_hash_lock);
   2449 		hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
   2450 			/* Only update the state if we're still NEW */
   2451 			if (new->f->state == I40E_FILTER_NEW)
   2452 				new->f->state = new->state;
   2453 			hlist_del(&new->hlist);
   2454 			kfree(new);
   2455 		}
   2456 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
   2457 		kfree(add_list);
   2458 		add_list = NULL;
   2459 	}
   2460 
   2461 	/* Determine the number of active and failed filters. */
   2462 	spin_lock_bh(&vsi->mac_filter_hash_lock);
   2463 	vsi->active_filters = 0;
   2464 	hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
   2465 		if (f->state == I40E_FILTER_ACTIVE)
   2466 			vsi->active_filters++;
   2467 		else if (f->state == I40E_FILTER_FAILED)
   2468 			failed_filters++;
   2469 	}
   2470 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
   2471 
   2472 	/* Check if we are able to exit overflow promiscuous mode. We can
   2473 	 * safely exit if we didn't just enter, we no longer have any failed
   2474 	 * filters, and we have reduced filters below the threshold value.
   2475 	 */
   2476 	if (old_overflow && !failed_filters &&
   2477 	    vsi->active_filters < vsi->promisc_threshold) {
   2478 		dev_info(&pf->pdev->dev,
   2479 			 "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
   2480 			 vsi_name);
   2481 		clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
   2482 		vsi->promisc_threshold = 0;
   2483 	}
   2484 
   2485 	/* if the VF is not trusted do not do promisc */
   2486 	if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
   2487 		clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
   2488 		goto out;
   2489 	}
   2490 
   2491 	new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
   2492 
   2493 	/* If we are entering overflow promiscuous, we need to calculate a new
   2494 	 * threshold for when we are safe to exit
   2495 	 */
   2496 	if (!old_overflow && new_overflow)
   2497 		vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
   2498 
   2499 	/* check for changes in promiscuous modes */
   2500 	if (changed_flags & IFF_ALLMULTI) {
   2501 		bool cur_multipromisc;
   2502 
   2503 		cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
   2504 		aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
   2505 							       vsi->seid,
   2506 							       cur_multipromisc,
   2507 							       NULL);
   2508 		if (aq_ret) {
   2509 			retval = i40e_aq_rc_to_posix(aq_ret,
   2510 						     hw->aq.asq_last_status);
   2511 			dev_info(&pf->pdev->dev,
   2512 				 "set multi promisc failed on %s, err %s aq_err %s\n",
   2513 				 vsi_name,
   2514 				 i40e_stat_str(hw, aq_ret),
   2515 				 i40e_aq_str(hw, hw->aq.asq_last_status));
   2516 		}
   2517 	}
   2518 
   2519 	if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
   2520 		bool cur_promisc;
   2521 
   2522 		cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
   2523 			       new_overflow);
   2524 		aq_ret = i40e_set_promiscuous(pf, cur_promisc);
   2525 		if (aq_ret) {
   2526 			retval = i40e_aq_rc_to_posix(aq_ret,
   2527 						     hw->aq.asq_last_status);
   2528 			dev_info(&pf->pdev->dev,
   2529 				 "Setting promiscuous %s failed on %s, err %s aq_err %s\n",
   2530 				 cur_promisc ? "on" : "off",
   2531 				 vsi_name,
   2532 				 i40e_stat_str(hw, aq_ret),
   2533 				 i40e_aq_str(hw, hw->aq.asq_last_status));
   2534 		}
   2535 	}
   2536 out:
   2537 	/* if something went wrong then set the changed flag so we try again */
   2538 	if (retval)
   2539 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
   2540 
   2541 	clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
   2542 	return retval;
   2543 
   2544 err_no_memory:
   2545 	/* Restore elements on the temporary add and delete lists */
   2546 	spin_lock_bh(&vsi->mac_filter_hash_lock);
   2547 err_no_memory_locked:
   2548 	i40e_undo_del_filter_entries(vsi, &tmp_del_list);
   2549 	i40e_undo_add_filter_entries(vsi, &tmp_add_list);
   2550 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
   2551 
   2552 	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
   2553 	clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
   2554 	return -ENOMEM;
   2555 }
   2556 
   2557 /**
   2558  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
   2559  * @pf: board private structure
   2560  **/
   2561 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
   2562 {
   2563 	int v;
   2564 
   2565 	if (!pf)
   2566 		return;
   2567 	if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
   2568 		return;
   2569 
   2570 	for (v = 0; v < pf->num_alloc_vsi; v++) {
   2571 		if (pf->vsi[v] &&
   2572 		    (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
   2573 			int ret = i40e_sync_vsi_filters(pf->vsi[v]);
   2574 
   2575 			if (ret) {
   2576 				/* come back and try again later */
   2577 				set_bit(__I40E_MACVLAN_SYNC_PENDING,
   2578 					pf->state);
   2579 				break;
   2580 			}
   2581 		}
   2582 	}
   2583 }
   2584 
   2585 /**
   2586  * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
   2587  * @vsi: the vsi
   2588  **/
   2589 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
   2590 {
   2591 	if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
   2592 		return I40E_RXBUFFER_2048;
   2593 	else
   2594 		return I40E_RXBUFFER_3072;
   2595 }
   2596 
   2597 /**
   2598  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
   2599  * @netdev: network interface device structure
   2600  * @new_mtu: new value for maximum frame size
   2601  *
   2602  * Returns 0 on success, negative on failure
   2603  **/
   2604 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
   2605 {
   2606 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   2607 	struct i40e_vsi *vsi = np->vsi;
   2608 	struct i40e_pf *pf = vsi->back;
   2609 
   2610 	if (i40e_enabled_xdp_vsi(vsi)) {
   2611 		int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
   2612 
   2613 		if (frame_size > i40e_max_xdp_frame_size(vsi))
   2614 			return -EINVAL;
   2615 	}
   2616 
   2617 	netdev_info(netdev, "changing MTU from %d to %d\n",
   2618 		    netdev->mtu, new_mtu);
   2619 	netdev->mtu = new_mtu;
   2620 	if (netif_running(netdev))
   2621 		i40e_vsi_reinit_locked(vsi);
   2622 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
   2623 	set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
   2624 	return 0;
   2625 }
   2626 
   2627 /**
   2628  * i40e_ioctl - Access the hwtstamp interface
   2629  * @netdev: network interface device structure
   2630  * @ifr: interface request data
   2631  * @cmd: ioctl command
   2632  **/
   2633 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
   2634 {
   2635 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   2636 	struct i40e_pf *pf = np->vsi->back;
   2637 
   2638 	switch (cmd) {
   2639 	case SIOCGHWTSTAMP:
   2640 		return i40e_ptp_get_ts_config(pf, ifr);
   2641 	case SIOCSHWTSTAMP:
   2642 		return i40e_ptp_set_ts_config(pf, ifr);
   2643 	default:
   2644 		return -EOPNOTSUPP;
   2645 	}
   2646 }
   2647 
   2648 /**
   2649  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
   2650  * @vsi: the vsi being adjusted
   2651  **/
   2652 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
   2653 {
   2654 	struct i40e_vsi_context ctxt;
   2655 	i40e_status ret;
   2656 
   2657 	if ((vsi->info.valid_sections &
   2658 	     cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
   2659 	    ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
   2660 		return;  /* already enabled */
   2661 
   2662 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
   2663 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
   2664 				    I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
   2665 
   2666 	ctxt.seid = vsi->seid;
   2667 	ctxt.info = vsi->info;
   2668 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
   2669 	if (ret) {
   2670 		dev_info(&vsi->back->pdev->dev,
   2671 			 "update vlan stripping failed, err %s aq_err %s\n",
   2672 			 i40e_stat_str(&vsi->back->hw, ret),
   2673 			 i40e_aq_str(&vsi->back->hw,
   2674 				     vsi->back->hw.aq.asq_last_status));
   2675 	}
   2676 }
   2677 
   2678 /**
   2679  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
   2680  * @vsi: the vsi being adjusted
   2681  **/
   2682 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
   2683 {
   2684 	struct i40e_vsi_context ctxt;
   2685 	i40e_status ret;
   2686 
   2687 	if ((vsi->info.valid_sections &
   2688 	     cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
   2689 	    ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
   2690 	     I40E_AQ_VSI_PVLAN_EMOD_MASK))
   2691 		return;  /* already disabled */
   2692 
   2693 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
   2694 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
   2695 				    I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
   2696 
   2697 	ctxt.seid = vsi->seid;
   2698 	ctxt.info = vsi->info;
   2699 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
   2700 	if (ret) {
   2701 		dev_info(&vsi->back->pdev->dev,
   2702 			 "update vlan stripping failed, err %s aq_err %s\n",
   2703 			 i40e_stat_str(&vsi->back->hw, ret),
   2704 			 i40e_aq_str(&vsi->back->hw,
   2705 				     vsi->back->hw.aq.asq_last_status));
   2706 	}
   2707 }
   2708 
   2709 /**
   2710  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
   2711  * @vsi: the vsi being configured
   2712  * @vid: vlan id to be added (0 = untagged only , -1 = any)
   2713  *
   2714  * This is a helper function for adding a new MAC/VLAN filter with the
   2715  * specified VLAN for each existing MAC address already in the hash table.
   2716  * This function does *not* perform any accounting to update filters based on
   2717  * VLAN mode.
   2718  *
   2719  * NOTE: this function expects to be called while under the
   2720  * mac_filter_hash_lock
   2721  **/
   2722 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
   2723 {
   2724 	struct i40e_mac_filter *f, *add_f;
   2725 	struct hlist_node *h;
   2726 	int bkt;
   2727 
   2728 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
   2729 		if (f->state == I40E_FILTER_REMOVE)
   2730 			continue;
   2731 		add_f = i40e_add_filter(vsi, f->macaddr, vid);
   2732 		if (!add_f) {
   2733 			dev_info(&vsi->back->pdev->dev,
   2734 				 "Could not add vlan filter %d for %pM\n",
   2735 				 vid, f->macaddr);
   2736 			return -ENOMEM;
   2737 		}
   2738 	}
   2739 
   2740 	return 0;
   2741 }
   2742 
   2743 /**
   2744  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
   2745  * @vsi: the VSI being configured
   2746  * @vid: VLAN id to be added
   2747  **/
   2748 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
   2749 {
   2750 	int err;
   2751 
   2752 	if (vsi->info.pvid)
   2753 		return -EINVAL;
   2754 
   2755 	/* The network stack will attempt to add VID=0, with the intention to
   2756 	 * receive priority tagged packets with a VLAN of 0. Our HW receives
   2757 	 * these packets by default when configured to receive untagged
   2758 	 * packets, so we don't need to add a filter for this case.
   2759 	 * Additionally, HW interprets adding a VID=0 filter as meaning to
   2760 	 * receive *only* tagged traffic and stops receiving untagged traffic.
   2761 	 * Thus, we do not want to actually add a filter for VID=0
   2762 	 */
   2763 	if (!vid)
   2764 		return 0;
   2765 
   2766 	/* Locked once because all functions invoked below iterates list*/
   2767 	spin_lock_bh(&vsi->mac_filter_hash_lock);
   2768 	err = i40e_add_vlan_all_mac(vsi, vid);
   2769 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
   2770 	if (err)
   2771 		return err;
   2772 
   2773 	/* schedule our worker thread which will take care of
   2774 	 * applying the new filter changes
   2775 	 */
   2776 	i40e_service_event_schedule(vsi->back);
   2777 	return 0;
   2778 }
   2779 
   2780 /**
   2781  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
   2782  * @vsi: the vsi being configured
   2783  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
   2784  *
   2785  * This function should be used to remove all VLAN filters which match the
   2786  * given VID. It does not schedule the service event and does not take the
   2787  * mac_filter_hash_lock so it may be combined with other operations under
   2788  * a single invocation of the mac_filter_hash_lock.
   2789  *
   2790  * NOTE: this function expects to be called while under the
   2791  * mac_filter_hash_lock
   2792  */
   2793 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
   2794 {
   2795 	struct i40e_mac_filter *f;
   2796 	struct hlist_node *h;
   2797 	int bkt;
   2798 
   2799 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
   2800 		if (f->vlan == vid)
   2801 			__i40e_del_filter(vsi, f);
   2802 	}
   2803 }
   2804 
   2805 /**
   2806  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
   2807  * @vsi: the VSI being configured
   2808  * @vid: VLAN id to be removed
   2809  **/
   2810 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
   2811 {
   2812 	if (!vid || vsi->info.pvid)
   2813 		return;
   2814 
   2815 	spin_lock_bh(&vsi->mac_filter_hash_lock);
   2816 	i40e_rm_vlan_all_mac(vsi, vid);
   2817 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
   2818 
   2819 	/* schedule our worker thread which will take care of
   2820 	 * applying the new filter changes
   2821 	 */
   2822 	i40e_service_event_schedule(vsi->back);
   2823 }
   2824 
   2825 /**
   2826  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
   2827  * @netdev: network interface to be adjusted
   2828  * @proto: unused protocol value
   2829  * @vid: vlan id to be added
   2830  *
   2831  * net_device_ops implementation for adding vlan ids
   2832  **/
   2833 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
   2834 				__always_unused __be16 proto, u16 vid)
   2835 {
   2836 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   2837 	struct i40e_vsi *vsi = np->vsi;
   2838 	int ret = 0;
   2839 
   2840 	if (vid >= VLAN_N_VID)
   2841 		return -EINVAL;
   2842 
   2843 	ret = i40e_vsi_add_vlan(vsi, vid);
   2844 	if (!ret)
   2845 		set_bit(vid, vsi->active_vlans);
   2846 
   2847 	return ret;
   2848 }
   2849 
   2850 /**
   2851  * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
   2852  * @netdev: network interface to be adjusted
   2853  * @proto: unused protocol value
   2854  * @vid: vlan id to be added
   2855  **/
   2856 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
   2857 				    __always_unused __be16 proto, u16 vid)
   2858 {
   2859 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   2860 	struct i40e_vsi *vsi = np->vsi;
   2861 
   2862 	if (vid >= VLAN_N_VID)
   2863 		return;
   2864 	set_bit(vid, vsi->active_vlans);
   2865 }
   2866 
   2867 /**
   2868  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
   2869  * @netdev: network interface to be adjusted
   2870  * @proto: unused protocol value
   2871  * @vid: vlan id to be removed
   2872  *
   2873  * net_device_ops implementation for removing vlan ids
   2874  **/
   2875 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
   2876 				 __always_unused __be16 proto, u16 vid)
   2877 {
   2878 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   2879 	struct i40e_vsi *vsi = np->vsi;
   2880 
   2881 	/* return code is ignored as there is nothing a user
   2882 	 * can do about failure to remove and a log message was
   2883 	 * already printed from the other function
   2884 	 */
   2885 	i40e_vsi_kill_vlan(vsi, vid);
   2886 
   2887 	clear_bit(vid, vsi->active_vlans);
   2888 
   2889 	return 0;
   2890 }
   2891 
   2892 /**
   2893  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
   2894  * @vsi: the vsi being brought back up
   2895  **/
   2896 static void i40e_restore_vlan(struct i40e_vsi *vsi)
   2897 {
   2898 	u16 vid;
   2899 
   2900 	if (!vsi->netdev)
   2901 		return;
   2902 
   2903 	if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
   2904 		i40e_vlan_stripping_enable(vsi);
   2905 	else
   2906 		i40e_vlan_stripping_disable(vsi);
   2907 
   2908 	for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
   2909 		i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
   2910 					vid);
   2911 }
   2912 
   2913 /**
   2914  * i40e_vsi_add_pvid - Add pvid for the VSI
   2915  * @vsi: the vsi being adjusted
   2916  * @vid: the vlan id to set as a PVID
   2917  **/
   2918 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
   2919 {
   2920 	struct i40e_vsi_context ctxt;
   2921 	i40e_status ret;
   2922 
   2923 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
   2924 	vsi->info.pvid = cpu_to_le16(vid);
   2925 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
   2926 				    I40E_AQ_VSI_PVLAN_INSERT_PVID |
   2927 				    I40E_AQ_VSI_PVLAN_EMOD_STR;
   2928 
   2929 	ctxt.seid = vsi->seid;
   2930 	ctxt.info = vsi->info;
   2931 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
   2932 	if (ret) {
   2933 		dev_info(&vsi->back->pdev->dev,
   2934 			 "add pvid failed, err %s aq_err %s\n",
   2935 			 i40e_stat_str(&vsi->back->hw, ret),
   2936 			 i40e_aq_str(&vsi->back->hw,
   2937 				     vsi->back->hw.aq.asq_last_status));
   2938 		return -ENOENT;
   2939 	}
   2940 
   2941 	return 0;
   2942 }
   2943 
   2944 /**
   2945  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
   2946  * @vsi: the vsi being adjusted
   2947  *
   2948  * Just use the vlan_rx_register() service to put it back to normal
   2949  **/
   2950 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
   2951 {
   2952 	i40e_vlan_stripping_disable(vsi);
   2953 
   2954 	vsi->info.pvid = 0;
   2955 }
   2956 
   2957 /**
   2958  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
   2959  * @vsi: ptr to the VSI
   2960  *
   2961  * If this function returns with an error, then it's possible one or
   2962  * more of the rings is populated (while the rest are not).  It is the
   2963  * callers duty to clean those orphaned rings.
   2964  *
   2965  * Return 0 on success, negative on failure
   2966  **/
   2967 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
   2968 {
   2969 	int i, err = 0;
   2970 
   2971 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
   2972 		err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
   2973 
   2974 	if (!i40e_enabled_xdp_vsi(vsi))
   2975 		return err;
   2976 
   2977 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
   2978 		err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
   2979 
   2980 	return err;
   2981 }
   2982 
   2983 /**
   2984  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
   2985  * @vsi: ptr to the VSI
   2986  *
   2987  * Free VSI's transmit software resources
   2988  **/
   2989 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
   2990 {
   2991 	int i;
   2992 
   2993 	if (vsi->tx_rings) {
   2994 		for (i = 0; i < vsi->num_queue_pairs; i++)
   2995 			if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
   2996 				i40e_free_tx_resources(vsi->tx_rings[i]);
   2997 	}
   2998 
   2999 	if (vsi->xdp_rings) {
   3000 		for (i = 0; i < vsi->num_queue_pairs; i++)
   3001 			if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
   3002 				i40e_free_tx_resources(vsi->xdp_rings[i]);
   3003 	}
   3004 }
   3005 
   3006 /**
   3007  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
   3008  * @vsi: ptr to the VSI
   3009  *
   3010  * If this function returns with an error, then it's possible one or
   3011  * more of the rings is populated (while the rest are not).  It is the
   3012  * callers duty to clean those orphaned rings.
   3013  *
   3014  * Return 0 on success, negative on failure
   3015  **/
   3016 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
   3017 {
   3018 	int i, err = 0;
   3019 
   3020 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
   3021 		err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
   3022 	return err;
   3023 }
   3024 
   3025 /**
   3026  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
   3027  * @vsi: ptr to the VSI
   3028  *
   3029  * Free all receive software resources
   3030  **/
   3031 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
   3032 {
   3033 	int i;
   3034 
   3035 	if (!vsi->rx_rings)
   3036 		return;
   3037 
   3038 	for (i = 0; i < vsi->num_queue_pairs; i++)
   3039 		if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
   3040 			i40e_free_rx_resources(vsi->rx_rings[i]);
   3041 }
   3042 
   3043 /**
   3044  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
   3045  * @ring: The Tx ring to configure
   3046  *
   3047  * This enables/disables XPS for a given Tx descriptor ring
   3048  * based on the TCs enabled for the VSI that ring belongs to.
   3049  **/
   3050 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
   3051 {
   3052 	int cpu;
   3053 
   3054 	if (!ring->q_vector || !ring->netdev || ring->ch)
   3055 		return;
   3056 
   3057 	/* We only initialize XPS once, so as not to overwrite user settings */
   3058 	if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
   3059 		return;
   3060 
   3061 	cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
   3062 	netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
   3063 			    ring->queue_index);
   3064 }
   3065 
   3066 /**
   3067  * i40e_xsk_umem - Retrieve the AF_XDP ZC if XDP and ZC is enabled
   3068  * @ring: The Tx or Rx ring
   3069  *
   3070  * Returns the UMEM or NULL.
   3071  **/
   3072 static struct xdp_umem *i40e_xsk_umem(struct i40e_ring *ring)
   3073 {
   3074 	bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
   3075 	int qid = ring->queue_index;
   3076 
   3077 	if (ring_is_xdp(ring))
   3078 		qid -= ring->vsi->alloc_queue_pairs;
   3079 
   3080 	if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
   3081 		return NULL;
   3082 
   3083 	return xdp_get_umem_from_qid(ring->vsi->netdev, qid);
   3084 }
   3085 
   3086 /**
   3087  * i40e_configure_tx_ring - Configure a transmit ring context and rest
   3088  * @ring: The Tx ring to configure
   3089  *
   3090  * Configure the Tx descriptor ring in the HMC context.
   3091  **/
   3092 static int i40e_configure_tx_ring(struct i40e_ring *ring)
   3093 {
   3094 	struct i40e_vsi *vsi = ring->vsi;
   3095 	u16 pf_q = vsi->base_queue + ring->queue_index;
   3096 	struct i40e_hw *hw = &vsi->back->hw;
   3097 	struct i40e_hmc_obj_txq tx_ctx;
   3098 	i40e_status err = 0;
   3099 	u32 qtx_ctl = 0;
   3100 
   3101 	if (ring_is_xdp(ring))
   3102 		ring->xsk_umem = i40e_xsk_umem(ring);
   3103 
   3104 	/* some ATR related tx ring init */
   3105 	if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
   3106 		ring->atr_sample_rate = vsi->back->atr_sample_rate;
   3107 		ring->atr_count = 0;
   3108 	} else {
   3109 		ring->atr_sample_rate = 0;
   3110 	}
   3111 
   3112 	/* configure XPS */
   3113 	i40e_config_xps_tx_ring(ring);
   3114 
   3115 	/* clear the context structure first */
   3116 	memset(&tx_ctx, 0, sizeof(tx_ctx));
   3117 
   3118 	tx_ctx.new_context = 1;
   3119 	tx_ctx.base = (ring->dma / 128);
   3120 	tx_ctx.qlen = ring->count;
   3121 	tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
   3122 					       I40E_FLAG_FD_ATR_ENABLED));
   3123 	tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
   3124 	/* FDIR VSI tx ring can still use RS bit and writebacks */
   3125 	if (vsi->type != I40E_VSI_FDIR)
   3126 		tx_ctx.head_wb_ena = 1;
   3127 	tx_ctx.head_wb_addr = ring->dma +
   3128 			      (ring->count * sizeof(struct i40e_tx_desc));
   3129 
   3130 	/* As part of VSI creation/update, FW allocates certain
   3131 	 * Tx arbitration queue sets for each TC enabled for
   3132 	 * the VSI. The FW returns the handles to these queue
   3133 	 * sets as part of the response buffer to Add VSI,
   3134 	 * Update VSI, etc. AQ commands. It is expected that
   3135 	 * these queue set handles be associated with the Tx
   3136 	 * queues by the driver as part of the TX queue context
   3137 	 * initialization. This has to be done regardless of
   3138 	 * DCB as by default everything is mapped to TC0.
   3139 	 */
   3140 
   3141 	if (ring->ch)
   3142 		tx_ctx.rdylist =
   3143 			le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
   3144 
   3145 	else
   3146 		tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
   3147 
   3148 	tx_ctx.rdylist_act = 0;
   3149 
   3150 	/* clear the context in the HMC */
   3151 	err = i40e_clear_lan_tx_queue_context(hw, pf_q);
   3152 	if (err) {
   3153 		dev_info(&vsi->back->pdev->dev,
   3154 			 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
   3155 			 ring->queue_index, pf_q, err);
   3156 		return -ENOMEM;
   3157 	}
   3158 
   3159 	/* set the context in the HMC */
   3160 	err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
   3161 	if (err) {
   3162 		dev_info(&vsi->back->pdev->dev,
   3163 			 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
   3164 			 ring->queue_index, pf_q, err);
   3165 		return -ENOMEM;
   3166 	}
   3167 
   3168 	/* Now associate this queue with this PCI function */
   3169 	if (ring->ch) {
   3170 		if (ring->ch->type == I40E_VSI_VMDQ2)
   3171 			qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
   3172 		else
   3173 			return -EINVAL;
   3174 
   3175 		qtx_ctl |= (ring->ch->vsi_number <<
   3176 			    I40E_QTX_CTL_VFVM_INDX_SHIFT) &
   3177 			    I40E_QTX_CTL_VFVM_INDX_MASK;
   3178 	} else {
   3179 		if (vsi->type == I40E_VSI_VMDQ2) {
   3180 			qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
   3181 			qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
   3182 				    I40E_QTX_CTL_VFVM_INDX_MASK;
   3183 		} else {
   3184 			qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
   3185 		}
   3186 	}
   3187 
   3188 	qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
   3189 		    I40E_QTX_CTL_PF_INDX_MASK);
   3190 	wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
   3191 	i40e_flush(hw);
   3192 
   3193 	/* cache tail off for easier writes later */
   3194 	ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
   3195 
   3196 	return 0;
   3197 }
   3198 
   3199 /**
   3200  * i40e_configure_rx_ring - Configure a receive ring context
   3201  * @ring: The Rx ring to configure
   3202  *
   3203  * Configure the Rx descriptor ring in the HMC context.
   3204  **/
   3205 static int i40e_configure_rx_ring(struct i40e_ring *ring)
   3206 {
   3207 	struct i40e_vsi *vsi = ring->vsi;
   3208 	u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
   3209 	u16 pf_q = vsi->base_queue + ring->queue_index;
   3210 	struct i40e_hw *hw = &vsi->back->hw;
   3211 	struct i40e_hmc_obj_rxq rx_ctx;
   3212 	i40e_status err = 0;
   3213 	bool ok;
   3214 	int ret;
   3215 
   3216 	bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
   3217 
   3218 	/* clear the context structure first */
   3219 	memset(&rx_ctx, 0, sizeof(rx_ctx));
   3220 
   3221 	if (ring->vsi->type == I40E_VSI_MAIN)
   3222 		xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
   3223 
   3224 	ring->xsk_umem = i40e_xsk_umem(ring);
   3225 	if (ring->xsk_umem) {
   3226 		ring->rx_buf_len = ring->xsk_umem->chunk_size_nohr -
   3227 				   XDP_PACKET_HEADROOM;
   3228 		/* For AF_XDP ZC, we disallow packets to span on
   3229 		 * multiple buffers, thus letting us skip that
   3230 		 * handling in the fast-path.
   3231 		 */
   3232 		chain_len = 1;
   3233 		ring->zca.free = i40e_zca_free;
   3234 		ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
   3235 						 MEM_TYPE_ZERO_COPY,
   3236 						 &ring->zca);
   3237 		if (ret)
   3238 			return ret;
   3239 		dev_info(&vsi->back->pdev->dev,
   3240 			 "Registered XDP mem model MEM_TYPE_ZERO_COPY on Rx ring %d\n",
   3241 			 ring->queue_index);
   3242 
   3243 	} else {
   3244 		ring->rx_buf_len = vsi->rx_buf_len;
   3245 		if (ring->vsi->type == I40E_VSI_MAIN) {
   3246 			ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
   3247 							 MEM_TYPE_PAGE_SHARED,
   3248 							 NULL);
   3249 			if (ret)
   3250 				return ret;
   3251 		}
   3252 	}
   3253 
   3254 	rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
   3255 				    BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
   3256 
   3257 	rx_ctx.base = (ring->dma / 128);
   3258 	rx_ctx.qlen = ring->count;
   3259 
   3260 	/* use 32 byte descriptors */
   3261 	rx_ctx.dsize = 1;
   3262 
   3263 	/* descriptor type is always zero
   3264 	 * rx_ctx.dtype = 0;
   3265 	 */
   3266 	rx_ctx.hsplit_0 = 0;
   3267 
   3268 	rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
   3269 	if (hw->revision_id == 0)
   3270 		rx_ctx.lrxqthresh = 0;
   3271 	else
   3272 		rx_ctx.lrxqthresh = 1;
   3273 	rx_ctx.crcstrip = 1;
   3274 	rx_ctx.l2tsel = 1;
   3275 	/* this controls whether VLAN is stripped from inner headers */
   3276 	rx_ctx.showiv = 0;
   3277 	/* set the prefena field to 1 because the manual says to */
   3278 	rx_ctx.prefena = 1;
   3279 
   3280 	/* clear the context in the HMC */
   3281 	err = i40e_clear_lan_rx_queue_context(hw, pf_q);
   3282 	if (err) {
   3283 		dev_info(&vsi->back->pdev->dev,
   3284 			 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
   3285 			 ring->queue_index, pf_q, err);
   3286 		return -ENOMEM;
   3287 	}
   3288 
   3289 	/* set the context in the HMC */
   3290 	err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
   3291 	if (err) {
   3292 		dev_info(&vsi->back->pdev->dev,
   3293 			 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
   3294 			 ring->queue_index, pf_q, err);
   3295 		return -ENOMEM;
   3296 	}
   3297 
   3298 	/* configure Rx buffer alignment */
   3299 	if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
   3300 		clear_ring_build_skb_enabled(ring);
   3301 	else
   3302 		set_ring_build_skb_enabled(ring);
   3303 
   3304 	/* cache tail for quicker writes, and clear the reg before use */
   3305 	ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
   3306 	writel(0, ring->tail);
   3307 
   3308 	ok = ring->xsk_umem ?
   3309 	     i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)) :
   3310 	     !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
   3311 	if (!ok) {
   3312 		/* Log this in case the user has forgotten to give the kernel
   3313 		 * any buffers, even later in the application.
   3314 		 */
   3315 		dev_info(&vsi->back->pdev->dev,
   3316 			 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
   3317 			 ring->xsk_umem ? "UMEM enabled " : "",
   3318 			 ring->queue_index, pf_q);
   3319 	}
   3320 
   3321 	return 0;
   3322 }
   3323 
   3324 /**
   3325  * i40e_vsi_configure_tx - Configure the VSI for Tx
   3326  * @vsi: VSI structure describing this set of rings and resources
   3327  *
   3328  * Configure the Tx VSI for operation.
   3329  **/
   3330 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
   3331 {
   3332 	int err = 0;
   3333 	u16 i;
   3334 
   3335 	for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
   3336 		err = i40e_configure_tx_ring(vsi->tx_rings[i]);
   3337 
   3338 	if (!i40e_enabled_xdp_vsi(vsi))
   3339 		return err;
   3340 
   3341 	for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
   3342 		err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
   3343 
   3344 	return err;
   3345 }
   3346 
   3347 /**
   3348  * i40e_vsi_configure_rx - Configure the VSI for Rx
   3349  * @vsi: the VSI being configured
   3350  *
   3351  * Configure the Rx VSI for operation.
   3352  **/
   3353 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
   3354 {
   3355 	int err = 0;
   3356 	u16 i;
   3357 
   3358 	if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
   3359 		vsi->max_frame = I40E_MAX_RXBUFFER;
   3360 		vsi->rx_buf_len = I40E_RXBUFFER_2048;
   3361 #if (PAGE_SIZE < 8192)
   3362 	} else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
   3363 		   (vsi->netdev->mtu <= ETH_DATA_LEN)) {
   3364 		vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
   3365 		vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
   3366 #endif
   3367 	} else {
   3368 		vsi->max_frame = I40E_MAX_RXBUFFER;
   3369 		vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
   3370 						       I40E_RXBUFFER_2048;
   3371 	}
   3372 
   3373 	/* set up individual rings */
   3374 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
   3375 		err = i40e_configure_rx_ring(vsi->rx_rings[i]);
   3376 
   3377 	return err;
   3378 }
   3379 
   3380 /**
   3381  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
   3382  * @vsi: ptr to the VSI
   3383  **/
   3384 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
   3385 {
   3386 	struct i40e_ring *tx_ring, *rx_ring;
   3387 	u16 qoffset, qcount;
   3388 	int i, n;
   3389 
   3390 	if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
   3391 		/* Reset the TC information */
   3392 		for (i = 0; i < vsi->num_queue_pairs; i++) {
   3393 			rx_ring = vsi->rx_rings[i];
   3394 			tx_ring = vsi->tx_rings[i];
   3395 			rx_ring->dcb_tc = 0;
   3396 			tx_ring->dcb_tc = 0;
   3397 		}
   3398 		return;
   3399 	}
   3400 
   3401 	for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
   3402 		if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
   3403 			continue;
   3404 
   3405 		qoffset = vsi->tc_config.tc_info[n].qoffset;
   3406 		qcount = vsi->tc_config.tc_info[n].qcount;
   3407 		for (i = qoffset; i < (qoffset + qcount); i++) {
   3408 			rx_ring = vsi->rx_rings[i];
   3409 			tx_ring = vsi->tx_rings[i];
   3410 			rx_ring->dcb_tc = n;
   3411 			tx_ring->dcb_tc = n;
   3412 		}
   3413 	}
   3414 }
   3415 
   3416 /**
   3417  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
   3418  * @vsi: ptr to the VSI
   3419  **/
   3420 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
   3421 {
   3422 	if (vsi->netdev)
   3423 		i40e_set_rx_mode(vsi->netdev);
   3424 }
   3425 
   3426 /**
   3427  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
   3428  * @vsi: Pointer to the targeted VSI
   3429  *
   3430  * This function replays the hlist on the hw where all the SB Flow Director
   3431  * filters were saved.
   3432  **/
   3433 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
   3434 {
   3435 	struct i40e_fdir_filter *filter;
   3436 	struct i40e_pf *pf = vsi->back;
   3437 	struct hlist_node *node;
   3438 
   3439 	if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
   3440 		return;
   3441 
   3442 	/* Reset FDir counters as we're replaying all existing filters */
   3443 	pf->fd_tcp4_filter_cnt = 0;
   3444 	pf->fd_udp4_filter_cnt = 0;
   3445 	pf->fd_sctp4_filter_cnt = 0;
   3446 	pf->fd_ip4_filter_cnt = 0;
   3447 
   3448 	hlist_for_each_entry_safe(filter, node,
   3449 				  &pf->fdir_filter_list, fdir_node) {
   3450 		i40e_add_del_fdir(vsi, filter, true);
   3451 	}
   3452 }
   3453 
   3454 /**
   3455  * i40e_vsi_configure - Set up the VSI for action
   3456  * @vsi: the VSI being configured
   3457  **/
   3458 static int i40e_vsi_configure(struct i40e_vsi *vsi)
   3459 {
   3460 	int err;
   3461 
   3462 	i40e_set_vsi_rx_mode(vsi);
   3463 	i40e_restore_vlan(vsi);
   3464 	i40e_vsi_config_dcb_rings(vsi);
   3465 	err = i40e_vsi_configure_tx(vsi);
   3466 	if (!err)
   3467 		err = i40e_vsi_configure_rx(vsi);
   3468 
   3469 	return err;
   3470 }
   3471 
   3472 /**
   3473  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
   3474  * @vsi: the VSI being configured
   3475  **/
   3476 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
   3477 {
   3478 	bool has_xdp = i40e_enabled_xdp_vsi(vsi);
   3479 	struct i40e_pf *pf = vsi->back;
   3480 	struct i40e_hw *hw = &pf->hw;
   3481 	u16 vector;
   3482 	int i, q;
   3483 	u32 qp;
   3484 
   3485 	/* The interrupt indexing is offset by 1 in the PFINT_ITRn
   3486 	 * and PFINT_LNKLSTn registers, e.g.:
   3487 	 *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
   3488 	 */
   3489 	qp = vsi->base_queue;
   3490 	vector = vsi->base_vector;
   3491 	for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
   3492 		struct i40e_q_vector *q_vector = vsi->q_vectors[i];
   3493 
   3494 		q_vector->rx.next_update = jiffies + 1;
   3495 		q_vector->rx.target_itr =
   3496 			ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
   3497 		wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
   3498 		     q_vector->rx.target_itr);
   3499 		q_vector->rx.current_itr = q_vector->rx.target_itr;
   3500 
   3501 		q_vector->tx.next_update = jiffies + 1;
   3502 		q_vector->tx.target_itr =
   3503 			ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
   3504 		wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
   3505 		     q_vector->tx.target_itr);
   3506 		q_vector->tx.current_itr = q_vector->tx.target_itr;
   3507 
   3508 		wr32(hw, I40E_PFINT_RATEN(vector - 1),
   3509 		     i40e_intrl_usec_to_reg(vsi->int_rate_limit));
   3510 
   3511 		/* Linked list for the queuepairs assigned to this vector */
   3512 		wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
   3513 		for (q = 0; q < q_vector->num_ringpairs; q++) {
   3514 			u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
   3515 			u32 val;
   3516 
   3517 			val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
   3518 			      (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
   3519 			      (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
   3520 			      (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
   3521 			      (I40E_QUEUE_TYPE_TX <<
   3522 			       I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
   3523 
   3524 			wr32(hw, I40E_QINT_RQCTL(qp), val);
   3525 
   3526 			if (has_xdp) {
   3527 				val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
   3528 				      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
   3529 				      (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
   3530 				      (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
   3531 				      (I40E_QUEUE_TYPE_TX <<
   3532 				       I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
   3533 
   3534 				wr32(hw, I40E_QINT_TQCTL(nextqp), val);
   3535 			}
   3536 
   3537 			val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
   3538 			      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
   3539 			      (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
   3540 			      ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
   3541 			      (I40E_QUEUE_TYPE_RX <<
   3542 			       I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
   3543 
   3544 			/* Terminate the linked list */
   3545 			if (q == (q_vector->num_ringpairs - 1))
   3546 				val |= (I40E_QUEUE_END_OF_LIST <<
   3547 					I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
   3548 
   3549 			wr32(hw, I40E_QINT_TQCTL(qp), val);
   3550 			qp++;
   3551 		}
   3552 	}
   3553 
   3554 	i40e_flush(hw);
   3555 }
   3556 
   3557 /**
   3558  * i40e_enable_misc_int_causes - enable the non-queue interrupts
   3559  * @pf: pointer to private device data structure
   3560  **/
   3561 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
   3562 {
   3563 	struct i40e_hw *hw = &pf->hw;
   3564 	u32 val;
   3565 
   3566 	/* clear things first */
   3567 	wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
   3568 	rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
   3569 
   3570 	val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
   3571 	      I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
   3572 	      I40E_PFINT_ICR0_ENA_GRST_MASK          |
   3573 	      I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
   3574 	      I40E_PFINT_ICR0_ENA_GPIO_MASK          |
   3575 	      I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
   3576 	      I40E_PFINT_ICR0_ENA_VFLR_MASK          |
   3577 	      I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
   3578 
   3579 	if (pf->flags & I40E_FLAG_IWARP_ENABLED)
   3580 		val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
   3581 
   3582 	if (pf->flags & I40E_FLAG_PTP)
   3583 		val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
   3584 
   3585 	wr32(hw, I40E_PFINT_ICR0_ENA, val);
   3586 
   3587 	/* SW_ITR_IDX = 0, but don't change INTENA */
   3588 	wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
   3589 					I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
   3590 
   3591 	/* OTHER_ITR_IDX = 0 */
   3592 	wr32(hw, I40E_PFINT_STAT_CTL0, 0);
   3593 }
   3594 
   3595 /**
   3596  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
   3597  * @vsi: the VSI being configured
   3598  **/
   3599 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
   3600 {
   3601 	u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
   3602 	struct i40e_q_vector *q_vector = vsi->q_vectors[0];
   3603 	struct i40e_pf *pf = vsi->back;
   3604 	struct i40e_hw *hw = &pf->hw;
   3605 	u32 val;
   3606 
   3607 	/* set the ITR configuration */
   3608 	q_vector->rx.next_update = jiffies + 1;
   3609 	q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
   3610 	wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr);
   3611 	q_vector->rx.current_itr = q_vector->rx.target_itr;
   3612 	q_vector->tx.next_update = jiffies + 1;
   3613 	q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
   3614 	wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr);
   3615 	q_vector->tx.current_itr = q_vector->tx.target_itr;
   3616 
   3617 	i40e_enable_misc_int_causes(pf);
   3618 
   3619 	/* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
   3620 	wr32(hw, I40E_PFINT_LNKLST0, 0);
   3621 
   3622 	/* Associate the queue pair to the vector and enable the queue int */
   3623 	val = I40E_QINT_RQCTL_CAUSE_ENA_MASK		       |
   3624 	      (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
   3625 	      (nextqp	   << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
   3626 	      (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
   3627 
   3628 	wr32(hw, I40E_QINT_RQCTL(0), val);
   3629 
   3630 	if (i40e_enabled_xdp_vsi(vsi)) {
   3631 		val = I40E_QINT_TQCTL_CAUSE_ENA_MASK		     |
   3632 		      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
   3633 		      (I40E_QUEUE_TYPE_TX
   3634 		       << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
   3635 
   3636 		wr32(hw, I40E_QINT_TQCTL(nextqp), val);
   3637 	}
   3638 
   3639 	val = I40E_QINT_TQCTL_CAUSE_ENA_MASK		      |
   3640 	      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
   3641 	      (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
   3642 
   3643 	wr32(hw, I40E_QINT_TQCTL(0), val);
   3644 	i40e_flush(hw);
   3645 }
   3646 
   3647 /**
   3648  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
   3649  * @pf: board private structure
   3650  **/
   3651 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
   3652 {
   3653 	struct i40e_hw *hw = &pf->hw;
   3654 
   3655 	wr32(hw, I40E_PFINT_DYN_CTL0,
   3656 	     I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
   3657 	i40e_flush(hw);
   3658 }
   3659 
   3660 /**
   3661  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
   3662  * @pf: board private structure
   3663  **/
   3664 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
   3665 {
   3666 	struct i40e_hw *hw = &pf->hw;
   3667 	u32 val;
   3668 
   3669 	val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
   3670 	      I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
   3671 	      (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
   3672 
   3673 	wr32(hw, I40E_PFINT_DYN_CTL0, val);
   3674 	i40e_flush(hw);
   3675 }
   3676 
   3677 /**
   3678  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
   3679  * @irq: interrupt number
   3680  * @data: pointer to a q_vector
   3681  **/
   3682 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
   3683 {
   3684 	struct i40e_q_vector *q_vector = data;
   3685 
   3686 	if (!q_vector->tx.ring && !q_vector->rx.ring)
   3687 		return IRQ_HANDLED;
   3688 
   3689 	napi_schedule_irqoff(&q_vector->napi);
   3690 
   3691 	return IRQ_HANDLED;
   3692 }
   3693 
   3694 /**
   3695  * i40e_irq_affinity_notify - Callback for affinity changes
   3696  * @notify: context as to what irq was changed
   3697  * @mask: the new affinity mask
   3698  *
   3699  * This is a callback function used by the irq_set_affinity_notifier function
   3700  * so that we may register to receive changes to the irq affinity masks.
   3701  **/
   3702 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
   3703 				     const cpumask_t *mask)
   3704 {
   3705 	struct i40e_q_vector *q_vector =
   3706 		container_of(notify, struct i40e_q_vector, affinity_notify);
   3707 
   3708 	cpumask_copy(&q_vector->affinity_mask, mask);
   3709 }
   3710 
   3711 /**
   3712  * i40e_irq_affinity_release - Callback for affinity notifier release
   3713  * @ref: internal core kernel usage
   3714  *
   3715  * This is a callback function used by the irq_set_affinity_notifier function
   3716  * to inform the current notification subscriber that they will no longer
   3717  * receive notifications.
   3718  **/
   3719 static void i40e_irq_affinity_release(struct kref *ref) {}
   3720 
   3721 /**
   3722  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
   3723  * @vsi: the VSI being configured
   3724  * @basename: name for the vector
   3725  *
   3726  * Allocates MSI-X vectors and requests interrupts from the kernel.
   3727  **/
   3728 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
   3729 {
   3730 	int q_vectors = vsi->num_q_vectors;
   3731 	struct i40e_pf *pf = vsi->back;
   3732 	int base = vsi->base_vector;
   3733 	int rx_int_idx = 0;
   3734 	int tx_int_idx = 0;
   3735 	int vector, err;
   3736 	int irq_num;
   3737 	int cpu;
   3738 
   3739 	for (vector = 0; vector < q_vectors; vector++) {
   3740 		struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
   3741 
   3742 		irq_num = pf->msix_entries[base + vector].vector;
   3743 
   3744 		if (q_vector->tx.ring && q_vector->rx.ring) {
   3745 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
   3746 				 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
   3747 			tx_int_idx++;
   3748 		} else if (q_vector->rx.ring) {
   3749 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
   3750 				 "%s-%s-%d", basename, "rx", rx_int_idx++);
   3751 		} else if (q_vector->tx.ring) {
   3752 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
   3753 				 "%s-%s-%d", basename, "tx", tx_int_idx++);
   3754 		} else {
   3755 			/* skip this unused q_vector */
   3756 			continue;
   3757 		}
   3758 		err = request_irq(irq_num,
   3759 				  vsi->irq_handler,
   3760 				  0,
   3761 				  q_vector->name,
   3762 				  q_vector);
   3763 		if (err) {
   3764 			dev_info(&pf->pdev->dev,
   3765 				 "MSIX request_irq failed, error: %d\n", err);
   3766 			goto free_queue_irqs;
   3767 		}
   3768 
   3769 		/* register for affinity change notifications */
   3770 		q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
   3771 		q_vector->affinity_notify.release = i40e_irq_affinity_release;
   3772 		irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
   3773 		/* Spread affinity hints out across online CPUs.
   3774 		 *
   3775 		 * get_cpu_mask returns a static constant mask with
   3776 		 * a permanent lifetime so it's ok to pass to
   3777 		 * irq_set_affinity_hint without making a copy.
   3778 		 */
   3779 		cpu = cpumask_local_spread(q_vector->v_idx, -1);
   3780 		irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
   3781 	}
   3782 
   3783 	vsi->irqs_ready = true;
   3784 	return 0;
   3785 
   3786 free_queue_irqs:
   3787 	while (vector) {
   3788 		vector--;
   3789 		irq_num = pf->msix_entries[base + vector].vector;
   3790 		irq_set_affinity_notifier(irq_num, NULL);
   3791 		irq_set_affinity_hint(irq_num, NULL);
   3792 		free_irq(irq_num, &vsi->q_vectors[vector]);
   3793 	}
   3794 	return err;
   3795 }
   3796 
   3797 /**
   3798  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
   3799  * @vsi: the VSI being un-configured
   3800  **/
   3801 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
   3802 {
   3803 	struct i40e_pf *pf = vsi->back;
   3804 	struct i40e_hw *hw = &pf->hw;
   3805 	int base = vsi->base_vector;
   3806 	int i;
   3807 
   3808 	/* disable interrupt causation from each queue */
   3809 	for (i = 0; i < vsi->num_queue_pairs; i++) {
   3810 		u32 val;
   3811 
   3812 		val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
   3813 		val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
   3814 		wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
   3815 
   3816 		val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
   3817 		val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
   3818 		wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
   3819 
   3820 		if (!i40e_enabled_xdp_vsi(vsi))
   3821 			continue;
   3822 		wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
   3823 	}
   3824 
   3825 	/* disable each interrupt */
   3826 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
   3827 		for (i = vsi->base_vector;
   3828 		     i < (vsi->num_q_vectors + vsi->base_vector); i++)
   3829 			wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
   3830 
   3831 		i40e_flush(hw);
   3832 		for (i = 0; i < vsi->num_q_vectors; i++)
   3833 			synchronize_irq(pf->msix_entries[i + base].vector);
   3834 	} else {
   3835 		/* Legacy and MSI mode - this stops all interrupt handling */
   3836 		wr32(hw, I40E_PFINT_ICR0_ENA, 0);
   3837 		wr32(hw, I40E_PFINT_DYN_CTL0, 0);
   3838 		i40e_flush(hw);
   3839 		synchronize_irq(pf->pdev->irq);
   3840 	}
   3841 }
   3842 
   3843 /**
   3844  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
   3845  * @vsi: the VSI being configured
   3846  **/
   3847 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
   3848 {
   3849 	struct i40e_pf *pf = vsi->back;
   3850 	int i;
   3851 
   3852 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
   3853 		for (i = 0; i < vsi->num_q_vectors; i++)
   3854 			i40e_irq_dynamic_enable(vsi, i);
   3855 	} else {
   3856 		i40e_irq_dynamic_enable_icr0(pf);
   3857 	}
   3858 
   3859 	i40e_flush(&pf->hw);
   3860 	return 0;
   3861 }
   3862 
   3863 /**
   3864  * i40e_free_misc_vector - Free the vector that handles non-queue events
   3865  * @pf: board private structure
   3866  **/
   3867 static void i40e_free_misc_vector(struct i40e_pf *pf)
   3868 {
   3869 	/* Disable ICR 0 */
   3870 	wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
   3871 	i40e_flush(&pf->hw);
   3872 
   3873 	if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
   3874 		synchronize_irq(pf->msix_entries[0].vector);
   3875 		free_irq(pf->msix_entries[0].vector, pf);
   3876 		clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
   3877 	}
   3878 }
   3879 
   3880 /**
   3881  * i40e_intr - MSI/Legacy and non-queue interrupt handler
   3882  * @irq: interrupt number
   3883  * @data: pointer to a q_vector
   3884  *
   3885  * This is the handler used for all MSI/Legacy interrupts, and deals
   3886  * with both queue and non-queue interrupts.  This is also used in
   3887  * MSIX mode to handle the non-queue interrupts.
   3888  **/
   3889 static irqreturn_t i40e_intr(int irq, void *data)
   3890 {
   3891 	struct i40e_pf *pf = (struct i40e_pf *)data;
   3892 	struct i40e_hw *hw = &pf->hw;
   3893 	irqreturn_t ret = IRQ_NONE;
   3894 	u32 icr0, icr0_remaining;
   3895 	u32 val, ena_mask;
   3896 
   3897 	icr0 = rd32(hw, I40E_PFINT_ICR0);
   3898 	ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
   3899 
   3900 	/* if sharing a legacy IRQ, we might get called w/o an intr pending */
   3901 	if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
   3902 		goto enable_intr;
   3903 
   3904 	/* if interrupt but no bits showing, must be SWINT */
   3905 	if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
   3906 	    (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
   3907 		pf->sw_int_count++;
   3908 
   3909 	if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
   3910 	    (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
   3911 		ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
   3912 		dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
   3913 		set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
   3914 	}
   3915 
   3916 	/* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
   3917 	if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
   3918 		struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
   3919 		struct i40e_q_vector *q_vector = vsi->q_vectors[0];
   3920 
   3921 		/* We do not have a way to disarm Queue causes while leaving
   3922 		 * interrupt enabled for all other causes, ideally
   3923 		 * interrupt should be disabled while we are in NAPI but
   3924 		 * this is not a performance path and napi_schedule()
   3925 		 * can deal with rescheduling.
   3926 		 */
   3927 		if (!test_bit(__I40E_DOWN, pf->state))
   3928 			napi_schedule_irqoff(&q_vector->napi);
   3929 	}
   3930 
   3931 	if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
   3932 		ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
   3933 		set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
   3934 		i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
   3935 	}
   3936 
   3937 	if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
   3938 		ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
   3939 		set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
   3940 	}
   3941 
   3942 	if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
   3943 		ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
   3944 		set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
   3945 	}
   3946 
   3947 	if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
   3948 		if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
   3949 			set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
   3950 		ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
   3951 		val = rd32(hw, I40E_GLGEN_RSTAT);
   3952 		val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
   3953 		       >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
   3954 		if (val == I40E_RESET_CORER) {
   3955 			pf->corer_count++;
   3956 		} else if (val == I40E_RESET_GLOBR) {
   3957 			pf->globr_count++;
   3958 		} else if (val == I40E_RESET_EMPR) {
   3959 			pf->empr_count++;
   3960 			set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
   3961 		}
   3962 	}
   3963 
   3964 	if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
   3965 		icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
   3966 		dev_info(&pf->pdev->dev, "HMC error interrupt\n");
   3967 		dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
   3968 			 rd32(hw, I40E_PFHMC_ERRORINFO),
   3969 			 rd32(hw, I40E_PFHMC_ERRORDATA));
   3970 	}
   3971 
   3972 	if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
   3973 		u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
   3974 
   3975 		if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
   3976 			icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
   3977 			i40e_ptp_tx_hwtstamp(pf);
   3978 		}
   3979 	}
   3980 
   3981 	/* If a critical error is pending we have no choice but to reset the
   3982 	 * device.
   3983 	 * Report and mask out any remaining unexpected interrupts.
   3984 	 */
   3985 	icr0_remaining = icr0 & ena_mask;
   3986 	if (icr0_remaining) {
   3987 		dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
   3988 			 icr0_remaining);
   3989 		if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
   3990 		    (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
   3991 		    (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
   3992 			dev_info(&pf->pdev->dev, "device will be reset\n");
   3993 			set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
   3994 			i40e_service_event_schedule(pf);
   3995 		}
   3996 		ena_mask &= ~icr0_remaining;
   3997 	}
   3998 	ret = IRQ_HANDLED;
   3999 
   4000 enable_intr:
   4001 	/* re-enable interrupt causes */
   4002 	wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
   4003 	if (!test_bit(__I40E_DOWN, pf->state)) {
   4004 		i40e_service_event_schedule(pf);
   4005 		i40e_irq_dynamic_enable_icr0(pf);
   4006 	}
   4007 
   4008 	return ret;
   4009 }
   4010 
   4011 /**
   4012  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
   4013  * @tx_ring:  tx ring to clean
   4014  * @budget:   how many cleans we're allowed
   4015  *
   4016  * Returns true if there's any budget left (e.g. the clean is finished)
   4017  **/
   4018 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
   4019 {
   4020 	struct i40e_vsi *vsi = tx_ring->vsi;
   4021 	u16 i = tx_ring->next_to_clean;
   4022 	struct i40e_tx_buffer *tx_buf;
   4023 	struct i40e_tx_desc *tx_desc;
   4024 
   4025 	tx_buf = &tx_ring->tx_bi[i];
   4026 	tx_desc = I40E_TX_DESC(tx_ring, i);
   4027 	i -= tx_ring->count;
   4028 
   4029 	do {
   4030 		struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
   4031 
   4032 		/* if next_to_watch is not set then there is no work pending */
   4033 		if (!eop_desc)
   4034 			break;
   4035 
   4036 		/* prevent any other reads prior to eop_desc */
   4037 		smp_rmb();
   4038 
   4039 		/* if the descriptor isn't done, no work yet to do */
   4040 		if (!(eop_desc->cmd_type_offset_bsz &
   4041 		      cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
   4042 			break;
   4043 
   4044 		/* clear next_to_watch to prevent false hangs */
   4045 		tx_buf->next_to_watch = NULL;
   4046 
   4047 		tx_desc->buffer_addr = 0;
   4048 		tx_desc->cmd_type_offset_bsz = 0;
   4049 		/* move past filter desc */
   4050 		tx_buf++;
   4051 		tx_desc++;
   4052 		i++;
   4053 		if (unlikely(!i)) {
   4054 			i -= tx_ring->count;
   4055 			tx_buf = tx_ring->tx_bi;
   4056 			tx_desc = I40E_TX_DESC(tx_ring, 0);
   4057 		}
   4058 		/* unmap skb header data */
   4059 		dma_unmap_single(tx_ring->dev,
   4060 				 dma_unmap_addr(tx_buf, dma),
   4061 				 dma_unmap_len(tx_buf, len),
   4062 				 DMA_TO_DEVICE);
   4063 		if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
   4064 			kfree(tx_buf->raw_buf);
   4065 
   4066 		tx_buf->raw_buf = NULL;
   4067 		tx_buf->tx_flags = 0;
   4068 		tx_buf->next_to_watch = NULL;
   4069 		dma_unmap_len_set(tx_buf, len, 0);
   4070 		tx_desc->buffer_addr = 0;
   4071 		tx_desc->cmd_type_offset_bsz = 0;
   4072 
   4073 		/* move us past the eop_desc for start of next FD desc */
   4074 		tx_buf++;
   4075 		tx_desc++;
   4076 		i++;
   4077 		if (unlikely(!i)) {
   4078 			i -= tx_ring->count;
   4079 			tx_buf = tx_ring->tx_bi;
   4080 			tx_desc = I40E_TX_DESC(tx_ring, 0);
   4081 		}
   4082 
   4083 		/* update budget accounting */
   4084 		budget--;
   4085 	} while (likely(budget));
   4086 
   4087 	i += tx_ring->count;
   4088 	tx_ring->next_to_clean = i;
   4089 
   4090 	if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
   4091 		i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
   4092 
   4093 	return budget > 0;
   4094 }
   4095 
   4096 /**
   4097  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
   4098  * @irq: interrupt number
   4099  * @data: pointer to a q_vector
   4100  **/
   4101 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
   4102 {
   4103 	struct i40e_q_vector *q_vector = data;
   4104 	struct i40e_vsi *vsi;
   4105 
   4106 	if (!q_vector->tx.ring)
   4107 		return IRQ_HANDLED;
   4108 
   4109 	vsi = q_vector->tx.ring->vsi;
   4110 	i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
   4111 
   4112 	return IRQ_HANDLED;
   4113 }
   4114 
   4115 /**
   4116  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
   4117  * @vsi: the VSI being configured
   4118  * @v_idx: vector index
   4119  * @qp_idx: queue pair index
   4120  **/
   4121 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
   4122 {
   4123 	struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
   4124 	struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
   4125 	struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
   4126 
   4127 	tx_ring->q_vector = q_vector;
   4128 	tx_ring->next = q_vector->tx.ring;
   4129 	q_vector->tx.ring = tx_ring;
   4130 	q_vector->tx.count++;
   4131 
   4132 	/* Place XDP Tx ring in the same q_vector ring list as regular Tx */
   4133 	if (i40e_enabled_xdp_vsi(vsi)) {
   4134 		struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
   4135 
   4136 		xdp_ring->q_vector = q_vector;
   4137 		xdp_ring->next = q_vector->tx.ring;
   4138 		q_vector->tx.ring = xdp_ring;
   4139 		q_vector->tx.count++;
   4140 	}
   4141 
   4142 	rx_ring->q_vector = q_vector;
   4143 	rx_ring->next = q_vector->rx.ring;
   4144 	q_vector->rx.ring = rx_ring;
   4145 	q_vector->rx.count++;
   4146 }
   4147 
   4148 /**
   4149  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
   4150  * @vsi: the VSI being configured
   4151  *
   4152  * This function maps descriptor rings to the queue-specific vectors
   4153  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
   4154  * one vector per queue pair, but on a constrained vector budget, we
   4155  * group the queue pairs as "efficiently" as possible.
   4156  **/
   4157 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
   4158 {
   4159 	int qp_remaining = vsi->num_queue_pairs;
   4160 	int q_vectors = vsi->num_q_vectors;
   4161 	int num_ringpairs;
   4162 	int v_start = 0;
   4163 	int qp_idx = 0;
   4164 
   4165 	/* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
   4166 	 * group them so there are multiple queues per vector.
   4167 	 * It is also important to go through all the vectors available to be
   4168 	 * sure that if we don't use all the vectors, that the remaining vectors
   4169 	 * are cleared. This is especially important when decreasing the
   4170 	 * number of queues in use.
   4171 	 */
   4172 	for (; v_start < q_vectors; v_start++) {
   4173 		struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
   4174 
   4175 		num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
   4176 
   4177 		q_vector->num_ringpairs = num_ringpairs;
   4178 		q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
   4179 
   4180 		q_vector->rx.count = 0;
   4181 		q_vector->tx.count = 0;
   4182 		q_vector->rx.ring = NULL;
   4183 		q_vector->tx.ring = NULL;
   4184 
   4185 		while (num_ringpairs--) {
   4186 			i40e_map_vector_to_qp(vsi, v_start, qp_idx);
   4187 			qp_idx++;
   4188 			qp_remaining--;
   4189 		}
   4190 	}
   4191 }
   4192 
   4193 /**
   4194  * i40e_vsi_request_irq - Request IRQ from the OS
   4195  * @vsi: the VSI being configured
   4196  * @basename: name for the vector
   4197  **/
   4198 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
   4199 {
   4200 	struct i40e_pf *pf = vsi->back;
   4201 	int err;
   4202 
   4203 	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
   4204 		err = i40e_vsi_request_irq_msix(vsi, basename);
   4205 	else if (pf->flags & I40E_FLAG_MSI_ENABLED)
   4206 		err = request_irq(pf->pdev->irq, i40e_intr, 0,
   4207 				  pf->int_name, pf);
   4208 	else
   4209 		err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
   4210 				  pf->int_name, pf);
   4211 
   4212 	if (err)
   4213 		dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
   4214 
   4215 	return err;
   4216 }
   4217 
   4218 #ifdef CONFIG_NET_POLL_CONTROLLER
   4219 /**
   4220  * i40e_netpoll - A Polling 'interrupt' handler
   4221  * @netdev: network interface device structure
   4222  *
   4223  * This is used by netconsole to send skbs without having to re-enable
   4224  * interrupts.  It's not called while the normal interrupt routine is executing.
   4225  **/
   4226 static void i40e_netpoll(struct net_device *netdev)
   4227 {
   4228 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   4229 	struct i40e_vsi *vsi = np->vsi;
   4230 	struct i40e_pf *pf = vsi->back;
   4231 	int i;
   4232 
   4233 	/* if interface is down do nothing */
   4234 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
   4235 		return;
   4236 
   4237 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
   4238 		for (i = 0; i < vsi->num_q_vectors; i++)
   4239 			i40e_msix_clean_rings(0, vsi->q_vectors[i]);
   4240 	} else {
   4241 		i40e_intr(pf->pdev->irq, netdev);
   4242 	}
   4243 }
   4244 #endif
   4245 
   4246 #define I40E_QTX_ENA_WAIT_COUNT 50
   4247 
   4248 /**
   4249  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
   4250  * @pf: the PF being configured
   4251  * @pf_q: the PF queue
   4252  * @enable: enable or disable state of the queue
   4253  *
   4254  * This routine will wait for the given Tx queue of the PF to reach the
   4255  * enabled or disabled state.
   4256  * Returns -ETIMEDOUT in case of failing to reach the requested state after
   4257  * multiple retries; else will return 0 in case of success.
   4258  **/
   4259 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
   4260 {
   4261 	int i;
   4262 	u32 tx_reg;
   4263 
   4264 	for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
   4265 		tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
   4266 		if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
   4267 			break;
   4268 
   4269 		usleep_range(10, 20);
   4270 	}
   4271 	if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
   4272 		return -ETIMEDOUT;
   4273 
   4274 	return 0;
   4275 }
   4276 
   4277 /**
   4278  * i40e_control_tx_q - Start or stop a particular Tx queue
   4279  * @pf: the PF structure
   4280  * @pf_q: the PF queue to configure
   4281  * @enable: start or stop the queue
   4282  *
   4283  * This function enables or disables a single queue. Note that any delay
   4284  * required after the operation is expected to be handled by the caller of
   4285  * this function.
   4286  **/
   4287 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
   4288 {
   4289 	struct i40e_hw *hw = &pf->hw;
   4290 	u32 tx_reg;
   4291 	int i;
   4292 
   4293 	/* warn the TX unit of coming changes */
   4294 	i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
   4295 	if (!enable)
   4296 		usleep_range(10, 20);
   4297 
   4298 	for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
   4299 		tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
   4300 		if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
   4301 		    ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
   4302 			break;
   4303 		usleep_range(1000, 2000);
   4304 	}
   4305 
   4306 	/* Skip if the queue is already in the requested state */
   4307 	if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
   4308 		return;
   4309 
   4310 	/* turn on/off the queue */
   4311 	if (enable) {
   4312 		wr32(hw, I40E_QTX_HEAD(pf_q), 0);
   4313 		tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
   4314 	} else {
   4315 		tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
   4316 	}
   4317 
   4318 	wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
   4319 }
   4320 
   4321 /**
   4322  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
   4323  * @seid: VSI SEID
   4324  * @pf: the PF structure
   4325  * @pf_q: the PF queue to configure
   4326  * @is_xdp: true if the queue is used for XDP
   4327  * @enable: start or stop the queue
   4328  **/
   4329 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
   4330 			   bool is_xdp, bool enable)
   4331 {
   4332 	int ret;
   4333 
   4334 	i40e_control_tx_q(pf, pf_q, enable);
   4335 
   4336 	/* wait for the change to finish */
   4337 	ret = i40e_pf_txq_wait(pf, pf_q, enable);
   4338 	if (ret) {
   4339 		dev_info(&pf->pdev->dev,
   4340 			 "VSI seid %d %sTx ring %d %sable timeout\n",
   4341 			 seid, (is_xdp ? "XDP " : ""), pf_q,
   4342 			 (enable ? "en" : "dis"));
   4343 	}
   4344 
   4345 	return ret;
   4346 }
   4347 
   4348 /**
   4349  * i40e_vsi_control_tx - Start or stop a VSI's rings
   4350  * @vsi: the VSI being configured
   4351  * @enable: start or stop the rings
   4352  **/
   4353 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
   4354 {
   4355 	struct i40e_pf *pf = vsi->back;
   4356 	int i, pf_q, ret = 0;
   4357 
   4358 	pf_q = vsi->base_queue;
   4359 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
   4360 		ret = i40e_control_wait_tx_q(vsi->seid, pf,
   4361 					     pf_q,
   4362 					     false /*is xdp*/, enable);
   4363 		if (ret)
   4364 			break;
   4365 
   4366 		if (!i40e_enabled_xdp_vsi(vsi))
   4367 			continue;
   4368 
   4369 		ret = i40e_control_wait_tx_q(vsi->seid, pf,
   4370 					     pf_q + vsi->alloc_queue_pairs,
   4371 					     true /*is xdp*/, enable);
   4372 		if (ret)
   4373 			break;
   4374 	}
   4375 	return ret;
   4376 }
   4377 
   4378 /**
   4379  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
   4380  * @pf: the PF being configured
   4381  * @pf_q: the PF queue
   4382  * @enable: enable or disable state of the queue
   4383  *
   4384  * This routine will wait for the given Rx queue of the PF to reach the
   4385  * enabled or disabled state.
   4386  * Returns -ETIMEDOUT in case of failing to reach the requested state after
   4387  * multiple retries; else will return 0 in case of success.
   4388  **/
   4389 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
   4390 {
   4391 	int i;
   4392 	u32 rx_reg;
   4393 
   4394 	for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
   4395 		rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
   4396 		if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
   4397 			break;
   4398 
   4399 		usleep_range(10, 20);
   4400 	}
   4401 	if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
   4402 		return -ETIMEDOUT;
   4403 
   4404 	return 0;
   4405 }
   4406 
   4407 /**
   4408  * i40e_control_rx_q - Start or stop a particular Rx queue
   4409  * @pf: the PF structure
   4410  * @pf_q: the PF queue to configure
   4411  * @enable: start or stop the queue
   4412  *
   4413  * This function enables or disables a single queue. Note that
   4414  * any delay required after the operation is expected to be
   4415  * handled by the caller of this function.
   4416  **/
   4417 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
   4418 {
   4419 	struct i40e_hw *hw = &pf->hw;
   4420 	u32 rx_reg;
   4421 	int i;
   4422 
   4423 	for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
   4424 		rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
   4425 		if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
   4426 		    ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
   4427 			break;
   4428 		usleep_range(1000, 2000);
   4429 	}
   4430 
   4431 	/* Skip if the queue is already in the requested state */
   4432 	if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
   4433 		return;
   4434 
   4435 	/* turn on/off the queue */
   4436 	if (enable)
   4437 		rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
   4438 	else
   4439 		rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
   4440 
   4441 	wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
   4442 }
   4443 
   4444 /**
   4445  * i40e_control_wait_rx_q
   4446  * @pf: the PF structure
   4447  * @pf_q: queue being configured
   4448  * @enable: start or stop the rings
   4449  *
   4450  * This function enables or disables a single queue along with waiting
   4451  * for the change to finish. The caller of this function should handle
   4452  * the delays needed in the case of disabling queues.
   4453  **/
   4454 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
   4455 {
   4456 	int ret = 0;
   4457 
   4458 	i40e_control_rx_q(pf, pf_q, enable);
   4459 
   4460 	/* wait for the change to finish */
   4461 	ret = i40e_pf_rxq_wait(pf, pf_q, enable);
   4462 	if (ret)
   4463 		return ret;
   4464 
   4465 	return ret;
   4466 }
   4467 
   4468 /**
   4469  * i40e_vsi_control_rx - Start or stop a VSI's rings
   4470  * @vsi: the VSI being configured
   4471  * @enable: start or stop the rings
   4472  **/
   4473 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
   4474 {
   4475 	struct i40e_pf *pf = vsi->back;
   4476 	int i, pf_q, ret = 0;
   4477 
   4478 	pf_q = vsi->base_queue;
   4479 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
   4480 		ret = i40e_control_wait_rx_q(pf, pf_q, enable);
   4481 		if (ret) {
   4482 			dev_info(&pf->pdev->dev,
   4483 				 "VSI seid %d Rx ring %d %sable timeout\n",
   4484 				 vsi->seid, pf_q, (enable ? "en" : "dis"));
   4485 			break;
   4486 		}
   4487 	}
   4488 
   4489 	/* Due to HW errata, on Rx disable only, the register can indicate done
   4490 	 * before it really is. Needs 50ms to be sure
   4491 	 */
   4492 	if (!enable)
   4493 		mdelay(50);
   4494 
   4495 	return ret;
   4496 }
   4497 
   4498 /**
   4499  * i40e_vsi_start_rings - Start a VSI's rings
   4500  * @vsi: the VSI being configured
   4501  **/
   4502 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
   4503 {
   4504 	int ret = 0;
   4505 
   4506 	/* do rx first for enable and last for disable */
   4507 	ret = i40e_vsi_control_rx(vsi, true);
   4508 	if (ret)
   4509 		return ret;
   4510 	ret = i40e_vsi_control_tx(vsi, true);
   4511 
   4512 	return ret;
   4513 }
   4514 
   4515 /**
   4516  * i40e_vsi_stop_rings - Stop a VSI's rings
   4517  * @vsi: the VSI being configured
   4518  **/
   4519 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
   4520 {
   4521 	/* When port TX is suspended, don't wait */
   4522 	if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
   4523 		return i40e_vsi_stop_rings_no_wait(vsi);
   4524 
   4525 	/* do rx first for enable and last for disable
   4526 	 * Ignore return value, we need to shutdown whatever we can
   4527 	 */
   4528 	i40e_vsi_control_tx(vsi, false);
   4529 	i40e_vsi_control_rx(vsi, false);
   4530 }
   4531 
   4532 /**
   4533  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
   4534  * @vsi: the VSI being shutdown
   4535  *
   4536  * This function stops all the rings for a VSI but does not delay to verify
   4537  * that rings have been disabled. It is expected that the caller is shutting
   4538  * down multiple VSIs at once and will delay together for all the VSIs after
   4539  * initiating the shutdown. This is particularly useful for shutting down lots
   4540  * of VFs together. Otherwise, a large delay can be incurred while configuring
   4541  * each VSI in serial.
   4542  **/
   4543 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
   4544 {
   4545 	struct i40e_pf *pf = vsi->back;
   4546 	int i, pf_q;
   4547 
   4548 	pf_q = vsi->base_queue;
   4549 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
   4550 		i40e_control_tx_q(pf, pf_q, false);
   4551 		i40e_control_rx_q(pf, pf_q, false);
   4552 	}
   4553 }
   4554 
   4555 /**
   4556  * i40e_vsi_free_irq - Free the irq association with the OS
   4557  * @vsi: the VSI being configured
   4558  **/
   4559 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
   4560 {
   4561 	struct i40e_pf *pf = vsi->back;
   4562 	struct i40e_hw *hw = &pf->hw;
   4563 	int base = vsi->base_vector;
   4564 	u32 val, qp;
   4565 	int i;
   4566 
   4567 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
   4568 		if (!vsi->q_vectors)
   4569 			return;
   4570 
   4571 		if (!vsi->irqs_ready)
   4572 			return;
   4573 
   4574 		vsi->irqs_ready = false;
   4575 		for (i = 0; i < vsi->num_q_vectors; i++) {
   4576 			int irq_num;
   4577 			u16 vector;
   4578 
   4579 			vector = i + base;
   4580 			irq_num = pf->msix_entries[vector].vector;
   4581 
   4582 			/* free only the irqs that were actually requested */
   4583 			if (!vsi->q_vectors[i] ||
   4584 			    !vsi->q_vectors[i]->num_ringpairs)
   4585 				continue;
   4586 
   4587 			/* clear the affinity notifier in the IRQ descriptor */
   4588 			irq_set_affinity_notifier(irq_num, NULL);
   4589 			/* remove our suggested affinity mask for this IRQ */
   4590 			irq_set_affinity_hint(irq_num, NULL);
   4591 			synchronize_irq(irq_num);
   4592 			free_irq(irq_num, vsi->q_vectors[i]);
   4593 
   4594 			/* Tear down the interrupt queue link list
   4595 			 *
   4596 			 * We know that they come in pairs and always
   4597 			 * the Rx first, then the Tx.  To clear the
   4598 			 * link list, stick the EOL value into the
   4599 			 * next_q field of the registers.
   4600 			 */
   4601 			val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
   4602 			qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
   4603 				>> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
   4604 			val |= I40E_QUEUE_END_OF_LIST
   4605 				<< I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
   4606 			wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
   4607 
   4608 			while (qp != I40E_QUEUE_END_OF_LIST) {
   4609 				u32 next;
   4610 
   4611 				val = rd32(hw, I40E_QINT_RQCTL(qp));
   4612 
   4613 				val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
   4614 					 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
   4615 					 I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
   4616 					 I40E_QINT_RQCTL_INTEVENT_MASK);
   4617 
   4618 				val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
   4619 					 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
   4620 
   4621 				wr32(hw, I40E_QINT_RQCTL(qp), val);
   4622 
   4623 				val = rd32(hw, I40E_QINT_TQCTL(qp));
   4624 
   4625 				next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
   4626 					>> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
   4627 
   4628 				val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
   4629 					 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
   4630 					 I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
   4631 					 I40E_QINT_TQCTL_INTEVENT_MASK);
   4632 
   4633 				val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
   4634 					 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
   4635 
   4636 				wr32(hw, I40E_QINT_TQCTL(qp), val);
   4637 				qp = next;
   4638 			}
   4639 		}
   4640 	} else {
   4641 		free_irq(pf->pdev->irq, pf);
   4642 
   4643 		val = rd32(hw, I40E_PFINT_LNKLST0);
   4644 		qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
   4645 			>> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
   4646 		val |= I40E_QUEUE_END_OF_LIST
   4647 			<< I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
   4648 		wr32(hw, I40E_PFINT_LNKLST0, val);
   4649 
   4650 		val = rd32(hw, I40E_QINT_RQCTL(qp));
   4651 		val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
   4652 			 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
   4653 			 I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
   4654 			 I40E_QINT_RQCTL_INTEVENT_MASK);
   4655 
   4656 		val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
   4657 			I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
   4658 
   4659 		wr32(hw, I40E_QINT_RQCTL(qp), val);
   4660 
   4661 		val = rd32(hw, I40E_QINT_TQCTL(qp));
   4662 
   4663 		val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
   4664 			 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
   4665 			 I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
   4666 			 I40E_QINT_TQCTL_INTEVENT_MASK);
   4667 
   4668 		val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
   4669 			I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
   4670 
   4671 		wr32(hw, I40E_QINT_TQCTL(qp), val);
   4672 	}
   4673 }
   4674 
   4675 /**
   4676  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
   4677  * @vsi: the VSI being configured
   4678  * @v_idx: Index of vector to be freed
   4679  *
   4680  * This function frees the memory allocated to the q_vector.  In addition if
   4681  * NAPI is enabled it will delete any references to the NAPI struct prior
   4682  * to freeing the q_vector.
   4683  **/
   4684 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
   4685 {
   4686 	struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
   4687 	struct i40e_ring *ring;
   4688 
   4689 	if (!q_vector)
   4690 		return;
   4691 
   4692 	/* disassociate q_vector from rings */
   4693 	i40e_for_each_ring(ring, q_vector->tx)
   4694 		ring->q_vector = NULL;
   4695 
   4696 	i40e_for_each_ring(ring, q_vector->rx)
   4697 		ring->q_vector = NULL;
   4698 
   4699 	/* only VSI w/ an associated netdev is set up w/ NAPI */
   4700 	if (vsi->netdev)
   4701 		netif_napi_del(&q_vector->napi);
   4702 
   4703 	vsi->q_vectors[v_idx] = NULL;
   4704 
   4705 	kfree_rcu(q_vector, rcu);
   4706 }
   4707 
   4708 /**
   4709  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
   4710  * @vsi: the VSI being un-configured
   4711  *
   4712  * This frees the memory allocated to the q_vectors and
   4713  * deletes references to the NAPI struct.
   4714  **/
   4715 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
   4716 {
   4717 	int v_idx;
   4718 
   4719 	for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
   4720 		i40e_free_q_vector(vsi, v_idx);
   4721 }
   4722 
   4723 /**
   4724  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
   4725  * @pf: board private structure
   4726  **/
   4727 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
   4728 {
   4729 	/* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
   4730 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
   4731 		pci_disable_msix(pf->pdev);
   4732 		kfree(pf->msix_entries);
   4733 		pf->msix_entries = NULL;
   4734 		kfree(pf->irq_pile);
   4735 		pf->irq_pile = NULL;
   4736 	} else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
   4737 		pci_disable_msi(pf->pdev);
   4738 	}
   4739 	pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
   4740 }
   4741 
   4742 /**
   4743  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
   4744  * @pf: board private structure
   4745  *
   4746  * We go through and clear interrupt specific resources and reset the structure
   4747  * to pre-load conditions
   4748  **/
   4749 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
   4750 {
   4751 	int i;
   4752 
   4753 	i40e_free_misc_vector(pf);
   4754 
   4755 	i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
   4756 		      I40E_IWARP_IRQ_PILE_ID);
   4757 
   4758 	i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
   4759 	for (i = 0; i < pf->num_alloc_vsi; i++)
   4760 		if (pf->vsi[i])
   4761 			i40e_vsi_free_q_vectors(pf->vsi[i]);
   4762 	i40e_reset_interrupt_capability(pf);
   4763 }
   4764 
   4765 /**
   4766  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
   4767  * @vsi: the VSI being configured
   4768  **/
   4769 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
   4770 {
   4771 	int q_idx;
   4772 
   4773 	if (!vsi->netdev)
   4774 		return;
   4775 
   4776 	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
   4777 		struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
   4778 
   4779 		if (q_vector->rx.ring || q_vector->tx.ring)
   4780 			napi_enable(&q_vector->napi);
   4781 	}
   4782 }
   4783 
   4784 /**
   4785  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
   4786  * @vsi: the VSI being configured
   4787  **/
   4788 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
   4789 {
   4790 	int q_idx;
   4791 
   4792 	if (!vsi->netdev)
   4793 		return;
   4794 
   4795 	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
   4796 		struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
   4797 
   4798 		if (q_vector->rx.ring || q_vector->tx.ring)
   4799 			napi_disable(&q_vector->napi);
   4800 	}
   4801 }
   4802 
   4803 /**
   4804  * i40e_vsi_close - Shut down a VSI
   4805  * @vsi: the vsi to be quelled
   4806  **/
   4807 static void i40e_vsi_close(struct i40e_vsi *vsi)
   4808 {
   4809 	struct i40e_pf *pf = vsi->back;
   4810 	if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
   4811 		i40e_down(vsi);
   4812 	i40e_vsi_free_irq(vsi);
   4813 	i40e_vsi_free_tx_resources(vsi);
   4814 	i40e_vsi_free_rx_resources(vsi);
   4815 	vsi->current_netdev_flags = 0;
   4816 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
   4817 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
   4818 		set_bit(__I40E_CLIENT_RESET, pf->state);
   4819 }
   4820 
   4821 /**
   4822  * i40e_quiesce_vsi - Pause a given VSI
   4823  * @vsi: the VSI being paused
   4824  **/
   4825 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
   4826 {
   4827 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
   4828 		return;
   4829 
   4830 	set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
   4831 	if (vsi->netdev && netif_running(vsi->netdev))
   4832 		vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
   4833 	else
   4834 		i40e_vsi_close(vsi);
   4835 }
   4836 
   4837 /**
   4838  * i40e_unquiesce_vsi - Resume a given VSI
   4839  * @vsi: the VSI being resumed
   4840  **/
   4841 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
   4842 {
   4843 	if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
   4844 		return;
   4845 
   4846 	if (vsi->netdev && netif_running(vsi->netdev))
   4847 		vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
   4848 	else
   4849 		i40e_vsi_open(vsi);   /* this clears the DOWN bit */
   4850 }
   4851 
   4852 /**
   4853  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
   4854  * @pf: the PF
   4855  **/
   4856 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
   4857 {
   4858 	int v;
   4859 
   4860 	for (v = 0; v < pf->num_alloc_vsi; v++) {
   4861 		if (pf->vsi[v])
   4862 			i40e_quiesce_vsi(pf->vsi[v]);
   4863 	}
   4864 }
   4865 
   4866 /**
   4867  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
   4868  * @pf: the PF
   4869  **/
   4870 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
   4871 {
   4872 	int v;
   4873 
   4874 	for (v = 0; v < pf->num_alloc_vsi; v++) {
   4875 		if (pf->vsi[v])
   4876 			i40e_unquiesce_vsi(pf->vsi[v]);
   4877 	}
   4878 }
   4879 
   4880 /**
   4881  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
   4882  * @vsi: the VSI being configured
   4883  *
   4884  * Wait until all queues on a given VSI have been disabled.
   4885  **/
   4886 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
   4887 {
   4888 	struct i40e_pf *pf = vsi->back;
   4889 	int i, pf_q, ret;
   4890 
   4891 	pf_q = vsi->base_queue;
   4892 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
   4893 		/* Check and wait for the Tx queue */
   4894 		ret = i40e_pf_txq_wait(pf, pf_q, false);
   4895 		if (ret) {
   4896 			dev_info(&pf->pdev->dev,
   4897 				 "VSI seid %d Tx ring %d disable timeout\n",
   4898 				 vsi->seid, pf_q);
   4899 			return ret;
   4900 		}
   4901 
   4902 		if (!i40e_enabled_xdp_vsi(vsi))
   4903 			goto wait_rx;
   4904 
   4905 		/* Check and wait for the XDP Tx queue */
   4906 		ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
   4907 				       false);
   4908 		if (ret) {
   4909 			dev_info(&pf->pdev->dev,
   4910 				 "VSI seid %d XDP Tx ring %d disable timeout\n",
   4911 				 vsi->seid, pf_q);
   4912 			return ret;
   4913 		}
   4914 wait_rx:
   4915 		/* Check and wait for the Rx queue */
   4916 		ret = i40e_pf_rxq_wait(pf, pf_q, false);
   4917 		if (ret) {
   4918 			dev_info(&pf->pdev->dev,
   4919 				 "VSI seid %d Rx ring %d disable timeout\n",
   4920 				 vsi->seid, pf_q);
   4921 			return ret;
   4922 		}
   4923 	}
   4924 
   4925 	return 0;
   4926 }
   4927 
   4928 #ifdef CONFIG_I40E_DCB
   4929 /**
   4930  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
   4931  * @pf: the PF
   4932  *
   4933  * This function waits for the queues to be in disabled state for all the
   4934  * VSIs that are managed by this PF.
   4935  **/
   4936 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
   4937 {
   4938 	int v, ret = 0;
   4939 
   4940 	for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
   4941 		if (pf->vsi[v]) {
   4942 			ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
   4943 			if (ret)
   4944 				break;
   4945 		}
   4946 	}
   4947 
   4948 	return ret;
   4949 }
   4950 
   4951 #endif
   4952 
   4953 /**
   4954  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
   4955  * @pf: pointer to PF
   4956  *
   4957  * Get TC map for ISCSI PF type that will include iSCSI TC
   4958  * and LAN TC.
   4959  **/
   4960 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
   4961 {
   4962 	struct i40e_dcb_app_priority_table app;
   4963 	struct i40e_hw *hw = &pf->hw;
   4964 	u8 enabled_tc = 1; /* TC0 is always enabled */
   4965 	u8 tc, i;
   4966 	/* Get the iSCSI APP TLV */
   4967 	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
   4968 
   4969 	for (i = 0; i < dcbcfg->numapps; i++) {
   4970 		app = dcbcfg->app[i];
   4971 		if (app.selector == I40E_APP_SEL_TCPIP &&
   4972 		    app.protocolid == I40E_APP_PROTOID_ISCSI) {
   4973 			tc = dcbcfg->etscfg.prioritytable[app.priority];
   4974 			enabled_tc |= BIT(tc);
   4975 			break;
   4976 		}
   4977 	}
   4978 
   4979 	return enabled_tc;
   4980 }
   4981 
   4982 /**
   4983  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
   4984  * @dcbcfg: the corresponding DCBx configuration structure
   4985  *
   4986  * Return the number of TCs from given DCBx configuration
   4987  **/
   4988 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
   4989 {
   4990 	int i, tc_unused = 0;
   4991 	u8 num_tc = 0;
   4992 	u8 ret = 0;
   4993 
   4994 	/* Scan the ETS Config Priority Table to find
   4995 	 * traffic class enabled for a given priority
   4996 	 * and create a bitmask of enabled TCs
   4997 	 */
   4998 	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
   4999 		num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
   5000 
   5001 	/* Now scan the bitmask to check for
   5002 	 * contiguous TCs starting with TC0
   5003 	 */
   5004 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   5005 		if (num_tc & BIT(i)) {
   5006 			if (!tc_unused) {
   5007 				ret++;
   5008 			} else {
   5009 				pr_err("Non-contiguous TC - Disabling DCB\n");
   5010 				return 1;
   5011 			}
   5012 		} else {
   5013 			tc_unused = 1;
   5014 		}
   5015 	}
   5016 
   5017 	/* There is always at least TC0 */
   5018 	if (!ret)
   5019 		ret = 1;
   5020 
   5021 	return ret;
   5022 }
   5023 
   5024 /**
   5025  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
   5026  * @dcbcfg: the corresponding DCBx configuration structure
   5027  *
   5028  * Query the current DCB configuration and return the number of
   5029  * traffic classes enabled from the given DCBX config
   5030  **/
   5031 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
   5032 {
   5033 	u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
   5034 	u8 enabled_tc = 1;
   5035 	u8 i;
   5036 
   5037 	for (i = 0; i < num_tc; i++)
   5038 		enabled_tc |= BIT(i);
   5039 
   5040 	return enabled_tc;
   5041 }
   5042 
   5043 /**
   5044  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
   5045  * @pf: PF being queried
   5046  *
   5047  * Query the current MQPRIO configuration and return the number of
   5048  * traffic classes enabled.
   5049  **/
   5050 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
   5051 {
   5052 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
   5053 	u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
   5054 	u8 enabled_tc = 1, i;
   5055 
   5056 	for (i = 1; i < num_tc; i++)
   5057 		enabled_tc |= BIT(i);
   5058 	return enabled_tc;
   5059 }
   5060 
   5061 /**
   5062  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
   5063  * @pf: PF being queried
   5064  *
   5065  * Return number of traffic classes enabled for the given PF
   5066  **/
   5067 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
   5068 {
   5069 	struct i40e_hw *hw = &pf->hw;
   5070 	u8 i, enabled_tc = 1;
   5071 	u8 num_tc = 0;
   5072 	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
   5073 
   5074 	if (pf->flags & I40E_FLAG_TC_MQPRIO)
   5075 		return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
   5076 
   5077 	/* If neither MQPRIO nor DCB is enabled, then always use single TC */
   5078 	if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
   5079 		return 1;
   5080 
   5081 	/* SFP mode will be enabled for all TCs on port */
   5082 	if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
   5083 		return i40e_dcb_get_num_tc(dcbcfg);
   5084 
   5085 	/* MFP mode return count of enabled TCs for this PF */
   5086 	if (pf->hw.func_caps.iscsi)
   5087 		enabled_tc =  i40e_get_iscsi_tc_map(pf);
   5088 	else
   5089 		return 1; /* Only TC0 */
   5090 
   5091 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   5092 		if (enabled_tc & BIT(i))
   5093 			num_tc++;
   5094 	}
   5095 	return num_tc;
   5096 }
   5097 
   5098 /**
   5099  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
   5100  * @pf: PF being queried
   5101  *
   5102  * Return a bitmap for enabled traffic classes for this PF.
   5103  **/
   5104 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
   5105 {
   5106 	if (pf->flags & I40E_FLAG_TC_MQPRIO)
   5107 		return i40e_mqprio_get_enabled_tc(pf);
   5108 
   5109 	/* If neither MQPRIO nor DCB is enabled for this PF then just return
   5110 	 * default TC
   5111 	 */
   5112 	if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
   5113 		return I40E_DEFAULT_TRAFFIC_CLASS;
   5114 
   5115 	/* SFP mode we want PF to be enabled for all TCs */
   5116 	if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
   5117 		return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
   5118 
   5119 	/* MFP enabled and iSCSI PF type */
   5120 	if (pf->hw.func_caps.iscsi)
   5121 		return i40e_get_iscsi_tc_map(pf);
   5122 	else
   5123 		return I40E_DEFAULT_TRAFFIC_CLASS;
   5124 }
   5125 
   5126 /**
   5127  * i40e_vsi_get_bw_info - Query VSI BW Information
   5128  * @vsi: the VSI being queried
   5129  *
   5130  * Returns 0 on success, negative value on failure
   5131  **/
   5132 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
   5133 {
   5134 	struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
   5135 	struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
   5136 	struct i40e_pf *pf = vsi->back;
   5137 	struct i40e_hw *hw = &pf->hw;
   5138 	i40e_status ret;
   5139 	u32 tc_bw_max;
   5140 	int i;
   5141 
   5142 	/* Get the VSI level BW configuration */
   5143 	ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
   5144 	if (ret) {
   5145 		dev_info(&pf->pdev->dev,
   5146 			 "couldn't get PF vsi bw config, err %s aq_err %s\n",
   5147 			 i40e_stat_str(&pf->hw, ret),
   5148 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   5149 		return -EINVAL;
   5150 	}
   5151 
   5152 	/* Get the VSI level BW configuration per TC */
   5153 	ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
   5154 					       NULL);
   5155 	if (ret) {
   5156 		dev_info(&pf->pdev->dev,
   5157 			 "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
   5158 			 i40e_stat_str(&pf->hw, ret),
   5159 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   5160 		return -EINVAL;
   5161 	}
   5162 
   5163 	if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
   5164 		dev_info(&pf->pdev->dev,
   5165 			 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
   5166 			 bw_config.tc_valid_bits,
   5167 			 bw_ets_config.tc_valid_bits);
   5168 		/* Still continuing */
   5169 	}
   5170 
   5171 	vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
   5172 	vsi->bw_max_quanta = bw_config.max_bw;
   5173 	tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
   5174 		    (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
   5175 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   5176 		vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
   5177 		vsi->bw_ets_limit_credits[i] =
   5178 					le16_to_cpu(bw_ets_config.credits[i]);
   5179 		/* 3 bits out of 4 for each TC */
   5180 		vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
   5181 	}
   5182 
   5183 	return 0;
   5184 }
   5185 
   5186 /**
   5187  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
   5188  * @vsi: the VSI being configured
   5189  * @enabled_tc: TC bitmap
   5190  * @bw_share: BW shared credits per TC
   5191  *
   5192  * Returns 0 on success, negative value on failure
   5193  **/
   5194 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
   5195 				       u8 *bw_share)
   5196 {
   5197 	struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
   5198 	struct i40e_pf *pf = vsi->back;
   5199 	i40e_status ret;
   5200 	int i;
   5201 
   5202 	/* There is no need to reset BW when mqprio mode is on.  */
   5203 	if (pf->flags & I40E_FLAG_TC_MQPRIO)
   5204 		return 0;
   5205 	if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
   5206 		ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
   5207 		if (ret)
   5208 			dev_info(&pf->pdev->dev,
   5209 				 "Failed to reset tx rate for vsi->seid %u\n",
   5210 				 vsi->seid);
   5211 		return ret;
   5212 	}
   5213 	bw_data.tc_valid_bits = enabled_tc;
   5214 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
   5215 		bw_data.tc_bw_credits[i] = bw_share[i];
   5216 
   5217 	ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
   5218 	if (ret) {
   5219 		dev_info(&pf->pdev->dev,
   5220 			 "AQ command Config VSI BW allocation per TC failed = %d\n",
   5221 			 pf->hw.aq.asq_last_status);
   5222 		return -EINVAL;
   5223 	}
   5224 
   5225 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
   5226 		vsi->info.qs_handle[i] = bw_data.qs_handles[i];
   5227 
   5228 	return 0;
   5229 }
   5230 
   5231 /**
   5232  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
   5233  * @vsi: the VSI being configured
   5234  * @enabled_tc: TC map to be enabled
   5235  *
   5236  **/
   5237 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
   5238 {
   5239 	struct net_device *netdev = vsi->netdev;
   5240 	struct i40e_pf *pf = vsi->back;
   5241 	struct i40e_hw *hw = &pf->hw;
   5242 	u8 netdev_tc = 0;
   5243 	int i;
   5244 	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
   5245 
   5246 	if (!netdev)
   5247 		return;
   5248 
   5249 	if (!enabled_tc) {
   5250 		netdev_reset_tc(netdev);
   5251 		return;
   5252 	}
   5253 
   5254 	/* Set up actual enabled TCs on the VSI */
   5255 	if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
   5256 		return;
   5257 
   5258 	/* set per TC queues for the VSI */
   5259 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   5260 		/* Only set TC queues for enabled tcs
   5261 		 *
   5262 		 * e.g. For a VSI that has TC0 and TC3 enabled the
   5263 		 * enabled_tc bitmap would be 0x00001001; the driver
   5264 		 * will set the numtc for netdev as 2 that will be
   5265 		 * referenced by the netdev layer as TC 0 and 1.
   5266 		 */
   5267 		if (vsi->tc_config.enabled_tc & BIT(i))
   5268 			netdev_set_tc_queue(netdev,
   5269 					vsi->tc_config.tc_info[i].netdev_tc,
   5270 					vsi->tc_config.tc_info[i].qcount,
   5271 					vsi->tc_config.tc_info[i].qoffset);
   5272 	}
   5273 
   5274 	if (pf->flags & I40E_FLAG_TC_MQPRIO)
   5275 		return;
   5276 
   5277 	/* Assign UP2TC map for the VSI */
   5278 	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
   5279 		/* Get the actual TC# for the UP */
   5280 		u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
   5281 		/* Get the mapped netdev TC# for the UP */
   5282 		netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
   5283 		netdev_set_prio_tc_map(netdev, i, netdev_tc);
   5284 	}
   5285 }
   5286 
   5287 /**
   5288  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
   5289  * @vsi: the VSI being configured
   5290  * @ctxt: the ctxt buffer returned from AQ VSI update param command
   5291  **/
   5292 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
   5293 				      struct i40e_vsi_context *ctxt)
   5294 {
   5295 	/* copy just the sections touched not the entire info
   5296 	 * since not all sections are valid as returned by
   5297 	 * update vsi params
   5298 	 */
   5299 	vsi->info.mapping_flags = ctxt->info.mapping_flags;
   5300 	memcpy(&vsi->info.queue_mapping,
   5301 	       &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
   5302 	memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
   5303 	       sizeof(vsi->info.tc_mapping));
   5304 }
   5305 
   5306 /**
   5307  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
   5308  * @vsi: VSI to be configured
   5309  * @enabled_tc: TC bitmap
   5310  *
   5311  * This configures a particular VSI for TCs that are mapped to the
   5312  * given TC bitmap. It uses default bandwidth share for TCs across
   5313  * VSIs to configure TC for a particular VSI.
   5314  *
   5315  * NOTE:
   5316  * It is expected that the VSI queues have been quisced before calling
   5317  * this function.
   5318  **/
   5319 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
   5320 {
   5321 	u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
   5322 	struct i40e_pf *pf = vsi->back;
   5323 	struct i40e_hw *hw = &pf->hw;
   5324 	struct i40e_vsi_context ctxt;
   5325 	int ret = 0;
   5326 	int i;
   5327 
   5328 	/* Check if enabled_tc is same as existing or new TCs */
   5329 	if (vsi->tc_config.enabled_tc == enabled_tc &&
   5330 	    vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
   5331 		return ret;
   5332 
   5333 	/* Enable ETS TCs with equal BW Share for now across all VSIs */
   5334 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   5335 		if (enabled_tc & BIT(i))
   5336 			bw_share[i] = 1;
   5337 	}
   5338 
   5339 	ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
   5340 	if (ret) {
   5341 		struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
   5342 
   5343 		dev_info(&pf->pdev->dev,
   5344 			 "Failed configuring TC map %d for VSI %d\n",
   5345 			 enabled_tc, vsi->seid);
   5346 		ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
   5347 						  &bw_config, NULL);
   5348 		if (ret) {
   5349 			dev_info(&pf->pdev->dev,
   5350 				 "Failed querying vsi bw info, err %s aq_err %s\n",
   5351 				 i40e_stat_str(hw, ret),
   5352 				 i40e_aq_str(hw, hw->aq.asq_last_status));
   5353 			goto out;
   5354 		}
   5355 		if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
   5356 			u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
   5357 
   5358 			if (!valid_tc)
   5359 				valid_tc = bw_config.tc_valid_bits;
   5360 			/* Always enable TC0, no matter what */
   5361 			valid_tc |= 1;
   5362 			dev_info(&pf->pdev->dev,
   5363 				 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
   5364 				 enabled_tc, bw_config.tc_valid_bits, valid_tc);
   5365 			enabled_tc = valid_tc;
   5366 		}
   5367 
   5368 		ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
   5369 		if (ret) {
   5370 			dev_err(&pf->pdev->dev,
   5371 				"Unable to  configure TC map %d for VSI %d\n",
   5372 				enabled_tc, vsi->seid);
   5373 			goto out;
   5374 		}
   5375 	}
   5376 
   5377 	/* Update Queue Pairs Mapping for currently enabled UPs */
   5378 	ctxt.seid = vsi->seid;
   5379 	ctxt.pf_num = vsi->back->hw.pf_id;
   5380 	ctxt.vf_num = 0;
   5381 	ctxt.uplink_seid = vsi->uplink_seid;
   5382 	ctxt.info = vsi->info;
   5383 	if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) {
   5384 		ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
   5385 		if (ret)
   5386 			goto out;
   5387 	} else {
   5388 		i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
   5389 	}
   5390 
   5391 	/* On destroying the qdisc, reset vsi->rss_size, as number of enabled
   5392 	 * queues changed.
   5393 	 */
   5394 	if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
   5395 		vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
   5396 				      vsi->num_queue_pairs);
   5397 		ret = i40e_vsi_config_rss(vsi);
   5398 		if (ret) {
   5399 			dev_info(&vsi->back->pdev->dev,
   5400 				 "Failed to reconfig rss for num_queues\n");
   5401 			return ret;
   5402 		}
   5403 		vsi->reconfig_rss = false;
   5404 	}
   5405 	if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
   5406 		ctxt.info.valid_sections |=
   5407 				cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
   5408 		ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
   5409 	}
   5410 
   5411 	/* Update the VSI after updating the VSI queue-mapping
   5412 	 * information
   5413 	 */
   5414 	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
   5415 	if (ret) {
   5416 		dev_info(&pf->pdev->dev,
   5417 			 "Update vsi tc config failed, err %s aq_err %s\n",
   5418 			 i40e_stat_str(hw, ret),
   5419 			 i40e_aq_str(hw, hw->aq.asq_last_status));
   5420 		goto out;
   5421 	}
   5422 	/* update the local VSI info with updated queue map */
   5423 	i40e_vsi_update_queue_map(vsi, &ctxt);
   5424 	vsi->info.valid_sections = 0;
   5425 
   5426 	/* Update current VSI BW information */
   5427 	ret = i40e_vsi_get_bw_info(vsi);
   5428 	if (ret) {
   5429 		dev_info(&pf->pdev->dev,
   5430 			 "Failed updating vsi bw info, err %s aq_err %s\n",
   5431 			 i40e_stat_str(hw, ret),
   5432 			 i40e_aq_str(hw, hw->aq.asq_last_status));
   5433 		goto out;
   5434 	}
   5435 
   5436 	/* Update the netdev TC setup */
   5437 	i40e_vsi_config_netdev_tc(vsi, enabled_tc);
   5438 out:
   5439 	return ret;
   5440 }
   5441 
   5442 /**
   5443  * i40e_get_link_speed - Returns link speed for the interface
   5444  * @vsi: VSI to be configured
   5445  *
   5446  **/
   5447 static int i40e_get_link_speed(struct i40e_vsi *vsi)
   5448 {
   5449 	struct i40e_pf *pf = vsi->back;
   5450 
   5451 	switch (pf->hw.phy.link_info.link_speed) {
   5452 	case I40E_LINK_SPEED_40GB:
   5453 		return 40000;
   5454 	case I40E_LINK_SPEED_25GB:
   5455 		return 25000;
   5456 	case I40E_LINK_SPEED_20GB:
   5457 		return 20000;
   5458 	case I40E_LINK_SPEED_10GB:
   5459 		return 10000;
   5460 	case I40E_LINK_SPEED_1GB:
   5461 		return 1000;
   5462 	default:
   5463 		return -EINVAL;
   5464 	}
   5465 }
   5466 
   5467 /**
   5468  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
   5469  * @vsi: VSI to be configured
   5470  * @seid: seid of the channel/VSI
   5471  * @max_tx_rate: max TX rate to be configured as BW limit
   5472  *
   5473  * Helper function to set BW limit for a given VSI
   5474  **/
   5475 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
   5476 {
   5477 	struct i40e_pf *pf = vsi->back;
   5478 	u64 credits = 0;
   5479 	int speed = 0;
   5480 	int ret = 0;
   5481 
   5482 	speed = i40e_get_link_speed(vsi);
   5483 	if (max_tx_rate > speed) {
   5484 		dev_err(&pf->pdev->dev,
   5485 			"Invalid max tx rate %llu specified for VSI seid %d.",
   5486 			max_tx_rate, seid);
   5487 		return -EINVAL;
   5488 	}
   5489 	if (max_tx_rate && max_tx_rate < 50) {
   5490 		dev_warn(&pf->pdev->dev,
   5491 			 "Setting max tx rate to minimum usable value of 50Mbps.\n");
   5492 		max_tx_rate = 50;
   5493 	}
   5494 
   5495 	/* Tx rate credits are in values of 50Mbps, 0 is disabled */
   5496 	credits = max_tx_rate;
   5497 	do_div(credits, I40E_BW_CREDIT_DIVISOR);
   5498 	ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
   5499 					  I40E_MAX_BW_INACTIVE_ACCUM, NULL);
   5500 	if (ret)
   5501 		dev_err(&pf->pdev->dev,
   5502 			"Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n",
   5503 			max_tx_rate, seid, i40e_stat_str(&pf->hw, ret),
   5504 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   5505 	return ret;
   5506 }
   5507 
   5508 /**
   5509  * i40e_remove_queue_channels - Remove queue channels for the TCs
   5510  * @vsi: VSI to be configured
   5511  *
   5512  * Remove queue channels for the TCs
   5513  **/
   5514 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
   5515 {
   5516 	enum i40e_admin_queue_err last_aq_status;
   5517 	struct i40e_cloud_filter *cfilter;
   5518 	struct i40e_channel *ch, *ch_tmp;
   5519 	struct i40e_pf *pf = vsi->back;
   5520 	struct hlist_node *node;
   5521 	int ret, i;
   5522 
   5523 	/* Reset rss size that was stored when reconfiguring rss for
   5524 	 * channel VSIs with non-power-of-2 queue count.
   5525 	 */
   5526 	vsi->current_rss_size = 0;
   5527 
   5528 	/* perform cleanup for channels if they exist */
   5529 	if (list_empty(&vsi->ch_list))
   5530 		return;
   5531 
   5532 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
   5533 		struct i40e_vsi *p_vsi;
   5534 
   5535 		list_del(&ch->list);
   5536 		p_vsi = ch->parent_vsi;
   5537 		if (!p_vsi || !ch->initialized) {
   5538 			kfree(ch);
   5539 			continue;
   5540 		}
   5541 		/* Reset queue contexts */
   5542 		for (i = 0; i < ch->num_queue_pairs; i++) {
   5543 			struct i40e_ring *tx_ring, *rx_ring;
   5544 			u16 pf_q;
   5545 
   5546 			pf_q = ch->base_queue + i;
   5547 			tx_ring = vsi->tx_rings[pf_q];
   5548 			tx_ring->ch = NULL;
   5549 
   5550 			rx_ring = vsi->rx_rings[pf_q];
   5551 			rx_ring->ch = NULL;
   5552 		}
   5553 
   5554 		/* Reset BW configured for this VSI via mqprio */
   5555 		ret = i40e_set_bw_limit(vsi, ch->seid, 0);
   5556 		if (ret)
   5557 			dev_info(&vsi->back->pdev->dev,
   5558 				 "Failed to reset tx rate for ch->seid %u\n",
   5559 				 ch->seid);
   5560 
   5561 		/* delete cloud filters associated with this channel */
   5562 		hlist_for_each_entry_safe(cfilter, node,
   5563 					  &pf->cloud_filter_list, cloud_node) {
   5564 			if (cfilter->seid != ch->seid)
   5565 				continue;
   5566 
   5567 			hash_del(&cfilter->cloud_node);
   5568 			if (cfilter->dst_port)
   5569 				ret = i40e_add_del_cloud_filter_big_buf(vsi,
   5570 									cfilter,
   5571 									false);
   5572 			else
   5573 				ret = i40e_add_del_cloud_filter(vsi, cfilter,
   5574 								false);
   5575 			last_aq_status = pf->hw.aq.asq_last_status;
   5576 			if (ret)
   5577 				dev_info(&pf->pdev->dev,
   5578 					 "Failed to delete cloud filter, err %s aq_err %s\n",
   5579 					 i40e_stat_str(&pf->hw, ret),
   5580 					 i40e_aq_str(&pf->hw, last_aq_status));
   5581 			kfree(cfilter);
   5582 		}
   5583 
   5584 		/* delete VSI from FW */
   5585 		ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
   5586 					     NULL);
   5587 		if (ret)
   5588 			dev_err(&vsi->back->pdev->dev,
   5589 				"unable to remove channel (%d) for parent VSI(%d)\n",
   5590 				ch->seid, p_vsi->seid);
   5591 		kfree(ch);
   5592 	}
   5593 	INIT_LIST_HEAD(&vsi->ch_list);
   5594 }
   5595 
   5596 /**
   5597  * i40e_is_any_channel - channel exist or not
   5598  * @vsi: ptr to VSI to which channels are associated with
   5599  *
   5600  * Returns true or false if channel(s) exist for associated VSI or not
   5601  **/
   5602 static bool i40e_is_any_channel(struct i40e_vsi *vsi)
   5603 {
   5604 	struct i40e_channel *ch, *ch_tmp;
   5605 
   5606 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
   5607 		if (ch->initialized)
   5608 			return true;
   5609 	}
   5610 
   5611 	return false;
   5612 }
   5613 
   5614 /**
   5615  * i40e_get_max_queues_for_channel
   5616  * @vsi: ptr to VSI to which channels are associated with
   5617  *
   5618  * Helper function which returns max value among the queue counts set on the
   5619  * channels/TCs created.
   5620  **/
   5621 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
   5622 {
   5623 	struct i40e_channel *ch, *ch_tmp;
   5624 	int max = 0;
   5625 
   5626 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
   5627 		if (!ch->initialized)
   5628 			continue;
   5629 		if (ch->num_queue_pairs > max)
   5630 			max = ch->num_queue_pairs;
   5631 	}
   5632 
   5633 	return max;
   5634 }
   5635 
   5636 /**
   5637  * i40e_validate_num_queues - validate num_queues w.r.t channel
   5638  * @pf: ptr to PF device
   5639  * @num_queues: number of queues
   5640  * @vsi: the parent VSI
   5641  * @reconfig_rss: indicates should the RSS be reconfigured or not
   5642  *
   5643  * This function validates number of queues in the context of new channel
   5644  * which is being established and determines if RSS should be reconfigured
   5645  * or not for parent VSI.
   5646  **/
   5647 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
   5648 				    struct i40e_vsi *vsi, bool *reconfig_rss)
   5649 {
   5650 	int max_ch_queues;
   5651 
   5652 	if (!reconfig_rss)
   5653 		return -EINVAL;
   5654 
   5655 	*reconfig_rss = false;
   5656 	if (vsi->current_rss_size) {
   5657 		if (num_queues > vsi->current_rss_size) {
   5658 			dev_dbg(&pf->pdev->dev,
   5659 				"Error: num_queues (%d) > vsi's current_size(%d)\n",
   5660 				num_queues, vsi->current_rss_size);
   5661 			return -EINVAL;
   5662 		} else if ((num_queues < vsi->current_rss_size) &&
   5663 			   (!is_power_of_2(num_queues))) {
   5664 			dev_dbg(&pf->pdev->dev,
   5665 				"Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
   5666 				num_queues, vsi->current_rss_size);
   5667 			return -EINVAL;
   5668 		}
   5669 	}
   5670 
   5671 	if (!is_power_of_2(num_queues)) {
   5672 		/* Find the max num_queues configured for channel if channel
   5673 		 * exist.
   5674 		 * if channel exist, then enforce 'num_queues' to be more than
   5675 		 * max ever queues configured for channel.
   5676 		 */
   5677 		max_ch_queues = i40e_get_max_queues_for_channel(vsi);
   5678 		if (num_queues < max_ch_queues) {
   5679 			dev_dbg(&pf->pdev->dev,
   5680 				"Error: num_queues (%d) < max queues configured for channel(%d)\n",
   5681 				num_queues, max_ch_queues);
   5682 			return -EINVAL;
   5683 		}
   5684 		*reconfig_rss = true;
   5685 	}
   5686 
   5687 	return 0;
   5688 }
   5689 
   5690 /**
   5691  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
   5692  * @vsi: the VSI being setup
   5693  * @rss_size: size of RSS, accordingly LUT gets reprogrammed
   5694  *
   5695  * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
   5696  **/
   5697 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
   5698 {
   5699 	struct i40e_pf *pf = vsi->back;
   5700 	u8 seed[I40E_HKEY_ARRAY_SIZE];
   5701 	struct i40e_hw *hw = &pf->hw;
   5702 	int local_rss_size;
   5703 	u8 *lut;
   5704 	int ret;
   5705 
   5706 	if (!vsi->rss_size)
   5707 		return -EINVAL;
   5708 
   5709 	if (rss_size > vsi->rss_size)
   5710 		return -EINVAL;
   5711 
   5712 	local_rss_size = min_t(int, vsi->rss_size, rss_size);
   5713 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
   5714 	if (!lut)
   5715 		return -ENOMEM;
   5716 
   5717 	/* Ignoring user configured lut if there is one */
   5718 	i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
   5719 
   5720 	/* Use user configured hash key if there is one, otherwise
   5721 	 * use default.
   5722 	 */
   5723 	if (vsi->rss_hkey_user)
   5724 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
   5725 	else
   5726 		netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
   5727 
   5728 	ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
   5729 	if (ret) {
   5730 		dev_info(&pf->pdev->dev,
   5731 			 "Cannot set RSS lut, err %s aq_err %s\n",
   5732 			 i40e_stat_str(hw, ret),
   5733 			 i40e_aq_str(hw, hw->aq.asq_last_status));
   5734 		kfree(lut);
   5735 		return ret;
   5736 	}
   5737 	kfree(lut);
   5738 
   5739 	/* Do the update w.r.t. storing rss_size */
   5740 	if (!vsi->orig_rss_size)
   5741 		vsi->orig_rss_size = vsi->rss_size;
   5742 	vsi->current_rss_size = local_rss_size;
   5743 
   5744 	return ret;
   5745 }
   5746 
   5747 /**
   5748  * i40e_channel_setup_queue_map - Setup a channel queue map
   5749  * @pf: ptr to PF device
   5750  * @vsi: the VSI being setup
   5751  * @ctxt: VSI context structure
   5752  * @ch: ptr to channel structure
   5753  *
   5754  * Setup queue map for a specific channel
   5755  **/
   5756 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
   5757 					 struct i40e_vsi_context *ctxt,
   5758 					 struct i40e_channel *ch)
   5759 {
   5760 	u16 qcount, qmap, sections = 0;
   5761 	u8 offset = 0;
   5762 	int pow;
   5763 
   5764 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
   5765 	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
   5766 
   5767 	qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
   5768 	ch->num_queue_pairs = qcount;
   5769 
   5770 	/* find the next higher power-of-2 of num queue pairs */
   5771 	pow = ilog2(qcount);
   5772 	if (!is_power_of_2(qcount))
   5773 		pow++;
   5774 
   5775 	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
   5776 		(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
   5777 
   5778 	/* Setup queue TC[0].qmap for given VSI context */
   5779 	ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
   5780 
   5781 	ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
   5782 	ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
   5783 	ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
   5784 	ctxt->info.valid_sections |= cpu_to_le16(sections);
   5785 }
   5786 
   5787 /**
   5788  * i40e_add_channel - add a channel by adding VSI
   5789  * @pf: ptr to PF device
   5790  * @uplink_seid: underlying HW switching element (VEB) ID
   5791  * @ch: ptr to channel structure
   5792  *
   5793  * Add a channel (VSI) using add_vsi and queue_map
   5794  **/
   5795 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
   5796 			    struct i40e_channel *ch)
   5797 {
   5798 	struct i40e_hw *hw = &pf->hw;
   5799 	struct i40e_vsi_context ctxt;
   5800 	u8 enabled_tc = 0x1; /* TC0 enabled */
   5801 	int ret;
   5802 
   5803 	if (ch->type != I40E_VSI_VMDQ2) {
   5804 		dev_info(&pf->pdev->dev,
   5805 			 "add new vsi failed, ch->type %d\n", ch->type);
   5806 		return -EINVAL;
   5807 	}
   5808 
   5809 	memset(&ctxt, 0, sizeof(ctxt));
   5810 	ctxt.pf_num = hw->pf_id;
   5811 	ctxt.vf_num = 0;
   5812 	ctxt.uplink_seid = uplink_seid;
   5813 	ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
   5814 	if (ch->type == I40E_VSI_VMDQ2)
   5815 		ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
   5816 
   5817 	if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
   5818 		ctxt.info.valid_sections |=
   5819 		     cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
   5820 		ctxt.info.switch_id =
   5821 		   cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
   5822 	}
   5823 
   5824 	/* Set queue map for a given VSI context */
   5825 	i40e_channel_setup_queue_map(pf, &ctxt, ch);
   5826 
   5827 	/* Now time to create VSI */
   5828 	ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
   5829 	if (ret) {
   5830 		dev_info(&pf->pdev->dev,
   5831 			 "add new vsi failed, err %s aq_err %s\n",
   5832 			 i40e_stat_str(&pf->hw, ret),
   5833 			 i40e_aq_str(&pf->hw,
   5834 				     pf->hw.aq.asq_last_status));
   5835 		return -ENOENT;
   5836 	}
   5837 
   5838 	/* Success, update channel */
   5839 	ch->enabled_tc = enabled_tc;
   5840 	ch->seid = ctxt.seid;
   5841 	ch->vsi_number = ctxt.vsi_number;
   5842 	ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx);
   5843 
   5844 	/* copy just the sections touched not the entire info
   5845 	 * since not all sections are valid as returned by
   5846 	 * update vsi params
   5847 	 */
   5848 	ch->info.mapping_flags = ctxt.info.mapping_flags;
   5849 	memcpy(&ch->info.queue_mapping,
   5850 	       &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
   5851 	memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
   5852 	       sizeof(ctxt.info.tc_mapping));
   5853 
   5854 	return 0;
   5855 }
   5856 
   5857 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
   5858 				  u8 *bw_share)
   5859 {
   5860 	struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
   5861 	i40e_status ret;
   5862 	int i;
   5863 
   5864 	bw_data.tc_valid_bits = ch->enabled_tc;
   5865 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
   5866 		bw_data.tc_bw_credits[i] = bw_share[i];
   5867 
   5868 	ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
   5869 				       &bw_data, NULL);
   5870 	if (ret) {
   5871 		dev_info(&vsi->back->pdev->dev,
   5872 			 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
   5873 			 vsi->back->hw.aq.asq_last_status, ch->seid);
   5874 		return -EINVAL;
   5875 	}
   5876 
   5877 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
   5878 		ch->info.qs_handle[i] = bw_data.qs_handles[i];
   5879 
   5880 	return 0;
   5881 }
   5882 
   5883 /**
   5884  * i40e_channel_config_tx_ring - config TX ring associated with new channel
   5885  * @pf: ptr to PF device
   5886  * @vsi: the VSI being setup
   5887  * @ch: ptr to channel structure
   5888  *
   5889  * Configure TX rings associated with channel (VSI) since queues are being
   5890  * from parent VSI.
   5891  **/
   5892 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
   5893 				       struct i40e_vsi *vsi,
   5894 				       struct i40e_channel *ch)
   5895 {
   5896 	i40e_status ret;
   5897 	int i;
   5898 	u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
   5899 
   5900 	/* Enable ETS TCs with equal BW Share for now across all VSIs */
   5901 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   5902 		if (ch->enabled_tc & BIT(i))
   5903 			bw_share[i] = 1;
   5904 	}
   5905 
   5906 	/* configure BW for new VSI */
   5907 	ret = i40e_channel_config_bw(vsi, ch, bw_share);
   5908 	if (ret) {
   5909 		dev_info(&vsi->back->pdev->dev,
   5910 			 "Failed configuring TC map %d for channel (seid %u)\n",
   5911 			 ch->enabled_tc, ch->seid);
   5912 		return ret;
   5913 	}
   5914 
   5915 	for (i = 0; i < ch->num_queue_pairs; i++) {
   5916 		struct i40e_ring *tx_ring, *rx_ring;
   5917 		u16 pf_q;
   5918 
   5919 		pf_q = ch->base_queue + i;
   5920 
   5921 		/* Get to TX ring ptr of main VSI, for re-setup TX queue
   5922 		 * context
   5923 		 */
   5924 		tx_ring = vsi->tx_rings[pf_q];
   5925 		tx_ring->ch = ch;
   5926 
   5927 		/* Get the RX ring ptr */
   5928 		rx_ring = vsi->rx_rings[pf_q];
   5929 		rx_ring->ch = ch;
   5930 	}
   5931 
   5932 	return 0;
   5933 }
   5934 
   5935 /**
   5936  * i40e_setup_hw_channel - setup new channel
   5937  * @pf: ptr to PF device
   5938  * @vsi: the VSI being setup
   5939  * @ch: ptr to channel structure
   5940  * @uplink_seid: underlying HW switching element (VEB) ID
   5941  * @type: type of channel to be created (VMDq2/VF)
   5942  *
   5943  * Setup new channel (VSI) based on specified type (VMDq2/VF)
   5944  * and configures TX rings accordingly
   5945  **/
   5946 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
   5947 					struct i40e_vsi *vsi,
   5948 					struct i40e_channel *ch,
   5949 					u16 uplink_seid, u8 type)
   5950 {
   5951 	int ret;
   5952 
   5953 	ch->initialized = false;
   5954 	ch->base_queue = vsi->next_base_queue;
   5955 	ch->type = type;
   5956 
   5957 	/* Proceed with creation of channel (VMDq2) VSI */
   5958 	ret = i40e_add_channel(pf, uplink_seid, ch);
   5959 	if (ret) {
   5960 		dev_info(&pf->pdev->dev,
   5961 			 "failed to add_channel using uplink_seid %u\n",
   5962 			 uplink_seid);
   5963 		return ret;
   5964 	}
   5965 
   5966 	/* Mark the successful creation of channel */
   5967 	ch->initialized = true;
   5968 
   5969 	/* Reconfigure TX queues using QTX_CTL register */
   5970 	ret = i40e_channel_config_tx_ring(pf, vsi, ch);
   5971 	if (ret) {
   5972 		dev_info(&pf->pdev->dev,
   5973 			 "failed to configure TX rings for channel %u\n",
   5974 			 ch->seid);
   5975 		return ret;
   5976 	}
   5977 
   5978 	/* update 'next_base_queue' */
   5979 	vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
   5980 	dev_dbg(&pf->pdev->dev,
   5981 		"Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
   5982 		ch->seid, ch->vsi_number, ch->stat_counter_idx,
   5983 		ch->num_queue_pairs,
   5984 		vsi->next_base_queue);
   5985 	return ret;
   5986 }
   5987 
   5988 /**
   5989  * i40e_setup_channel - setup new channel using uplink element
   5990  * @pf: ptr to PF device
   5991  * @type: type of channel to be created (VMDq2/VF)
   5992  * @uplink_seid: underlying HW switching element (VEB) ID
   5993  * @ch: ptr to channel structure
   5994  *
   5995  * Setup new channel (VSI) based on specified type (VMDq2/VF)
   5996  * and uplink switching element (uplink_seid)
   5997  **/
   5998 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
   5999 			       struct i40e_channel *ch)
   6000 {
   6001 	u8 vsi_type;
   6002 	u16 seid;
   6003 	int ret;
   6004 
   6005 	if (vsi->type == I40E_VSI_MAIN) {
   6006 		vsi_type = I40E_VSI_VMDQ2;
   6007 	} else {
   6008 		dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
   6009 			vsi->type);
   6010 		return false;
   6011 	}
   6012 
   6013 	/* underlying switching element */
   6014 	seid = pf->vsi[pf->lan_vsi]->uplink_seid;
   6015 
   6016 	/* create channel (VSI), configure TX rings */
   6017 	ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
   6018 	if (ret) {
   6019 		dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
   6020 		return false;
   6021 	}
   6022 
   6023 	return ch->initialized ? true : false;
   6024 }
   6025 
   6026 /**
   6027  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
   6028  * @vsi: ptr to VSI which has PF backing
   6029  *
   6030  * Sets up switch mode correctly if it needs to be changed and perform
   6031  * what are allowed modes.
   6032  **/
   6033 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
   6034 {
   6035 	u8 mode;
   6036 	struct i40e_pf *pf = vsi->back;
   6037 	struct i40e_hw *hw = &pf->hw;
   6038 	int ret;
   6039 
   6040 	ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
   6041 	if (ret)
   6042 		return -EINVAL;
   6043 
   6044 	if (hw->dev_caps.switch_mode) {
   6045 		/* if switch mode is set, support mode2 (non-tunneled for
   6046 		 * cloud filter) for now
   6047 		 */
   6048 		u32 switch_mode = hw->dev_caps.switch_mode &
   6049 				  I40E_SWITCH_MODE_MASK;
   6050 		if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
   6051 			if (switch_mode == I40E_CLOUD_FILTER_MODE2)
   6052 				return 0;
   6053 			dev_err(&pf->pdev->dev,
   6054 				"Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
   6055 				hw->dev_caps.switch_mode);
   6056 			return -EINVAL;
   6057 		}
   6058 	}
   6059 
   6060 	/* Set Bit 7 to be valid */
   6061 	mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
   6062 
   6063 	/* Set L4type for TCP support */
   6064 	mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
   6065 
   6066 	/* Set cloud filter mode */
   6067 	mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
   6068 
   6069 	/* Prep mode field for set_switch_config */
   6070 	ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
   6071 					pf->last_sw_conf_valid_flags,
   6072 					mode, NULL);
   6073 	if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
   6074 		dev_err(&pf->pdev->dev,
   6075 			"couldn't set switch config bits, err %s aq_err %s\n",
   6076 			i40e_stat_str(hw, ret),
   6077 			i40e_aq_str(hw,
   6078 				    hw->aq.asq_last_status));
   6079 
   6080 	return ret;
   6081 }
   6082 
   6083 /**
   6084  * i40e_create_queue_channel - function to create channel
   6085  * @vsi: VSI to be configured
   6086  * @ch: ptr to channel (it contains channel specific params)
   6087  *
   6088  * This function creates channel (VSI) using num_queues specified by user,
   6089  * reconfigs RSS if needed.
   6090  **/
   6091 int i40e_create_queue_channel(struct i40e_vsi *vsi,
   6092 			      struct i40e_channel *ch)
   6093 {
   6094 	struct i40e_pf *pf = vsi->back;
   6095 	bool reconfig_rss;
   6096 	int err;
   6097 
   6098 	if (!ch)
   6099 		return -EINVAL;
   6100 
   6101 	if (!ch->num_queue_pairs) {
   6102 		dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
   6103 			ch->num_queue_pairs);
   6104 		return -EINVAL;
   6105 	}
   6106 
   6107 	/* validate user requested num_queues for channel */
   6108 	err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
   6109 				       &reconfig_rss);
   6110 	if (err) {
   6111 		dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
   6112 			 ch->num_queue_pairs);
   6113 		return -EINVAL;
   6114 	}
   6115 
   6116 	/* By default we are in VEPA mode, if this is the first VF/VMDq
   6117 	 * VSI to be added switch to VEB mode.
   6118 	 */
   6119 	if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) ||
   6120 	    (!i40e_is_any_channel(vsi))) {
   6121 		if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) {
   6122 			dev_dbg(&pf->pdev->dev,
   6123 				"Failed to create channel. Override queues (%u) not power of 2\n",
   6124 				vsi->tc_config.tc_info[0].qcount);
   6125 			return -EINVAL;
   6126 		}
   6127 
   6128 		if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
   6129 			pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
   6130 
   6131 			if (vsi->type == I40E_VSI_MAIN) {
   6132 				if (pf->flags & I40E_FLAG_TC_MQPRIO)
   6133 					i40e_do_reset(pf, I40E_PF_RESET_FLAG,
   6134 						      true);
   6135 				else
   6136 					i40e_do_reset_safe(pf,
   6137 							   I40E_PF_RESET_FLAG);
   6138 			}
   6139 		}
   6140 		/* now onwards for main VSI, number of queues will be value
   6141 		 * of TC0's queue count
   6142 		 */
   6143 	}
   6144 
   6145 	/* By this time, vsi->cnt_q_avail shall be set to non-zero and
   6146 	 * it should be more than num_queues
   6147 	 */
   6148 	if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
   6149 		dev_dbg(&pf->pdev->dev,
   6150 			"Error: cnt_q_avail (%u) less than num_queues %d\n",
   6151 			vsi->cnt_q_avail, ch->num_queue_pairs);
   6152 		return -EINVAL;
   6153 	}
   6154 
   6155 	/* reconfig_rss only if vsi type is MAIN_VSI */
   6156 	if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
   6157 		err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
   6158 		if (err) {
   6159 			dev_info(&pf->pdev->dev,
   6160 				 "Error: unable to reconfig rss for num_queues (%u)\n",
   6161 				 ch->num_queue_pairs);
   6162 			return -EINVAL;
   6163 		}
   6164 	}
   6165 
   6166 	if (!i40e_setup_channel(pf, vsi, ch)) {
   6167 		dev_info(&pf->pdev->dev, "Failed to setup channel\n");
   6168 		return -EINVAL;
   6169 	}
   6170 
   6171 	dev_info(&pf->pdev->dev,
   6172 		 "Setup channel (id:%u) utilizing num_queues %d\n",
   6173 		 ch->seid, ch->num_queue_pairs);
   6174 
   6175 	/* configure VSI for BW limit */
   6176 	if (ch->max_tx_rate) {
   6177 		u64 credits = ch->max_tx_rate;
   6178 
   6179 		if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
   6180 			return -EINVAL;
   6181 
   6182 		do_div(credits, I40E_BW_CREDIT_DIVISOR);
   6183 		dev_dbg(&pf->pdev->dev,
   6184 			"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
   6185 			ch->max_tx_rate,
   6186 			credits,
   6187 			ch->seid);
   6188 	}
   6189 
   6190 	/* in case of VF, this will be main SRIOV VSI */
   6191 	ch->parent_vsi = vsi;
   6192 
   6193 	/* and update main_vsi's count for queue_available to use */
   6194 	vsi->cnt_q_avail -= ch->num_queue_pairs;
   6195 
   6196 	return 0;
   6197 }
   6198 
   6199 /**
   6200  * i40e_configure_queue_channels - Add queue channel for the given TCs
   6201  * @vsi: VSI to be configured
   6202  *
   6203  * Configures queue channel mapping to the given TCs
   6204  **/
   6205 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
   6206 {
   6207 	struct i40e_channel *ch;
   6208 	u64 max_rate = 0;
   6209 	int ret = 0, i;
   6210 
   6211 	/* Create app vsi with the TCs. Main VSI with TC0 is already set up */
   6212 	vsi->tc_seid_map[0] = vsi->seid;
   6213 	for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   6214 		if (vsi->tc_config.enabled_tc & BIT(i)) {
   6215 			ch = kzalloc(sizeof(*ch), GFP_KERNEL);
   6216 			if (!ch) {
   6217 				ret = -ENOMEM;
   6218 				goto err_free;
   6219 			}
   6220 
   6221 			INIT_LIST_HEAD(&ch->list);
   6222 			ch->num_queue_pairs =
   6223 				vsi->tc_config.tc_info[i].qcount;
   6224 			ch->base_queue =
   6225 				vsi->tc_config.tc_info[i].qoffset;
   6226 
   6227 			/* Bandwidth limit through tc interface is in bytes/s,
   6228 			 * change to Mbit/s
   6229 			 */
   6230 			max_rate = vsi->mqprio_qopt.max_rate[i];
   6231 			do_div(max_rate, I40E_BW_MBPS_DIVISOR);
   6232 			ch->max_tx_rate = max_rate;
   6233 
   6234 			list_add_tail(&ch->list, &vsi->ch_list);
   6235 
   6236 			ret = i40e_create_queue_channel(vsi, ch);
   6237 			if (ret) {
   6238 				dev_err(&vsi->back->pdev->dev,
   6239 					"Failed creating queue channel with TC%d: queues %d\n",
   6240 					i, ch->num_queue_pairs);
   6241 				goto err_free;
   6242 			}
   6243 			vsi->tc_seid_map[i] = ch->seid;
   6244 		}
   6245 	}
   6246 	return ret;
   6247 
   6248 err_free:
   6249 	i40e_remove_queue_channels(vsi);
   6250 	return ret;
   6251 }
   6252 
   6253 /**
   6254  * i40e_veb_config_tc - Configure TCs for given VEB
   6255  * @veb: given VEB
   6256  * @enabled_tc: TC bitmap
   6257  *
   6258  * Configures given TC bitmap for VEB (switching) element
   6259  **/
   6260 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
   6261 {
   6262 	struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
   6263 	struct i40e_pf *pf = veb->pf;
   6264 	int ret = 0;
   6265 	int i;
   6266 
   6267 	/* No TCs or already enabled TCs just return */
   6268 	if (!enabled_tc || veb->enabled_tc == enabled_tc)
   6269 		return ret;
   6270 
   6271 	bw_data.tc_valid_bits = enabled_tc;
   6272 	/* bw_data.absolute_credits is not set (relative) */
   6273 
   6274 	/* Enable ETS TCs with equal BW Share for now */
   6275 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   6276 		if (enabled_tc & BIT(i))
   6277 			bw_data.tc_bw_share_credits[i] = 1;
   6278 	}
   6279 
   6280 	ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
   6281 						   &bw_data, NULL);
   6282 	if (ret) {
   6283 		dev_info(&pf->pdev->dev,
   6284 			 "VEB bw config failed, err %s aq_err %s\n",
   6285 			 i40e_stat_str(&pf->hw, ret),
   6286 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   6287 		goto out;
   6288 	}
   6289 
   6290 	/* Update the BW information */
   6291 	ret = i40e_veb_get_bw_info(veb);
   6292 	if (ret) {
   6293 		dev_info(&pf->pdev->dev,
   6294 			 "Failed getting veb bw config, err %s aq_err %s\n",
   6295 			 i40e_stat_str(&pf->hw, ret),
   6296 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   6297 	}
   6298 
   6299 out:
   6300 	return ret;
   6301 }
   6302 
   6303 #ifdef CONFIG_I40E_DCB
   6304 /**
   6305  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
   6306  * @pf: PF struct
   6307  *
   6308  * Reconfigure VEB/VSIs on a given PF; it is assumed that
   6309  * the caller would've quiesce all the VSIs before calling
   6310  * this function
   6311  **/
   6312 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
   6313 {
   6314 	u8 tc_map = 0;
   6315 	int ret;
   6316 	u8 v;
   6317 
   6318 	/* Enable the TCs available on PF to all VEBs */
   6319 	tc_map = i40e_pf_get_tc_map(pf);
   6320 	for (v = 0; v < I40E_MAX_VEB; v++) {
   6321 		if (!pf->veb[v])
   6322 			continue;
   6323 		ret = i40e_veb_config_tc(pf->veb[v], tc_map);
   6324 		if (ret) {
   6325 			dev_info(&pf->pdev->dev,
   6326 				 "Failed configuring TC for VEB seid=%d\n",
   6327 				 pf->veb[v]->seid);
   6328 			/* Will try to configure as many components */
   6329 		}
   6330 	}
   6331 
   6332 	/* Update each VSI */
   6333 	for (v = 0; v < pf->num_alloc_vsi; v++) {
   6334 		if (!pf->vsi[v])
   6335 			continue;
   6336 
   6337 		/* - Enable all TCs for the LAN VSI
   6338 		 * - For all others keep them at TC0 for now
   6339 		 */
   6340 		if (v == pf->lan_vsi)
   6341 			tc_map = i40e_pf_get_tc_map(pf);
   6342 		else
   6343 			tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
   6344 
   6345 		ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
   6346 		if (ret) {
   6347 			dev_info(&pf->pdev->dev,
   6348 				 "Failed configuring TC for VSI seid=%d\n",
   6349 				 pf->vsi[v]->seid);
   6350 			/* Will try to configure as many components */
   6351 		} else {
   6352 			/* Re-configure VSI vectors based on updated TC map */
   6353 			i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
   6354 			if (pf->vsi[v]->netdev)
   6355 				i40e_dcbnl_set_all(pf->vsi[v]);
   6356 		}
   6357 	}
   6358 }
   6359 
   6360 /**
   6361  * i40e_resume_port_tx - Resume port Tx
   6362  * @pf: PF struct
   6363  *
   6364  * Resume a port's Tx and issue a PF reset in case of failure to
   6365  * resume.
   6366  **/
   6367 static int i40e_resume_port_tx(struct i40e_pf *pf)
   6368 {
   6369 	struct i40e_hw *hw = &pf->hw;
   6370 	int ret;
   6371 
   6372 	ret = i40e_aq_resume_port_tx(hw, NULL);
   6373 	if (ret) {
   6374 		dev_info(&pf->pdev->dev,
   6375 			 "Resume Port Tx failed, err %s aq_err %s\n",
   6376 			  i40e_stat_str(&pf->hw, ret),
   6377 			  i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   6378 		/* Schedule PF reset to recover */
   6379 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
   6380 		i40e_service_event_schedule(pf);
   6381 	}
   6382 
   6383 	return ret;
   6384 }
   6385 
   6386 /**
   6387  * i40e_init_pf_dcb - Initialize DCB configuration
   6388  * @pf: PF being configured
   6389  *
   6390  * Query the current DCB configuration and cache it
   6391  * in the hardware structure
   6392  **/
   6393 static int i40e_init_pf_dcb(struct i40e_pf *pf)
   6394 {
   6395 	struct i40e_hw *hw = &pf->hw;
   6396 	int err = 0;
   6397 
   6398 	/* Do not enable DCB for SW1 and SW2 images even if the FW is capable
   6399 	 * Also do not enable DCBx if FW LLDP agent is disabled
   6400 	 */
   6401 	if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) ||
   6402 	    (pf->flags & I40E_FLAG_DISABLE_FW_LLDP))
   6403 		goto out;
   6404 
   6405 	/* Get the initial DCB configuration */
   6406 	err = i40e_init_dcb(hw);
   6407 	if (!err) {
   6408 		/* Device/Function is not DCBX capable */
   6409 		if ((!hw->func_caps.dcb) ||
   6410 		    (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
   6411 			dev_info(&pf->pdev->dev,
   6412 				 "DCBX offload is not supported or is disabled for this PF.\n");
   6413 		} else {
   6414 			/* When status is not DISABLED then DCBX in FW */
   6415 			pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
   6416 				       DCB_CAP_DCBX_VER_IEEE;
   6417 
   6418 			pf->flags |= I40E_FLAG_DCB_CAPABLE;
   6419 			/* Enable DCB tagging only when more than one TC
   6420 			 * or explicitly disable if only one TC
   6421 			 */
   6422 			if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
   6423 				pf->flags |= I40E_FLAG_DCB_ENABLED;
   6424 			else
   6425 				pf->flags &= ~I40E_FLAG_DCB_ENABLED;
   6426 			dev_dbg(&pf->pdev->dev,
   6427 				"DCBX offload is supported for this PF.\n");
   6428 		}
   6429 	} else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
   6430 		dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
   6431 		pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
   6432 	} else {
   6433 		dev_info(&pf->pdev->dev,
   6434 			 "Query for DCB configuration failed, err %s aq_err %s\n",
   6435 			 i40e_stat_str(&pf->hw, err),
   6436 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   6437 	}
   6438 
   6439 out:
   6440 	return err;
   6441 }
   6442 #endif /* CONFIG_I40E_DCB */
   6443 #define SPEED_SIZE 14
   6444 #define FC_SIZE 8
   6445 /**
   6446  * i40e_print_link_message - print link up or down
   6447  * @vsi: the VSI for which link needs a message
   6448  * @isup: true of link is up, false otherwise
   6449  */
   6450 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
   6451 {
   6452 	enum i40e_aq_link_speed new_speed;
   6453 	struct i40e_pf *pf = vsi->back;
   6454 	char *speed = "Unknown";
   6455 	char *fc = "Unknown";
   6456 	char *fec = "";
   6457 	char *req_fec = "";
   6458 	char *an = "";
   6459 
   6460 	if (isup)
   6461 		new_speed = pf->hw.phy.link_info.link_speed;
   6462 	else
   6463 		new_speed = I40E_LINK_SPEED_UNKNOWN;
   6464 
   6465 	if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
   6466 		return;
   6467 	vsi->current_isup = isup;
   6468 	vsi->current_speed = new_speed;
   6469 	if (!isup) {
   6470 		netdev_info(vsi->netdev, "NIC Link is Down\n");
   6471 		return;
   6472 	}
   6473 
   6474 	/* Warn user if link speed on NPAR enabled partition is not at
   6475 	 * least 10GB
   6476 	 */
   6477 	if (pf->hw.func_caps.npar_enable &&
   6478 	    (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
   6479 	     pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
   6480 		netdev_warn(vsi->netdev,
   6481 			    "The partition detected link speed that is less than 10Gbps\n");
   6482 
   6483 	switch (pf->hw.phy.link_info.link_speed) {
   6484 	case I40E_LINK_SPEED_40GB:
   6485 		speed = "40 G";
   6486 		break;
   6487 	case I40E_LINK_SPEED_20GB:
   6488 		speed = "20 G";
   6489 		break;
   6490 	case I40E_LINK_SPEED_25GB:
   6491 		speed = "25 G";
   6492 		break;
   6493 	case I40E_LINK_SPEED_10GB:
   6494 		speed = "10 G";
   6495 		break;
   6496 	case I40E_LINK_SPEED_1GB:
   6497 		speed = "1000 M";
   6498 		break;
   6499 	case I40E_LINK_SPEED_100MB:
   6500 		speed = "100 M";
   6501 		break;
   6502 	default:
   6503 		break;
   6504 	}
   6505 
   6506 	switch (pf->hw.fc.current_mode) {
   6507 	case I40E_FC_FULL:
   6508 		fc = "RX/TX";
   6509 		break;
   6510 	case I40E_FC_TX_PAUSE:
   6511 		fc = "TX";
   6512 		break;
   6513 	case I40E_FC_RX_PAUSE:
   6514 		fc = "RX";
   6515 		break;
   6516 	default:
   6517 		fc = "None";
   6518 		break;
   6519 	}
   6520 
   6521 	if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
   6522 		req_fec = ", Requested FEC: None";
   6523 		fec = ", FEC: None";
   6524 		an = ", Autoneg: False";
   6525 
   6526 		if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
   6527 			an = ", Autoneg: True";
   6528 
   6529 		if (pf->hw.phy.link_info.fec_info &
   6530 		    I40E_AQ_CONFIG_FEC_KR_ENA)
   6531 			fec = ", FEC: CL74 FC-FEC/BASE-R";
   6532 		else if (pf->hw.phy.link_info.fec_info &
   6533 			 I40E_AQ_CONFIG_FEC_RS_ENA)
   6534 			fec = ", FEC: CL108 RS-FEC";
   6535 
   6536 		/* 'CL108 RS-FEC' should be displayed when RS is requested, or
   6537 		 * both RS and FC are requested
   6538 		 */
   6539 		if (vsi->back->hw.phy.link_info.req_fec_info &
   6540 		    (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
   6541 			if (vsi->back->hw.phy.link_info.req_fec_info &
   6542 			    I40E_AQ_REQUEST_FEC_RS)
   6543 				req_fec = ", Requested FEC: CL108 RS-FEC";
   6544 			else
   6545 				req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R";
   6546 		}
   6547 	}
   6548 
   6549 	netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n",
   6550 		    speed, req_fec, fec, an, fc);
   6551 }
   6552 
   6553 /**
   6554  * i40e_up_complete - Finish the last steps of bringing up a connection
   6555  * @vsi: the VSI being configured
   6556  **/
   6557 static int i40e_up_complete(struct i40e_vsi *vsi)
   6558 {
   6559 	struct i40e_pf *pf = vsi->back;
   6560 	int err;
   6561 
   6562 	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
   6563 		i40e_vsi_configure_msix(vsi);
   6564 	else
   6565 		i40e_configure_msi_and_legacy(vsi);
   6566 
   6567 	/* start rings */
   6568 	err = i40e_vsi_start_rings(vsi);
   6569 	if (err)
   6570 		return err;
   6571 
   6572 	clear_bit(__I40E_VSI_DOWN, vsi->state);
   6573 	i40e_napi_enable_all(vsi);
   6574 	i40e_vsi_enable_irq(vsi);
   6575 
   6576 	if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
   6577 	    (vsi->netdev)) {
   6578 		i40e_print_link_message(vsi, true);
   6579 		netif_tx_start_all_queues(vsi->netdev);
   6580 		netif_carrier_on(vsi->netdev);
   6581 	}
   6582 
   6583 	/* replay FDIR SB filters */
   6584 	if (vsi->type == I40E_VSI_FDIR) {
   6585 		/* reset fd counters */
   6586 		pf->fd_add_err = 0;
   6587 		pf->fd_atr_cnt = 0;
   6588 		i40e_fdir_filter_restore(vsi);
   6589 	}
   6590 
   6591 	/* On the next run of the service_task, notify any clients of the new
   6592 	 * opened netdev
   6593 	 */
   6594 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
   6595 	i40e_service_event_schedule(pf);
   6596 
   6597 	return 0;
   6598 }
   6599 
   6600 /**
   6601  * i40e_vsi_reinit_locked - Reset the VSI
   6602  * @vsi: the VSI being configured
   6603  *
   6604  * Rebuild the ring structs after some configuration
   6605  * has changed, e.g. MTU size.
   6606  **/
   6607 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
   6608 {
   6609 	struct i40e_pf *pf = vsi->back;
   6610 
   6611 	WARN_ON(in_interrupt());
   6612 	while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
   6613 		usleep_range(1000, 2000);
   6614 	i40e_down(vsi);
   6615 
   6616 	i40e_up(vsi);
   6617 	clear_bit(__I40E_CONFIG_BUSY, pf->state);
   6618 }
   6619 
   6620 /**
   6621  * i40e_up - Bring the connection back up after being down
   6622  * @vsi: the VSI being configured
   6623  **/
   6624 int i40e_up(struct i40e_vsi *vsi)
   6625 {
   6626 	int err;
   6627 
   6628 	err = i40e_vsi_configure(vsi);
   6629 	if (!err)
   6630 		err = i40e_up_complete(vsi);
   6631 
   6632 	return err;
   6633 }
   6634 
   6635 /**
   6636  * i40e_force_link_state - Force the link status
   6637  * @pf: board private structure
   6638  * @is_up: whether the link state should be forced up or down
   6639  **/
   6640 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up)
   6641 {
   6642 	struct i40e_aq_get_phy_abilities_resp abilities;
   6643 	struct i40e_aq_set_phy_config config = {0};
   6644 	struct i40e_hw *hw = &pf->hw;
   6645 	i40e_status err;
   6646 	u64 mask;
   6647 	u8 speed;
   6648 
   6649 	/* Card might've been put in an unstable state by other drivers
   6650 	 * and applications, which causes incorrect speed values being
   6651 	 * set on startup. In order to clear speed registers, we call
   6652 	 * get_phy_capabilities twice, once to get initial state of
   6653 	 * available speeds, and once to get current PHY config.
   6654 	 */
   6655 	err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
   6656 					   NULL);
   6657 	if (err) {
   6658 		dev_err(&pf->pdev->dev,
   6659 			"failed to get phy cap., ret =  %s last_status =  %s\n",
   6660 			i40e_stat_str(hw, err),
   6661 			i40e_aq_str(hw, hw->aq.asq_last_status));
   6662 		return err;
   6663 	}
   6664 	speed = abilities.link_speed;
   6665 
   6666 	/* Get the current phy config */
   6667 	err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
   6668 					   NULL);
   6669 	if (err) {
   6670 		dev_err(&pf->pdev->dev,
   6671 			"failed to get phy cap., ret =  %s last_status =  %s\n",
   6672 			i40e_stat_str(hw, err),
   6673 			i40e_aq_str(hw, hw->aq.asq_last_status));
   6674 		return err;
   6675 	}
   6676 
   6677 	/* If link needs to go up, but was not forced to go down,
   6678 	 * and its speed values are OK, no need for a flap
   6679 	 */
   6680 	if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
   6681 		return I40E_SUCCESS;
   6682 
   6683 	/* To force link we need to set bits for all supported PHY types,
   6684 	 * but there are now more than 32, so we need to split the bitmap
   6685 	 * across two fields.
   6686 	 */
   6687 	mask = I40E_PHY_TYPES_BITMASK;
   6688 	config.phy_type = is_up ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
   6689 	config.phy_type_ext = is_up ? (u8)((mask >> 32) & 0xff) : 0;
   6690 	/* Copy the old settings, except of phy_type */
   6691 	config.abilities = abilities.abilities;
   6692 	if (abilities.link_speed != 0)
   6693 		config.link_speed = abilities.link_speed;
   6694 	else
   6695 		config.link_speed = speed;
   6696 	config.eee_capability = abilities.eee_capability;
   6697 	config.eeer = abilities.eeer_val;
   6698 	config.low_power_ctrl = abilities.d3_lpan;
   6699 	config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
   6700 			    I40E_AQ_PHY_FEC_CONFIG_MASK;
   6701 	err = i40e_aq_set_phy_config(hw, &config, NULL);
   6702 
   6703 	if (err) {
   6704 		dev_err(&pf->pdev->dev,
   6705 			"set phy config ret =  %s last_status =  %s\n",
   6706 			i40e_stat_str(&pf->hw, err),
   6707 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   6708 		return err;
   6709 	}
   6710 
   6711 	/* Update the link info */
   6712 	err = i40e_update_link_info(hw);
   6713 	if (err) {
   6714 		/* Wait a little bit (on 40G cards it sometimes takes a really
   6715 		 * long time for link to come back from the atomic reset)
   6716 		 * and try once more
   6717 		 */
   6718 		msleep(1000);
   6719 		i40e_update_link_info(hw);
   6720 	}
   6721 
   6722 	i40e_aq_set_link_restart_an(hw, true, NULL);
   6723 
   6724 	return I40E_SUCCESS;
   6725 }
   6726 
   6727 /**
   6728  * i40e_down - Shutdown the connection processing
   6729  * @vsi: the VSI being stopped
   6730  **/
   6731 void i40e_down(struct i40e_vsi *vsi)
   6732 {
   6733 	int i;
   6734 
   6735 	/* It is assumed that the caller of this function
   6736 	 * sets the vsi->state __I40E_VSI_DOWN bit.
   6737 	 */
   6738 	if (vsi->netdev) {
   6739 		netif_carrier_off(vsi->netdev);
   6740 		netif_tx_disable(vsi->netdev);
   6741 	}
   6742 	i40e_vsi_disable_irq(vsi);
   6743 	i40e_vsi_stop_rings(vsi);
   6744 	if (vsi->type == I40E_VSI_MAIN &&
   6745 	    vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED)
   6746 		i40e_force_link_state(vsi->back, false);
   6747 	i40e_napi_disable_all(vsi);
   6748 
   6749 	for (i = 0; i < vsi->num_queue_pairs; i++) {
   6750 		i40e_clean_tx_ring(vsi->tx_rings[i]);
   6751 		if (i40e_enabled_xdp_vsi(vsi)) {
   6752 			/* Make sure that in-progress ndo_xdp_xmit
   6753 			 * calls are completed.
   6754 			 */
   6755 			synchronize_rcu();
   6756 			i40e_clean_tx_ring(vsi->xdp_rings[i]);
   6757 		}
   6758 		i40e_clean_rx_ring(vsi->rx_rings[i]);
   6759 	}
   6760 
   6761 }
   6762 
   6763 /**
   6764  * i40e_validate_mqprio_qopt- validate queue mapping info
   6765  * @vsi: the VSI being configured
   6766  * @mqprio_qopt: queue parametrs
   6767  **/
   6768 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
   6769 				     struct tc_mqprio_qopt_offload *mqprio_qopt)
   6770 {
   6771 	u64 sum_max_rate = 0;
   6772 	u64 max_rate = 0;
   6773 	int i;
   6774 
   6775 	if (mqprio_qopt->qopt.offset[0] != 0 ||
   6776 	    mqprio_qopt->qopt.num_tc < 1 ||
   6777 	    mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
   6778 		return -EINVAL;
   6779 	for (i = 0; ; i++) {
   6780 		if (!mqprio_qopt->qopt.count[i])
   6781 			return -EINVAL;
   6782 		if (mqprio_qopt->min_rate[i]) {
   6783 			dev_err(&vsi->back->pdev->dev,
   6784 				"Invalid min tx rate (greater than 0) specified\n");
   6785 			return -EINVAL;
   6786 		}
   6787 		max_rate = mqprio_qopt->max_rate[i];
   6788 		do_div(max_rate, I40E_BW_MBPS_DIVISOR);
   6789 		sum_max_rate += max_rate;
   6790 
   6791 		if (i >= mqprio_qopt->qopt.num_tc - 1)
   6792 			break;
   6793 		if (mqprio_qopt->qopt.offset[i + 1] !=
   6794 		    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
   6795 			return -EINVAL;
   6796 	}
   6797 	if (vsi->num_queue_pairs <
   6798 	    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
   6799 		return -EINVAL;
   6800 	}
   6801 	if (sum_max_rate > i40e_get_link_speed(vsi)) {
   6802 		dev_err(&vsi->back->pdev->dev,
   6803 			"Invalid max tx rate specified\n");
   6804 		return -EINVAL;
   6805 	}
   6806 	return 0;
   6807 }
   6808 
   6809 /**
   6810  * i40e_vsi_set_default_tc_config - set default values for tc configuration
   6811  * @vsi: the VSI being configured
   6812  **/
   6813 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
   6814 {
   6815 	u16 qcount;
   6816 	int i;
   6817 
   6818 	/* Only TC0 is enabled */
   6819 	vsi->tc_config.numtc = 1;
   6820 	vsi->tc_config.enabled_tc = 1;
   6821 	qcount = min_t(int, vsi->alloc_queue_pairs,
   6822 		       i40e_pf_get_max_q_per_tc(vsi->back));
   6823 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
   6824 		/* For the TC that is not enabled set the offset to to default
   6825 		 * queue and allocate one queue for the given TC.
   6826 		 */
   6827 		vsi->tc_config.tc_info[i].qoffset = 0;
   6828 		if (i == 0)
   6829 			vsi->tc_config.tc_info[i].qcount = qcount;
   6830 		else
   6831 			vsi->tc_config.tc_info[i].qcount = 1;
   6832 		vsi->tc_config.tc_info[i].netdev_tc = 0;
   6833 	}
   6834 }
   6835 
   6836 /**
   6837  * i40e_setup_tc - configure multiple traffic classes
   6838  * @netdev: net device to configure
   6839  * @type_data: tc offload data
   6840  **/
   6841 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
   6842 {
   6843 	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
   6844 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   6845 	struct i40e_vsi *vsi = np->vsi;
   6846 	struct i40e_pf *pf = vsi->back;
   6847 	u8 enabled_tc = 0, num_tc, hw;
   6848 	bool need_reset = false;
   6849 	int ret = -EINVAL;
   6850 	u16 mode;
   6851 	int i;
   6852 
   6853 	num_tc = mqprio_qopt->qopt.num_tc;
   6854 	hw = mqprio_qopt->qopt.hw;
   6855 	mode = mqprio_qopt->mode;
   6856 	if (!hw) {
   6857 		pf->flags &= ~I40E_FLAG_TC_MQPRIO;
   6858 		memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
   6859 		goto config_tc;
   6860 	}
   6861 
   6862 	/* Check if MFP enabled */
   6863 	if (pf->flags & I40E_FLAG_MFP_ENABLED) {
   6864 		netdev_info(netdev,
   6865 			    "Configuring TC not supported in MFP mode\n");
   6866 		return ret;
   6867 	}
   6868 	switch (mode) {
   6869 	case TC_MQPRIO_MODE_DCB:
   6870 		pf->flags &= ~I40E_FLAG_TC_MQPRIO;
   6871 
   6872 		/* Check if DCB enabled to continue */
   6873 		if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
   6874 			netdev_info(netdev,
   6875 				    "DCB is not enabled for adapter\n");
   6876 			return ret;
   6877 		}
   6878 
   6879 		/* Check whether tc count is within enabled limit */
   6880 		if (num_tc > i40e_pf_get_num_tc(pf)) {
   6881 			netdev_info(netdev,
   6882 				    "TC count greater than enabled on link for adapter\n");
   6883 			return ret;
   6884 		}
   6885 		break;
   6886 	case TC_MQPRIO_MODE_CHANNEL:
   6887 		if (pf->flags & I40E_FLAG_DCB_ENABLED) {
   6888 			netdev_info(netdev,
   6889 				    "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
   6890 			return ret;
   6891 		}
   6892 		if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
   6893 			return ret;
   6894 		ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
   6895 		if (ret)
   6896 			return ret;
   6897 		memcpy(&vsi->mqprio_qopt, mqprio_qopt,
   6898 		       sizeof(*mqprio_qopt));
   6899 		pf->flags |= I40E_FLAG_TC_MQPRIO;
   6900 		pf->flags &= ~I40E_FLAG_DCB_ENABLED;
   6901 		break;
   6902 	default:
   6903 		return -EINVAL;
   6904 	}
   6905 
   6906 config_tc:
   6907 	/* Generate TC map for number of tc requested */
   6908 	for (i = 0; i < num_tc; i++)
   6909 		enabled_tc |= BIT(i);
   6910 
   6911 	/* Requesting same TC configuration as already enabled */
   6912 	if (enabled_tc == vsi->tc_config.enabled_tc &&
   6913 	    mode != TC_MQPRIO_MODE_CHANNEL)
   6914 		return 0;
   6915 
   6916 	/* Quiesce VSI queues */
   6917 	i40e_quiesce_vsi(vsi);
   6918 
   6919 	if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO))
   6920 		i40e_remove_queue_channels(vsi);
   6921 
   6922 	/* Configure VSI for enabled TCs */
   6923 	ret = i40e_vsi_config_tc(vsi, enabled_tc);
   6924 	if (ret) {
   6925 		netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
   6926 			    vsi->seid);
   6927 		need_reset = true;
   6928 		goto exit;
   6929 	}
   6930 
   6931 	if (pf->flags & I40E_FLAG_TC_MQPRIO) {
   6932 		if (vsi->mqprio_qopt.max_rate[0]) {
   6933 			u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
   6934 
   6935 			do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
   6936 			ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
   6937 			if (!ret) {
   6938 				u64 credits = max_tx_rate;
   6939 
   6940 				do_div(credits, I40E_BW_CREDIT_DIVISOR);
   6941 				dev_dbg(&vsi->back->pdev->dev,
   6942 					"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
   6943 					max_tx_rate,
   6944 					credits,
   6945 					vsi->seid);
   6946 			} else {
   6947 				need_reset = true;
   6948 				goto exit;
   6949 			}
   6950 		}
   6951 		ret = i40e_configure_queue_channels(vsi);
   6952 		if (ret) {
   6953 			netdev_info(netdev,
   6954 				    "Failed configuring queue channels\n");
   6955 			need_reset = true;
   6956 			goto exit;
   6957 		}
   6958 	}
   6959 
   6960 exit:
   6961 	/* Reset the configuration data to defaults, only TC0 is enabled */
   6962 	if (need_reset) {
   6963 		i40e_vsi_set_default_tc_config(vsi);
   6964 		need_reset = false;
   6965 	}
   6966 
   6967 	/* Unquiesce VSI */
   6968 	i40e_unquiesce_vsi(vsi);
   6969 	return ret;
   6970 }
   6971 
   6972 /**
   6973  * i40e_set_cld_element - sets cloud filter element data
   6974  * @filter: cloud filter rule
   6975  * @cld: ptr to cloud filter element data
   6976  *
   6977  * This is helper function to copy data into cloud filter element
   6978  **/
   6979 static inline void
   6980 i40e_set_cld_element(struct i40e_cloud_filter *filter,
   6981 		     struct i40e_aqc_cloud_filters_element_data *cld)
   6982 {
   6983 	int i, j;
   6984 	u32 ipa;
   6985 
   6986 	memset(cld, 0, sizeof(*cld));
   6987 	ether_addr_copy(cld->outer_mac, filter->dst_mac);
   6988 	ether_addr_copy(cld->inner_mac, filter->src_mac);
   6989 
   6990 	if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
   6991 		return;
   6992 
   6993 	if (filter->n_proto == ETH_P_IPV6) {
   6994 #define IPV6_MAX_INDEX	(ARRAY_SIZE(filter->dst_ipv6) - 1)
   6995 		for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6);
   6996 		     i++, j += 2) {
   6997 			ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
   6998 			ipa = cpu_to_le32(ipa);
   6999 			memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa));
   7000 		}
   7001 	} else {
   7002 		ipa = be32_to_cpu(filter->dst_ipv4);
   7003 		memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
   7004 	}
   7005 
   7006 	cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
   7007 
   7008 	/* tenant_id is not supported by FW now, once the support is enabled
   7009 	 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
   7010 	 */
   7011 	if (filter->tenant_id)
   7012 		return;
   7013 }
   7014 
   7015 /**
   7016  * i40e_add_del_cloud_filter - Add/del cloud filter
   7017  * @vsi: pointer to VSI
   7018  * @filter: cloud filter rule
   7019  * @add: if true, add, if false, delete
   7020  *
   7021  * Add or delete a cloud filter for a specific flow spec.
   7022  * Returns 0 if the filter were successfully added.
   7023  **/
   7024 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
   7025 			      struct i40e_cloud_filter *filter, bool add)
   7026 {
   7027 	struct i40e_aqc_cloud_filters_element_data cld_filter;
   7028 	struct i40e_pf *pf = vsi->back;
   7029 	int ret;
   7030 	static const u16 flag_table[128] = {
   7031 		[I40E_CLOUD_FILTER_FLAGS_OMAC]  =
   7032 			I40E_AQC_ADD_CLOUD_FILTER_OMAC,
   7033 		[I40E_CLOUD_FILTER_FLAGS_IMAC]  =
   7034 			I40E_AQC_ADD_CLOUD_FILTER_IMAC,
   7035 		[I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN]  =
   7036 			I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
   7037 		[I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
   7038 			I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
   7039 		[I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
   7040 			I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
   7041 		[I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
   7042 			I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
   7043 		[I40E_CLOUD_FILTER_FLAGS_IIP] =
   7044 			I40E_AQC_ADD_CLOUD_FILTER_IIP,
   7045 	};
   7046 
   7047 	if (filter->flags >= ARRAY_SIZE(flag_table))
   7048 		return I40E_ERR_CONFIG;
   7049 
   7050 	/* copy element needed to add cloud filter from filter */
   7051 	i40e_set_cld_element(filter, &cld_filter);
   7052 
   7053 	if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
   7054 		cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
   7055 					     I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
   7056 
   7057 	if (filter->n_proto == ETH_P_IPV6)
   7058 		cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
   7059 						I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
   7060 	else
   7061 		cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
   7062 						I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
   7063 
   7064 	if (add)
   7065 		ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
   7066 						&cld_filter, 1);
   7067 	else
   7068 		ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
   7069 						&cld_filter, 1);
   7070 	if (ret)
   7071 		dev_dbg(&pf->pdev->dev,
   7072 			"Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
   7073 			add ? "add" : "delete", filter->dst_port, ret,
   7074 			pf->hw.aq.asq_last_status);
   7075 	else
   7076 		dev_info(&pf->pdev->dev,
   7077 			 "%s cloud filter for VSI: %d\n",
   7078 			 add ? "Added" : "Deleted", filter->seid);
   7079 	return ret;
   7080 }
   7081 
   7082 /**
   7083  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
   7084  * @vsi: pointer to VSI
   7085  * @filter: cloud filter rule
   7086  * @add: if true, add, if false, delete
   7087  *
   7088  * Add or delete a cloud filter for a specific flow spec using big buffer.
   7089  * Returns 0 if the filter were successfully added.
   7090  **/
   7091 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
   7092 				      struct i40e_cloud_filter *filter,
   7093 				      bool add)
   7094 {
   7095 	struct i40e_aqc_cloud_filters_element_bb cld_filter;
   7096 	struct i40e_pf *pf = vsi->back;
   7097 	int ret;
   7098 
   7099 	/* Both (src/dst) valid mac_addr are not supported */
   7100 	if ((is_valid_ether_addr(filter->dst_mac) &&
   7101 	     is_valid_ether_addr(filter->src_mac)) ||
   7102 	    (is_multicast_ether_addr(filter->dst_mac) &&
   7103 	     is_multicast_ether_addr(filter->src_mac)))
   7104 		return -EOPNOTSUPP;
   7105 
   7106 	/* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
   7107 	 * ports are not supported via big buffer now.
   7108 	 */
   7109 	if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
   7110 		return -EOPNOTSUPP;
   7111 
   7112 	/* adding filter using src_port/src_ip is not supported at this stage */
   7113 	if (filter->src_port || filter->src_ipv4 ||
   7114 	    !ipv6_addr_any(&filter->ip.v6.src_ip6))
   7115 		return -EOPNOTSUPP;
   7116 
   7117 	/* copy element needed to add cloud filter from filter */
   7118 	i40e_set_cld_element(filter, &cld_filter.element);
   7119 
   7120 	if (is_valid_ether_addr(filter->dst_mac) ||
   7121 	    is_valid_ether_addr(filter->src_mac) ||
   7122 	    is_multicast_ether_addr(filter->dst_mac) ||
   7123 	    is_multicast_ether_addr(filter->src_mac)) {
   7124 		/* MAC + IP : unsupported mode */
   7125 		if (filter->dst_ipv4)
   7126 			return -EOPNOTSUPP;
   7127 
   7128 		/* since we validated that L4 port must be valid before
   7129 		 * we get here, start with respective "flags" value
   7130 		 * and update if vlan is present or not
   7131 		 */
   7132 		cld_filter.element.flags =
   7133 			cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
   7134 
   7135 		if (filter->vlan_id) {
   7136 			cld_filter.element.flags =
   7137 			cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
   7138 		}
   7139 
   7140 	} else if (filter->dst_ipv4 ||
   7141 		   !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
   7142 		cld_filter.element.flags =
   7143 				cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
   7144 		if (filter->n_proto == ETH_P_IPV6)
   7145 			cld_filter.element.flags |=
   7146 				cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
   7147 		else
   7148 			cld_filter.element.flags |=
   7149 				cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
   7150 	} else {
   7151 		dev_err(&pf->pdev->dev,
   7152 			"either mac or ip has to be valid for cloud filter\n");
   7153 		return -EINVAL;
   7154 	}
   7155 
   7156 	/* Now copy L4 port in Byte 6..7 in general fields */
   7157 	cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
   7158 						be16_to_cpu(filter->dst_port);
   7159 
   7160 	if (add) {
   7161 		/* Validate current device switch mode, change if necessary */
   7162 		ret = i40e_validate_and_set_switch_mode(vsi);
   7163 		if (ret) {
   7164 			dev_err(&pf->pdev->dev,
   7165 				"failed to set switch mode, ret %d\n",
   7166 				ret);
   7167 			return ret;
   7168 		}
   7169 
   7170 		ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
   7171 						   &cld_filter, 1);
   7172 	} else {
   7173 		ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
   7174 						   &cld_filter, 1);
   7175 	}
   7176 
   7177 	if (ret)
   7178 		dev_dbg(&pf->pdev->dev,
   7179 			"Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
   7180 			add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
   7181 	else
   7182 		dev_info(&pf->pdev->dev,
   7183 			 "%s cloud filter for VSI: %d, L4 port: %d\n",
   7184 			 add ? "add" : "delete", filter->seid,
   7185 			 ntohs(filter->dst_port));
   7186 	return ret;
   7187 }
   7188 
   7189 /**
   7190  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
   7191  * @vsi: Pointer to VSI
   7192  * @cls_flower: Pointer to struct tc_cls_flower_offload
   7193  * @filter: Pointer to cloud filter structure
   7194  *
   7195  **/
   7196 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
   7197 				 struct tc_cls_flower_offload *f,
   7198 				 struct i40e_cloud_filter *filter)
   7199 {
   7200 	struct flow_rule *rule = tc_cls_flower_offload_flow_rule(f);
   7201 	struct flow_dissector *dissector = rule->match.dissector;
   7202 	u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
   7203 	struct i40e_pf *pf = vsi->back;
   7204 	u8 field_flags = 0;
   7205 
   7206 	if (dissector->used_keys &
   7207 	    ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
   7208 	      BIT(FLOW_DISSECTOR_KEY_BASIC) |
   7209 	      BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
   7210 	      BIT(FLOW_DISSECTOR_KEY_VLAN) |
   7211 	      BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
   7212 	      BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
   7213 	      BIT(FLOW_DISSECTOR_KEY_PORTS) |
   7214 	      BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
   7215 		dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n",
   7216 			dissector->used_keys);
   7217 		return -EOPNOTSUPP;
   7218 	}
   7219 
   7220 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
   7221 		struct flow_match_enc_keyid match;
   7222 
   7223 		flow_rule_match_enc_keyid(rule, &match);
   7224 		if (match.mask->keyid != 0)
   7225 			field_flags |= I40E_CLOUD_FIELD_TEN_ID;
   7226 
   7227 		filter->tenant_id = be32_to_cpu(match.key->keyid);
   7228 	}
   7229 
   7230 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
   7231 		struct flow_match_basic match;
   7232 
   7233 		flow_rule_match_basic(rule, &match);
   7234 		n_proto_key = ntohs(match.key->n_proto);
   7235 		n_proto_mask = ntohs(match.mask->n_proto);
   7236 
   7237 		if (n_proto_key == ETH_P_ALL) {
   7238 			n_proto_key = 0;
   7239 			n_proto_mask = 0;
   7240 		}
   7241 		filter->n_proto = n_proto_key & n_proto_mask;
   7242 		filter->ip_proto = match.key->ip_proto;
   7243 	}
   7244 
   7245 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
   7246 		struct flow_match_eth_addrs match;
   7247 
   7248 		flow_rule_match_eth_addrs(rule, &match);
   7249 
   7250 		/* use is_broadcast and is_zero to check for all 0xf or 0 */
   7251 		if (!is_zero_ether_addr(match.mask->dst)) {
   7252 			if (is_broadcast_ether_addr(match.mask->dst)) {
   7253 				field_flags |= I40E_CLOUD_FIELD_OMAC;
   7254 			} else {
   7255 				dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
   7256 					match.mask->dst);
   7257 				return I40E_ERR_CONFIG;
   7258 			}
   7259 		}
   7260 
   7261 		if (!is_zero_ether_addr(match.mask->src)) {
   7262 			if (is_broadcast_ether_addr(match.mask->src)) {
   7263 				field_flags |= I40E_CLOUD_FIELD_IMAC;
   7264 			} else {
   7265 				dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
   7266 					match.mask->src);
   7267 				return I40E_ERR_CONFIG;
   7268 			}
   7269 		}
   7270 		ether_addr_copy(filter->dst_mac, match.key->dst);
   7271 		ether_addr_copy(filter->src_mac, match.key->src);
   7272 	}
   7273 
   7274 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
   7275 		struct flow_match_vlan match;
   7276 
   7277 		flow_rule_match_vlan(rule, &match);
   7278 		if (match.mask->vlan_id) {
   7279 			if (match.mask->vlan_id == VLAN_VID_MASK) {
   7280 				field_flags |= I40E_CLOUD_FIELD_IVLAN;
   7281 
   7282 			} else {
   7283 				dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
   7284 					match.mask->vlan_id);
   7285 				return I40E_ERR_CONFIG;
   7286 			}
   7287 		}
   7288 
   7289 		filter->vlan_id = cpu_to_be16(match.key->vlan_id);
   7290 	}
   7291 
   7292 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
   7293 		struct flow_match_control match;
   7294 
   7295 		flow_rule_match_control(rule, &match);
   7296 		addr_type = match.key->addr_type;
   7297 	}
   7298 
   7299 	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
   7300 		struct flow_match_ipv4_addrs match;
   7301 
   7302 		flow_rule_match_ipv4_addrs(rule, &match);
   7303 		if (match.mask->dst) {
   7304 			if (match.mask->dst == cpu_to_be32(0xffffffff)) {
   7305 				field_flags |= I40E_CLOUD_FIELD_IIP;
   7306 			} else {
   7307 				dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
   7308 					&match.mask->dst);
   7309 				return I40E_ERR_CONFIG;
   7310 			}
   7311 		}
   7312 
   7313 		if (match.mask->src) {
   7314 			if (match.mask->src == cpu_to_be32(0xffffffff)) {
   7315 				field_flags |= I40E_CLOUD_FIELD_IIP;
   7316 			} else {
   7317 				dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
   7318 					&match.mask->src);
   7319 				return I40E_ERR_CONFIG;
   7320 			}
   7321 		}
   7322 
   7323 		if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
   7324 			dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
   7325 			return I40E_ERR_CONFIG;
   7326 		}
   7327 		filter->dst_ipv4 = match.key->dst;
   7328 		filter->src_ipv4 = match.key->src;
   7329 	}
   7330 
   7331 	if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
   7332 		struct flow_match_ipv6_addrs match;
   7333 
   7334 		flow_rule_match_ipv6_addrs(rule, &match);
   7335 
   7336 		/* src and dest IPV6 address should not be LOOPBACK
   7337 		 * (0:0:0:0:0:0:0:1), which can be represented as ::1
   7338 		 */
   7339 		if (ipv6_addr_loopback(&match.key->dst) ||
   7340 		    ipv6_addr_loopback(&match.key->src)) {
   7341 			dev_err(&pf->pdev->dev,
   7342 				"Bad ipv6, addr is LOOPBACK\n");
   7343 			return I40E_ERR_CONFIG;
   7344 		}
   7345 		if (!ipv6_addr_any(&match.mask->dst) ||
   7346 		    !ipv6_addr_any(&match.mask->src))
   7347 			field_flags |= I40E_CLOUD_FIELD_IIP;
   7348 
   7349 		memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
   7350 		       sizeof(filter->src_ipv6));
   7351 		memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
   7352 		       sizeof(filter->dst_ipv6));
   7353 	}
   7354 
   7355 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
   7356 		struct flow_match_ports match;
   7357 
   7358 		flow_rule_match_ports(rule, &match);
   7359 		if (match.mask->src) {
   7360 			if (match.mask->src == cpu_to_be16(0xffff)) {
   7361 				field_flags |= I40E_CLOUD_FIELD_IIP;
   7362 			} else {
   7363 				dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
   7364 					be16_to_cpu(match.mask->src));
   7365 				return I40E_ERR_CONFIG;
   7366 			}
   7367 		}
   7368 
   7369 		if (match.mask->dst) {
   7370 			if (match.mask->dst == cpu_to_be16(0xffff)) {
   7371 				field_flags |= I40E_CLOUD_FIELD_IIP;
   7372 			} else {
   7373 				dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
   7374 					be16_to_cpu(match.mask->dst));
   7375 				return I40E_ERR_CONFIG;
   7376 			}
   7377 		}
   7378 
   7379 		filter->dst_port = match.key->dst;
   7380 		filter->src_port = match.key->src;
   7381 
   7382 		switch (filter->ip_proto) {
   7383 		case IPPROTO_TCP:
   7384 		case IPPROTO_UDP:
   7385 			break;
   7386 		default:
   7387 			dev_err(&pf->pdev->dev,
   7388 				"Only UDP and TCP transport are supported\n");
   7389 			return -EINVAL;
   7390 		}
   7391 	}
   7392 	filter->flags = field_flags;
   7393 	return 0;
   7394 }
   7395 
   7396 /**
   7397  * i40e_handle_tclass: Forward to a traffic class on the device
   7398  * @vsi: Pointer to VSI
   7399  * @tc: traffic class index on the device
   7400  * @filter: Pointer to cloud filter structure
   7401  *
   7402  **/
   7403 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
   7404 			      struct i40e_cloud_filter *filter)
   7405 {
   7406 	struct i40e_channel *ch, *ch_tmp;
   7407 
   7408 	/* direct to a traffic class on the same device */
   7409 	if (tc == 0) {
   7410 		filter->seid = vsi->seid;
   7411 		return 0;
   7412 	} else if (vsi->tc_config.enabled_tc & BIT(tc)) {
   7413 		if (!filter->dst_port) {
   7414 			dev_err(&vsi->back->pdev->dev,
   7415 				"Specify destination port to direct to traffic class that is not default\n");
   7416 			return -EINVAL;
   7417 		}
   7418 		if (list_empty(&vsi->ch_list))
   7419 			return -EINVAL;
   7420 		list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
   7421 					 list) {
   7422 			if (ch->seid == vsi->tc_seid_map[tc])
   7423 				filter->seid = ch->seid;
   7424 		}
   7425 		return 0;
   7426 	}
   7427 	dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
   7428 	return -EINVAL;
   7429 }
   7430 
   7431 /**
   7432  * i40e_configure_clsflower - Configure tc flower filters
   7433  * @vsi: Pointer to VSI
   7434  * @cls_flower: Pointer to struct tc_cls_flower_offload
   7435  *
   7436  **/
   7437 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
   7438 				    struct tc_cls_flower_offload *cls_flower)
   7439 {
   7440 	int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
   7441 	struct i40e_cloud_filter *filter = NULL;
   7442 	struct i40e_pf *pf = vsi->back;
   7443 	int err = 0;
   7444 
   7445 	if (tc < 0) {
   7446 		dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
   7447 		return -EOPNOTSUPP;
   7448 	}
   7449 
   7450 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
   7451 	    test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
   7452 		return -EBUSY;
   7453 
   7454 	if (pf->fdir_pf_active_filters ||
   7455 	    (!hlist_empty(&pf->fdir_filter_list))) {
   7456 		dev_err(&vsi->back->pdev->dev,
   7457 			"Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
   7458 		return -EINVAL;
   7459 	}
   7460 
   7461 	if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
   7462 		dev_err(&vsi->back->pdev->dev,
   7463 			"Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
   7464 		vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
   7465 		vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
   7466 	}
   7467 
   7468 	filter = kzalloc(sizeof(*filter), GFP_KERNEL);
   7469 	if (!filter)
   7470 		return -ENOMEM;
   7471 
   7472 	filter->cookie = cls_flower->cookie;
   7473 
   7474 	err = i40e_parse_cls_flower(vsi, cls_flower, filter);
   7475 	if (err < 0)
   7476 		goto err;
   7477 
   7478 	err = i40e_handle_tclass(vsi, tc, filter);
   7479 	if (err < 0)
   7480 		goto err;
   7481 
   7482 	/* Add cloud filter */
   7483 	if (filter->dst_port)
   7484 		err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
   7485 	else
   7486 		err = i40e_add_del_cloud_filter(vsi, filter, true);
   7487 
   7488 	if (err) {
   7489 		dev_err(&pf->pdev->dev,
   7490 			"Failed to add cloud filter, err %s\n",
   7491 			i40e_stat_str(&pf->hw, err));
   7492 		goto err;
   7493 	}
   7494 
   7495 	/* add filter to the ordered list */
   7496 	INIT_HLIST_NODE(&filter->cloud_node);
   7497 
   7498 	hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
   7499 
   7500 	pf->num_cloud_filters++;
   7501 
   7502 	return err;
   7503 err:
   7504 	kfree(filter);
   7505 	return err;
   7506 }
   7507 
   7508 /**
   7509  * i40e_find_cloud_filter - Find the could filter in the list
   7510  * @vsi: Pointer to VSI
   7511  * @cookie: filter specific cookie
   7512  *
   7513  **/
   7514 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
   7515 							unsigned long *cookie)
   7516 {
   7517 	struct i40e_cloud_filter *filter = NULL;
   7518 	struct hlist_node *node2;
   7519 
   7520 	hlist_for_each_entry_safe(filter, node2,
   7521 				  &vsi->back->cloud_filter_list, cloud_node)
   7522 		if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
   7523 			return filter;
   7524 	return NULL;
   7525 }
   7526 
   7527 /**
   7528  * i40e_delete_clsflower - Remove tc flower filters
   7529  * @vsi: Pointer to VSI
   7530  * @cls_flower: Pointer to struct tc_cls_flower_offload
   7531  *
   7532  **/
   7533 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
   7534 				 struct tc_cls_flower_offload *cls_flower)
   7535 {
   7536 	struct i40e_cloud_filter *filter = NULL;
   7537 	struct i40e_pf *pf = vsi->back;
   7538 	int err = 0;
   7539 
   7540 	filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
   7541 
   7542 	if (!filter)
   7543 		return -EINVAL;
   7544 
   7545 	hash_del(&filter->cloud_node);
   7546 
   7547 	if (filter->dst_port)
   7548 		err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
   7549 	else
   7550 		err = i40e_add_del_cloud_filter(vsi, filter, false);
   7551 
   7552 	kfree(filter);
   7553 	if (err) {
   7554 		dev_err(&pf->pdev->dev,
   7555 			"Failed to delete cloud filter, err %s\n",
   7556 			i40e_stat_str(&pf->hw, err));
   7557 		return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
   7558 	}
   7559 
   7560 	pf->num_cloud_filters--;
   7561 	if (!pf->num_cloud_filters)
   7562 		if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
   7563 		    !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
   7564 			pf->flags |= I40E_FLAG_FD_SB_ENABLED;
   7565 			pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
   7566 			pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
   7567 		}
   7568 	return 0;
   7569 }
   7570 
   7571 /**
   7572  * i40e_setup_tc_cls_flower - flower classifier offloads
   7573  * @netdev: net device to configure
   7574  * @type_data: offload data
   7575  **/
   7576 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
   7577 				    struct tc_cls_flower_offload *cls_flower)
   7578 {
   7579 	struct i40e_vsi *vsi = np->vsi;
   7580 
   7581 	switch (cls_flower->command) {
   7582 	case TC_CLSFLOWER_REPLACE:
   7583 		return i40e_configure_clsflower(vsi, cls_flower);
   7584 	case TC_CLSFLOWER_DESTROY:
   7585 		return i40e_delete_clsflower(vsi, cls_flower);
   7586 	case TC_CLSFLOWER_STATS:
   7587 		return -EOPNOTSUPP;
   7588 	default:
   7589 		return -EOPNOTSUPP;
   7590 	}
   7591 }
   7592 
   7593 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
   7594 				  void *cb_priv)
   7595 {
   7596 	struct i40e_netdev_priv *np = cb_priv;
   7597 
   7598 	if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
   7599 		return -EOPNOTSUPP;
   7600 
   7601 	switch (type) {
   7602 	case TC_SETUP_CLSFLOWER:
   7603 		return i40e_setup_tc_cls_flower(np, type_data);
   7604 
   7605 	default:
   7606 		return -EOPNOTSUPP;
   7607 	}
   7608 }
   7609 
   7610 static int i40e_setup_tc_block(struct net_device *dev,
   7611 			       struct tc_block_offload *f)
   7612 {
   7613 	struct i40e_netdev_priv *np = netdev_priv(dev);
   7614 
   7615 	if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
   7616 		return -EOPNOTSUPP;
   7617 
   7618 	switch (f->command) {
   7619 	case TC_BLOCK_BIND:
   7620 		return tcf_block_cb_register(f->block, i40e_setup_tc_block_cb,
   7621 					     np, np, f->extack);
   7622 	case TC_BLOCK_UNBIND:
   7623 		tcf_block_cb_unregister(f->block, i40e_setup_tc_block_cb, np);
   7624 		return 0;
   7625 	default:
   7626 		return -EOPNOTSUPP;
   7627 	}
   7628 }
   7629 
   7630 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
   7631 			   void *type_data)
   7632 {
   7633 	switch (type) {
   7634 	case TC_SETUP_QDISC_MQPRIO:
   7635 		return i40e_setup_tc(netdev, type_data);
   7636 	case TC_SETUP_BLOCK:
   7637 		return i40e_setup_tc_block(netdev, type_data);
   7638 	default:
   7639 		return -EOPNOTSUPP;
   7640 	}
   7641 }
   7642 
   7643 /**
   7644  * i40e_open - Called when a network interface is made active
   7645  * @netdev: network interface device structure
   7646  *
   7647  * The open entry point is called when a network interface is made
   7648  * active by the system (IFF_UP).  At this point all resources needed
   7649  * for transmit and receive operations are allocated, the interrupt
   7650  * handler is registered with the OS, the netdev watchdog subtask is
   7651  * enabled, and the stack is notified that the interface is ready.
   7652  *
   7653  * Returns 0 on success, negative value on failure
   7654  **/
   7655 int i40e_open(struct net_device *netdev)
   7656 {
   7657 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   7658 	struct i40e_vsi *vsi = np->vsi;
   7659 	struct i40e_pf *pf = vsi->back;
   7660 	int err;
   7661 
   7662 	/* disallow open during test or if eeprom is broken */
   7663 	if (test_bit(__I40E_TESTING, pf->state) ||
   7664 	    test_bit(__I40E_BAD_EEPROM, pf->state))
   7665 		return -EBUSY;
   7666 
   7667 	netif_carrier_off(netdev);
   7668 
   7669 	if (i40e_force_link_state(pf, true))
   7670 		return -EAGAIN;
   7671 
   7672 	err = i40e_vsi_open(vsi);
   7673 	if (err)
   7674 		return err;
   7675 
   7676 	/* configure global TSO hardware offload settings */
   7677 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
   7678 						       TCP_FLAG_FIN) >> 16);
   7679 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
   7680 						       TCP_FLAG_FIN |
   7681 						       TCP_FLAG_CWR) >> 16);
   7682 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
   7683 
   7684 	udp_tunnel_get_rx_info(netdev);
   7685 
   7686 	return 0;
   7687 }
   7688 
   7689 /**
   7690  * i40e_vsi_open -
   7691  * @vsi: the VSI to open
   7692  *
   7693  * Finish initialization of the VSI.
   7694  *
   7695  * Returns 0 on success, negative value on failure
   7696  *
   7697  * Note: expects to be called while under rtnl_lock()
   7698  **/
   7699 int i40e_vsi_open(struct i40e_vsi *vsi)
   7700 {
   7701 	struct i40e_pf *pf = vsi->back;
   7702 	char int_name[I40E_INT_NAME_STR_LEN];
   7703 	int err;
   7704 
   7705 	/* allocate descriptors */
   7706 	err = i40e_vsi_setup_tx_resources(vsi);
   7707 	if (err)
   7708 		goto err_setup_tx;
   7709 	err = i40e_vsi_setup_rx_resources(vsi);
   7710 	if (err)
   7711 		goto err_setup_rx;
   7712 
   7713 	err = i40e_vsi_configure(vsi);
   7714 	if (err)
   7715 		goto err_setup_rx;
   7716 
   7717 	if (vsi->netdev) {
   7718 		snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
   7719 			 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
   7720 		err = i40e_vsi_request_irq(vsi, int_name);
   7721 		if (err)
   7722 			goto err_setup_rx;
   7723 
   7724 		/* Notify the stack of the actual queue counts. */
   7725 		err = netif_set_real_num_tx_queues(vsi->netdev,
   7726 						   vsi->num_queue_pairs);
   7727 		if (err)
   7728 			goto err_set_queues;
   7729 
   7730 		err = netif_set_real_num_rx_queues(vsi->netdev,
   7731 						   vsi->num_queue_pairs);
   7732 		if (err)
   7733 			goto err_set_queues;
   7734 
   7735 	} else if (vsi->type == I40E_VSI_FDIR) {
   7736 		snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
   7737 			 dev_driver_string(&pf->pdev->dev),
   7738 			 dev_name(&pf->pdev->dev));
   7739 		err = i40e_vsi_request_irq(vsi, int_name);
   7740 
   7741 	} else {
   7742 		err = -EINVAL;
   7743 		goto err_setup_rx;
   7744 	}
   7745 
   7746 	err = i40e_up_complete(vsi);
   7747 	if (err)
   7748 		goto err_up_complete;
   7749 
   7750 	return 0;
   7751 
   7752 err_up_complete:
   7753 	i40e_down(vsi);
   7754 err_set_queues:
   7755 	i40e_vsi_free_irq(vsi);
   7756 err_setup_rx:
   7757 	i40e_vsi_free_rx_resources(vsi);
   7758 err_setup_tx:
   7759 	i40e_vsi_free_tx_resources(vsi);
   7760 	if (vsi == pf->vsi[pf->lan_vsi])
   7761 		i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
   7762 
   7763 	return err;
   7764 }
   7765 
   7766 /**
   7767  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
   7768  * @pf: Pointer to PF
   7769  *
   7770  * This function destroys the hlist where all the Flow Director
   7771  * filters were saved.
   7772  **/
   7773 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
   7774 {
   7775 	struct i40e_fdir_filter *filter;
   7776 	struct i40e_flex_pit *pit_entry, *tmp;
   7777 	struct hlist_node *node2;
   7778 
   7779 	hlist_for_each_entry_safe(filter, node2,
   7780 				  &pf->fdir_filter_list, fdir_node) {
   7781 		hlist_del(&filter->fdir_node);
   7782 		kfree(filter);
   7783 	}
   7784 
   7785 	list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
   7786 		list_del(&pit_entry->list);
   7787 		kfree(pit_entry);
   7788 	}
   7789 	INIT_LIST_HEAD(&pf->l3_flex_pit_list);
   7790 
   7791 	list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
   7792 		list_del(&pit_entry->list);
   7793 		kfree(pit_entry);
   7794 	}
   7795 	INIT_LIST_HEAD(&pf->l4_flex_pit_list);
   7796 
   7797 	pf->fdir_pf_active_filters = 0;
   7798 	pf->fd_tcp4_filter_cnt = 0;
   7799 	pf->fd_udp4_filter_cnt = 0;
   7800 	pf->fd_sctp4_filter_cnt = 0;
   7801 	pf->fd_ip4_filter_cnt = 0;
   7802 
   7803 	/* Reprogram the default input set for TCP/IPv4 */
   7804 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
   7805 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
   7806 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
   7807 
   7808 	/* Reprogram the default input set for UDP/IPv4 */
   7809 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
   7810 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
   7811 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
   7812 
   7813 	/* Reprogram the default input set for SCTP/IPv4 */
   7814 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
   7815 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
   7816 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
   7817 
   7818 	/* Reprogram the default input set for Other/IPv4 */
   7819 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
   7820 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
   7821 
   7822 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
   7823 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
   7824 }
   7825 
   7826 /**
   7827  * i40e_cloud_filter_exit - Cleans up the cloud filters
   7828  * @pf: Pointer to PF
   7829  *
   7830  * This function destroys the hlist where all the cloud filters
   7831  * were saved.
   7832  **/
   7833 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
   7834 {
   7835 	struct i40e_cloud_filter *cfilter;
   7836 	struct hlist_node *node;
   7837 
   7838 	hlist_for_each_entry_safe(cfilter, node,
   7839 				  &pf->cloud_filter_list, cloud_node) {
   7840 		hlist_del(&cfilter->cloud_node);
   7841 		kfree(cfilter);
   7842 	}
   7843 	pf->num_cloud_filters = 0;
   7844 
   7845 	if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
   7846 	    !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
   7847 		pf->flags |= I40E_FLAG_FD_SB_ENABLED;
   7848 		pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
   7849 		pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
   7850 	}
   7851 }
   7852 
   7853 /**
   7854  * i40e_close - Disables a network interface
   7855  * @netdev: network interface device structure
   7856  *
   7857  * The close entry point is called when an interface is de-activated
   7858  * by the OS.  The hardware is still under the driver's control, but
   7859  * this netdev interface is disabled.
   7860  *
   7861  * Returns 0, this is not allowed to fail
   7862  **/
   7863 int i40e_close(struct net_device *netdev)
   7864 {
   7865 	struct i40e_netdev_priv *np = netdev_priv(netdev);
   7866 	struct i40e_vsi *vsi = np->vsi;
   7867 
   7868 	i40e_vsi_close(vsi);
   7869 
   7870 	return 0;
   7871 }
   7872 
   7873 /**
   7874  * i40e_do_reset - Start a PF or Core Reset sequence
   7875  * @pf: board private structure
   7876  * @reset_flags: which reset is requested
   7877  * @lock_acquired: indicates whether or not the lock has been acquired
   7878  * before this function was called.
   7879  *
   7880  * The essential difference in resets is that the PF Reset
   7881  * doesn't clear the packet buffers, doesn't reset the PE
   7882  * firmware, and doesn't bother the other PFs on the chip.
   7883  **/
   7884 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
   7885 {
   7886 	u32 val;
   7887 
   7888 	WARN_ON(in_interrupt());
   7889 
   7890 
   7891 	/* do the biggest reset indicated */
   7892 	if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
   7893 
   7894 		/* Request a Global Reset
   7895 		 *
   7896 		 * This will start the chip's countdown to the actual full
   7897 		 * chip reset event, and a warning interrupt to be sent
   7898 		 * to all PFs, including the requestor.  Our handler
   7899 		 * for the warning interrupt will deal with the shutdown
   7900 		 * and recovery of the switch setup.
   7901 		 */
   7902 		dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
   7903 		val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
   7904 		val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
   7905 		wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
   7906 
   7907 	} else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
   7908 
   7909 		/* Request a Core Reset
   7910 		 *
   7911 		 * Same as Global Reset, except does *not* include the MAC/PHY
   7912 		 */
   7913 		dev_dbg(&pf->pdev->dev, "CoreR requested\n");
   7914 		val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
   7915 		val |= I40E_GLGEN_RTRIG_CORER_MASK;
   7916 		wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
   7917 		i40e_flush(&pf->hw);
   7918 
   7919 	} else if (reset_flags & I40E_PF_RESET_FLAG) {
   7920 
   7921 		/* Request a PF Reset
   7922 		 *
   7923 		 * Resets only the PF-specific registers
   7924 		 *
   7925 		 * This goes directly to the tear-down and rebuild of
   7926 		 * the switch, since we need to do all the recovery as
   7927 		 * for the Core Reset.
   7928 		 */
   7929 		dev_dbg(&pf->pdev->dev, "PFR requested\n");
   7930 		i40e_handle_reset_warning(pf, lock_acquired);
   7931 
   7932 	} else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
   7933 		int v;
   7934 
   7935 		/* Find the VSI(s) that requested a re-init */
   7936 		dev_info(&pf->pdev->dev,
   7937 			 "VSI reinit requested\n");
   7938 		for (v = 0; v < pf->num_alloc_vsi; v++) {
   7939 			struct i40e_vsi *vsi = pf->vsi[v];
   7940 
   7941 			if (vsi != NULL &&
   7942 			    test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
   7943 					       vsi->state))
   7944 				i40e_vsi_reinit_locked(pf->vsi[v]);
   7945 		}
   7946 	} else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
   7947 		int v;
   7948 
   7949 		/* Find the VSI(s) that needs to be brought down */
   7950 		dev_info(&pf->pdev->dev, "VSI down requested\n");
   7951 		for (v = 0; v < pf->num_alloc_vsi; v++) {
   7952 			struct i40e_vsi *vsi = pf->vsi[v];
   7953 
   7954 			if (vsi != NULL &&
   7955 			    test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
   7956 					       vsi->state)) {
   7957 				set_bit(__I40E_VSI_DOWN, vsi->state);
   7958 				i40e_down(vsi);
   7959 			}
   7960 		}
   7961 	} else {
   7962 		dev_info(&pf->pdev->dev,
   7963 			 "bad reset request 0x%08x\n", reset_flags);
   7964 	}
   7965 }
   7966 
   7967 #ifdef CONFIG_I40E_DCB
   7968 /**
   7969  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
   7970  * @pf: board private structure
   7971  * @old_cfg: current DCB config
   7972  * @new_cfg: new DCB config
   7973  **/
   7974 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
   7975 			    struct i40e_dcbx_config *old_cfg,
   7976 			    struct i40e_dcbx_config *new_cfg)
   7977 {
   7978 	bool need_reconfig = false;
   7979 
   7980 	/* Check if ETS configuration has changed */
   7981 	if (memcmp(&new_cfg->etscfg,
   7982 		   &old_cfg->etscfg,
   7983 		   sizeof(new_cfg->etscfg))) {
   7984 		/* If Priority Table has changed reconfig is needed */
   7985 		if (memcmp(&new_cfg->etscfg.prioritytable,
   7986 			   &old_cfg->etscfg.prioritytable,
   7987 			   sizeof(new_cfg->etscfg.prioritytable))) {
   7988 			need_reconfig = true;
   7989 			dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
   7990 		}
   7991 
   7992 		if (memcmp(&new_cfg->etscfg.tcbwtable,
   7993 			   &old_cfg->etscfg.tcbwtable,
   7994 			   sizeof(new_cfg->etscfg.tcbwtable)))
   7995 			dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
   7996 
   7997 		if (memcmp(&new_cfg->etscfg.tsatable,
   7998 			   &old_cfg->etscfg.tsatable,
   7999 			   sizeof(new_cfg->etscfg.tsatable)))
   8000 			dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
   8001 	}
   8002 
   8003 	/* Check if PFC configuration has changed */
   8004 	if (memcmp(&new_cfg->pfc,
   8005 		   &old_cfg->pfc,
   8006 		   sizeof(new_cfg->pfc))) {
   8007 		need_reconfig = true;
   8008 		dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
   8009 	}
   8010 
   8011 	/* Check if APP Table has changed */
   8012 	if (memcmp(&new_cfg->app,
   8013 		   &old_cfg->app,
   8014 		   sizeof(new_cfg->app))) {
   8015 		need_reconfig = true;
   8016 		dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
   8017 	}
   8018 
   8019 	dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
   8020 	return need_reconfig;
   8021 }
   8022 
   8023 /**
   8024  * i40e_handle_lldp_event - Handle LLDP Change MIB event
   8025  * @pf: board private structure
   8026  * @e: event info posted on ARQ
   8027  **/
   8028 static int i40e_handle_lldp_event(struct i40e_pf *pf,
   8029 				  struct i40e_arq_event_info *e)
   8030 {
   8031 	struct i40e_aqc_lldp_get_mib *mib =
   8032 		(struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
   8033 	struct i40e_hw *hw = &pf->hw;
   8034 	struct i40e_dcbx_config tmp_dcbx_cfg;
   8035 	bool need_reconfig = false;
   8036 	int ret = 0;
   8037 	u8 type;
   8038 
   8039 	/* Not DCB capable or capability disabled */
   8040 	if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
   8041 		return ret;
   8042 
   8043 	/* Ignore if event is not for Nearest Bridge */
   8044 	type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
   8045 		& I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
   8046 	dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
   8047 	if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
   8048 		return ret;
   8049 
   8050 	/* Check MIB Type and return if event for Remote MIB update */
   8051 	type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
   8052 	dev_dbg(&pf->pdev->dev,
   8053 		"LLDP event mib type %s\n", type ? "remote" : "local");
   8054 	if (type == I40E_AQ_LLDP_MIB_REMOTE) {
   8055 		/* Update the remote cached instance and return */
   8056 		ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
   8057 				I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
   8058 				&hw->remote_dcbx_config);
   8059 		goto exit;
   8060 	}
   8061 
   8062 	/* Store the old configuration */
   8063 	tmp_dcbx_cfg = hw->local_dcbx_config;
   8064 
   8065 	/* Reset the old DCBx configuration data */
   8066 	memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
   8067 	/* Get updated DCBX data from firmware */
   8068 	ret = i40e_get_dcb_config(&pf->hw);
   8069 	if (ret) {
   8070 		dev_info(&pf->pdev->dev,
   8071 			 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
   8072 			 i40e_stat_str(&pf->hw, ret),
   8073 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   8074 		goto exit;
   8075 	}
   8076 
   8077 	/* No change detected in DCBX configs */
   8078 	if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
   8079 		    sizeof(tmp_dcbx_cfg))) {
   8080 		dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
   8081 		goto exit;
   8082 	}
   8083 
   8084 	need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
   8085 					       &hw->local_dcbx_config);
   8086 
   8087 	i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
   8088 
   8089 	if (!need_reconfig)
   8090 		goto exit;
   8091 
   8092 	/* Enable DCB tagging only when more than one TC */
   8093 	if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
   8094 		pf->flags |= I40E_FLAG_DCB_ENABLED;
   8095 	else
   8096 		pf->flags &= ~I40E_FLAG_DCB_ENABLED;
   8097 
   8098 	set_bit(__I40E_PORT_SUSPENDED, pf->state);
   8099 	/* Reconfiguration needed quiesce all VSIs */
   8100 	i40e_pf_quiesce_all_vsi(pf);
   8101 
   8102 	/* Changes in configuration update VEB/VSI */
   8103 	i40e_dcb_reconfigure(pf);
   8104 
   8105 	ret = i40e_resume_port_tx(pf);
   8106 
   8107 	clear_bit(__I40E_PORT_SUSPENDED, pf->state);
   8108 	/* In case of error no point in resuming VSIs */
   8109 	if (ret)
   8110 		goto exit;
   8111 
   8112 	/* Wait for the PF's queues to be disabled */
   8113 	ret = i40e_pf_wait_queues_disabled(pf);
   8114 	if (ret) {
   8115 		/* Schedule PF reset to recover */
   8116 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
   8117 		i40e_service_event_schedule(pf);
   8118 	} else {
   8119 		i40e_pf_unquiesce_all_vsi(pf);
   8120 		set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
   8121 		set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
   8122 	}
   8123 
   8124 exit:
   8125 	return ret;
   8126 }
   8127 #endif /* CONFIG_I40E_DCB */
   8128 
   8129 /**
   8130  * i40e_do_reset_safe - Protected reset path for userland calls.
   8131  * @pf: board private structure
   8132  * @reset_flags: which reset is requested
   8133  *
   8134  **/
   8135 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
   8136 {
   8137 	rtnl_lock();
   8138 	i40e_do_reset(pf, reset_flags, true);
   8139 	rtnl_unlock();
   8140 }
   8141 
   8142 /**
   8143  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
   8144  * @pf: board private structure
   8145  * @e: event info posted on ARQ
   8146  *
   8147  * Handler for LAN Queue Overflow Event generated by the firmware for PF
   8148  * and VF queues
   8149  **/
   8150 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
   8151 					   struct i40e_arq_event_info *e)
   8152 {
   8153 	struct i40e_aqc_lan_overflow *data =
   8154 		(struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
   8155 	u32 queue = le32_to_cpu(data->prtdcb_rupto);
   8156 	u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
   8157 	struct i40e_hw *hw = &pf->hw;
   8158 	struct i40e_vf *vf;
   8159 	u16 vf_id;
   8160 
   8161 	dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
   8162 		queue, qtx_ctl);
   8163 
   8164 	/* Queue belongs to VF, find the VF and issue VF reset */
   8165 	if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
   8166 	    >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
   8167 		vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
   8168 			 >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
   8169 		vf_id -= hw->func_caps.vf_base_id;
   8170 		vf = &pf->vf[vf_id];
   8171 		i40e_vc_notify_vf_reset(vf);
   8172 		/* Allow VF to process pending reset notification */
   8173 		msleep(20);
   8174 		i40e_reset_vf(vf, false);
   8175 	}
   8176 }
   8177 
   8178 /**
   8179  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
   8180  * @pf: board private structure
   8181  **/
   8182 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
   8183 {
   8184 	u32 val, fcnt_prog;
   8185 
   8186 	val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
   8187 	fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
   8188 	return fcnt_prog;
   8189 }
   8190 
   8191 /**
   8192  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
   8193  * @pf: board private structure
   8194  **/
   8195 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
   8196 {
   8197 	u32 val, fcnt_prog;
   8198 
   8199 	val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
   8200 	fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
   8201 		    ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
   8202 		      I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
   8203 	return fcnt_prog;
   8204 }
   8205 
   8206 /**
   8207  * i40e_get_global_fd_count - Get total FD filters programmed on device
   8208  * @pf: board private structure
   8209  **/
   8210 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
   8211 {
   8212 	u32 val, fcnt_prog;
   8213 
   8214 	val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
   8215 	fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
   8216 		    ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
   8217 		     I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
   8218 	return fcnt_prog;
   8219 }
   8220 
   8221 /**
   8222  * i40e_reenable_fdir_sb - Restore FDir SB capability
   8223  * @pf: board private structure
   8224  **/
   8225 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
   8226 {
   8227 	if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
   8228 		if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
   8229 		    (I40E_DEBUG_FD & pf->hw.debug_mask))
   8230 			dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
   8231 }
   8232 
   8233 /**
   8234  * i40e_reenable_fdir_atr - Restore FDir ATR capability
   8235  * @pf: board private structure
   8236  **/
   8237 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
   8238 {
   8239 	if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
   8240 		/* ATR uses the same filtering logic as SB rules. It only
   8241 		 * functions properly if the input set mask is at the default
   8242 		 * settings. It is safe to restore the default input set
   8243 		 * because there are no active TCPv4 filter rules.
   8244 		 */
   8245 		i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
   8246 					I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
   8247 					I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
   8248 
   8249 		if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
   8250 		    (I40E_DEBUG_FD & pf->hw.debug_mask))
   8251 			dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
   8252 	}
   8253 }
   8254 
   8255 /**
   8256  * i40e_delete_invalid_filter - Delete an invalid FDIR filter
   8257  * @pf: board private structure
   8258  * @filter: FDir filter to remove
   8259  */
   8260 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
   8261 				       struct i40e_fdir_filter *filter)
   8262 {
   8263 	/* Update counters */
   8264 	pf->fdir_pf_active_filters--;
   8265 	pf->fd_inv = 0;
   8266 
   8267 	switch (filter->flow_type) {
   8268 	case TCP_V4_FLOW:
   8269 		pf->fd_tcp4_filter_cnt--;
   8270 		break;
   8271 	case UDP_V4_FLOW:
   8272 		pf->fd_udp4_filter_cnt--;
   8273 		break;
   8274 	case SCTP_V4_FLOW:
   8275 		pf->fd_sctp4_filter_cnt--;
   8276 		break;
   8277 	case IP_USER_FLOW:
   8278 		switch (filter->ip4_proto) {
   8279 		case IPPROTO_TCP:
   8280 			pf->fd_tcp4_filter_cnt--;
   8281 			break;
   8282 		case IPPROTO_UDP:
   8283 			pf->fd_udp4_filter_cnt--;
   8284 			break;
   8285 		case IPPROTO_SCTP:
   8286 			pf->fd_sctp4_filter_cnt--;
   8287 			break;
   8288 		case IPPROTO_IP:
   8289 			pf->fd_ip4_filter_cnt--;
   8290 			break;
   8291 		}
   8292 		break;
   8293 	}
   8294 
   8295 	/* Remove the filter from the list and free memory */
   8296 	hlist_del(&filter->fdir_node);
   8297 	kfree(filter);
   8298 }
   8299 
   8300 /**
   8301  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
   8302  * @pf: board private structure
   8303  **/
   8304 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
   8305 {
   8306 	struct i40e_fdir_filter *filter;
   8307 	u32 fcnt_prog, fcnt_avail;
   8308 	struct hlist_node *node;
   8309 
   8310 	if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
   8311 		return;
   8312 
   8313 	/* Check if we have enough room to re-enable FDir SB capability. */
   8314 	fcnt_prog = i40e_get_global_fd_count(pf);
   8315 	fcnt_avail = pf->fdir_pf_filter_count;
   8316 	if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
   8317 	    (pf->fd_add_err == 0) ||
   8318 	    (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
   8319 		i40e_reenable_fdir_sb(pf);
   8320 
   8321 	/* We should wait for even more space before re-enabling ATR.
   8322 	 * Additionally, we cannot enable ATR as long as we still have TCP SB
   8323 	 * rules active.
   8324 	 */
   8325 	if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
   8326 	    (pf->fd_tcp4_filter_cnt == 0))
   8327 		i40e_reenable_fdir_atr(pf);
   8328 
   8329 	/* if hw had a problem adding a filter, delete it */
   8330 	if (pf->fd_inv > 0) {
   8331 		hlist_for_each_entry_safe(filter, node,
   8332 					  &pf->fdir_filter_list, fdir_node)
   8333 			if (filter->fd_id == pf->fd_inv)
   8334 				i40e_delete_invalid_filter(pf, filter);
   8335 	}
   8336 }
   8337 
   8338 #define I40E_MIN_FD_FLUSH_INTERVAL 10
   8339 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
   8340 /**
   8341  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
   8342  * @pf: board private structure
   8343  **/
   8344 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
   8345 {
   8346 	unsigned long min_flush_time;
   8347 	int flush_wait_retry = 50;
   8348 	bool disable_atr = false;
   8349 	int fd_room;
   8350 	int reg;
   8351 
   8352 	if (!time_after(jiffies, pf->fd_flush_timestamp +
   8353 				 (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
   8354 		return;
   8355 
   8356 	/* If the flush is happening too quick and we have mostly SB rules we
   8357 	 * should not re-enable ATR for some time.
   8358 	 */
   8359 	min_flush_time = pf->fd_flush_timestamp +
   8360 			 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
   8361 	fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
   8362 
   8363 	if (!(time_after(jiffies, min_flush_time)) &&
   8364 	    (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
   8365 		if (I40E_DEBUG_FD & pf->hw.debug_mask)
   8366 			dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
   8367 		disable_atr = true;
   8368 	}
   8369 
   8370 	pf->fd_flush_timestamp = jiffies;
   8371 	set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
   8372 	/* flush all filters */
   8373 	wr32(&pf->hw, I40E_PFQF_CTL_1,
   8374 	     I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
   8375 	i40e_flush(&pf->hw);
   8376 	pf->fd_flush_cnt++;
   8377 	pf->fd_add_err = 0;
   8378 	do {
   8379 		/* Check FD flush status every 5-6msec */
   8380 		usleep_range(5000, 6000);
   8381 		reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
   8382 		if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
   8383 			break;
   8384 	} while (flush_wait_retry--);
   8385 	if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
   8386 		dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
   8387 	} else {
   8388 		/* replay sideband filters */
   8389 		i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
   8390 		if (!disable_atr && !pf->fd_tcp4_filter_cnt)
   8391 			clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
   8392 		clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
   8393 		if (I40E_DEBUG_FD & pf->hw.debug_mask)
   8394 			dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
   8395 	}
   8396 }
   8397 
   8398 /**
   8399  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
   8400  * @pf: board private structure
   8401  **/
   8402 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
   8403 {
   8404 	return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
   8405 }
   8406 
   8407 /* We can see up to 256 filter programming desc in transit if the filters are
   8408  * being applied really fast; before we see the first
   8409  * filter miss error on Rx queue 0. Accumulating enough error messages before
   8410  * reacting will make sure we don't cause flush too often.
   8411  */
   8412 #define I40E_MAX_FD_PROGRAM_ERROR 256
   8413 
   8414 /**
   8415  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
   8416  * @pf: board private structure
   8417  **/
   8418 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
   8419 {
   8420 
   8421 	/* if interface is down do nothing */
   8422 	if (test_bit(__I40E_DOWN, pf->state))
   8423 		return;
   8424 
   8425 	if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
   8426 		i40e_fdir_flush_and_replay(pf);
   8427 
   8428 	i40e_fdir_check_and_reenable(pf);
   8429 
   8430 }
   8431 
   8432 /**
   8433  * i40e_vsi_link_event - notify VSI of a link event
   8434  * @vsi: vsi to be notified
   8435  * @link_up: link up or down
   8436  **/
   8437 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
   8438 {
   8439 	if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
   8440 		return;
   8441 
   8442 	switch (vsi->type) {
   8443 	case I40E_VSI_MAIN:
   8444 		if (!vsi->netdev || !vsi->netdev_registered)
   8445 			break;
   8446 
   8447 		if (link_up) {
   8448 			netif_carrier_on(vsi->netdev);
   8449 			netif_tx_wake_all_queues(vsi->netdev);
   8450 		} else {
   8451 			netif_carrier_off(vsi->netdev);
   8452 			netif_tx_stop_all_queues(vsi->netdev);
   8453 		}
   8454 		break;
   8455 
   8456 	case I40E_VSI_SRIOV:
   8457 	case I40E_VSI_VMDQ2:
   8458 	case I40E_VSI_CTRL:
   8459 	case I40E_VSI_IWARP:
   8460 	case I40E_VSI_MIRROR:
   8461 	default:
   8462 		/* there is no notification for other VSIs */
   8463 		break;
   8464 	}
   8465 }
   8466 
   8467 /**
   8468  * i40e_veb_link_event - notify elements on the veb of a link event
   8469  * @veb: veb to be notified
   8470  * @link_up: link up or down
   8471  **/
   8472 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
   8473 {
   8474 	struct i40e_pf *pf;
   8475 	int i;
   8476 
   8477 	if (!veb || !veb->pf)
   8478 		return;
   8479 	pf = veb->pf;
   8480 
   8481 	/* depth first... */
   8482 	for (i = 0; i < I40E_MAX_VEB; i++)
   8483 		if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
   8484 			i40e_veb_link_event(pf->veb[i], link_up);
   8485 
   8486 	/* ... now the local VSIs */
   8487 	for (i = 0; i < pf->num_alloc_vsi; i++)
   8488 		if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
   8489 			i40e_vsi_link_event(pf->vsi[i], link_up);
   8490 }
   8491 
   8492 /**
   8493  * i40e_link_event - Update netif_carrier status
   8494  * @pf: board private structure
   8495  **/
   8496 static void i40e_link_event(struct i40e_pf *pf)
   8497 {
   8498 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
   8499 	u8 new_link_speed, old_link_speed;
   8500 	i40e_status status;
   8501 	bool new_link, old_link;
   8502 
   8503 	/* set this to force the get_link_status call to refresh state */
   8504 	pf->hw.phy.get_link_info = true;
   8505 	old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
   8506 	status = i40e_get_link_status(&pf->hw, &new_link);
   8507 
   8508 	/* On success, disable temp link polling */
   8509 	if (status == I40E_SUCCESS) {
   8510 		clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
   8511 	} else {
   8512 		/* Enable link polling temporarily until i40e_get_link_status
   8513 		 * returns I40E_SUCCESS
   8514 		 */
   8515 		set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
   8516 		dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
   8517 			status);
   8518 		return;
   8519 	}
   8520 
   8521 	old_link_speed = pf->hw.phy.link_info_old.link_speed;
   8522 	new_link_speed = pf->hw.phy.link_info.link_speed;
   8523 
   8524 	if (new_link == old_link &&
   8525 	    new_link_speed == old_link_speed &&
   8526 	    (test_bit(__I40E_VSI_DOWN, vsi->state) ||
   8527 	     new_link == netif_carrier_ok(vsi->netdev)))
   8528 		return;
   8529 
   8530 	i40e_print_link_message(vsi, new_link);
   8531 
   8532 	/* Notify the base of the switch tree connected to
   8533 	 * the link.  Floating VEBs are not notified.
   8534 	 */
   8535 	if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
   8536 		i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
   8537 	else
   8538 		i40e_vsi_link_event(vsi, new_link);
   8539 
   8540 	if (pf->vf)
   8541 		i40e_vc_notify_link_state(pf);
   8542 
   8543 	if (pf->flags & I40E_FLAG_PTP)
   8544 		i40e_ptp_set_increment(pf);
   8545 }
   8546 
   8547 /**
   8548  * i40e_watchdog_subtask - periodic checks not using event driven response
   8549  * @pf: board private structure
   8550  **/
   8551 static void i40e_watchdog_subtask(struct i40e_pf *pf)
   8552 {
   8553 	int i;
   8554 
   8555 	/* if interface is down do nothing */
   8556 	if (test_bit(__I40E_DOWN, pf->state) ||
   8557 	    test_bit(__I40E_CONFIG_BUSY, pf->state))
   8558 		return;
   8559 
   8560 	/* make sure we don't do these things too often */
   8561 	if (time_before(jiffies, (pf->service_timer_previous +
   8562 				  pf->service_timer_period)))
   8563 		return;
   8564 	pf->service_timer_previous = jiffies;
   8565 
   8566 	if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
   8567 	    test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
   8568 		i40e_link_event(pf);
   8569 
   8570 	/* Update the stats for active netdevs so the network stack
   8571 	 * can look at updated numbers whenever it cares to
   8572 	 */
   8573 	for (i = 0; i < pf->num_alloc_vsi; i++)
   8574 		if (pf->vsi[i] && pf->vsi[i]->netdev)
   8575 			i40e_update_stats(pf->vsi[i]);
   8576 
   8577 	if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
   8578 		/* Update the stats for the active switching components */
   8579 		for (i = 0; i < I40E_MAX_VEB; i++)
   8580 			if (pf->veb[i])
   8581 				i40e_update_veb_stats(pf->veb[i]);
   8582 	}
   8583 
   8584 	i40e_ptp_rx_hang(pf);
   8585 	i40e_ptp_tx_hang(pf);
   8586 }
   8587 
   8588 /**
   8589  * i40e_reset_subtask - Set up for resetting the device and driver
   8590  * @pf: board private structure
   8591  **/
   8592 static void i40e_reset_subtask(struct i40e_pf *pf)
   8593 {
   8594 	u32 reset_flags = 0;
   8595 
   8596 	if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
   8597 		reset_flags |= BIT(__I40E_REINIT_REQUESTED);
   8598 		clear_bit(__I40E_REINIT_REQUESTED, pf->state);
   8599 	}
   8600 	if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
   8601 		reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
   8602 		clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
   8603 	}
   8604 	if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
   8605 		reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
   8606 		clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
   8607 	}
   8608 	if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
   8609 		reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
   8610 		clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
   8611 	}
   8612 	if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
   8613 		reset_flags |= BIT(__I40E_DOWN_REQUESTED);
   8614 		clear_bit(__I40E_DOWN_REQUESTED, pf->state);
   8615 	}
   8616 
   8617 	/* If there's a recovery already waiting, it takes
   8618 	 * precedence before starting a new reset sequence.
   8619 	 */
   8620 	if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
   8621 		i40e_prep_for_reset(pf, false);
   8622 		i40e_reset(pf);
   8623 		i40e_rebuild(pf, false, false);
   8624 	}
   8625 
   8626 	/* If we're already down or resetting, just bail */
   8627 	if (reset_flags &&
   8628 	    !test_bit(__I40E_DOWN, pf->state) &&
   8629 	    !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
   8630 		i40e_do_reset(pf, reset_flags, false);
   8631 	}
   8632 }
   8633 
   8634 /**
   8635  * i40e_handle_link_event - Handle link event
   8636  * @pf: board private structure
   8637  * @e: event info posted on ARQ
   8638  **/
   8639 static void i40e_handle_link_event(struct i40e_pf *pf,
   8640 				   struct i40e_arq_event_info *e)
   8641 {
   8642 	struct i40e_aqc_get_link_status *status =
   8643 		(struct i40e_aqc_get_link_status *)&e->desc.params.raw;
   8644 
   8645 	/* Do a new status request to re-enable LSE reporting
   8646 	 * and load new status information into the hw struct
   8647 	 * This completely ignores any state information
   8648 	 * in the ARQ event info, instead choosing to always
   8649 	 * issue the AQ update link status command.
   8650 	 */
   8651 	i40e_link_event(pf);
   8652 
   8653 	/* Check if module meets thermal requirements */
   8654 	if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
   8655 		dev_err(&pf->pdev->dev,
   8656 			"Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
   8657 		dev_err(&pf->pdev->dev,
   8658 			"Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
   8659 	} else {
   8660 		/* check for unqualified module, if link is down, suppress
   8661 		 * the message if link was forced to be down.
   8662 		 */
   8663 		if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
   8664 		    (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
   8665 		    (!(status->link_info & I40E_AQ_LINK_UP)) &&
   8666 		    (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
   8667 			dev_err(&pf->pdev->dev,
   8668 				"Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
   8669 			dev_err(&pf->pdev->dev,
   8670 				"Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
   8671 		}
   8672 	}
   8673 }
   8674 
   8675 /**
   8676  * i40e_clean_adminq_subtask - Clean the AdminQ rings
   8677  * @pf: board private structure
   8678  **/
   8679 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
   8680 {
   8681 	struct i40e_arq_event_info event;
   8682 	struct i40e_hw *hw = &pf->hw;
   8683 	u16 pending, i = 0;
   8684 	i40e_status ret;
   8685 	u16 opcode;
   8686 	u32 oldval;
   8687 	u32 val;
   8688 
   8689 	/* Do not run clean AQ when PF reset fails */
   8690 	if (test_bit(__I40E_RESET_FAILED, pf->state))
   8691 		return;
   8692 
   8693 	/* check for error indications */
   8694 	val = rd32(&pf->hw, pf->hw.aq.arq.len);
   8695 	oldval = val;
   8696 	if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
   8697 		if (hw->debug_mask & I40E_DEBUG_AQ)
   8698 			dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
   8699 		val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
   8700 	}
   8701 	if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
   8702 		if (hw->debug_mask & I40E_DEBUG_AQ)
   8703 			dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
   8704 		val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
   8705 		pf->arq_overflows++;
   8706 	}
   8707 	if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
   8708 		if (hw->debug_mask & I40E_DEBUG_AQ)
   8709 			dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
   8710 		val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
   8711 	}
   8712 	if (oldval != val)
   8713 		wr32(&pf->hw, pf->hw.aq.arq.len, val);
   8714 
   8715 	val = rd32(&pf->hw, pf->hw.aq.asq.len);
   8716 	oldval = val;
   8717 	if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
   8718 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)
   8719 			dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
   8720 		val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
   8721 	}
   8722 	if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
   8723 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)
   8724 			dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
   8725 		val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
   8726 	}
   8727 	if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
   8728 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)
   8729 			dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
   8730 		val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
   8731 	}
   8732 	if (oldval != val)
   8733 		wr32(&pf->hw, pf->hw.aq.asq.len, val);
   8734 
   8735 	event.buf_len = I40E_MAX_AQ_BUF_SIZE;
   8736 	event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
   8737 	if (!event.msg_buf)
   8738 		return;
   8739 
   8740 	do {
   8741 		ret = i40e_clean_arq_element(hw, &event, &pending);
   8742 		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
   8743 			break;
   8744 		else if (ret) {
   8745 			dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
   8746 			break;
   8747 		}
   8748 
   8749 		opcode = le16_to_cpu(event.desc.opcode);
   8750 		switch (opcode) {
   8751 
   8752 		case i40e_aqc_opc_get_link_status:
   8753 			i40e_handle_link_event(pf, &event);
   8754 			break;
   8755 		case i40e_aqc_opc_send_msg_to_pf:
   8756 			ret = i40e_vc_process_vf_msg(pf,
   8757 					le16_to_cpu(event.desc.retval),
   8758 					le32_to_cpu(event.desc.cookie_high),
   8759 					le32_to_cpu(event.desc.cookie_low),
   8760 					event.msg_buf,
   8761 					event.msg_len);
   8762 			break;
   8763 		case i40e_aqc_opc_lldp_update_mib:
   8764 			dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
   8765 #ifdef CONFIG_I40E_DCB
   8766 			rtnl_lock();
   8767 			ret = i40e_handle_lldp_event(pf, &event);
   8768 			rtnl_unlock();
   8769 #endif /* CONFIG_I40E_DCB */
   8770 			break;
   8771 		case i40e_aqc_opc_event_lan_overflow:
   8772 			dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
   8773 			i40e_handle_lan_overflow_event(pf, &event);
   8774 			break;
   8775 		case i40e_aqc_opc_send_msg_to_peer:
   8776 			dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
   8777 			break;
   8778 		case i40e_aqc_opc_nvm_erase:
   8779 		case i40e_aqc_opc_nvm_update:
   8780 		case i40e_aqc_opc_oem_post_update:
   8781 			i40e_debug(&pf->hw, I40E_DEBUG_NVM,
   8782 				   "ARQ NVM operation 0x%04x completed\n",
   8783 				   opcode);
   8784 			break;
   8785 		default:
   8786 			dev_info(&pf->pdev->dev,
   8787 				 "ARQ: Unknown event 0x%04x ignored\n",
   8788 				 opcode);
   8789 			break;
   8790 		}
   8791 	} while (i++ < pf->adminq_work_limit);
   8792 
   8793 	if (i < pf->adminq_work_limit)
   8794 		clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
   8795 
   8796 	/* re-enable Admin queue interrupt cause */
   8797 	val = rd32(hw, I40E_PFINT_ICR0_ENA);
   8798 	val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
   8799 	wr32(hw, I40E_PFINT_ICR0_ENA, val);
   8800 	i40e_flush(hw);
   8801 
   8802 	kfree(event.msg_buf);
   8803 }
   8804 
   8805 /**
   8806  * i40e_verify_eeprom - make sure eeprom is good to use
   8807  * @pf: board private structure
   8808  **/
   8809 static void i40e_verify_eeprom(struct i40e_pf *pf)
   8810 {
   8811 	int err;
   8812 
   8813 	err = i40e_diag_eeprom_test(&pf->hw);
   8814 	if (err) {
   8815 		/* retry in case of garbage read */
   8816 		err = i40e_diag_eeprom_test(&pf->hw);
   8817 		if (err) {
   8818 			dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
   8819 				 err);
   8820 			set_bit(__I40E_BAD_EEPROM, pf->state);
   8821 		}
   8822 	}
   8823 
   8824 	if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
   8825 		dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
   8826 		clear_bit(__I40E_BAD_EEPROM, pf->state);
   8827 	}
   8828 }
   8829 
   8830 /**
   8831  * i40e_enable_pf_switch_lb
   8832  * @pf: pointer to the PF structure
   8833  *
   8834  * enable switch loop back or die - no point in a return value
   8835  **/
   8836 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
   8837 {
   8838 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
   8839 	struct i40e_vsi_context ctxt;
   8840 	int ret;
   8841 
   8842 	ctxt.seid = pf->main_vsi_seid;
   8843 	ctxt.pf_num = pf->hw.pf_id;
   8844 	ctxt.vf_num = 0;
   8845 	ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
   8846 	if (ret) {
   8847 		dev_info(&pf->pdev->dev,
   8848 			 "couldn't get PF vsi config, err %s aq_err %s\n",
   8849 			 i40e_stat_str(&pf->hw, ret),
   8850 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   8851 		return;
   8852 	}
   8853 	ctxt.flags = I40E_AQ_VSI_TYPE_PF;
   8854 	ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
   8855 	ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
   8856 
   8857 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
   8858 	if (ret) {
   8859 		dev_info(&pf->pdev->dev,
   8860 			 "update vsi switch failed, err %s aq_err %s\n",
   8861 			 i40e_stat_str(&pf->hw, ret),
   8862 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   8863 	}
   8864 }
   8865 
   8866 /**
   8867  * i40e_disable_pf_switch_lb
   8868  * @pf: pointer to the PF structure
   8869  *
   8870  * disable switch loop back or die - no point in a return value
   8871  **/
   8872 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
   8873 {
   8874 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
   8875 	struct i40e_vsi_context ctxt;
   8876 	int ret;
   8877 
   8878 	ctxt.seid = pf->main_vsi_seid;
   8879 	ctxt.pf_num = pf->hw.pf_id;
   8880 	ctxt.vf_num = 0;
   8881 	ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
   8882 	if (ret) {
   8883 		dev_info(&pf->pdev->dev,
   8884 			 "couldn't get PF vsi config, err %s aq_err %s\n",
   8885 			 i40e_stat_str(&pf->hw, ret),
   8886 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   8887 		return;
   8888 	}
   8889 	ctxt.flags = I40E_AQ_VSI_TYPE_PF;
   8890 	ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
   8891 	ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
   8892 
   8893 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
   8894 	if (ret) {
   8895 		dev_info(&pf->pdev->dev,
   8896 			 "update vsi switch failed, err %s aq_err %s\n",
   8897 			 i40e_stat_str(&pf->hw, ret),
   8898 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   8899 	}
   8900 }
   8901 
   8902 /**
   8903  * i40e_config_bridge_mode - Configure the HW bridge mode
   8904  * @veb: pointer to the bridge instance
   8905  *
   8906  * Configure the loop back mode for the LAN VSI that is downlink to the
   8907  * specified HW bridge instance. It is expected this function is called
   8908  * when a new HW bridge is instantiated.
   8909  **/
   8910 static void i40e_config_bridge_mode(struct i40e_veb *veb)
   8911 {
   8912 	struct i40e_pf *pf = veb->pf;
   8913 
   8914 	if (pf->hw.debug_mask & I40E_DEBUG_LAN)
   8915 		dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
   8916 			 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
   8917 	if (veb->bridge_mode & BRIDGE_MODE_VEPA)
   8918 		i40e_disable_pf_switch_lb(pf);
   8919 	else
   8920 		i40e_enable_pf_switch_lb(pf);
   8921 }
   8922 
   8923 /**
   8924  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
   8925  * @veb: pointer to the VEB instance
   8926  *
   8927  * This is a recursive function that first builds the attached VSIs then
   8928  * recurses in to build the next layer of VEB.  We track the connections
   8929  * through our own index numbers because the seid's from the HW could
   8930  * change across the reset.
   8931  **/
   8932 static int i40e_reconstitute_veb(struct i40e_veb *veb)
   8933 {
   8934 	struct i40e_vsi *ctl_vsi = NULL;
   8935 	struct i40e_pf *pf = veb->pf;
   8936 	int v, veb_idx;
   8937 	int ret;
   8938 
   8939 	/* build VSI that owns this VEB, temporarily attached to base VEB */
   8940 	for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
   8941 		if (pf->vsi[v] &&
   8942 		    pf->vsi[v]->veb_idx == veb->idx &&
   8943 		    pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
   8944 			ctl_vsi = pf->vsi[v];
   8945 			break;
   8946 		}
   8947 	}
   8948 	if (!ctl_vsi) {
   8949 		dev_info(&pf->pdev->dev,
   8950 			 "missing owner VSI for veb_idx %d\n", veb->idx);
   8951 		ret = -ENOENT;
   8952 		goto end_reconstitute;
   8953 	}
   8954 	if (ctl_vsi != pf->vsi[pf->lan_vsi])
   8955 		ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
   8956 	ret = i40e_add_vsi(ctl_vsi);
   8957 	if (ret) {
   8958 		dev_info(&pf->pdev->dev,
   8959 			 "rebuild of veb_idx %d owner VSI failed: %d\n",
   8960 			 veb->idx, ret);
   8961 		goto end_reconstitute;
   8962 	}
   8963 	i40e_vsi_reset_stats(ctl_vsi);
   8964 
   8965 	/* create the VEB in the switch and move the VSI onto the VEB */
   8966 	ret = i40e_add_veb(veb, ctl_vsi);
   8967 	if (ret)
   8968 		goto end_reconstitute;
   8969 
   8970 	if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
   8971 		veb->bridge_mode = BRIDGE_MODE_VEB;
   8972 	else
   8973 		veb->bridge_mode = BRIDGE_MODE_VEPA;
   8974 	i40e_config_bridge_mode(veb);
   8975 
   8976 	/* create the remaining VSIs attached to this VEB */
   8977 	for (v = 0; v < pf->num_alloc_vsi; v++) {
   8978 		if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
   8979 			continue;
   8980 
   8981 		if (pf->vsi[v]->veb_idx == veb->idx) {
   8982 			struct i40e_vsi *vsi = pf->vsi[v];
   8983 
   8984 			vsi->uplink_seid = veb->seid;
   8985 			ret = i40e_add_vsi(vsi);
   8986 			if (ret) {
   8987 				dev_info(&pf->pdev->dev,
   8988 					 "rebuild of vsi_idx %d failed: %d\n",
   8989 					 v, ret);
   8990 				goto end_reconstitute;
   8991 			}
   8992 			i40e_vsi_reset_stats(vsi);
   8993 		}
   8994 	}
   8995 
   8996 	/* create any VEBs attached to this VEB - RECURSION */
   8997 	for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
   8998 		if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
   8999 			pf->veb[veb_idx]->uplink_seid = veb->seid;
   9000 			ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
   9001 			if (ret)
   9002 				break;
   9003 		}
   9004 	}
   9005 
   9006 end_reconstitute:
   9007 	return ret;
   9008 }
   9009 
   9010 /**
   9011  * i40e_get_capabilities - get info about the HW
   9012  * @pf: the PF struct
   9013  **/
   9014 static int i40e_get_capabilities(struct i40e_pf *pf,
   9015 				 enum i40e_admin_queue_opc list_type)
   9016 {
   9017 	struct i40e_aqc_list_capabilities_element_resp *cap_buf;
   9018 	u16 data_size;
   9019 	int buf_len;
   9020 	int err;
   9021 
   9022 	buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
   9023 	do {
   9024 		cap_buf = kzalloc(buf_len, GFP_KERNEL);
   9025 		if (!cap_buf)
   9026 			return -ENOMEM;
   9027 
   9028 		/* this loads the data into the hw struct for us */
   9029 		err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
   9030 						    &data_size, list_type,
   9031 						    NULL);
   9032 		/* data loaded, buffer no longer needed */
   9033 		kfree(cap_buf);
   9034 
   9035 		if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
   9036 			/* retry with a larger buffer */
   9037 			buf_len = data_size;
   9038 		} else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
   9039 			dev_info(&pf->pdev->dev,
   9040 				 "capability discovery failed, err %s aq_err %s\n",
   9041 				 i40e_stat_str(&pf->hw, err),
   9042 				 i40e_aq_str(&pf->hw,
   9043 					     pf->hw.aq.asq_last_status));
   9044 			return -ENODEV;
   9045 		}
   9046 	} while (err);
   9047 
   9048 	if (pf->hw.debug_mask & I40E_DEBUG_USER) {
   9049 		if (list_type == i40e_aqc_opc_list_func_capabilities) {
   9050 			dev_info(&pf->pdev->dev,
   9051 				 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
   9052 				 pf->hw.pf_id, pf->hw.func_caps.num_vfs,
   9053 				 pf->hw.func_caps.num_msix_vectors,
   9054 				 pf->hw.func_caps.num_msix_vectors_vf,
   9055 				 pf->hw.func_caps.fd_filters_guaranteed,
   9056 				 pf->hw.func_caps.fd_filters_best_effort,
   9057 				 pf->hw.func_caps.num_tx_qp,
   9058 				 pf->hw.func_caps.num_vsis);
   9059 		} else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
   9060 			dev_info(&pf->pdev->dev,
   9061 				 "switch_mode=0x%04x, function_valid=0x%08x\n",
   9062 				 pf->hw.dev_caps.switch_mode,
   9063 				 pf->hw.dev_caps.valid_functions);
   9064 			dev_info(&pf->pdev->dev,
   9065 				 "SR-IOV=%d, num_vfs for all function=%u\n",
   9066 				 pf->hw.dev_caps.sr_iov_1_1,
   9067 				 pf->hw.dev_caps.num_vfs);
   9068 			dev_info(&pf->pdev->dev,
   9069 				 "num_vsis=%u, num_rx:%u, num_tx=%u\n",
   9070 				 pf->hw.dev_caps.num_vsis,
   9071 				 pf->hw.dev_caps.num_rx_qp,
   9072 				 pf->hw.dev_caps.num_tx_qp);
   9073 		}
   9074 	}
   9075 	if (list_type == i40e_aqc_opc_list_func_capabilities) {
   9076 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
   9077 		       + pf->hw.func_caps.num_vfs)
   9078 		if (pf->hw.revision_id == 0 &&
   9079 		    pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
   9080 			dev_info(&pf->pdev->dev,
   9081 				 "got num_vsis %d, setting num_vsis to %d\n",
   9082 				 pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
   9083 			pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
   9084 		}
   9085 	}
   9086 	return 0;
   9087 }
   9088 
   9089 static int i40e_vsi_clear(struct i40e_vsi *vsi);
   9090 
   9091 /**
   9092  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
   9093  * @pf: board private structure
   9094  **/
   9095 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
   9096 {
   9097 	struct i40e_vsi *vsi;
   9098 
   9099 	/* quick workaround for an NVM issue that leaves a critical register
   9100 	 * uninitialized
   9101 	 */
   9102 	if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
   9103 		static const u32 hkey[] = {
   9104 			0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
   9105 			0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
   9106 			0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
   9107 			0x95b3a76d};
   9108 		int i;
   9109 
   9110 		for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
   9111 			wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
   9112 	}
   9113 
   9114 	if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
   9115 		return;
   9116 
   9117 	/* find existing VSI and see if it needs configuring */
   9118 	vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
   9119 
   9120 	/* create a new VSI if none exists */
   9121 	if (!vsi) {
   9122 		vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
   9123 				     pf->vsi[pf->lan_vsi]->seid, 0);
   9124 		if (!vsi) {
   9125 			dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
   9126 			pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
   9127 			pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
   9128 			return;
   9129 		}
   9130 	}
   9131 
   9132 	i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
   9133 }
   9134 
   9135 /**
   9136  * i40e_fdir_teardown - release the Flow Director resources
   9137  * @pf: board private structure
   9138  **/
   9139 static void i40e_fdir_teardown(struct i40e_pf *pf)
   9140 {
   9141 	struct i40e_vsi *vsi;
   9142 
   9143 	i40e_fdir_filter_exit(pf);
   9144 	vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
   9145 	if (vsi)
   9146 		i40e_vsi_release(vsi);
   9147 }
   9148 
   9149 /**
   9150  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
   9151  * @vsi: PF main vsi
   9152  * @seid: seid of main or channel VSIs
   9153  *
   9154  * Rebuilds cloud filters associated with main VSI and channel VSIs if they
   9155  * existed before reset
   9156  **/
   9157 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
   9158 {
   9159 	struct i40e_cloud_filter *cfilter;
   9160 	struct i40e_pf *pf = vsi->back;
   9161 	struct hlist_node *node;
   9162 	i40e_status ret;
   9163 
   9164 	/* Add cloud filters back if they exist */
   9165 	hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
   9166 				  cloud_node) {
   9167 		if (cfilter->seid != seid)
   9168 			continue;
   9169 
   9170 		if (cfilter->dst_port)
   9171 			ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
   9172 								true);
   9173 		else
   9174 			ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
   9175 
   9176 		if (ret) {
   9177 			dev_dbg(&pf->pdev->dev,
   9178 				"Failed to rebuild cloud filter, err %s aq_err %s\n",
   9179 				i40e_stat_str(&pf->hw, ret),
   9180 				i40e_aq_str(&pf->hw,
   9181 					    pf->hw.aq.asq_last_status));
   9182 			return ret;
   9183 		}
   9184 	}
   9185 	return 0;
   9186 }
   9187 
   9188 /**
   9189  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
   9190  * @vsi: PF main vsi
   9191  *
   9192  * Rebuilds channel VSIs if they existed before reset
   9193  **/
   9194 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
   9195 {
   9196 	struct i40e_channel *ch, *ch_tmp;
   9197 	i40e_status ret;
   9198 
   9199 	if (list_empty(&vsi->ch_list))
   9200 		return 0;
   9201 
   9202 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
   9203 		if (!ch->initialized)
   9204 			break;
   9205 		/* Proceed with creation of channel (VMDq2) VSI */
   9206 		ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
   9207 		if (ret) {
   9208 			dev_info(&vsi->back->pdev->dev,
   9209 				 "failed to rebuild channels using uplink_seid %u\n",
   9210 				 vsi->uplink_seid);
   9211 			return ret;
   9212 		}
   9213 		/* Reconfigure TX queues using QTX_CTL register */
   9214 		ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
   9215 		if (ret) {
   9216 			dev_info(&vsi->back->pdev->dev,
   9217 				 "failed to configure TX rings for channel %u\n",
   9218 				 ch->seid);
   9219 			return ret;
   9220 		}
   9221 		/* update 'next_base_queue' */
   9222 		vsi->next_base_queue = vsi->next_base_queue +
   9223 							ch->num_queue_pairs;
   9224 		if (ch->max_tx_rate) {
   9225 			u64 credits = ch->max_tx_rate;
   9226 
   9227 			if (i40e_set_bw_limit(vsi, ch->seid,
   9228 					      ch->max_tx_rate))
   9229 				return -EINVAL;
   9230 
   9231 			do_div(credits, I40E_BW_CREDIT_DIVISOR);
   9232 			dev_dbg(&vsi->back->pdev->dev,
   9233 				"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
   9234 				ch->max_tx_rate,
   9235 				credits,
   9236 				ch->seid);
   9237 		}
   9238 		ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
   9239 		if (ret) {
   9240 			dev_dbg(&vsi->back->pdev->dev,
   9241 				"Failed to rebuild cloud filters for channel VSI %u\n",
   9242 				ch->seid);
   9243 			return ret;
   9244 		}
   9245 	}
   9246 	return 0;
   9247 }
   9248 
   9249 /**
   9250  * i40e_prep_for_reset - prep for the core to reset
   9251  * @pf: board private structure
   9252  * @lock_acquired: indicates whether or not the lock has been acquired
   9253  * before this function was called.
   9254  *
   9255  * Close up the VFs and other things in prep for PF Reset.
   9256   **/
   9257 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired)
   9258 {
   9259 	struct i40e_hw *hw = &pf->hw;
   9260 	i40e_status ret = 0;
   9261 	u32 v;
   9262 
   9263 	clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
   9264 	if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
   9265 		return;
   9266 	if (i40e_check_asq_alive(&pf->hw))
   9267 		i40e_vc_notify_reset(pf);
   9268 
   9269 	dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
   9270 
   9271 	/* quiesce the VSIs and their queues that are not already DOWN */
   9272 	/* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */
   9273 	if (!lock_acquired)
   9274 		rtnl_lock();
   9275 	i40e_pf_quiesce_all_vsi(pf);
   9276 	if (!lock_acquired)
   9277 		rtnl_unlock();
   9278 
   9279 	for (v = 0; v < pf->num_alloc_vsi; v++) {
   9280 		if (pf->vsi[v])
   9281 			pf->vsi[v]->seid = 0;
   9282 	}
   9283 
   9284 	i40e_shutdown_adminq(&pf->hw);
   9285 
   9286 	/* call shutdown HMC */
   9287 	if (hw->hmc.hmc_obj) {
   9288 		ret = i40e_shutdown_lan_hmc(hw);
   9289 		if (ret)
   9290 			dev_warn(&pf->pdev->dev,
   9291 				 "shutdown_lan_hmc failed: %d\n", ret);
   9292 	}
   9293 }
   9294 
   9295 /**
   9296  * i40e_send_version - update firmware with driver version
   9297  * @pf: PF struct
   9298  */
   9299 static void i40e_send_version(struct i40e_pf *pf)
   9300 {
   9301 	struct i40e_driver_version dv;
   9302 
   9303 	dv.major_version = DRV_VERSION_MAJOR;
   9304 	dv.minor_version = DRV_VERSION_MINOR;
   9305 	dv.build_version = DRV_VERSION_BUILD;
   9306 	dv.subbuild_version = 0;
   9307 	strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
   9308 	i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
   9309 }
   9310 
   9311 /**
   9312  * i40e_get_oem_version - get OEM specific version information
   9313  * @hw: pointer to the hardware structure
   9314  **/
   9315 static void i40e_get_oem_version(struct i40e_hw *hw)
   9316 {
   9317 	u16 block_offset = 0xffff;
   9318 	u16 block_length = 0;
   9319 	u16 capabilities = 0;
   9320 	u16 gen_snap = 0;
   9321 	u16 release = 0;
   9322 
   9323 #define I40E_SR_NVM_OEM_VERSION_PTR		0x1B
   9324 #define I40E_NVM_OEM_LENGTH_OFFSET		0x00
   9325 #define I40E_NVM_OEM_CAPABILITIES_OFFSET	0x01
   9326 #define I40E_NVM_OEM_GEN_OFFSET			0x02
   9327 #define I40E_NVM_OEM_RELEASE_OFFSET		0x03
   9328 #define I40E_NVM_OEM_CAPABILITIES_MASK		0x000F
   9329 #define I40E_NVM_OEM_LENGTH			3
   9330 
   9331 	/* Check if pointer to OEM version block is valid. */
   9332 	i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
   9333 	if (block_offset == 0xffff)
   9334 		return;
   9335 
   9336 	/* Check if OEM version block has correct length. */
   9337 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
   9338 			   &block_length);
   9339 	if (block_length < I40E_NVM_OEM_LENGTH)
   9340 		return;
   9341 
   9342 	/* Check if OEM version format is as expected. */
   9343 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
   9344 			   &capabilities);
   9345 	if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
   9346 		return;
   9347 
   9348 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
   9349 			   &gen_snap);
   9350 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
   9351 			   &release);
   9352 	hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
   9353 	hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
   9354 }
   9355 
   9356 /**
   9357  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
   9358  * @pf: board private structure
   9359  **/
   9360 static int i40e_reset(struct i40e_pf *pf)
   9361 {
   9362 	struct i40e_hw *hw = &pf->hw;
   9363 	i40e_status ret;
   9364 
   9365 	ret = i40e_pf_reset(hw);
   9366 	if (ret) {
   9367 		dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
   9368 		set_bit(__I40E_RESET_FAILED, pf->state);
   9369 		clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
   9370 	} else {
   9371 		pf->pfr_count++;
   9372 	}
   9373 	return ret;
   9374 }
   9375 
   9376 /**
   9377  * i40e_rebuild - rebuild using a saved config
   9378  * @pf: board private structure
   9379  * @reinit: if the Main VSI needs to re-initialized.
   9380  * @lock_acquired: indicates whether or not the lock has been acquired
   9381  * before this function was called.
   9382  **/
   9383 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
   9384 {
   9385 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
   9386 	struct i40e_hw *hw = &pf->hw;
   9387 	u8 set_fc_aq_fail = 0;
   9388 	i40e_status ret;
   9389 	u32 val;
   9390 	int v;
   9391 
   9392 	if (test_bit(__I40E_DOWN, pf->state))
   9393 		goto clear_recovery;
   9394 	dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
   9395 
   9396 	/* rebuild the basics for the AdminQ, HMC, and initial HW switch */
   9397 	ret = i40e_init_adminq(&pf->hw);
   9398 	if (ret) {
   9399 		dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
   9400 			 i40e_stat_str(&pf->hw, ret),
   9401 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   9402 		goto clear_recovery;
   9403 	}
   9404 	i40e_get_oem_version(&pf->hw);
   9405 
   9406 	if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
   9407 	    ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) ||
   9408 	     hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) {
   9409 		/* The following delay is necessary for 4.33 firmware and older
   9410 		 * to recover after EMP reset. 200 ms should suffice but we
   9411 		 * put here 300 ms to be sure that FW is ready to operate
   9412 		 * after reset.
   9413 		 */
   9414 		mdelay(300);
   9415 	}
   9416 
   9417 	/* re-verify the eeprom if we just had an EMP reset */
   9418 	if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
   9419 		i40e_verify_eeprom(pf);
   9420 
   9421 	i40e_clear_pxe_mode(hw);
   9422 	ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
   9423 	if (ret)
   9424 		goto end_core_reset;
   9425 
   9426 	ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
   9427 				hw->func_caps.num_rx_qp, 0, 0);
   9428 	if (ret) {
   9429 		dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
   9430 		goto end_core_reset;
   9431 	}
   9432 	ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
   9433 	if (ret) {
   9434 		dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
   9435 		goto end_core_reset;
   9436 	}
   9437 
   9438 	/* Enable FW to write a default DCB config on link-up */
   9439 	i40e_aq_set_dcb_parameters(hw, true, NULL);
   9440 
   9441 #ifdef CONFIG_I40E_DCB
   9442 	ret = i40e_init_pf_dcb(pf);
   9443 	if (ret) {
   9444 		dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
   9445 		pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
   9446 		/* Continue without DCB enabled */
   9447 	}
   9448 #endif /* CONFIG_I40E_DCB */
   9449 	/* do basic switch setup */
   9450 	if (!lock_acquired)
   9451 		rtnl_lock();
   9452 	ret = i40e_setup_pf_switch(pf, reinit);
   9453 	if (ret)
   9454 		goto end_unlock;
   9455 
   9456 	/* The driver only wants link up/down and module qualification
   9457 	 * reports from firmware.  Note the negative logic.
   9458 	 */
   9459 	ret = i40e_aq_set_phy_int_mask(&pf->hw,
   9460 				       ~(I40E_AQ_EVENT_LINK_UPDOWN |
   9461 					 I40E_AQ_EVENT_MEDIA_NA |
   9462 					 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
   9463 	if (ret)
   9464 		dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
   9465 			 i40e_stat_str(&pf->hw, ret),
   9466 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   9467 
   9468 	/* make sure our flow control settings are restored */
   9469 	ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
   9470 	if (ret)
   9471 		dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n",
   9472 			i40e_stat_str(&pf->hw, ret),
   9473 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   9474 
   9475 	/* Rebuild the VSIs and VEBs that existed before reset.
   9476 	 * They are still in our local switch element arrays, so only
   9477 	 * need to rebuild the switch model in the HW.
   9478 	 *
   9479 	 * If there were VEBs but the reconstitution failed, we'll try
   9480 	 * try to recover minimal use by getting the basic PF VSI working.
   9481 	 */
   9482 	if (vsi->uplink_seid != pf->mac_seid) {
   9483 		dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
   9484 		/* find the one VEB connected to the MAC, and find orphans */
   9485 		for (v = 0; v < I40E_MAX_VEB; v++) {
   9486 			if (!pf->veb[v])
   9487 				continue;
   9488 
   9489 			if (pf->veb[v]->uplink_seid == pf->mac_seid ||
   9490 			    pf->veb[v]->uplink_seid == 0) {
   9491 				ret = i40e_reconstitute_veb(pf->veb[v]);
   9492 
   9493 				if (!ret)
   9494 					continue;
   9495 
   9496 				/* If Main VEB failed, we're in deep doodoo,
   9497 				 * so give up rebuilding the switch and set up
   9498 				 * for minimal rebuild of PF VSI.
   9499 				 * If orphan failed, we'll report the error
   9500 				 * but try to keep going.
   9501 				 */
   9502 				if (pf->veb[v]->uplink_seid == pf->mac_seid) {
   9503 					dev_info(&pf->pdev->dev,
   9504 						 "rebuild of switch failed: %d, will try to set up simple PF connection\n",
   9505 						 ret);
   9506 					vsi->uplink_seid = pf->mac_seid;
   9507 					break;
   9508 				} else if (pf->veb[v]->uplink_seid == 0) {
   9509 					dev_info(&pf->pdev->dev,
   9510 						 "rebuild of orphan VEB failed: %d\n",
   9511 						 ret);
   9512 				}
   9513 			}
   9514 		}
   9515 	}
   9516 
   9517 	if (vsi->uplink_seid == pf->mac_seid) {
   9518 		dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
   9519 		/* no VEB, so rebuild only the Main VSI */
   9520 		ret = i40e_add_vsi(vsi);
   9521 		if (ret) {
   9522 			dev_info(&pf->pdev->dev,
   9523 				 "rebuild of Main VSI failed: %d\n", ret);
   9524 			goto end_unlock;
   9525 		}
   9526 	}
   9527 
   9528 	if (vsi->mqprio_qopt.max_rate[0]) {
   9529 		u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
   9530 		u64 credits = 0;
   9531 
   9532 		do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
   9533 		ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
   9534 		if (ret)
   9535 			goto end_unlock;
   9536 
   9537 		credits = max_tx_rate;
   9538 		do_div(credits, I40E_BW_CREDIT_DIVISOR);
   9539 		dev_dbg(&vsi->back->pdev->dev,
   9540 			"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
   9541 			max_tx_rate,
   9542 			credits,
   9543 			vsi->seid);
   9544 	}
   9545 
   9546 	ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
   9547 	if (ret)
   9548 		goto end_unlock;
   9549 
   9550 	/* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
   9551 	 * for this main VSI if they exist
   9552 	 */
   9553 	ret = i40e_rebuild_channels(vsi);
   9554 	if (ret)
   9555 		goto end_unlock;
   9556 
   9557 	/* Reconfigure hardware for allowing smaller MSS in the case
   9558 	 * of TSO, so that we avoid the MDD being fired and causing
   9559 	 * a reset in the case of small MSS+TSO.
   9560 	 */
   9561 #define I40E_REG_MSS          0x000E64DC
   9562 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
   9563 #define I40E_64BYTE_MSS       0x400000
   9564 	val = rd32(hw, I40E_REG_MSS);
   9565 	if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
   9566 		val &= ~I40E_REG_MSS_MIN_MASK;
   9567 		val |= I40E_64BYTE_MSS;
   9568 		wr32(hw, I40E_REG_MSS, val);
   9569 	}
   9570 
   9571 	if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
   9572 		msleep(75);
   9573 		ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
   9574 		if (ret)
   9575 			dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
   9576 				 i40e_stat_str(&pf->hw, ret),
   9577 				 i40e_aq_str(&pf->hw,
   9578 					     pf->hw.aq.asq_last_status));
   9579 	}
   9580 	/* reinit the misc interrupt */
   9581 	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
   9582 		ret = i40e_setup_misc_vector(pf);
   9583 
   9584 	/* Add a filter to drop all Flow control frames from any VSI from being
   9585 	 * transmitted. By doing so we stop a malicious VF from sending out
   9586 	 * PAUSE or PFC frames and potentially controlling traffic for other
   9587 	 * PF/VF VSIs.
   9588 	 * The FW can still send Flow control frames if enabled.
   9589 	 */
   9590 	i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
   9591 						       pf->main_vsi_seid);
   9592 
   9593 	/* restart the VSIs that were rebuilt and running before the reset */
   9594 	i40e_pf_unquiesce_all_vsi(pf);
   9595 
   9596 	/* Release the RTNL lock before we start resetting VFs */
   9597 	if (!lock_acquired)
   9598 		rtnl_unlock();
   9599 
   9600 	/* Restore promiscuous settings */
   9601 	ret = i40e_set_promiscuous(pf, pf->cur_promisc);
   9602 	if (ret)
   9603 		dev_warn(&pf->pdev->dev,
   9604 			 "Failed to restore promiscuous setting: %s, err %s aq_err %s\n",
   9605 			 pf->cur_promisc ? "on" : "off",
   9606 			 i40e_stat_str(&pf->hw, ret),
   9607 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
   9608 
   9609 	i40e_reset_all_vfs(pf, true);
   9610 
   9611 	/* tell the firmware that we're starting */
   9612 	i40e_send_version(pf);
   9613 
   9614 	/* We've already released the lock, so don't do it again */
   9615 	goto end_core_reset;
   9616 
   9617 end_unlock:
   9618 	if (!lock_acquired)
   9619 		rtnl_unlock();
   9620 end_core_reset:
   9621 	clear_bit(__I40E_RESET_FAILED, pf->state);
   9622 clear_recovery:
   9623 	clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
   9624 	clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
   9625 }
   9626 
   9627 /**
   9628  * i40e_reset_and_rebuild - reset and rebuild using a saved config
   9629  * @pf: board private structure
   9630  * @reinit: if the Main VSI needs to re-initialized.
   9631  * @lock_acquired: indicates whether or not the lock has been acquired
   9632  * before this function was called.
   9633  **/
   9634 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
   9635 				   bool lock_acquired)
   9636 {
   9637 	int ret;
   9638 	/* Now we wait for GRST to settle out.
   9639 	 * We don't have to delete the VEBs or VSIs from the hw switch
   9640 	 * because the reset will make them disappear.
   9641 	 */
   9642 	ret = i40e_reset(pf);
   9643 	if (!ret)
   9644 		i40e_rebuild(pf, reinit, lock_acquired);
   9645 }
   9646 
   9647 /**
   9648  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
   9649  * @pf: board private structure
   9650  *
   9651  * Close up the VFs and other things in prep for a Core Reset,
   9652  * then get ready to rebuild the world.
   9653  * @lock_acquired: indicates whether or not the lock has been acquired
   9654  * before this function was called.
   9655  **/
   9656 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
   9657 {
   9658 	i40e_prep_for_reset(pf, lock_acquired);
   9659 	i40e_reset_and_rebuild(pf, false, lock_acquired);
   9660 }
   9661 
   9662 /**
   9663  * i40e_handle_mdd_event
   9664  * @pf: pointer to the PF structure
   9665  *
   9666  * Called from the MDD irq handler to identify possibly malicious vfs
   9667  **/
   9668 static void i40e_handle_mdd_event(struct i40e_pf *pf)
   9669 {
   9670 	struct i40e_hw *hw = &pf->hw;
   9671 	bool mdd_detected = false;
   9672 	bool pf_mdd_detected = false;
   9673 	struct i40e_vf *vf;
   9674 	u32 reg;
   9675 	int i;
   9676 
   9677 	if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
   9678 		return;
   9679 
   9680 	/* find what triggered the MDD event */
   9681 	reg = rd32(hw, I40E_GL_MDET_TX);
   9682 	if (reg & I40E_GL_MDET_TX_VALID_MASK) {
   9683 		u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
   9684 				I40E_GL_MDET_TX_PF_NUM_SHIFT;
   9685 		u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
   9686 				I40E_GL_MDET_TX_VF_NUM_SHIFT;
   9687 		u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
   9688 				I40E_GL_MDET_TX_EVENT_SHIFT;
   9689 		u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
   9690 				I40E_GL_MDET_TX_QUEUE_SHIFT) -
   9691 				pf->hw.func_caps.base_queue;
   9692 		if (netif_msg_tx_err(pf))
   9693 			dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
   9694 				 event, queue, pf_num, vf_num);
   9695 		wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
   9696 		mdd_detected = true;
   9697 	}
   9698 	reg = rd32(hw, I40E_GL_MDET_RX);
   9699 	if (reg & I40E_GL_MDET_RX_VALID_MASK) {
   9700 		u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
   9701 				I40E_GL_MDET_RX_FUNCTION_SHIFT;
   9702 		u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
   9703 				I40E_GL_MDET_RX_EVENT_SHIFT;
   9704 		u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
   9705 				I40E_GL_MDET_RX_QUEUE_SHIFT) -
   9706 				pf->hw.func_caps.base_queue;
   9707 		if (netif_msg_rx_err(pf))
   9708 			dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
   9709 				 event, queue, func);
   9710 		wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
   9711 		mdd_detected = true;
   9712 	}
   9713 
   9714 	if (mdd_detected) {
   9715 		reg = rd32(hw, I40E_PF_MDET_TX);
   9716 		if (reg & I40E_PF_MDET_TX_VALID_MASK) {
   9717 			wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
   9718 			dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
   9719 			pf_mdd_detected = true;
   9720 		}
   9721 		reg = rd32(hw, I40E_PF_MDET_RX);
   9722 		if (reg & I40E_PF_MDET_RX_VALID_MASK) {
   9723 			wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
   9724 			dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
   9725 			pf_mdd_detected = true;
   9726 		}
   9727 		/* Queue belongs to the PF, initiate a reset */
   9728 		if (pf_mdd_detected) {
   9729 			set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
   9730 			i40e_service_event_schedule(pf);
   9731 		}
   9732 	}
   9733 
   9734 	/* see if one of the VFs needs its hand slapped */
   9735 	for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
   9736 		vf = &(pf->vf[i]);
   9737 		reg = rd32(hw, I40E_VP_MDET_TX(i));
   9738 		if (reg & I40E_VP_MDET_TX_VALID_MASK) {
   9739 			wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
   9740 			vf->num_mdd_events++;
   9741 			dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
   9742 				 i);
   9743 		}
   9744 
   9745 		reg = rd32(hw, I40E_VP_MDET_RX(i));
   9746 		if (reg & I40E_VP_MDET_RX_VALID_MASK) {
   9747 			wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
   9748 			vf->num_mdd_events++;
   9749 			dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
   9750 				 i);
   9751 		}
   9752 
   9753 		if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
   9754 			dev_info(&pf->pdev->dev,
   9755 				 "Too many MDD events on VF %d, disabled\n", i);
   9756 			dev_info(&pf->pdev->dev,
   9757 				 "Use PF Control I/F to re-enable the VF\n");
   9758 			set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
   9759 		}
   9760 	}
   9761 
   9762 	/* re-enable mdd interrupt cause */
   9763 	clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
   9764 	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
   9765 	reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
   9766 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
   9767 	i40e_flush(hw);
   9768 }
   9769 
   9770 static const char *i40e_tunnel_name(u8 type)
   9771 {
   9772 	switch (type) {
   9773 	case UDP_TUNNEL_TYPE_VXLAN:
   9774 		return "vxlan";
   9775 	case UDP_TUNNEL_TYPE_GENEVE:
   9776 		return "geneve";
   9777 	default:
   9778 		return "unknown";
   9779 	}
   9780 }
   9781 
   9782 /**
   9783  * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters
   9784  * @pf: board private structure
   9785  **/
   9786 static void i40e_sync_udp_filters(struct i40e_pf *pf)
   9787 {
   9788 	int i;
   9789 
   9790 	/* loop through and set pending bit for all active UDP filters */
   9791 	for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
   9792 		if (pf->udp_ports[i].port)
   9793 			pf->pending_udp_bitmap |= BIT_ULL(i);
   9794 	}
   9795 
   9796 	set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
   9797 }
   9798 
   9799 /**
   9800  * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW
   9801  * @pf: board private structure
   9802  **/
   9803 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
   9804 {
   9805 	struct i40e_hw *hw = &pf->hw;
   9806 	u8 filter_index, type;
   9807 	u16 port;
   9808 	int i;
   9809 
   9810 	if (!test_and_clear_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state))
   9811 		return;
   9812 
   9813 	/* acquire RTNL to maintain state of flags and port requests */
   9814 	rtnl_lock();
   9815 
   9816 	for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
   9817 		if (pf->pending_udp_bitmap & BIT_ULL(i)) {
   9818 			struct i40e_udp_port_config *udp_port;
   9819 			i40e_status ret = 0;
   9820 
   9821 			udp_port = &pf->udp_ports[i];
   9822 			pf->pending_udp_bitmap &= ~BIT_ULL(i);
   9823 
   9824 			port = READ_ONCE(udp_port->port);
   9825 			type = READ_ONCE(udp_port->type);
   9826 			filter_index = READ_ONCE(udp_port->filter_index);
   9827 
   9828 			/* release RTNL while we wait on AQ command */
   9829 			rtnl_unlock();
   9830 
   9831 			if (port)
   9832 				ret = i40e_aq_add_udp_tunnel(hw, port,
   9833 							     type,
   9834 							     &filter_index,
   9835 							     NULL);
   9836 			else if (filter_index != I40E_UDP_PORT_INDEX_UNUSED)
   9837 				ret = i40e_aq_del_udp_tunnel(hw, filter_index,
   9838 							     NULL);
   9839 
   9840 			/* reacquire RTNL so we can update filter_index */
   9841 			rtnl_lock();
   9842 
   9843 			if (ret) {
   9844 				dev_info(&pf->pdev->dev,
   9845 					 "%s %s port %d, index %d failed, err %s aq_err %s\n",
   9846 					 i40e_tunnel_name(type),
   9847 					 port ? "add" : "delete",
   9848 					 port,
   9849 					 filter_index,
   9850 					 i40e_stat_str(&pf->hw, ret),
   9851 					 i40e_aq_str(&pf->hw,
   9852 						     pf->hw.aq.asq_last_status));
   9853 				if (port) {
   9854 					/* failed to add, just reset port,
   9855 					 * drop pending bit for any deletion
   9856 					 */
   9857 					udp_port->port = 0;
   9858 					pf->pending_udp_bitmap &= ~BIT_ULL(i);
   9859 				}
   9860 			} else if (port) {
   9861 				/* record filter index on success */
   9862 				udp_port->filter_index = filter_index;
   9863 			}
   9864 		}
   9865 	}
   9866 
   9867 	rtnl_unlock();
   9868 }
   9869 
   9870 /**
   9871  * i40e_service_task - Run the driver's async subtasks
   9872  * @work: pointer to work_struct containing our data
   9873  **/
   9874 static void i40e_service_task(struct work_struct *work)
   9875 {
   9876 	struct i40e_pf *pf = container_of(work,
   9877 					  struct i40e_pf,
   9878 					  service_task);
   9879 	unsigned long start_time = jiffies;
   9880 
   9881 	/* don't bother with service tasks if a reset is in progress */
   9882 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
   9883 		return;
   9884 
   9885 	if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
   9886 		return;
   9887 
   9888 	i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
   9889 	i40e_sync_filters_subtask(pf);
   9890 	i40e_reset_subtask(pf);
   9891 	i40e_handle_mdd_event(pf);
   9892 	i40e_vc_process_vflr_event(pf);
   9893 	i40e_watchdog_subtask(pf);
   9894 	i40e_fdir_reinit_subtask(pf);
   9895 	if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
   9896 		/* Client subtask will reopen next time through. */
   9897 		i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true);
   9898 	} else {
   9899 		i40e_client_subtask(pf);
   9900 		if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
   9901 				       pf->state))
   9902 			i40e_notify_client_of_l2_param_changes(
   9903 							pf->vsi[pf->lan_vsi]);
   9904 	}
   9905 	i40e_sync_filters_subtask(pf);
   9906 	i40e_sync_udp_filters_subtask(pf);
   9907 	i40e_clean_adminq_subtask(pf);
   9908 
   9909 	/* flush memory to make sure state is correct before next watchdog */
   9910 	smp_mb__before_atomic();
   9911 	clear_bit(__I40E_SERVICE_SCHED, pf->state);
   9912 
   9913 	/* If the tasks have taken longer than one timer cycle or there
   9914 	 * is more work to be done, reschedule the service task now
   9915 	 * rather than wait for the timer to tick again.
   9916 	 */
   9917 	if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
   9918 	    test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)		 ||
   9919 	    test_bit(__I40E_MDD_EVENT_PENDING, pf->state)		 ||
   9920 	    test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
   9921 		i40e_service_event_schedule(pf);
   9922 }
   9923 
   9924 /**
   9925  * i40e_service_timer - timer callback
   9926  * @data: pointer to PF struct
   9927  **/
   9928 static void i40e_service_timer(struct timer_list *t)
   9929 {
   9930 	struct i40e_pf *pf = from_timer(pf, t, service_timer);
   9931 
   9932 	mod_timer(&pf->service_timer,
   9933 		  round_jiffies(jiffies + pf->service_timer_period));
   9934 	i40e_service_event_schedule(pf);
   9935 }
   9936 
   9937 /**
   9938  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
   9939  * @vsi: the VSI being configured
   9940  **/
   9941 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
   9942 {
   9943 	struct i40e_pf *pf = vsi->back;
   9944 
   9945 	switch (vsi->type) {
   9946 	case I40E_VSI_MAIN:
   9947 		vsi->alloc_queue_pairs = pf->num_lan_qps;
   9948 		vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
   9949 				      I40E_REQ_DESCRIPTOR_MULTIPLE);
   9950 		if (pf->flags & I40E_FLAG_MSIX_ENABLED)
   9951 			vsi->num_q_vectors = pf->num_lan_msix;
   9952 		else
   9953 			vsi->num_q_vectors = 1;
   9954 
   9955 		break;
   9956 
   9957 	case I40E_VSI_FDIR:
   9958 		vsi->alloc_queue_pairs = 1;
   9959 		vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
   9960 				      I40E_REQ_DESCRIPTOR_MULTIPLE);
   9961 		vsi->num_q_vectors = pf->num_fdsb_msix;
   9962 		break;
   9963 
   9964 	case I40E_VSI_VMDQ2:
   9965 		vsi->alloc_queue_pairs = pf->num_vmdq_qps;
   9966 		vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
   9967 				      I40E_REQ_DESCRIPTOR_MULTIPLE);
   9968 		vsi->num_q_vectors = pf->num_vmdq_msix;
   9969 		break;
   9970 
   9971 	case I40E_VSI_SRIOV:
   9972 		vsi->alloc_queue_pairs = pf->num_vf_qps;
   9973 		vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
   9974 				      I40E_REQ_DESCRIPTOR_MULTIPLE);
   9975 		break;
   9976 
   9977 	default:
   9978 		WARN_ON(1);
   9979 		return -ENODATA;
   9980 	}
   9981 
   9982 	return 0;
   9983 }
   9984 
   9985 /**
   9986  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
   9987  * @vsi: VSI pointer
   9988  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
   9989  *
   9990  * On error: returns error code (negative)
   9991  * On success: returns 0
   9992  **/
   9993 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
   9994 {
   9995 	struct i40e_ring **next_rings;
   9996 	int size;
   9997 	int ret = 0;
   9998 
   9999 	/* allocate memory for both Tx, XDP Tx and Rx ring pointers */
  10000 	size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
  10001 	       (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
  10002 	vsi->tx_rings = kzalloc(size, GFP_KERNEL);
  10003 	if (!vsi->tx_rings)
  10004 		return -ENOMEM;
  10005 	next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
  10006 	if (i40e_enabled_xdp_vsi(vsi)) {
  10007 		vsi->xdp_rings = next_rings;
  10008 		next_rings += vsi->alloc_queue_pairs;
  10009 	}
  10010 	vsi->rx_rings = next_rings;
  10011 
  10012 	if (alloc_qvectors) {
  10013 		/* allocate memory for q_vector pointers */
  10014 		size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
  10015 		vsi->q_vectors = kzalloc(size, GFP_KERNEL);
  10016 		if (!vsi->q_vectors) {
  10017 			ret = -ENOMEM;
  10018 			goto err_vectors;
  10019 		}
  10020 	}
  10021 	return ret;
  10022 
  10023 err_vectors:
  10024 	kfree(vsi->tx_rings);
  10025 	return ret;
  10026 }
  10027 
  10028 /**
  10029  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
  10030  * @pf: board private structure
  10031  * @type: type of VSI
  10032  *
  10033  * On error: returns error code (negative)
  10034  * On success: returns vsi index in PF (positive)
  10035  **/
  10036 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
  10037 {
  10038 	int ret = -ENODEV;
  10039 	struct i40e_vsi *vsi;
  10040 	int vsi_idx;
  10041 	int i;
  10042 
  10043 	/* Need to protect the allocation of the VSIs at the PF level */
  10044 	mutex_lock(&pf->switch_mutex);
  10045 
  10046 	/* VSI list may be fragmented if VSI creation/destruction has
  10047 	 * been happening.  We can afford to do a quick scan to look
  10048 	 * for any free VSIs in the list.
  10049 	 *
  10050 	 * find next empty vsi slot, looping back around if necessary
  10051 	 */
  10052 	i = pf->next_vsi;
  10053 	while (i < pf->num_alloc_vsi && pf->vsi[i])
  10054 		i++;
  10055 	if (i >= pf->num_alloc_vsi) {
  10056 		i = 0;
  10057 		while (i < pf->next_vsi && pf->vsi[i])
  10058 			i++;
  10059 	}
  10060 
  10061 	if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
  10062 		vsi_idx = i;             /* Found one! */
  10063 	} else {
  10064 		ret = -ENODEV;
  10065 		goto unlock_pf;  /* out of VSI slots! */
  10066 	}
  10067 	pf->next_vsi = ++i;
  10068 
  10069 	vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
  10070 	if (!vsi) {
  10071 		ret = -ENOMEM;
  10072 		goto unlock_pf;
  10073 	}
  10074 	vsi->type = type;
  10075 	vsi->back = pf;
  10076 	set_bit(__I40E_VSI_DOWN, vsi->state);
  10077 	vsi->flags = 0;
  10078 	vsi->idx = vsi_idx;
  10079 	vsi->int_rate_limit = 0;
  10080 	vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
  10081 				pf->rss_table_size : 64;
  10082 	vsi->netdev_registered = false;
  10083 	vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
  10084 	hash_init(vsi->mac_filter_hash);
  10085 	vsi->irqs_ready = false;
  10086 
  10087 	if (type == I40E_VSI_MAIN) {
  10088 		vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
  10089 		if (!vsi->af_xdp_zc_qps)
  10090 			goto err_rings;
  10091 	}
  10092 
  10093 	ret = i40e_set_num_rings_in_vsi(vsi);
  10094 	if (ret)
  10095 		goto err_rings;
  10096 
  10097 	ret = i40e_vsi_alloc_arrays(vsi, true);
  10098 	if (ret)
  10099 		goto err_rings;
  10100 
  10101 	/* Setup default MSIX irq handler for VSI */
  10102 	i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
  10103 
  10104 	/* Initialize VSI lock */
  10105 	spin_lock_init(&vsi->mac_filter_hash_lock);
  10106 	pf->vsi[vsi_idx] = vsi;
  10107 	ret = vsi_idx;
  10108 	goto unlock_pf;
  10109 
  10110 err_rings:
  10111 	bitmap_free(vsi->af_xdp_zc_qps);
  10112 	pf->next_vsi = i - 1;
  10113 	kfree(vsi);
  10114 unlock_pf:
  10115 	mutex_unlock(&pf->switch_mutex);
  10116 	return ret;
  10117 }
  10118 
  10119 /**
  10120  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
  10121  * @vsi: VSI pointer
  10122  * @free_qvectors: a bool to specify if q_vectors need to be freed.
  10123  *
  10124  * On error: returns error code (negative)
  10125  * On success: returns 0
  10126  **/
  10127 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
  10128 {
  10129 	/* free the ring and vector containers */
  10130 	if (free_qvectors) {
  10131 		kfree(vsi->q_vectors);
  10132 		vsi->q_vectors = NULL;
  10133 	}
  10134 	kfree(vsi->tx_rings);
  10135 	vsi->tx_rings = NULL;
  10136 	vsi->rx_rings = NULL;
  10137 	vsi->xdp_rings = NULL;
  10138 }
  10139 
  10140 /**
  10141  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
  10142  * and lookup table
  10143  * @vsi: Pointer to VSI structure
  10144  */
  10145 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
  10146 {
  10147 	if (!vsi)
  10148 		return;
  10149 
  10150 	kfree(vsi->rss_hkey_user);
  10151 	vsi->rss_hkey_user = NULL;
  10152 
  10153 	kfree(vsi->rss_lut_user);
  10154 	vsi->rss_lut_user = NULL;
  10155 }
  10156 
  10157 /**
  10158  * i40e_vsi_clear - Deallocate the VSI provided
  10159  * @vsi: the VSI being un-configured
  10160  **/
  10161 static int i40e_vsi_clear(struct i40e_vsi *vsi)
  10162 {
  10163 	struct i40e_pf *pf;
  10164 
  10165 	if (!vsi)
  10166 		return 0;
  10167 
  10168 	if (!vsi->back)
  10169 		goto free_vsi;
  10170 	pf = vsi->back;
  10171 
  10172 	mutex_lock(&pf->switch_mutex);
  10173 	if (!pf->vsi[vsi->idx]) {
  10174 		dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
  10175 			vsi->idx, vsi->idx, vsi->type);
  10176 		goto unlock_vsi;
  10177 	}
  10178 
  10179 	if (pf->vsi[vsi->idx] != vsi) {
  10180 		dev_err(&pf->pdev->dev,
  10181 			"pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
  10182 			pf->vsi[vsi->idx]->idx,
  10183 			pf->vsi[vsi->idx]->type,
  10184 			vsi->idx, vsi->type);
  10185 		goto unlock_vsi;
  10186 	}
  10187 
  10188 	/* updates the PF for this cleared vsi */
  10189 	i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
  10190 	i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
  10191 
  10192 	bitmap_free(vsi->af_xdp_zc_qps);
  10193 	i40e_vsi_free_arrays(vsi, true);
  10194 	i40e_clear_rss_config_user(vsi);
  10195 
  10196 	pf->vsi[vsi->idx] = NULL;
  10197 	if (vsi->idx < pf->next_vsi)
  10198 		pf->next_vsi = vsi->idx;
  10199 
  10200 unlock_vsi:
  10201 	mutex_unlock(&pf->switch_mutex);
  10202 free_vsi:
  10203 	kfree(vsi);
  10204 
  10205 	return 0;
  10206 }
  10207 
  10208 /**
  10209  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
  10210  * @vsi: the VSI being cleaned
  10211  **/
  10212 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
  10213 {
  10214 	int i;
  10215 
  10216 	if (vsi->tx_rings && vsi->tx_rings[0]) {
  10217 		for (i = 0; i < vsi->alloc_queue_pairs; i++) {
  10218 			kfree_rcu(vsi->tx_rings[i], rcu);
  10219 			vsi->tx_rings[i] = NULL;
  10220 			vsi->rx_rings[i] = NULL;
  10221 			if (vsi->xdp_rings)
  10222 				vsi->xdp_rings[i] = NULL;
  10223 		}
  10224 	}
  10225 }
  10226 
  10227 /**
  10228  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
  10229  * @vsi: the VSI being configured
  10230  **/
  10231 static int i40e_alloc_rings(struct i40e_vsi *vsi)
  10232 {
  10233 	int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
  10234 	struct i40e_pf *pf = vsi->back;
  10235 	struct i40e_ring *ring;
  10236 
  10237 	/* Set basic values in the rings to be used later during open() */
  10238 	for (i = 0; i < vsi->alloc_queue_pairs; i++) {
  10239 		/* allocate space for both Tx and Rx in one shot */
  10240 		ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
  10241 		if (!ring)
  10242 			goto err_out;
  10243 
  10244 		ring->queue_index = i;
  10245 		ring->reg_idx = vsi->base_queue + i;
  10246 		ring->ring_active = false;
  10247 		ring->vsi = vsi;
  10248 		ring->netdev = vsi->netdev;
  10249 		ring->dev = &pf->pdev->dev;
  10250 		ring->count = vsi->num_desc;
  10251 		ring->size = 0;
  10252 		ring->dcb_tc = 0;
  10253 		if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
  10254 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
  10255 		ring->itr_setting = pf->tx_itr_default;
  10256 		vsi->tx_rings[i] = ring++;
  10257 
  10258 		if (!i40e_enabled_xdp_vsi(vsi))
  10259 			goto setup_rx;
  10260 
  10261 		ring->queue_index = vsi->alloc_queue_pairs + i;
  10262 		ring->reg_idx = vsi->base_queue + ring->queue_index;
  10263 		ring->ring_active = false;
  10264 		ring->vsi = vsi;
  10265 		ring->netdev = NULL;
  10266 		ring->dev = &pf->pdev->dev;
  10267 		ring->count = vsi->num_desc;
  10268 		ring->size = 0;
  10269 		ring->dcb_tc = 0;
  10270 		if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
  10271 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
  10272 		set_ring_xdp(ring);
  10273 		ring->itr_setting = pf->tx_itr_default;
  10274 		vsi->xdp_rings[i] = ring++;
  10275 
  10276 setup_rx:
  10277 		ring->queue_index = i;
  10278 		ring->reg_idx = vsi->base_queue + i;
  10279 		ring->ring_active = false;
  10280 		ring->vsi = vsi;
  10281 		ring->netdev = vsi->netdev;
  10282 		ring->dev = &pf->pdev->dev;
  10283 		ring->count = vsi->num_desc;
  10284 		ring->size = 0;
  10285 		ring->dcb_tc = 0;
  10286 		ring->itr_setting = pf->rx_itr_default;
  10287 		vsi->rx_rings[i] = ring;
  10288 	}
  10289 
  10290 	return 0;
  10291 
  10292 err_out:
  10293 	i40e_vsi_clear_rings(vsi);
  10294 	return -ENOMEM;
  10295 }
  10296 
  10297 /**
  10298  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
  10299  * @pf: board private structure
  10300  * @vectors: the number of MSI-X vectors to request
  10301  *
  10302  * Returns the number of vectors reserved, or error
  10303  **/
  10304 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
  10305 {
  10306 	vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
  10307 					I40E_MIN_MSIX, vectors);
  10308 	if (vectors < 0) {
  10309 		dev_info(&pf->pdev->dev,
  10310 			 "MSI-X vector reservation failed: %d\n", vectors);
  10311 		vectors = 0;
  10312 	}
  10313 
  10314 	return vectors;
  10315 }
  10316 
  10317 /**
  10318  * i40e_init_msix - Setup the MSIX capability
  10319  * @pf: board private structure
  10320  *
  10321  * Work with the OS to set up the MSIX vectors needed.
  10322  *
  10323  * Returns the number of vectors reserved or negative on failure
  10324  **/
  10325 static int i40e_init_msix(struct i40e_pf *pf)
  10326 {
  10327 	struct i40e_hw *hw = &pf->hw;
  10328 	int cpus, extra_vectors;
  10329 	int vectors_left;
  10330 	int v_budget, i;
  10331 	int v_actual;
  10332 	int iwarp_requested = 0;
  10333 
  10334 	if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
  10335 		return -ENODEV;
  10336 
  10337 	/* The number of vectors we'll request will be comprised of:
  10338 	 *   - Add 1 for "other" cause for Admin Queue events, etc.
  10339 	 *   - The number of LAN queue pairs
  10340 	 *	- Queues being used for RSS.
  10341 	 *		We don't need as many as max_rss_size vectors.
  10342 	 *		use rss_size instead in the calculation since that
  10343 	 *		is governed by number of cpus in the system.
  10344 	 *	- assumes symmetric Tx/Rx pairing
  10345 	 *   - The number of VMDq pairs
  10346 	 *   - The CPU count within the NUMA node if iWARP is enabled
  10347 	 * Once we count this up, try the request.
  10348 	 *
  10349 	 * If we can't get what we want, we'll simplify to nearly nothing
  10350 	 * and try again.  If that still fails, we punt.
  10351 	 */
  10352 	vectors_left = hw->func_caps.num_msix_vectors;
  10353 	v_budget = 0;
  10354 
  10355 	/* reserve one vector for miscellaneous handler */
  10356 	if (vectors_left) {
  10357 		v_budget++;
  10358 		vectors_left--;
  10359 	}
  10360 
  10361 	/* reserve some vectors for the main PF traffic queues. Initially we
  10362 	 * only reserve at most 50% of the available vectors, in the case that
  10363 	 * the number of online CPUs is large. This ensures that we can enable
  10364 	 * extra features as well. Once we've enabled the other features, we
  10365 	 * will use any remaining vectors to reach as close as we can to the
  10366 	 * number of online CPUs.
  10367 	 */
  10368 	cpus = num_online_cpus();
  10369 	pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
  10370 	vectors_left -= pf->num_lan_msix;
  10371 
  10372 	/* reserve one vector for sideband flow director */
  10373 	if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
  10374 		if (vectors_left) {
  10375 			pf->num_fdsb_msix = 1;
  10376 			v_budget++;
  10377 			vectors_left--;
  10378 		} else {
  10379 			pf->num_fdsb_msix = 0;
  10380 		}
  10381 	}
  10382 
  10383 	/* can we reserve enough for iWARP? */
  10384 	if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
  10385 		iwarp_requested = pf->num_iwarp_msix;
  10386 
  10387 		if (!vectors_left)
  10388 			pf->num_iwarp_msix = 0;
  10389 		else if (vectors_left < pf->num_iwarp_msix)
  10390 			pf->num_iwarp_msix = 1;
  10391 		v_budget += pf->num_iwarp_msix;
  10392 		vectors_left -= pf->num_iwarp_msix;
  10393 	}
  10394 
  10395 	/* any vectors left over go for VMDq support */
  10396 	if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
  10397 		if (!vectors_left) {
  10398 			pf->num_vmdq_msix = 0;
  10399 			pf->num_vmdq_qps = 0;
  10400 		} else {
  10401 			int vmdq_vecs_wanted =
  10402 				pf->num_vmdq_vsis * pf->num_vmdq_qps;
  10403 			int vmdq_vecs =
  10404 				min_t(int, vectors_left, vmdq_vecs_wanted);
  10405 
  10406 			/* if we're short on vectors for what's desired, we limit
  10407 			 * the queues per vmdq.  If this is still more than are
  10408 			 * available, the user will need to change the number of
  10409 			 * queues/vectors used by the PF later with the ethtool
  10410 			 * channels command
  10411 			 */
  10412 			if (vectors_left < vmdq_vecs_wanted) {
  10413 				pf->num_vmdq_qps = 1;
  10414 				vmdq_vecs_wanted = pf->num_vmdq_vsis;
  10415 				vmdq_vecs = min_t(int,
  10416 						  vectors_left,
  10417 						  vmdq_vecs_wanted);
  10418 			}
  10419 			pf->num_vmdq_msix = pf->num_vmdq_qps;
  10420 
  10421 			v_budget += vmdq_vecs;
  10422 			vectors_left -= vmdq_vecs;
  10423 		}
  10424 	}
  10425 
  10426 	/* On systems with a large number of SMP cores, we previously limited
  10427 	 * the number of vectors for num_lan_msix to be at most 50% of the
  10428 	 * available vectors, to allow for other features. Now, we add back
  10429 	 * the remaining vectors. However, we ensure that the total
  10430 	 * num_lan_msix will not exceed num_online_cpus(). To do this, we
  10431 	 * calculate the number of vectors we can add without going over the
  10432 	 * cap of CPUs. For systems with a small number of CPUs this will be
  10433 	 * zero.
  10434 	 */
  10435 	extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
  10436 	pf->num_lan_msix += extra_vectors;
  10437 	vectors_left -= extra_vectors;
  10438 
  10439 	WARN(vectors_left < 0,
  10440 	     "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
  10441 
  10442 	v_budget += pf->num_lan_msix;
  10443 	pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
  10444 				   GFP_KERNEL);
  10445 	if (!pf->msix_entries)
  10446 		return -ENOMEM;
  10447 
  10448 	for (i = 0; i < v_budget; i++)
  10449 		pf->msix_entries[i].entry = i;
  10450 	v_actual = i40e_reserve_msix_vectors(pf, v_budget);
  10451 
  10452 	if (v_actual < I40E_MIN_MSIX) {
  10453 		pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
  10454 		kfree(pf->msix_entries);
  10455 		pf->msix_entries = NULL;
  10456 		pci_disable_msix(pf->pdev);
  10457 		return -ENODEV;
  10458 
  10459 	} else if (v_actual == I40E_MIN_MSIX) {
  10460 		/* Adjust for minimal MSIX use */
  10461 		pf->num_vmdq_vsis = 0;
  10462 		pf->num_vmdq_qps = 0;
  10463 		pf->num_lan_qps = 1;
  10464 		pf->num_lan_msix = 1;
  10465 
  10466 	} else if (v_actual != v_budget) {
  10467 		/* If we have limited resources, we will start with no vectors
  10468 		 * for the special features and then allocate vectors to some
  10469 		 * of these features based on the policy and at the end disable
  10470 		 * the features that did not get any vectors.
  10471 		 */
  10472 		int vec;
  10473 
  10474 		dev_info(&pf->pdev->dev,
  10475 			 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
  10476 			 v_actual, v_budget);
  10477 		/* reserve the misc vector */
  10478 		vec = v_actual - 1;
  10479 
  10480 		/* Scale vector usage down */
  10481 		pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
  10482 		pf->num_vmdq_vsis = 1;
  10483 		pf->num_vmdq_qps = 1;
  10484 
  10485 		/* partition out the remaining vectors */
  10486 		switch (vec) {
  10487 		case 2:
  10488 			pf->num_lan_msix = 1;
  10489 			break;
  10490 		case 3:
  10491 			if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
  10492 				pf->num_lan_msix = 1;
  10493 				pf->num_iwarp_msix = 1;
  10494 			} else {
  10495 				pf->num_lan_msix = 2;
  10496 			}
  10497 			break;
  10498 		default:
  10499 			if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
  10500 				pf->num_iwarp_msix = min_t(int, (vec / 3),
  10501 						 iwarp_requested);
  10502 				pf->num_vmdq_vsis = min_t(int, (vec / 3),
  10503 						  I40E_DEFAULT_NUM_VMDQ_VSI);
  10504 			} else {
  10505 				pf->num_vmdq_vsis = min_t(int, (vec / 2),
  10506 						  I40E_DEFAULT_NUM_VMDQ_VSI);
  10507 			}
  10508 			if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
  10509 				pf->num_fdsb_msix = 1;
  10510 				vec--;
  10511 			}
  10512 			pf->num_lan_msix = min_t(int,
  10513 			       (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
  10514 							      pf->num_lan_msix);
  10515 			pf->num_lan_qps = pf->num_lan_msix;
  10516 			break;
  10517 		}
  10518 	}
  10519 
  10520 	if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
  10521 	    (pf->num_fdsb_msix == 0)) {
  10522 		dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
  10523 		pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
  10524 		pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
  10525 	}
  10526 	if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
  10527 	    (pf->num_vmdq_msix == 0)) {
  10528 		dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
  10529 		pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
  10530 	}
  10531 
  10532 	if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
  10533 	    (pf->num_iwarp_msix == 0)) {
  10534 		dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
  10535 		pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
  10536 	}
  10537 	i40e_debug(&pf->hw, I40E_DEBUG_INIT,
  10538 		   "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
  10539 		   pf->num_lan_msix,
  10540 		   pf->num_vmdq_msix * pf->num_vmdq_vsis,
  10541 		   pf->num_fdsb_msix,
  10542 		   pf->num_iwarp_msix);
  10543 
  10544 	return v_actual;
  10545 }
  10546 
  10547 /**
  10548  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
  10549  * @vsi: the VSI being configured
  10550  * @v_idx: index of the vector in the vsi struct
  10551  * @cpu: cpu to be used on affinity_mask
  10552  *
  10553  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
  10554  **/
  10555 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
  10556 {
  10557 	struct i40e_q_vector *q_vector;
  10558 
  10559 	/* allocate q_vector */
  10560 	q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
  10561 	if (!q_vector)
  10562 		return -ENOMEM;
  10563 
  10564 	q_vector->vsi = vsi;
  10565 	q_vector->v_idx = v_idx;
  10566 	cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
  10567 
  10568 	if (vsi->netdev)
  10569 		netif_napi_add(vsi->netdev, &q_vector->napi,
  10570 			       i40e_napi_poll, NAPI_POLL_WEIGHT);
  10571 
  10572 	/* tie q_vector and vsi together */
  10573 	vsi->q_vectors[v_idx] = q_vector;
  10574 
  10575 	return 0;
  10576 }
  10577 
  10578 /**
  10579  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
  10580  * @vsi: the VSI being configured
  10581  *
  10582  * We allocate one q_vector per queue interrupt.  If allocation fails we
  10583  * return -ENOMEM.
  10584  **/
  10585 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
  10586 {
  10587 	struct i40e_pf *pf = vsi->back;
  10588 	int err, v_idx, num_q_vectors, current_cpu;
  10589 
  10590 	/* if not MSIX, give the one vector only to the LAN VSI */
  10591 	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
  10592 		num_q_vectors = vsi->num_q_vectors;
  10593 	else if (vsi == pf->vsi[pf->lan_vsi])
  10594 		num_q_vectors = 1;
  10595 	else
  10596 		return -EINVAL;
  10597 
  10598 	current_cpu = cpumask_first(cpu_online_mask);
  10599 
  10600 	for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
  10601 		err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu);
  10602 		if (err)
  10603 			goto err_out;
  10604 		current_cpu = cpumask_next(current_cpu, cpu_online_mask);
  10605 		if (unlikely(current_cpu >= nr_cpu_ids))
  10606 			current_cpu = cpumask_first(cpu_online_mask);
  10607 	}
  10608 
  10609 	return 0;
  10610 
  10611 err_out:
  10612 	while (v_idx--)
  10613 		i40e_free_q_vector(vsi, v_idx);
  10614 
  10615 	return err;
  10616 }
  10617 
  10618 /**
  10619  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
  10620  * @pf: board private structure to initialize
  10621  **/
  10622 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
  10623 {
  10624 	int vectors = 0;
  10625 	ssize_t size;
  10626 
  10627 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
  10628 		vectors = i40e_init_msix(pf);
  10629 		if (vectors < 0) {
  10630 			pf->flags &= ~(I40E_FLAG_MSIX_ENABLED	|
  10631 				       I40E_FLAG_IWARP_ENABLED	|
  10632 				       I40E_FLAG_RSS_ENABLED	|
  10633 				       I40E_FLAG_DCB_CAPABLE	|
  10634 				       I40E_FLAG_DCB_ENABLED	|
  10635 				       I40E_FLAG_SRIOV_ENABLED	|
  10636 				       I40E_FLAG_FD_SB_ENABLED	|
  10637 				       I40E_FLAG_FD_ATR_ENABLED	|
  10638 				       I40E_FLAG_VMDQ_ENABLED);
  10639 			pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
  10640 
  10641 			/* rework the queue expectations without MSIX */
  10642 			i40e_determine_queue_usage(pf);
  10643 		}
  10644 	}
  10645 
  10646 	if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
  10647 	    (pf->flags & I40E_FLAG_MSI_ENABLED)) {
  10648 		dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
  10649 		vectors = pci_enable_msi(pf->pdev);
  10650 		if (vectors < 0) {
  10651 			dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
  10652 				 vectors);
  10653 			pf->flags &= ~I40E_FLAG_MSI_ENABLED;
  10654 		}
  10655 		vectors = 1;  /* one MSI or Legacy vector */
  10656 	}
  10657 
  10658 	if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
  10659 		dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
  10660 
  10661 	/* set up vector assignment tracking */
  10662 	size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
  10663 	pf->irq_pile = kzalloc(size, GFP_KERNEL);
  10664 	if (!pf->irq_pile)
  10665 		return -ENOMEM;
  10666 
  10667 	pf->irq_pile->num_entries = vectors;
  10668 	pf->irq_pile->search_hint = 0;
  10669 
  10670 	/* track first vector for misc interrupts, ignore return */
  10671 	(void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
  10672 
  10673 	return 0;
  10674 }
  10675 
  10676 /**
  10677  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
  10678  * @pf: private board data structure
  10679  *
  10680  * Restore the interrupt scheme that was cleared when we suspended the
  10681  * device. This should be called during resume to re-allocate the q_vectors
  10682  * and reacquire IRQs.
  10683  */
  10684 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
  10685 {
  10686 	int err, i;
  10687 
  10688 	/* We cleared the MSI and MSI-X flags when disabling the old interrupt
  10689 	 * scheme. We need to re-enabled them here in order to attempt to
  10690 	 * re-acquire the MSI or MSI-X vectors
  10691 	 */
  10692 	pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
  10693 
  10694 	err = i40e_init_interrupt_scheme(pf);
  10695 	if (err)
  10696 		return err;
  10697 
  10698 	/* Now that we've re-acquired IRQs, we need to remap the vectors and
  10699 	 * rings together again.
  10700 	 */
  10701 	for (i = 0; i < pf->num_alloc_vsi; i++) {
  10702 		if (pf->vsi[i]) {
  10703 			err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
  10704 			if (err)
  10705 				goto err_unwind;
  10706 			i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
  10707 		}
  10708 	}
  10709 
  10710 	err = i40e_setup_misc_vector(pf);
  10711 	if (err)
  10712 		goto err_unwind;
  10713 
  10714 	if (pf->flags & I40E_FLAG_IWARP_ENABLED)
  10715 		i40e_client_update_msix_info(pf);
  10716 
  10717 	return 0;
  10718 
  10719 err_unwind:
  10720 	while (i--) {
  10721 		if (pf->vsi[i])
  10722 			i40e_vsi_free_q_vectors(pf->vsi[i]);
  10723 	}
  10724 
  10725 	return err;
  10726 }
  10727 
  10728 /**
  10729  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
  10730  * @pf: board private structure
  10731  *
  10732  * This sets up the handler for MSIX 0, which is used to manage the
  10733  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
  10734  * when in MSI or Legacy interrupt mode.
  10735  **/
  10736 static int i40e_setup_misc_vector(struct i40e_pf *pf)
  10737 {
  10738 	struct i40e_hw *hw = &pf->hw;
  10739 	int err = 0;
  10740 
  10741 	/* Only request the IRQ once, the first time through. */
  10742 	if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
  10743 		err = request_irq(pf->msix_entries[0].vector,
  10744 				  i40e_intr, 0, pf->int_name, pf);
  10745 		if (err) {
  10746 			clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
  10747 			dev_info(&pf->pdev->dev,
  10748 				 "request_irq for %s failed: %d\n",
  10749 				 pf->int_name, err);
  10750 			return -EFAULT;
  10751 		}
  10752 	}
  10753 
  10754 	i40e_enable_misc_int_causes(pf);
  10755 
  10756 	/* associate no queues to the misc vector */
  10757 	wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
  10758 	wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
  10759 
  10760 	i40e_flush(hw);
  10761 
  10762 	i40e_irq_dynamic_enable_icr0(pf);
  10763 
  10764 	return err;
  10765 }
  10766 
  10767 /**
  10768  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
  10769  * @vsi: Pointer to vsi structure
  10770  * @seed: Buffter to store the hash keys
  10771  * @lut: Buffer to store the lookup table entries
  10772  * @lut_size: Size of buffer to store the lookup table entries
  10773  *
  10774  * Return 0 on success, negative on failure
  10775  */
  10776 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
  10777 			   u8 *lut, u16 lut_size)
  10778 {
  10779 	struct i40e_pf *pf = vsi->back;
  10780 	struct i40e_hw *hw = &pf->hw;
  10781 	int ret = 0;
  10782 
  10783 	if (seed) {
  10784 		ret = i40e_aq_get_rss_key(hw, vsi->id,
  10785 			(struct i40e_aqc_get_set_rss_key_data *)seed);
  10786 		if (ret) {
  10787 			dev_info(&pf->pdev->dev,
  10788 				 "Cannot get RSS key, err %s aq_err %s\n",
  10789 				 i40e_stat_str(&pf->hw, ret),
  10790 				 i40e_aq_str(&pf->hw,
  10791 					     pf->hw.aq.asq_last_status));
  10792 			return ret;
  10793 		}
  10794 	}
  10795 
  10796 	if (lut) {
  10797 		bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
  10798 
  10799 		ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
  10800 		if (ret) {
  10801 			dev_info(&pf->pdev->dev,
  10802 				 "Cannot get RSS lut, err %s aq_err %s\n",
  10803 				 i40e_stat_str(&pf->hw, ret),
  10804 				 i40e_aq_str(&pf->hw,
  10805 					     pf->hw.aq.asq_last_status));
  10806 			return ret;
  10807 		}
  10808 	}
  10809 
  10810 	return ret;
  10811 }
  10812 
  10813 /**
  10814  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
  10815  * @vsi: Pointer to vsi structure
  10816  * @seed: RSS hash seed
  10817  * @lut: Lookup table
  10818  * @lut_size: Lookup table size
  10819  *
  10820  * Returns 0 on success, negative on failure
  10821  **/
  10822 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
  10823 			       const u8 *lut, u16 lut_size)
  10824 {
  10825 	struct i40e_pf *pf = vsi->back;
  10826 	struct i40e_hw *hw = &pf->hw;
  10827 	u16 vf_id = vsi->vf_id;
  10828 	u8 i;
  10829 
  10830 	/* Fill out hash function seed */
  10831 	if (seed) {
  10832 		u32 *seed_dw = (u32 *)seed;
  10833 
  10834 		if (vsi->type == I40E_VSI_MAIN) {
  10835 			for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
  10836 				wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
  10837 		} else if (vsi->type == I40E_VSI_SRIOV) {
  10838 			for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
  10839 				wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
  10840 		} else {
  10841 			dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
  10842 		}
  10843 	}
  10844 
  10845 	if (lut) {
  10846 		u32 *lut_dw = (u32 *)lut;
  10847 
  10848 		if (vsi->type == I40E_VSI_MAIN) {
  10849 			if (lut_size != I40E_HLUT_ARRAY_SIZE)
  10850 				return -EINVAL;
  10851 			for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
  10852 				wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
  10853 		} else if (vsi->type == I40E_VSI_SRIOV) {
  10854 			if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
  10855 				return -EINVAL;
  10856 			for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
  10857 				wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
  10858 		} else {
  10859 			dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
  10860 		}
  10861 	}
  10862 	i40e_flush(hw);
  10863 
  10864 	return 0;
  10865 }
  10866 
  10867 /**
  10868  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
  10869  * @vsi: Pointer to VSI structure
  10870  * @seed: Buffer to store the keys
  10871  * @lut: Buffer to store the lookup table entries
  10872  * @lut_size: Size of buffer to store the lookup table entries
  10873  *
  10874  * Returns 0 on success, negative on failure
  10875  */
  10876 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
  10877 			    u8 *lut, u16 lut_size)
  10878 {
  10879 	struct i40e_pf *pf = vsi->back;
  10880 	struct i40e_hw *hw = &pf->hw;
  10881 	u16 i;
  10882 
  10883 	if (seed) {
  10884 		u32 *seed_dw = (u32 *)seed;
  10885 
  10886 		for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
  10887 			seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
  10888 	}
  10889 	if (lut) {
  10890 		u32 *lut_dw = (u32 *)lut;
  10891 
  10892 		if (lut_size != I40E_HLUT_ARRAY_SIZE)
  10893 			return -EINVAL;
  10894 		for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
  10895 			lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
  10896 	}
  10897 
  10898 	return 0;
  10899 }
  10900 
  10901 /**
  10902  * i40e_config_rss - Configure RSS keys and lut
  10903  * @vsi: Pointer to VSI structure
  10904  * @seed: RSS hash seed
  10905  * @lut: Lookup table
  10906  * @lut_size: Lookup table size
  10907  *
  10908  * Returns 0 on success, negative on failure
  10909  */
  10910 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
  10911 {
  10912 	struct i40e_pf *pf = vsi->back;
  10913 
  10914 	if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
  10915 		return i40e_config_rss_aq(vsi, seed, lut, lut_size);
  10916 	else
  10917 		return i40e_config_rss_reg(vsi, seed, lut, lut_size);
  10918 }
  10919 
  10920 /**
  10921  * i40e_get_rss - Get RSS keys and lut
  10922  * @vsi: Pointer to VSI structure
  10923  * @seed: Buffer to store the keys
  10924  * @lut: Buffer to store the lookup table entries
  10925  * @lut_size: Size of buffer to store the lookup table entries
  10926  *
  10927  * Returns 0 on success, negative on failure
  10928  */
  10929 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
  10930 {
  10931 	struct i40e_pf *pf = vsi->back;
  10932 
  10933 	if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
  10934 		return i40e_get_rss_aq(vsi, seed, lut, lut_size);
  10935 	else
  10936 		return i40e_get_rss_reg(vsi, seed, lut, lut_size);
  10937 }
  10938 
  10939 /**
  10940  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
  10941  * @pf: Pointer to board private structure
  10942  * @lut: Lookup table
  10943  * @rss_table_size: Lookup table size
  10944  * @rss_size: Range of queue number for hashing
  10945  */
  10946 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
  10947 		       u16 rss_table_size, u16 rss_size)
  10948 {
  10949 	u16 i;
  10950 
  10951 	for (i = 0; i < rss_table_size; i++)
  10952 		lut[i] = i % rss_size;
  10953 }
  10954 
  10955 /**
  10956  * i40e_pf_config_rss - Prepare for RSS if used
  10957  * @pf: board private structure
  10958  **/
  10959 static int i40e_pf_config_rss(struct i40e_pf *pf)
  10960 {
  10961 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
  10962 	u8 seed[I40E_HKEY_ARRAY_SIZE];
  10963 	u8 *lut;
  10964 	struct i40e_hw *hw = &pf->hw;
  10965 	u32 reg_val;
  10966 	u64 hena;
  10967 	int ret;
  10968 
  10969 	/* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
  10970 	hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
  10971 		((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
  10972 	hena |= i40e_pf_get_default_rss_hena(pf);
  10973 
  10974 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
  10975 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
  10976 
  10977 	/* Determine the RSS table size based on the hardware capabilities */
  10978 	reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
  10979 	reg_val = (pf->rss_table_size == 512) ?
  10980 			(reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
  10981 			(reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
  10982 	i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
  10983 
  10984 	/* Determine the RSS size of the VSI */
  10985 	if (!vsi->rss_size) {
  10986 		u16 qcount;
  10987 		/* If the firmware does something weird during VSI init, we
  10988 		 * could end up with zero TCs. Check for that to avoid
  10989 		 * divide-by-zero. It probably won't pass traffic, but it also
  10990 		 * won't panic.
  10991 		 */
  10992 		qcount = vsi->num_queue_pairs /
  10993 			 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
  10994 		vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
  10995 	}
  10996 	if (!vsi->rss_size)
  10997 		return -EINVAL;
  10998 
  10999 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
  11000 	if (!lut)
  11001 		return -ENOMEM;
  11002 
  11003 	/* Use user configured lut if there is one, otherwise use default */
  11004 	if (vsi->rss_lut_user)
  11005 		memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
  11006 	else
  11007 		i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
  11008 
  11009 	/* Use user configured hash key if there is one, otherwise
  11010 	 * use default.
  11011 	 */
  11012 	if (vsi->rss_hkey_user)
  11013 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
  11014 	else
  11015 		netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
  11016 	ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
  11017 	kfree(lut);
  11018 
  11019 	return ret;
  11020 }
  11021 
  11022 /**
  11023  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
  11024  * @pf: board private structure
  11025  * @queue_count: the requested queue count for rss.
  11026  *
  11027  * returns 0 if rss is not enabled, if enabled returns the final rss queue
  11028  * count which may be different from the requested queue count.
  11029  * Note: expects to be called while under rtnl_lock()
  11030  **/
  11031 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
  11032 {
  11033 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
  11034 	int new_rss_size;
  11035 
  11036 	if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
  11037 		return 0;
  11038 
  11039 	queue_count = min_t(int, queue_count, num_online_cpus());
  11040 	new_rss_size = min_t(int, queue_count, pf->rss_size_max);
  11041 
  11042 	if (queue_count != vsi->num_queue_pairs) {
  11043 		u16 qcount;
  11044 
  11045 		vsi->req_queue_pairs = queue_count;
  11046 		i40e_prep_for_reset(pf, true);
  11047 
  11048 		pf->alloc_rss_size = new_rss_size;
  11049 
  11050 		i40e_reset_and_rebuild(pf, true, true);
  11051 
  11052 		/* Discard the user configured hash keys and lut, if less
  11053 		 * queues are enabled.
  11054 		 */
  11055 		if (queue_count < vsi->rss_size) {
  11056 			i40e_clear_rss_config_user(vsi);
  11057 			dev_dbg(&pf->pdev->dev,
  11058 				"discard user configured hash keys and lut\n");
  11059 		}
  11060 
  11061 		/* Reset vsi->rss_size, as number of enabled queues changed */
  11062 		qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
  11063 		vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
  11064 
  11065 		i40e_pf_config_rss(pf);
  11066 	}
  11067 	dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
  11068 		 vsi->req_queue_pairs, pf->rss_size_max);
  11069 	return pf->alloc_rss_size;
  11070 }
  11071 
  11072 /**
  11073  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
  11074  * @pf: board private structure
  11075  **/
  11076 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
  11077 {
  11078 	i40e_status status;
  11079 	bool min_valid, max_valid;
  11080 	u32 max_bw, min_bw;
  11081 
  11082 	status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
  11083 					   &min_valid, &max_valid);
  11084 
  11085 	if (!status) {
  11086 		if (min_valid)
  11087 			pf->min_bw = min_bw;
  11088 		if (max_valid)
  11089 			pf->max_bw = max_bw;
  11090 	}
  11091 
  11092 	return status;
  11093 }
  11094 
  11095 /**
  11096  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
  11097  * @pf: board private structure
  11098  **/
  11099 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
  11100 {
  11101 	struct i40e_aqc_configure_partition_bw_data bw_data;
  11102 	i40e_status status;
  11103 
  11104 	/* Set the valid bit for this PF */
  11105 	bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
  11106 	bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
  11107 	bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
  11108 
  11109 	/* Set the new bandwidths */
  11110 	status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
  11111 
  11112 	return status;
  11113 }
  11114 
  11115 /**
  11116  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
  11117  * @pf: board private structure
  11118  **/
  11119 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
  11120 {
  11121 	/* Commit temporary BW setting to permanent NVM image */
  11122 	enum i40e_admin_queue_err last_aq_status;
  11123 	i40e_status ret;
  11124 	u16 nvm_word;
  11125 
  11126 	if (pf->hw.partition_id != 1) {
  11127 		dev_info(&pf->pdev->dev,
  11128 			 "Commit BW only works on partition 1! This is partition %d",
  11129 			 pf->hw.partition_id);
  11130 		ret = I40E_NOT_SUPPORTED;
  11131 		goto bw_commit_out;
  11132 	}
  11133 
  11134 	/* Acquire NVM for read access */
  11135 	ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
  11136 	last_aq_status = pf->hw.aq.asq_last_status;
  11137 	if (ret) {
  11138 		dev_info(&pf->pdev->dev,
  11139 			 "Cannot acquire NVM for read access, err %s aq_err %s\n",
  11140 			 i40e_stat_str(&pf->hw, ret),
  11141 			 i40e_aq_str(&pf->hw, last_aq_status));
  11142 		goto bw_commit_out;
  11143 	}
  11144 
  11145 	/* Read word 0x10 of NVM - SW compatibility word 1 */
  11146 	ret = i40e_aq_read_nvm(&pf->hw,
  11147 			       I40E_SR_NVM_CONTROL_WORD,
  11148 			       0x10, sizeof(nvm_word), &nvm_word,
  11149 			       false, NULL);
  11150 	/* Save off last admin queue command status before releasing
  11151 	 * the NVM
  11152 	 */
  11153 	last_aq_status = pf->hw.aq.asq_last_status;
  11154 	i40e_release_nvm(&pf->hw);
  11155 	if (ret) {
  11156 		dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
  11157 			 i40e_stat_str(&pf->hw, ret),
  11158 			 i40e_aq_str(&pf->hw, last_aq_status));
  11159 		goto bw_commit_out;
  11160 	}
  11161 
  11162 	/* Wait a bit for NVM release to complete */
  11163 	msleep(50);
  11164 
  11165 	/* Acquire NVM for write access */
  11166 	ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
  11167 	last_aq_status = pf->hw.aq.asq_last_status;
  11168 	if (ret) {
  11169 		dev_info(&pf->pdev->dev,
  11170 			 "Cannot acquire NVM for write access, err %s aq_err %s\n",
  11171 			 i40e_stat_str(&pf->hw, ret),
  11172 			 i40e_aq_str(&pf->hw, last_aq_status));
  11173 		goto bw_commit_out;
  11174 	}
  11175 	/* Write it back out unchanged to initiate update NVM,
  11176 	 * which will force a write of the shadow (alt) RAM to
  11177 	 * the NVM - thus storing the bandwidth values permanently.
  11178 	 */
  11179 	ret = i40e_aq_update_nvm(&pf->hw,
  11180 				 I40E_SR_NVM_CONTROL_WORD,
  11181 				 0x10, sizeof(nvm_word),
  11182 				 &nvm_word, true, 0, NULL);
  11183 	/* Save off last admin queue command status before releasing
  11184 	 * the NVM
  11185 	 */
  11186 	last_aq_status = pf->hw.aq.asq_last_status;
  11187 	i40e_release_nvm(&pf->hw);
  11188 	if (ret)
  11189 		dev_info(&pf->pdev->dev,
  11190 			 "BW settings NOT SAVED, err %s aq_err %s\n",
  11191 			 i40e_stat_str(&pf->hw, ret),
  11192 			 i40e_aq_str(&pf->hw, last_aq_status));
  11193 bw_commit_out:
  11194 
  11195 	return ret;
  11196 }
  11197 
  11198 /**
  11199  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
  11200  * @pf: board private structure to initialize
  11201  *
  11202  * i40e_sw_init initializes the Adapter private data structure.
  11203  * Fields are initialized based on PCI device information and
  11204  * OS network device settings (MTU size).
  11205  **/
  11206 static int i40e_sw_init(struct i40e_pf *pf)
  11207 {
  11208 	int err = 0;
  11209 	int size;
  11210 
  11211 	/* Set default capability flags */
  11212 	pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
  11213 		    I40E_FLAG_MSI_ENABLED     |
  11214 		    I40E_FLAG_MSIX_ENABLED;
  11215 
  11216 	/* Set default ITR */
  11217 	pf->rx_itr_default = I40E_ITR_RX_DEF;
  11218 	pf->tx_itr_default = I40E_ITR_TX_DEF;
  11219 
  11220 	/* Depending on PF configurations, it is possible that the RSS
  11221 	 * maximum might end up larger than the available queues
  11222 	 */
  11223 	pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
  11224 	pf->alloc_rss_size = 1;
  11225 	pf->rss_table_size = pf->hw.func_caps.rss_table_size;
  11226 	pf->rss_size_max = min_t(int, pf->rss_size_max,
  11227 				 pf->hw.func_caps.num_tx_qp);
  11228 	if (pf->hw.func_caps.rss) {
  11229 		pf->flags |= I40E_FLAG_RSS_ENABLED;
  11230 		pf->alloc_rss_size = min_t(int, pf->rss_size_max,
  11231 					   num_online_cpus());
  11232 	}
  11233 
  11234 	/* MFP mode enabled */
  11235 	if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
  11236 		pf->flags |= I40E_FLAG_MFP_ENABLED;
  11237 		dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
  11238 		if (i40e_get_partition_bw_setting(pf)) {
  11239 			dev_warn(&pf->pdev->dev,
  11240 				 "Could not get partition bw settings\n");
  11241 		} else {
  11242 			dev_info(&pf->pdev->dev,
  11243 				 "Partition BW Min = %8.8x, Max = %8.8x\n",
  11244 				 pf->min_bw, pf->max_bw);
  11245 
  11246 			/* nudge the Tx scheduler */
  11247 			i40e_set_partition_bw_setting(pf);
  11248 		}
  11249 	}
  11250 
  11251 	if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
  11252 	    (pf->hw.func_caps.fd_filters_best_effort > 0)) {
  11253 		pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
  11254 		pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
  11255 		if (pf->flags & I40E_FLAG_MFP_ENABLED &&
  11256 		    pf->hw.num_partitions > 1)
  11257 			dev_info(&pf->pdev->dev,
  11258 				 "Flow Director Sideband mode Disabled in MFP mode\n");
  11259 		else
  11260 			pf->flags |= I40E_FLAG_FD_SB_ENABLED;
  11261 		pf->fdir_pf_filter_count =
  11262 				 pf->hw.func_caps.fd_filters_guaranteed;
  11263 		pf->hw.fdir_shared_filter_count =
  11264 				 pf->hw.func_caps.fd_filters_best_effort;
  11265 	}
  11266 
  11267 	if (pf->hw.mac.type == I40E_MAC_X722) {
  11268 		pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
  11269 				    I40E_HW_128_QP_RSS_CAPABLE |
  11270 				    I40E_HW_ATR_EVICT_CAPABLE |
  11271 				    I40E_HW_WB_ON_ITR_CAPABLE |
  11272 				    I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
  11273 				    I40E_HW_NO_PCI_LINK_CHECK |
  11274 				    I40E_HW_USE_SET_LLDP_MIB |
  11275 				    I40E_HW_GENEVE_OFFLOAD_CAPABLE |
  11276 				    I40E_HW_PTP_L4_CAPABLE |
  11277 				    I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
  11278 				    I40E_HW_OUTER_UDP_CSUM_CAPABLE);
  11279 
  11280 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
  11281 		if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
  11282 		    I40E_FDEVICT_PCTYPE_DEFAULT) {
  11283 			dev_warn(&pf->pdev->dev,
  11284 				 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
  11285 			pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
  11286 		}
  11287 	} else if ((pf->hw.aq.api_maj_ver > 1) ||
  11288 		   ((pf->hw.aq.api_maj_ver == 1) &&
  11289 		    (pf->hw.aq.api_min_ver > 4))) {
  11290 		/* Supported in FW API version higher than 1.4 */
  11291 		pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
  11292 	}
  11293 
  11294 	/* Enable HW ATR eviction if possible */
  11295 	if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
  11296 		pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
  11297 
  11298 	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
  11299 	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
  11300 	    (pf->hw.aq.fw_maj_ver < 4))) {
  11301 		pf->hw_features |= I40E_HW_RESTART_AUTONEG;
  11302 		/* No DCB support  for FW < v4.33 */
  11303 		pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
  11304 	}
  11305 
  11306 	/* Disable FW LLDP if FW < v4.3 */
  11307 	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
  11308 	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
  11309 	    (pf->hw.aq.fw_maj_ver < 4)))
  11310 		pf->hw_features |= I40E_HW_STOP_FW_LLDP;
  11311 
  11312 	/* Use the FW Set LLDP MIB API if FW > v4.40 */
  11313 	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
  11314 	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
  11315 	    (pf->hw.aq.fw_maj_ver >= 5)))
  11316 		pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
  11317 
  11318 	/* Enable PTP L4 if FW > v6.0 */
  11319 	if (pf->hw.mac.type == I40E_MAC_XL710 &&
  11320 	    pf->hw.aq.fw_maj_ver >= 6)
  11321 		pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
  11322 
  11323 	if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
  11324 		pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
  11325 		pf->flags |= I40E_FLAG_VMDQ_ENABLED;
  11326 		pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
  11327 	}
  11328 
  11329 	if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
  11330 		pf->flags |= I40E_FLAG_IWARP_ENABLED;
  11331 		/* IWARP needs one extra vector for CQP just like MISC.*/
  11332 		pf->num_iwarp_msix = (int)num_online_cpus() + 1;
  11333 	}
  11334 	/* Stopping FW LLDP engine is supported on XL710 and X722
  11335 	 * starting from FW versions determined in i40e_init_adminq.
  11336 	 * Stopping the FW LLDP engine is not supported on XL710
  11337 	 * if NPAR is functioning so unset this hw flag in this case.
  11338 	 */
  11339 	if (pf->hw.mac.type == I40E_MAC_XL710 &&
  11340 	    pf->hw.func_caps.npar_enable &&
  11341 	    (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
  11342 		pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE;
  11343 
  11344 #ifdef CONFIG_PCI_IOV
  11345 	if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
  11346 		pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
  11347 		pf->flags |= I40E_FLAG_SRIOV_ENABLED;
  11348 		pf->num_req_vfs = min_t(int,
  11349 					pf->hw.func_caps.num_vfs,
  11350 					I40E_MAX_VF_COUNT);
  11351 	}
  11352 #endif /* CONFIG_PCI_IOV */
  11353 	pf->eeprom_version = 0xDEAD;
  11354 	pf->lan_veb = I40E_NO_VEB;
  11355 	pf->lan_vsi = I40E_NO_VSI;
  11356 
  11357 	/* By default FW has this off for performance reasons */
  11358 	pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
  11359 
  11360 	/* set up queue assignment tracking */
  11361 	size = sizeof(struct i40e_lump_tracking)
  11362 		+ (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
  11363 	pf->qp_pile = kzalloc(size, GFP_KERNEL);
  11364 	if (!pf->qp_pile) {
  11365 		err = -ENOMEM;
  11366 		goto sw_init_done;
  11367 	}
  11368 	pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
  11369 	pf->qp_pile->search_hint = 0;
  11370 
  11371 	pf->tx_timeout_recovery_level = 1;
  11372 
  11373 	mutex_init(&pf->switch_mutex);
  11374 
  11375 sw_init_done:
  11376 	return err;
  11377 }
  11378 
  11379 /**
  11380  * i40e_set_ntuple - set the ntuple feature flag and take action
  11381  * @pf: board private structure to initialize
  11382  * @features: the feature set that the stack is suggesting
  11383  *
  11384  * returns a bool to indicate if reset needs to happen
  11385  **/
  11386 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
  11387 {
  11388 	bool need_reset = false;
  11389 
  11390 	/* Check if Flow Director n-tuple support was enabled or disabled.  If
  11391 	 * the state changed, we need to reset.
  11392 	 */
  11393 	if (features & NETIF_F_NTUPLE) {
  11394 		/* Enable filters and mark for reset */
  11395 		if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
  11396 			need_reset = true;
  11397 		/* enable FD_SB only if there is MSI-X vector and no cloud
  11398 		 * filters exist
  11399 		 */
  11400 		if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
  11401 			pf->flags |= I40E_FLAG_FD_SB_ENABLED;
  11402 			pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
  11403 		}
  11404 	} else {
  11405 		/* turn off filters, mark for reset and clear SW filter list */
  11406 		if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
  11407 			need_reset = true;
  11408 			i40e_fdir_filter_exit(pf);
  11409 		}
  11410 		pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
  11411 		clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
  11412 		pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
  11413 
  11414 		/* reset fd counters */
  11415 		pf->fd_add_err = 0;
  11416 		pf->fd_atr_cnt = 0;
  11417 		/* if ATR was auto disabled it can be re-enabled. */
  11418 		if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
  11419 			if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
  11420 			    (I40E_DEBUG_FD & pf->hw.debug_mask))
  11421 				dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
  11422 	}
  11423 	return need_reset;
  11424 }
  11425 
  11426 /**
  11427  * i40e_clear_rss_lut - clear the rx hash lookup table
  11428  * @vsi: the VSI being configured
  11429  **/
  11430 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
  11431 {
  11432 	struct i40e_pf *pf = vsi->back;
  11433 	struct i40e_hw *hw = &pf->hw;
  11434 	u16 vf_id = vsi->vf_id;
  11435 	u8 i;
  11436 
  11437 	if (vsi->type == I40E_VSI_MAIN) {
  11438 		for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
  11439 			wr32(hw, I40E_PFQF_HLUT(i), 0);
  11440 	} else if (vsi->type == I40E_VSI_SRIOV) {
  11441 		for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
  11442 			i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
  11443 	} else {
  11444 		dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
  11445 	}
  11446 }
  11447 
  11448 /**
  11449  * i40e_set_features - set the netdev feature flags
  11450  * @netdev: ptr to the netdev being adjusted
  11451  * @features: the feature set that the stack is suggesting
  11452  * Note: expects to be called while under rtnl_lock()
  11453  **/
  11454 static int i40e_set_features(struct net_device *netdev,
  11455 			     netdev_features_t features)
  11456 {
  11457 	struct i40e_netdev_priv *np = netdev_priv(netdev);
  11458 	struct i40e_vsi *vsi = np->vsi;
  11459 	struct i40e_pf *pf = vsi->back;
  11460 	bool need_reset;
  11461 
  11462 	if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
  11463 		i40e_pf_config_rss(pf);
  11464 	else if (!(features & NETIF_F_RXHASH) &&
  11465 		 netdev->features & NETIF_F_RXHASH)
  11466 		i40e_clear_rss_lut(vsi);
  11467 
  11468 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
  11469 		i40e_vlan_stripping_enable(vsi);
  11470 	else
  11471 		i40e_vlan_stripping_disable(vsi);
  11472 
  11473 	if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
  11474 		dev_err(&pf->pdev->dev,
  11475 			"Offloaded tc filters active, can't turn hw_tc_offload off");
  11476 		return -EINVAL;
  11477 	}
  11478 
  11479 	need_reset = i40e_set_ntuple(pf, features);
  11480 
  11481 	if (need_reset)
  11482 		i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
  11483 
  11484 	return 0;
  11485 }
  11486 
  11487 /**
  11488  * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
  11489  * @pf: board private structure
  11490  * @port: The UDP port to look up
  11491  *
  11492  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
  11493  **/
  11494 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port)
  11495 {
  11496 	u8 i;
  11497 
  11498 	for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
  11499 		/* Do not report ports with pending deletions as
  11500 		 * being available.
  11501 		 */
  11502 		if (!port && (pf->pending_udp_bitmap & BIT_ULL(i)))
  11503 			continue;
  11504 		if (pf->udp_ports[i].port == port)
  11505 			return i;
  11506 	}
  11507 
  11508 	return i;
  11509 }
  11510 
  11511 /**
  11512  * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up
  11513  * @netdev: This physical port's netdev
  11514  * @ti: Tunnel endpoint information
  11515  **/
  11516 static void i40e_udp_tunnel_add(struct net_device *netdev,
  11517 				struct udp_tunnel_info *ti)
  11518 {
  11519 	struct i40e_netdev_priv *np = netdev_priv(netdev);
  11520 	struct i40e_vsi *vsi = np->vsi;
  11521 	struct i40e_pf *pf = vsi->back;
  11522 	u16 port = ntohs(ti->port);
  11523 	u8 next_idx;
  11524 	u8 idx;
  11525 
  11526 	idx = i40e_get_udp_port_idx(pf, port);
  11527 
  11528 	/* Check if port already exists */
  11529 	if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
  11530 		netdev_info(netdev, "port %d already offloaded\n", port);
  11531 		return;
  11532 	}
  11533 
  11534 	/* Now check if there is space to add the new port */
  11535 	next_idx = i40e_get_udp_port_idx(pf, 0);
  11536 
  11537 	if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
  11538 		netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
  11539 			    port);
  11540 		return;
  11541 	}
  11542 
  11543 	switch (ti->type) {
  11544 	case UDP_TUNNEL_TYPE_VXLAN:
  11545 		pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
  11546 		break;
  11547 	case UDP_TUNNEL_TYPE_GENEVE:
  11548 		if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE))
  11549 			return;
  11550 		pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
  11551 		break;
  11552 	default:
  11553 		return;
  11554 	}
  11555 
  11556 	/* New port: add it and mark its index in the bitmap */
  11557 	pf->udp_ports[next_idx].port = port;
  11558 	pf->udp_ports[next_idx].filter_index = I40E_UDP_PORT_INDEX_UNUSED;
  11559 	pf->pending_udp_bitmap |= BIT_ULL(next_idx);
  11560 	set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
  11561 }
  11562 
  11563 /**
  11564  * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away
  11565  * @netdev: This physical port's netdev
  11566  * @ti: Tunnel endpoint information
  11567  **/
  11568 static void i40e_udp_tunnel_del(struct net_device *netdev,
  11569 				struct udp_tunnel_info *ti)
  11570 {
  11571 	struct i40e_netdev_priv *np = netdev_priv(netdev);
  11572 	struct i40e_vsi *vsi = np->vsi;
  11573 	struct i40e_pf *pf = vsi->back;
  11574 	u16 port = ntohs(ti->port);
  11575 	u8 idx;
  11576 
  11577 	idx = i40e_get_udp_port_idx(pf, port);
  11578 
  11579 	/* Check if port already exists */
  11580 	if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
  11581 		goto not_found;
  11582 
  11583 	switch (ti->type) {
  11584 	case UDP_TUNNEL_TYPE_VXLAN:
  11585 		if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
  11586 			goto not_found;
  11587 		break;
  11588 	case UDP_TUNNEL_TYPE_GENEVE:
  11589 		if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
  11590 			goto not_found;
  11591 		break;
  11592 	default:
  11593 		goto not_found;
  11594 	}
  11595 
  11596 	/* if port exists, set it to 0 (mark for deletion)
  11597 	 * and make it pending
  11598 	 */
  11599 	pf->udp_ports[idx].port = 0;
  11600 
  11601 	/* Toggle pending bit instead of setting it. This way if we are
  11602 	 * deleting a port that has yet to be added we just clear the pending
  11603 	 * bit and don't have to worry about it.
  11604 	 */
  11605 	pf->pending_udp_bitmap ^= BIT_ULL(idx);
  11606 	set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
  11607 
  11608 	return;
  11609 not_found:
  11610 	netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
  11611 		    port);
  11612 }
  11613 
  11614 static int i40e_get_phys_port_id(struct net_device *netdev,
  11615 				 struct netdev_phys_item_id *ppid)
  11616 {
  11617 	struct i40e_netdev_priv *np = netdev_priv(netdev);
  11618 	struct i40e_pf *pf = np->vsi->back;
  11619 	struct i40e_hw *hw = &pf->hw;
  11620 
  11621 	if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
  11622 		return -EOPNOTSUPP;
  11623 
  11624 	ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
  11625 	memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
  11626 
  11627 	return 0;
  11628 }
  11629 
  11630 /**
  11631  * i40e_ndo_fdb_add - add an entry to the hardware database
  11632  * @ndm: the input from the stack
  11633  * @tb: pointer to array of nladdr (unused)
  11634  * @dev: the net device pointer
  11635  * @addr: the MAC address entry being added
  11636  * @vid: VLAN ID
  11637  * @flags: instructions from stack about fdb operation
  11638  */
  11639 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
  11640 			    struct net_device *dev,
  11641 			    const unsigned char *addr, u16 vid,
  11642 			    u16 flags,
  11643 			    struct netlink_ext_ack *extack)
  11644 {
  11645 	struct i40e_netdev_priv *np = netdev_priv(dev);
  11646 	struct i40e_pf *pf = np->vsi->back;
  11647 	int err = 0;
  11648 
  11649 	if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
  11650 		return -EOPNOTSUPP;
  11651 
  11652 	if (vid) {
  11653 		pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
  11654 		return -EINVAL;
  11655 	}
  11656 
  11657 	/* Hardware does not support aging addresses so if a
  11658 	 * ndm_state is given only allow permanent addresses
  11659 	 */
  11660 	if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
  11661 		netdev_info(dev, "FDB only supports static addresses\n");
  11662 		return -EINVAL;
  11663 	}
  11664 
  11665 	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
  11666 		err = dev_uc_add_excl(dev, addr);
  11667 	else if (is_multicast_ether_addr(addr))
  11668 		err = dev_mc_add_excl(dev, addr);
  11669 	else
  11670 		err = -EINVAL;
  11671 
  11672 	/* Only return duplicate errors if NLM_F_EXCL is set */
  11673 	if (err == -EEXIST && !(flags & NLM_F_EXCL))
  11674 		err = 0;
  11675 
  11676 	return err;
  11677 }
  11678 
  11679 /**
  11680  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
  11681  * @dev: the netdev being configured
  11682  * @nlh: RTNL message
  11683  * @flags: bridge flags
  11684  * @extack: netlink extended ack
  11685  *
  11686  * Inserts a new hardware bridge if not already created and
  11687  * enables the bridging mode requested (VEB or VEPA). If the
  11688  * hardware bridge has already been inserted and the request
  11689  * is to change the mode then that requires a PF reset to
  11690  * allow rebuild of the components with required hardware
  11691  * bridge mode enabled.
  11692  *
  11693  * Note: expects to be called while under rtnl_lock()
  11694  **/
  11695 static int i40e_ndo_bridge_setlink(struct net_device *dev,
  11696 				   struct nlmsghdr *nlh,
  11697 				   u16 flags,
  11698 				   struct netlink_ext_ack *extack)
  11699 {
  11700 	struct i40e_netdev_priv *np = netdev_priv(dev);
  11701 	struct i40e_vsi *vsi = np->vsi;
  11702 	struct i40e_pf *pf = vsi->back;
  11703 	struct i40e_veb *veb = NULL;
  11704 	struct nlattr *attr, *br_spec;
  11705 	int i, rem;
  11706 
  11707 	/* Only for PF VSI for now */
  11708 	if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
  11709 		return -EOPNOTSUPP;
  11710 
  11711 	/* Find the HW bridge for PF VSI */
  11712 	for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
  11713 		if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
  11714 			veb = pf->veb[i];
  11715 	}
  11716 
  11717 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
  11718 
  11719 	nla_for_each_nested(attr, br_spec, rem) {
  11720 		__u16 mode;
  11721 
  11722 		if (nla_type(attr) != IFLA_BRIDGE_MODE)
  11723 			continue;
  11724 
  11725 		mode = nla_get_u16(attr);
  11726 		if ((mode != BRIDGE_MODE_VEPA) &&
  11727 		    (mode != BRIDGE_MODE_VEB))
  11728 			return -EINVAL;
  11729 
  11730 		/* Insert a new HW bridge */
  11731 		if (!veb) {
  11732 			veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
  11733 					     vsi->tc_config.enabled_tc);
  11734 			if (veb) {
  11735 				veb->bridge_mode = mode;
  11736 				i40e_config_bridge_mode(veb);
  11737 			} else {
  11738 				/* No Bridge HW offload available */
  11739 				return -ENOENT;
  11740 			}
  11741 			break;
  11742 		} else if (mode != veb->bridge_mode) {
  11743 			/* Existing HW bridge but different mode needs reset */
  11744 			veb->bridge_mode = mode;
  11745 			/* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
  11746 			if (mode == BRIDGE_MODE_VEB)
  11747 				pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
  11748 			else
  11749 				pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
  11750 			i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
  11751 			break;
  11752 		}
  11753 	}
  11754 
  11755 	return 0;
  11756 }
  11757 
  11758 /**
  11759  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
  11760  * @skb: skb buff
  11761  * @pid: process id
  11762  * @seq: RTNL message seq #
  11763  * @dev: the netdev being configured
  11764  * @filter_mask: unused
  11765  * @nlflags: netlink flags passed in
  11766  *
  11767  * Return the mode in which the hardware bridge is operating in
  11768  * i.e VEB or VEPA.
  11769  **/
  11770 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
  11771 				   struct net_device *dev,
  11772 				   u32 __always_unused filter_mask,
  11773 				   int nlflags)
  11774 {
  11775 	struct i40e_netdev_priv *np = netdev_priv(dev);
  11776 	struct i40e_vsi *vsi = np->vsi;
  11777 	struct i40e_pf *pf = vsi->back;
  11778 	struct i40e_veb *veb = NULL;
  11779 	int i;
  11780 
  11781 	/* Only for PF VSI for now */
  11782 	if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
  11783 		return -EOPNOTSUPP;
  11784 
  11785 	/* Find the HW bridge for the PF VSI */
  11786 	for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
  11787 		if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
  11788 			veb = pf->veb[i];
  11789 	}
  11790 
  11791 	if (!veb)
  11792 		return 0;
  11793 
  11794 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
  11795 				       0, 0, nlflags, filter_mask, NULL);
  11796 }
  11797 
  11798 /**
  11799  * i40e_features_check - Validate encapsulated packet conforms to limits
  11800  * @skb: skb buff
  11801  * @dev: This physical port's netdev
  11802  * @features: Offload features that the stack believes apply
  11803  **/
  11804 static netdev_features_t i40e_features_check(struct sk_buff *skb,
  11805 					     struct net_device *dev,
  11806 					     netdev_features_t features)
  11807 {
  11808 	size_t len;
  11809 
  11810 	/* No point in doing any of this if neither checksum nor GSO are
  11811 	 * being requested for this frame.  We can rule out both by just
  11812 	 * checking for CHECKSUM_PARTIAL
  11813 	 */
  11814 	if (skb->ip_summed != CHECKSUM_PARTIAL)
  11815 		return features;
  11816 
  11817 	/* We cannot support GSO if the MSS is going to be less than
  11818 	 * 64 bytes.  If it is then we need to drop support for GSO.
  11819 	 */
  11820 	if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
  11821 		features &= ~NETIF_F_GSO_MASK;
  11822 
  11823 	/* MACLEN can support at most 63 words */
  11824 	len = skb_network_header(skb) - skb->data;
  11825 	if (len & ~(63 * 2))
  11826 		goto out_err;
  11827 
  11828 	/* IPLEN and EIPLEN can support at most 127 dwords */
  11829 	len = skb_transport_header(skb) - skb_network_header(skb);
  11830 	if (len & ~(127 * 4))
  11831 		goto out_err;
  11832 
  11833 	if (skb->encapsulation) {
  11834 		/* L4TUNLEN can support 127 words */
  11835 		len = skb_inner_network_header(skb) - skb_transport_header(skb);
  11836 		if (len & ~(127 * 2))
  11837 			goto out_err;
  11838 
  11839 		/* IPLEN can support at most 127 dwords */
  11840 		len = skb_inner_transport_header(skb) -
  11841 		      skb_inner_network_header(skb);
  11842 		if (len & ~(127 * 4))
  11843 			goto out_err;
  11844 	}
  11845 
  11846 	/* No need to validate L4LEN as TCP is the only protocol with a
  11847 	 * a flexible value and we support all possible values supported
  11848 	 * by TCP, which is at most 15 dwords
  11849 	 */
  11850 
  11851 	return features;
  11852 out_err:
  11853 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
  11854 }
  11855 
  11856 /**
  11857  * i40e_xdp_setup - add/remove an XDP program
  11858  * @vsi: VSI to changed
  11859  * @prog: XDP program
  11860  **/
  11861 static int i40e_xdp_setup(struct i40e_vsi *vsi,
  11862 			  struct bpf_prog *prog)
  11863 {
  11864 	int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
  11865 	struct i40e_pf *pf = vsi->back;
  11866 	struct bpf_prog *old_prog;
  11867 	bool need_reset;
  11868 	int i;
  11869 
  11870 	/* Don't allow frames that span over multiple buffers */
  11871 	if (frame_size > vsi->rx_buf_len)
  11872 		return -EINVAL;
  11873 
  11874 	if (!i40e_enabled_xdp_vsi(vsi) && !prog)
  11875 		return 0;
  11876 
  11877 	/* When turning XDP on->off/off->on we reset and rebuild the rings. */
  11878 	need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
  11879 
  11880 	if (need_reset)
  11881 		i40e_prep_for_reset(pf, true);
  11882 
  11883 	old_prog = xchg(&vsi->xdp_prog, prog);
  11884 
  11885 	if (need_reset)
  11886 		i40e_reset_and_rebuild(pf, true, true);
  11887 
  11888 	for (i = 0; i < vsi->num_queue_pairs; i++)
  11889 		WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
  11890 
  11891 	if (old_prog)
  11892 		bpf_prog_put(old_prog);
  11893 
  11894 	/* Kick start the NAPI context if there is an AF_XDP socket open
  11895 	 * on that queue id. This so that receiving will start.
  11896 	 */
  11897 	if (need_reset && prog)
  11898 		for (i = 0; i < vsi->num_queue_pairs; i++)
  11899 			if (vsi->xdp_rings[i]->xsk_umem)
  11900 				(void)i40e_xsk_async_xmit(vsi->netdev, i);
  11901 
  11902 	return 0;
  11903 }
  11904 
  11905 /**
  11906  * i40e_enter_busy_conf - Enters busy config state
  11907  * @vsi: vsi
  11908  *
  11909  * Returns 0 on success, <0 for failure.
  11910  **/
  11911 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
  11912 {
  11913 	struct i40e_pf *pf = vsi->back;
  11914 	int timeout = 50;
  11915 
  11916 	while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
  11917 		timeout--;
  11918 		if (!timeout)
  11919 			return -EBUSY;
  11920 		usleep_range(1000, 2000);
  11921 	}
  11922 
  11923 	return 0;
  11924 }
  11925 
  11926 /**
  11927  * i40e_exit_busy_conf - Exits busy config state
  11928  * @vsi: vsi
  11929  **/
  11930 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
  11931 {
  11932 	struct i40e_pf *pf = vsi->back;
  11933 
  11934 	clear_bit(__I40E_CONFIG_BUSY, pf->state);
  11935 }
  11936 
  11937 /**
  11938  * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
  11939  * @vsi: vsi
  11940  * @queue_pair: queue pair
  11941  **/
  11942 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
  11943 {
  11944 	memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
  11945 	       sizeof(vsi->rx_rings[queue_pair]->rx_stats));
  11946 	memset(&vsi->tx_rings[queue_pair]->stats, 0,
  11947 	       sizeof(vsi->tx_rings[queue_pair]->stats));
  11948 	if (i40e_enabled_xdp_vsi(vsi)) {
  11949 		memset(&vsi->xdp_rings[queue_pair]->stats, 0,
  11950 		       sizeof(vsi->xdp_rings[queue_pair]->stats));
  11951 	}
  11952 }
  11953 
  11954 /**
  11955  * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
  11956  * @vsi: vsi
  11957  * @queue_pair: queue pair
  11958  **/
  11959 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
  11960 {
  11961 	i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
  11962 	if (i40e_enabled_xdp_vsi(vsi)) {
  11963 		/* Make sure that in-progress ndo_xdp_xmit calls are
  11964 		 * completed.
  11965 		 */
  11966 		synchronize_rcu();
  11967 		i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
  11968 	}
  11969 	i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
  11970 }
  11971 
  11972 /**
  11973  * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
  11974  * @vsi: vsi
  11975  * @queue_pair: queue pair
  11976  * @enable: true for enable, false for disable
  11977  **/
  11978 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
  11979 					bool enable)
  11980 {
  11981 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
  11982 	struct i40e_q_vector *q_vector = rxr->q_vector;
  11983 
  11984 	if (!vsi->netdev)
  11985 		return;
  11986 
  11987 	/* All rings in a qp belong to the same qvector. */
  11988 	if (q_vector->rx.ring || q_vector->tx.ring) {
  11989 		if (enable)
  11990 			napi_enable(&q_vector->napi);
  11991 		else
  11992 			napi_disable(&q_vector->napi);
  11993 	}
  11994 }
  11995 
  11996 /**
  11997  * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
  11998  * @vsi: vsi
  11999  * @queue_pair: queue pair
  12000  * @enable: true for enable, false for disable
  12001  *
  12002  * Returns 0 on success, <0 on failure.
  12003  **/
  12004 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
  12005 					bool enable)
  12006 {
  12007 	struct i40e_pf *pf = vsi->back;
  12008 	int pf_q, ret = 0;
  12009 
  12010 	pf_q = vsi->base_queue + queue_pair;
  12011 	ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
  12012 				     false /*is xdp*/, enable);
  12013 	if (ret) {
  12014 		dev_info(&pf->pdev->dev,
  12015 			 "VSI seid %d Tx ring %d %sable timeout\n",
  12016 			 vsi->seid, pf_q, (enable ? "en" : "dis"));
  12017 		return ret;
  12018 	}
  12019 
  12020 	i40e_control_rx_q(pf, pf_q, enable);
  12021 	ret = i40e_pf_rxq_wait(pf, pf_q, enable);
  12022 	if (ret) {
  12023 		dev_info(&pf->pdev->dev,
  12024 			 "VSI seid %d Rx ring %d %sable timeout\n",
  12025 			 vsi->seid, pf_q, (enable ? "en" : "dis"));
  12026 		return ret;
  12027 	}
  12028 
  12029 	/* Due to HW errata, on Rx disable only, the register can
  12030 	 * indicate done before it really is. Needs 50ms to be sure
  12031 	 */
  12032 	if (!enable)
  12033 		mdelay(50);
  12034 
  12035 	if (!i40e_enabled_xdp_vsi(vsi))
  12036 		return ret;
  12037 
  12038 	ret = i40e_control_wait_tx_q(vsi->seid, pf,
  12039 				     pf_q + vsi->alloc_queue_pairs,
  12040 				     true /*is xdp*/, enable);
  12041 	if (ret) {
  12042 		dev_info(&pf->pdev->dev,
  12043 			 "VSI seid %d XDP Tx ring %d %sable timeout\n",
  12044 			 vsi->seid, pf_q, (enable ? "en" : "dis"));
  12045 	}
  12046 
  12047 	return ret;
  12048 }
  12049 
  12050 /**
  12051  * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
  12052  * @vsi: vsi
  12053  * @queue_pair: queue_pair
  12054  **/
  12055 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
  12056 {
  12057 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
  12058 	struct i40e_pf *pf = vsi->back;
  12059 	struct i40e_hw *hw = &pf->hw;
  12060 
  12061 	/* All rings in a qp belong to the same qvector. */
  12062 	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
  12063 		i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
  12064 	else
  12065 		i40e_irq_dynamic_enable_icr0(pf);
  12066 
  12067 	i40e_flush(hw);
  12068 }
  12069 
  12070 /**
  12071  * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
  12072  * @vsi: vsi
  12073  * @queue_pair: queue_pair
  12074  **/
  12075 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
  12076 {
  12077 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
  12078 	struct i40e_pf *pf = vsi->back;
  12079 	struct i40e_hw *hw = &pf->hw;
  12080 
  12081 	/* For simplicity, instead of removing the qp interrupt causes
  12082 	 * from the interrupt linked list, we simply disable the interrupt, and
  12083 	 * leave the list intact.
  12084 	 *
  12085 	 * All rings in a qp belong to the same qvector.
  12086 	 */
  12087 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
  12088 		u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
  12089 
  12090 		wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
  12091 		i40e_flush(hw);
  12092 		synchronize_irq(pf->msix_entries[intpf].vector);
  12093 	} else {
  12094 		/* Legacy and MSI mode - this stops all interrupt handling */
  12095 		wr32(hw, I40E_PFINT_ICR0_ENA, 0);
  12096 		wr32(hw, I40E_PFINT_DYN_CTL0, 0);
  12097 		i40e_flush(hw);
  12098 		synchronize_irq(pf->pdev->irq);
  12099 	}
  12100 }
  12101 
  12102 /**
  12103  * i40e_queue_pair_disable - Disables a queue pair
  12104  * @vsi: vsi
  12105  * @queue_pair: queue pair
  12106  *
  12107  * Returns 0 on success, <0 on failure.
  12108  **/
  12109 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
  12110 {
  12111 	int err;
  12112 
  12113 	err = i40e_enter_busy_conf(vsi);
  12114 	if (err)
  12115 		return err;
  12116 
  12117 	i40e_queue_pair_disable_irq(vsi, queue_pair);
  12118 	err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
  12119 	i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
  12120 	i40e_queue_pair_clean_rings(vsi, queue_pair);
  12121 	i40e_queue_pair_reset_stats(vsi, queue_pair);
  12122 
  12123 	return err;
  12124 }
  12125 
  12126 /**
  12127  * i40e_queue_pair_enable - Enables a queue pair
  12128  * @vsi: vsi
  12129  * @queue_pair: queue pair
  12130  *
  12131  * Returns 0 on success, <0 on failure.
  12132  **/
  12133 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
  12134 {
  12135 	int err;
  12136 
  12137 	err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
  12138 	if (err)
  12139 		return err;
  12140 
  12141 	if (i40e_enabled_xdp_vsi(vsi)) {
  12142 		err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
  12143 		if (err)
  12144 			return err;
  12145 	}
  12146 
  12147 	err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
  12148 	if (err)
  12149 		return err;
  12150 
  12151 	err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
  12152 	i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
  12153 	i40e_queue_pair_enable_irq(vsi, queue_pair);
  12154 
  12155 	i40e_exit_busy_conf(vsi);
  12156 
  12157 	return err;
  12158 }
  12159 
  12160 /**
  12161  * i40e_xdp - implements ndo_bpf for i40e
  12162  * @dev: netdevice
  12163  * @xdp: XDP command
  12164  **/
  12165 static int i40e_xdp(struct net_device *dev,
  12166 		    struct netdev_bpf *xdp)
  12167 {
  12168 	struct i40e_netdev_priv *np = netdev_priv(dev);
  12169 	struct i40e_vsi *vsi = np->vsi;
  12170 
  12171 	if (vsi->type != I40E_VSI_MAIN)
  12172 		return -EINVAL;
  12173 
  12174 	switch (xdp->command) {
  12175 	case XDP_SETUP_PROG:
  12176 		return i40e_xdp_setup(vsi, xdp->prog);
  12177 	case XDP_QUERY_PROG:
  12178 		xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;
  12179 		return 0;
  12180 	case XDP_SETUP_XSK_UMEM:
  12181 		return i40e_xsk_umem_setup(vsi, xdp->xsk.umem,
  12182 					   xdp->xsk.queue_id);
  12183 	default:
  12184 		return -EINVAL;
  12185 	}
  12186 }
  12187 
  12188 static const struct net_device_ops i40e_netdev_ops = {
  12189 	.ndo_open		= i40e_open,
  12190 	.ndo_stop		= i40e_close,
  12191 	.ndo_start_xmit		= i40e_lan_xmit_frame,
  12192 	.ndo_get_stats64	= i40e_get_netdev_stats_struct,
  12193 	.ndo_set_rx_mode	= i40e_set_rx_mode,
  12194 	.ndo_validate_addr	= eth_validate_addr,
  12195 	.ndo_set_mac_address	= i40e_set_mac,
  12196 	.ndo_change_mtu		= i40e_change_mtu,
  12197 	.ndo_do_ioctl		= i40e_ioctl,
  12198 	.ndo_tx_timeout		= i40e_tx_timeout,
  12199 	.ndo_vlan_rx_add_vid	= i40e_vlan_rx_add_vid,
  12200 	.ndo_vlan_rx_kill_vid	= i40e_vlan_rx_kill_vid,
  12201 #ifdef CONFIG_NET_POLL_CONTROLLER
  12202 	.ndo_poll_controller	= i40e_netpoll,
  12203 #endif
  12204 	.ndo_setup_tc		= __i40e_setup_tc,
  12205 	.ndo_set_features	= i40e_set_features,
  12206 	.ndo_set_vf_mac		= i40e_ndo_set_vf_mac,
  12207 	.ndo_set_vf_vlan	= i40e_ndo_set_vf_port_vlan,
  12208 	.ndo_set_vf_rate	= i40e_ndo_set_vf_bw,
  12209 	.ndo_get_vf_config	= i40e_ndo_get_vf_config,
  12210 	.ndo_set_vf_link_state	= i40e_ndo_set_vf_link_state,
  12211 	.ndo_set_vf_spoofchk	= i40e_ndo_set_vf_spoofchk,
  12212 	.ndo_set_vf_trust	= i40e_ndo_set_vf_trust,
  12213 	.ndo_udp_tunnel_add	= i40e_udp_tunnel_add,
  12214 	.ndo_udp_tunnel_del	= i40e_udp_tunnel_del,
  12215 	.ndo_get_phys_port_id	= i40e_get_phys_port_id,
  12216 	.ndo_fdb_add		= i40e_ndo_fdb_add,
  12217 	.ndo_features_check	= i40e_features_check,
  12218 	.ndo_bridge_getlink	= i40e_ndo_bridge_getlink,
  12219 	.ndo_bridge_setlink	= i40e_ndo_bridge_setlink,
  12220 	.ndo_bpf		= i40e_xdp,
  12221 	.ndo_xdp_xmit		= i40e_xdp_xmit,
  12222 	.ndo_xsk_async_xmit	= i40e_xsk_async_xmit,
  12223 };
  12224 
  12225 /**
  12226  * i40e_config_netdev - Setup the netdev flags
  12227  * @vsi: the VSI being configured
  12228  *
  12229  * Returns 0 on success, negative value on failure
  12230  **/
  12231 static int i40e_config_netdev(struct i40e_vsi *vsi)
  12232 {
  12233 	struct i40e_pf *pf = vsi->back;
  12234 	struct i40e_hw *hw = &pf->hw;
  12235 	struct i40e_netdev_priv *np;
  12236 	struct net_device *netdev;
  12237 	u8 broadcast[ETH_ALEN];
  12238 	u8 mac_addr[ETH_ALEN];
  12239 	int etherdev_size;
  12240 	netdev_features_t hw_enc_features;
  12241 	netdev_features_t hw_features;
  12242 
  12243 	etherdev_size = sizeof(struct i40e_netdev_priv);
  12244 	netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
  12245 	if (!netdev)
  12246 		return -ENOMEM;
  12247 
  12248 	vsi->netdev = netdev;
  12249 	np = netdev_priv(netdev);
  12250 	np->vsi = vsi;
  12251 
  12252 	hw_enc_features = NETIF_F_SG			|
  12253 			  NETIF_F_IP_CSUM		|
  12254 			  NETIF_F_IPV6_CSUM		|
  12255 			  NETIF_F_HIGHDMA		|
  12256 			  NETIF_F_SOFT_FEATURES		|
  12257 			  NETIF_F_TSO			|
  12258 			  NETIF_F_TSO_ECN		|
  12259 			  NETIF_F_TSO6			|
  12260 			  NETIF_F_GSO_GRE		|
  12261 			  NETIF_F_GSO_GRE_CSUM		|
  12262 			  NETIF_F_GSO_PARTIAL		|
  12263 			  NETIF_F_GSO_IPXIP4		|
  12264 			  NETIF_F_GSO_IPXIP6		|
  12265 			  NETIF_F_GSO_UDP_TUNNEL	|
  12266 			  NETIF_F_GSO_UDP_TUNNEL_CSUM	|
  12267 			  NETIF_F_SCTP_CRC		|
  12268 			  NETIF_F_RXHASH		|
  12269 			  NETIF_F_RXCSUM		|
  12270 			  0;
  12271 
  12272 	if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
  12273 		netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
  12274 
  12275 	netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
  12276 
  12277 	netdev->hw_enc_features |= hw_enc_features;
  12278 
  12279 	/* record features VLANs can make use of */
  12280 	netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
  12281 
  12282 	hw_features = hw_enc_features		|
  12283 		      NETIF_F_HW_VLAN_CTAG_TX	|
  12284 		      NETIF_F_HW_VLAN_CTAG_RX;
  12285 
  12286 	if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
  12287 		hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
  12288 
  12289 	netdev->hw_features |= hw_features;
  12290 
  12291 	netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
  12292 	netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
  12293 
  12294 	if (vsi->type == I40E_VSI_MAIN) {
  12295 		SET_NETDEV_DEV(netdev, &pf->pdev->dev);
  12296 		ether_addr_copy(mac_addr, hw->mac.perm_addr);
  12297 		/* The following steps are necessary for two reasons. First,
  12298 		 * some older NVM configurations load a default MAC-VLAN
  12299 		 * filter that will accept any tagged packet, and we want to
  12300 		 * replace this with a normal filter. Additionally, it is
  12301 		 * possible our MAC address was provided by the platform using
  12302 		 * Open Firmware or similar.
  12303 		 *
  12304 		 * Thus, we need to remove the default filter and install one
  12305 		 * specific to the MAC address.
  12306 		 */
  12307 		i40e_rm_default_mac_filter(vsi, mac_addr);
  12308 		spin_lock_bh(&vsi->mac_filter_hash_lock);
  12309 		i40e_add_mac_filter(vsi, mac_addr);
  12310 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
  12311 	} else {
  12312 		/* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
  12313 		 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
  12314 		 * the end, which is 4 bytes long, so force truncation of the
  12315 		 * original name by IFNAMSIZ - 4
  12316 		 */
  12317 		snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
  12318 			 IFNAMSIZ - 4,
  12319 			 pf->vsi[pf->lan_vsi]->netdev->name);
  12320 		eth_random_addr(mac_addr);
  12321 
  12322 		spin_lock_bh(&vsi->mac_filter_hash_lock);
  12323 		i40e_add_mac_filter(vsi, mac_addr);
  12324 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
  12325 	}
  12326 
  12327 	/* Add the broadcast filter so that we initially will receive
  12328 	 * broadcast packets. Note that when a new VLAN is first added the
  12329 	 * driver will convert all filters marked I40E_VLAN_ANY into VLAN
  12330 	 * specific filters as part of transitioning into "vlan" operation.
  12331 	 * When more VLANs are added, the driver will copy each existing MAC
  12332 	 * filter and add it for the new VLAN.
  12333 	 *
  12334 	 * Broadcast filters are handled specially by
  12335 	 * i40e_sync_filters_subtask, as the driver must to set the broadcast
  12336 	 * promiscuous bit instead of adding this directly as a MAC/VLAN
  12337 	 * filter. The subtask will update the correct broadcast promiscuous
  12338 	 * bits as VLANs become active or inactive.
  12339 	 */
  12340 	eth_broadcast_addr(broadcast);
  12341 	spin_lock_bh(&vsi->mac_filter_hash_lock);
  12342 	i40e_add_mac_filter(vsi, broadcast);
  12343 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
  12344 
  12345 	ether_addr_copy(netdev->dev_addr, mac_addr);
  12346 	ether_addr_copy(netdev->perm_addr, mac_addr);
  12347 
  12348 	/* i40iw_net_event() reads 16 bytes from neigh->primary_key */
  12349 	netdev->neigh_priv_len = sizeof(u32) * 4;
  12350 
  12351 	netdev->priv_flags |= IFF_UNICAST_FLT;
  12352 	netdev->priv_flags |= IFF_SUPP_NOFCS;
  12353 	/* Setup netdev TC information */
  12354 	i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
  12355 
  12356 	netdev->netdev_ops = &i40e_netdev_ops;
  12357 	netdev->watchdog_timeo = 5 * HZ;
  12358 	i40e_set_ethtool_ops(netdev);
  12359 
  12360 	/* MTU range: 68 - 9706 */
  12361 	netdev->min_mtu = ETH_MIN_MTU;
  12362 	netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
  12363 
  12364 	return 0;
  12365 }
  12366 
  12367 /**
  12368  * i40e_vsi_delete - Delete a VSI from the switch
  12369  * @vsi: the VSI being removed
  12370  *
  12371  * Returns 0 on success, negative value on failure
  12372  **/
  12373 static void i40e_vsi_delete(struct i40e_vsi *vsi)
  12374 {
  12375 	/* remove default VSI is not allowed */
  12376 	if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
  12377 		return;
  12378 
  12379 	i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
  12380 }
  12381 
  12382 /**
  12383  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
  12384  * @vsi: the VSI being queried
  12385  *
  12386  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
  12387  **/
  12388 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
  12389 {
  12390 	struct i40e_veb *veb;
  12391 	struct i40e_pf *pf = vsi->back;
  12392 
  12393 	/* Uplink is not a bridge so default to VEB */
  12394 	if (vsi->veb_idx == I40E_NO_VEB)
  12395 		return 1;
  12396 
  12397 	veb = pf->veb[vsi->veb_idx];
  12398 	if (!veb) {
  12399 		dev_info(&pf->pdev->dev,
  12400 			 "There is no veb associated with the bridge\n");
  12401 		return -ENOENT;
  12402 	}
  12403 
  12404 	/* Uplink is a bridge in VEPA mode */
  12405 	if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
  12406 		return 0;
  12407 	} else {
  12408 		/* Uplink is a bridge in VEB mode */
  12409 		return 1;
  12410 	}
  12411 
  12412 	/* VEPA is now default bridge, so return 0 */
  12413 	return 0;
  12414 }
  12415 
  12416 /**
  12417  * i40e_add_vsi - Add a VSI to the switch
  12418  * @vsi: the VSI being configured
  12419  *
  12420  * This initializes a VSI context depending on the VSI type to be added and
  12421  * passes it down to the add_vsi aq command.
  12422  **/
  12423 static int i40e_add_vsi(struct i40e_vsi *vsi)
  12424 {
  12425 	int ret = -ENODEV;
  12426 	struct i40e_pf *pf = vsi->back;
  12427 	struct i40e_hw *hw = &pf->hw;
  12428 	struct i40e_vsi_context ctxt;
  12429 	struct i40e_mac_filter *f;
  12430 	struct hlist_node *h;
  12431 	int bkt;
  12432 
  12433 	u8 enabled_tc = 0x1; /* TC0 enabled */
  12434 	int f_count = 0;
  12435 
  12436 	memset(&ctxt, 0, sizeof(ctxt));
  12437 	switch (vsi->type) {
  12438 	case I40E_VSI_MAIN:
  12439 		/* The PF's main VSI is already setup as part of the
  12440 		 * device initialization, so we'll not bother with
  12441 		 * the add_vsi call, but we will retrieve the current
  12442 		 * VSI context.
  12443 		 */
  12444 		ctxt.seid = pf->main_vsi_seid;
  12445 		ctxt.pf_num = pf->hw.pf_id;
  12446 		ctxt.vf_num = 0;
  12447 		ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
  12448 		ctxt.flags = I40E_AQ_VSI_TYPE_PF;
  12449 		if (ret) {
  12450 			dev_info(&pf->pdev->dev,
  12451 				 "couldn't get PF vsi config, err %s aq_err %s\n",
  12452 				 i40e_stat_str(&pf->hw, ret),
  12453 				 i40e_aq_str(&pf->hw,
  12454 					     pf->hw.aq.asq_last_status));
  12455 			return -ENOENT;
  12456 		}
  12457 		vsi->info = ctxt.info;
  12458 		vsi->info.valid_sections = 0;
  12459 
  12460 		vsi->seid = ctxt.seid;
  12461 		vsi->id = ctxt.vsi_number;
  12462 
  12463 		enabled_tc = i40e_pf_get_tc_map(pf);
  12464 
  12465 		/* Source pruning is enabled by default, so the flag is
  12466 		 * negative logic - if it's set, we need to fiddle with
  12467 		 * the VSI to disable source pruning.
  12468 		 */
  12469 		if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
  12470 			memset(&ctxt, 0, sizeof(ctxt));
  12471 			ctxt.seid = pf->main_vsi_seid;
  12472 			ctxt.pf_num = pf->hw.pf_id;
  12473 			ctxt.vf_num = 0;
  12474 			ctxt.info.valid_sections |=
  12475 				     cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
  12476 			ctxt.info.switch_id =
  12477 				   cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
  12478 			ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
  12479 			if (ret) {
  12480 				dev_info(&pf->pdev->dev,
  12481 					 "update vsi failed, err %s aq_err %s\n",
  12482 					 i40e_stat_str(&pf->hw, ret),
  12483 					 i40e_aq_str(&pf->hw,
  12484 						     pf->hw.aq.asq_last_status));
  12485 				ret = -ENOENT;
  12486 				goto err;
  12487 			}
  12488 		}
  12489 
  12490 		/* MFP mode setup queue map and update VSI */
  12491 		if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
  12492 		    !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
  12493 			memset(&ctxt, 0, sizeof(ctxt));
  12494 			ctxt.seid = pf->main_vsi_seid;
  12495 			ctxt.pf_num = pf->hw.pf_id;
  12496 			ctxt.vf_num = 0;
  12497 			i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
  12498 			ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
  12499 			if (ret) {
  12500 				dev_info(&pf->pdev->dev,
  12501 					 "update vsi failed, err %s aq_err %s\n",
  12502 					 i40e_stat_str(&pf->hw, ret),
  12503 					 i40e_aq_str(&pf->hw,
  12504 						    pf->hw.aq.asq_last_status));
  12505 				ret = -ENOENT;
  12506 				goto err;
  12507 			}
  12508 			/* update the local VSI info queue map */
  12509 			i40e_vsi_update_queue_map(vsi, &ctxt);
  12510 			vsi->info.valid_sections = 0;
  12511 		} else {
  12512 			/* Default/Main VSI is only enabled for TC0
  12513 			 * reconfigure it to enable all TCs that are
  12514 			 * available on the port in SFP mode.
  12515 			 * For MFP case the iSCSI PF would use this
  12516 			 * flow to enable LAN+iSCSI TC.
  12517 			 */
  12518 			ret = i40e_vsi_config_tc(vsi, enabled_tc);
  12519 			if (ret) {
  12520 				/* Single TC condition is not fatal,
  12521 				 * message and continue
  12522 				 */
  12523 				dev_info(&pf->pdev->dev,
  12524 					 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
  12525 					 enabled_tc,
  12526 					 i40e_stat_str(&pf->hw, ret),
  12527 					 i40e_aq_str(&pf->hw,
  12528 						    pf->hw.aq.asq_last_status));
  12529 			}
  12530 		}
  12531 		break;
  12532 
  12533 	case I40E_VSI_FDIR:
  12534 		ctxt.pf_num = hw->pf_id;
  12535 		ctxt.vf_num = 0;
  12536 		ctxt.uplink_seid = vsi->uplink_seid;
  12537 		ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
  12538 		ctxt.flags = I40E_AQ_VSI_TYPE_PF;
  12539 		if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
  12540 		    (i40e_is_vsi_uplink_mode_veb(vsi))) {
  12541 			ctxt.info.valid_sections |=
  12542 			     cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
  12543 			ctxt.info.switch_id =
  12544 			   cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
  12545 		}
  12546 		i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
  12547 		break;
  12548 
  12549 	case I40E_VSI_VMDQ2:
  12550 		ctxt.pf_num = hw->pf_id;
  12551 		ctxt.vf_num = 0;
  12552 		ctxt.uplink_seid = vsi->uplink_seid;
  12553 		ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
  12554 		ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
  12555 
  12556 		/* This VSI is connected to VEB so the switch_id
  12557 		 * should be set to zero by default.
  12558 		 */
  12559 		if (i40e_is_vsi_uplink_mode_veb(vsi)) {
  12560 			ctxt.info.valid_sections |=
  12561 				cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
  12562 			ctxt.info.switch_id =
  12563 				cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
  12564 		}
  12565 
  12566 		/* Setup the VSI tx/rx queue map for TC0 only for now */
  12567 		i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
  12568 		break;
  12569 
  12570 	case I40E_VSI_SRIOV:
  12571 		ctxt.pf_num = hw->pf_id;
  12572 		ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
  12573 		ctxt.uplink_seid = vsi->uplink_seid;
  12574 		ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
  12575 		ctxt.flags = I40E_AQ_VSI_TYPE_VF;
  12576 
  12577 		/* This VSI is connected to VEB so the switch_id
  12578 		 * should be set to zero by default.
  12579 		 */
  12580 		if (i40e_is_vsi_uplink_mode_veb(vsi)) {
  12581 			ctxt.info.valid_sections |=
  12582 				cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
  12583 			ctxt.info.switch_id =
  12584 				cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
  12585 		}
  12586 
  12587 		if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
  12588 			ctxt.info.valid_sections |=
  12589 				cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
  12590 			ctxt.info.queueing_opt_flags |=
  12591 				(I40E_AQ_VSI_QUE_OPT_TCP_ENA |
  12592 				 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
  12593 		}
  12594 
  12595 		ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
  12596 		ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
  12597 		if (pf->vf[vsi->vf_id].spoofchk) {