whiterose

linux unikernel
Log | Files | Refs | README | LICENSE | git clone https://git.ne02ptzero.me/git/whiterose

i40e_xsk.c (20920B)


      1 // SPDX-License-Identifier: GPL-2.0
      2 /* Copyright(c) 2018 Intel Corporation. */
      3 
      4 #include <linux/bpf_trace.h>
      5 #include <net/xdp_sock.h>
      6 #include <net/xdp.h>
      7 
      8 #include "i40e.h"
      9 #include "i40e_txrx_common.h"
     10 #include "i40e_xsk.h"
     11 
     12 /**
     13  * i40e_xsk_umem_dma_map - DMA maps all UMEM memory for the netdev
     14  * @vsi: Current VSI
     15  * @umem: UMEM to DMA map
     16  *
     17  * Returns 0 on success, <0 on failure
     18  **/
     19 static int i40e_xsk_umem_dma_map(struct i40e_vsi *vsi, struct xdp_umem *umem)
     20 {
     21 	struct i40e_pf *pf = vsi->back;
     22 	struct device *dev;
     23 	unsigned int i, j;
     24 	dma_addr_t dma;
     25 
     26 	dev = &pf->pdev->dev;
     27 	for (i = 0; i < umem->npgs; i++) {
     28 		dma = dma_map_page_attrs(dev, umem->pgs[i], 0, PAGE_SIZE,
     29 					 DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
     30 		if (dma_mapping_error(dev, dma))
     31 			goto out_unmap;
     32 
     33 		umem->pages[i].dma = dma;
     34 	}
     35 
     36 	return 0;
     37 
     38 out_unmap:
     39 	for (j = 0; j < i; j++) {
     40 		dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
     41 				     DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
     42 		umem->pages[i].dma = 0;
     43 	}
     44 
     45 	return -1;
     46 }
     47 
     48 /**
     49  * i40e_xsk_umem_dma_unmap - DMA unmaps all UMEM memory for the netdev
     50  * @vsi: Current VSI
     51  * @umem: UMEM to DMA map
     52  **/
     53 static void i40e_xsk_umem_dma_unmap(struct i40e_vsi *vsi, struct xdp_umem *umem)
     54 {
     55 	struct i40e_pf *pf = vsi->back;
     56 	struct device *dev;
     57 	unsigned int i;
     58 
     59 	dev = &pf->pdev->dev;
     60 
     61 	for (i = 0; i < umem->npgs; i++) {
     62 		dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
     63 				     DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
     64 
     65 		umem->pages[i].dma = 0;
     66 	}
     67 }
     68 
     69 /**
     70  * i40e_xsk_umem_enable - Enable/associate a UMEM to a certain ring/qid
     71  * @vsi: Current VSI
     72  * @umem: UMEM
     73  * @qid: Rx ring to associate UMEM to
     74  *
     75  * Returns 0 on success, <0 on failure
     76  **/
     77 static int i40e_xsk_umem_enable(struct i40e_vsi *vsi, struct xdp_umem *umem,
     78 				u16 qid)
     79 {
     80 	struct net_device *netdev = vsi->netdev;
     81 	struct xdp_umem_fq_reuse *reuseq;
     82 	bool if_running;
     83 	int err;
     84 
     85 	if (vsi->type != I40E_VSI_MAIN)
     86 		return -EINVAL;
     87 
     88 	if (qid >= vsi->num_queue_pairs)
     89 		return -EINVAL;
     90 
     91 	if (qid >= netdev->real_num_rx_queues ||
     92 	    qid >= netdev->real_num_tx_queues)
     93 		return -EINVAL;
     94 
     95 	reuseq = xsk_reuseq_prepare(vsi->rx_rings[0]->count);
     96 	if (!reuseq)
     97 		return -ENOMEM;
     98 
     99 	xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
    100 
    101 	err = i40e_xsk_umem_dma_map(vsi, umem);
    102 	if (err)
    103 		return err;
    104 
    105 	set_bit(qid, vsi->af_xdp_zc_qps);
    106 
    107 	if_running = netif_running(vsi->netdev) && i40e_enabled_xdp_vsi(vsi);
    108 
    109 	if (if_running) {
    110 		err = i40e_queue_pair_disable(vsi, qid);
    111 		if (err)
    112 			return err;
    113 
    114 		err = i40e_queue_pair_enable(vsi, qid);
    115 		if (err)
    116 			return err;
    117 
    118 		/* Kick start the NAPI context so that receiving will start */
    119 		err = i40e_xsk_async_xmit(vsi->netdev, qid);
    120 		if (err)
    121 			return err;
    122 	}
    123 
    124 	return 0;
    125 }
    126 
    127 /**
    128  * i40e_xsk_umem_disable - Disassociate a UMEM from a certain ring/qid
    129  * @vsi: Current VSI
    130  * @qid: Rx ring to associate UMEM to
    131  *
    132  * Returns 0 on success, <0 on failure
    133  **/
    134 static int i40e_xsk_umem_disable(struct i40e_vsi *vsi, u16 qid)
    135 {
    136 	struct net_device *netdev = vsi->netdev;
    137 	struct xdp_umem *umem;
    138 	bool if_running;
    139 	int err;
    140 
    141 	umem = xdp_get_umem_from_qid(netdev, qid);
    142 	if (!umem)
    143 		return -EINVAL;
    144 
    145 	if_running = netif_running(vsi->netdev) && i40e_enabled_xdp_vsi(vsi);
    146 
    147 	if (if_running) {
    148 		err = i40e_queue_pair_disable(vsi, qid);
    149 		if (err)
    150 			return err;
    151 	}
    152 
    153 	clear_bit(qid, vsi->af_xdp_zc_qps);
    154 	i40e_xsk_umem_dma_unmap(vsi, umem);
    155 
    156 	if (if_running) {
    157 		err = i40e_queue_pair_enable(vsi, qid);
    158 		if (err)
    159 			return err;
    160 	}
    161 
    162 	return 0;
    163 }
    164 
    165 /**
    166  * i40e_xsk_umem_setup - Enable/disassociate a UMEM to/from a ring/qid
    167  * @vsi: Current VSI
    168  * @umem: UMEM to enable/associate to a ring, or NULL to disable
    169  * @qid: Rx ring to (dis)associate UMEM (from)to
    170  *
    171  * This function enables or disables a UMEM to a certain ring.
    172  *
    173  * Returns 0 on success, <0 on failure
    174  **/
    175 int i40e_xsk_umem_setup(struct i40e_vsi *vsi, struct xdp_umem *umem,
    176 			u16 qid)
    177 {
    178 	return umem ? i40e_xsk_umem_enable(vsi, umem, qid) :
    179 		i40e_xsk_umem_disable(vsi, qid);
    180 }
    181 
    182 /**
    183  * i40e_run_xdp_zc - Executes an XDP program on an xdp_buff
    184  * @rx_ring: Rx ring
    185  * @xdp: xdp_buff used as input to the XDP program
    186  *
    187  * This function enables or disables a UMEM to a certain ring.
    188  *
    189  * Returns any of I40E_XDP_{PASS, CONSUMED, TX, REDIR}
    190  **/
    191 static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
    192 {
    193 	int err, result = I40E_XDP_PASS;
    194 	struct i40e_ring *xdp_ring;
    195 	struct bpf_prog *xdp_prog;
    196 	u32 act;
    197 
    198 	rcu_read_lock();
    199 	/* NB! xdp_prog will always be !NULL, due to the fact that
    200 	 * this path is enabled by setting an XDP program.
    201 	 */
    202 	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
    203 	act = bpf_prog_run_xdp(xdp_prog, xdp);
    204 	xdp->handle += xdp->data - xdp->data_hard_start;
    205 	switch (act) {
    206 	case XDP_PASS:
    207 		break;
    208 	case XDP_TX:
    209 		xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index];
    210 		result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring);
    211 		break;
    212 	case XDP_REDIRECT:
    213 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
    214 		result = !err ? I40E_XDP_REDIR : I40E_XDP_CONSUMED;
    215 		break;
    216 	default:
    217 		bpf_warn_invalid_xdp_action(act);
    218 	case XDP_ABORTED:
    219 		trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
    220 		/* fallthrough -- handle aborts by dropping packet */
    221 	case XDP_DROP:
    222 		result = I40E_XDP_CONSUMED;
    223 		break;
    224 	}
    225 	rcu_read_unlock();
    226 	return result;
    227 }
    228 
    229 /**
    230  * i40e_alloc_buffer_zc - Allocates an i40e_rx_buffer
    231  * @rx_ring: Rx ring
    232  * @bi: Rx buffer to populate
    233  *
    234  * This function allocates an Rx buffer. The buffer can come from fill
    235  * queue, or via the recycle queue (next_to_alloc).
    236  *
    237  * Returns true for a successful allocation, false otherwise
    238  **/
    239 static bool i40e_alloc_buffer_zc(struct i40e_ring *rx_ring,
    240 				 struct i40e_rx_buffer *bi)
    241 {
    242 	struct xdp_umem *umem = rx_ring->xsk_umem;
    243 	void *addr = bi->addr;
    244 	u64 handle, hr;
    245 
    246 	if (addr) {
    247 		rx_ring->rx_stats.page_reuse_count++;
    248 		return true;
    249 	}
    250 
    251 	if (!xsk_umem_peek_addr(umem, &handle)) {
    252 		rx_ring->rx_stats.alloc_page_failed++;
    253 		return false;
    254 	}
    255 
    256 	hr = umem->headroom + XDP_PACKET_HEADROOM;
    257 
    258 	bi->dma = xdp_umem_get_dma(umem, handle);
    259 	bi->dma += hr;
    260 
    261 	bi->addr = xdp_umem_get_data(umem, handle);
    262 	bi->addr += hr;
    263 
    264 	bi->handle = handle + umem->headroom;
    265 
    266 	xsk_umem_discard_addr(umem);
    267 	return true;
    268 }
    269 
    270 /**
    271  * i40e_alloc_buffer_slow_zc - Allocates an i40e_rx_buffer
    272  * @rx_ring: Rx ring
    273  * @bi: Rx buffer to populate
    274  *
    275  * This function allocates an Rx buffer. The buffer can come from fill
    276  * queue, or via the reuse queue.
    277  *
    278  * Returns true for a successful allocation, false otherwise
    279  **/
    280 static bool i40e_alloc_buffer_slow_zc(struct i40e_ring *rx_ring,
    281 				      struct i40e_rx_buffer *bi)
    282 {
    283 	struct xdp_umem *umem = rx_ring->xsk_umem;
    284 	u64 handle, hr;
    285 
    286 	if (!xsk_umem_peek_addr_rq(umem, &handle)) {
    287 		rx_ring->rx_stats.alloc_page_failed++;
    288 		return false;
    289 	}
    290 
    291 	handle &= rx_ring->xsk_umem->chunk_mask;
    292 
    293 	hr = umem->headroom + XDP_PACKET_HEADROOM;
    294 
    295 	bi->dma = xdp_umem_get_dma(umem, handle);
    296 	bi->dma += hr;
    297 
    298 	bi->addr = xdp_umem_get_data(umem, handle);
    299 	bi->addr += hr;
    300 
    301 	bi->handle = handle + umem->headroom;
    302 
    303 	xsk_umem_discard_addr_rq(umem);
    304 	return true;
    305 }
    306 
    307 static __always_inline bool
    308 __i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count,
    309 			   bool alloc(struct i40e_ring *rx_ring,
    310 				      struct i40e_rx_buffer *bi))
    311 {
    312 	u16 ntu = rx_ring->next_to_use;
    313 	union i40e_rx_desc *rx_desc;
    314 	struct i40e_rx_buffer *bi;
    315 	bool ok = true;
    316 
    317 	rx_desc = I40E_RX_DESC(rx_ring, ntu);
    318 	bi = &rx_ring->rx_bi[ntu];
    319 	do {
    320 		if (!alloc(rx_ring, bi)) {
    321 			ok = false;
    322 			goto no_buffers;
    323 		}
    324 
    325 		dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 0,
    326 						 rx_ring->rx_buf_len,
    327 						 DMA_BIDIRECTIONAL);
    328 
    329 		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
    330 
    331 		rx_desc++;
    332 		bi++;
    333 		ntu++;
    334 
    335 		if (unlikely(ntu == rx_ring->count)) {
    336 			rx_desc = I40E_RX_DESC(rx_ring, 0);
    337 			bi = rx_ring->rx_bi;
    338 			ntu = 0;
    339 		}
    340 
    341 		rx_desc->wb.qword1.status_error_len = 0;
    342 		count--;
    343 	} while (count);
    344 
    345 no_buffers:
    346 	if (rx_ring->next_to_use != ntu)
    347 		i40e_release_rx_desc(rx_ring, ntu);
    348 
    349 	return ok;
    350 }
    351 
    352 /**
    353  * i40e_alloc_rx_buffers_zc - Allocates a number of Rx buffers
    354  * @rx_ring: Rx ring
    355  * @count: The number of buffers to allocate
    356  *
    357  * This function allocates a number of Rx buffers from the reuse queue
    358  * or fill ring and places them on the Rx ring.
    359  *
    360  * Returns true for a successful allocation, false otherwise
    361  **/
    362 bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count)
    363 {
    364 	return __i40e_alloc_rx_buffers_zc(rx_ring, count,
    365 					  i40e_alloc_buffer_slow_zc);
    366 }
    367 
    368 /**
    369  * i40e_alloc_rx_buffers_fast_zc - Allocates a number of Rx buffers
    370  * @rx_ring: Rx ring
    371  * @count: The number of buffers to allocate
    372  *
    373  * This function allocates a number of Rx buffers from the fill ring
    374  * or the internal recycle mechanism and places them on the Rx ring.
    375  *
    376  * Returns true for a successful allocation, false otherwise
    377  **/
    378 static bool i40e_alloc_rx_buffers_fast_zc(struct i40e_ring *rx_ring, u16 count)
    379 {
    380 	return __i40e_alloc_rx_buffers_zc(rx_ring, count,
    381 					  i40e_alloc_buffer_zc);
    382 }
    383 
    384 /**
    385  * i40e_get_rx_buffer_zc - Return the current Rx buffer
    386  * @rx_ring: Rx ring
    387  * @size: The size of the rx buffer (read from descriptor)
    388  *
    389  * This function returns the current, received Rx buffer, and also
    390  * does DMA synchronization.  the Rx ring.
    391  *
    392  * Returns the received Rx buffer
    393  **/
    394 static struct i40e_rx_buffer *i40e_get_rx_buffer_zc(struct i40e_ring *rx_ring,
    395 						    const unsigned int size)
    396 {
    397 	struct i40e_rx_buffer *bi;
    398 
    399 	bi = &rx_ring->rx_bi[rx_ring->next_to_clean];
    400 
    401 	/* we are reusing so sync this buffer for CPU use */
    402 	dma_sync_single_range_for_cpu(rx_ring->dev,
    403 				      bi->dma, 0,
    404 				      size,
    405 				      DMA_BIDIRECTIONAL);
    406 
    407 	return bi;
    408 }
    409 
    410 /**
    411  * i40e_reuse_rx_buffer_zc - Recycle an Rx buffer
    412  * @rx_ring: Rx ring
    413  * @old_bi: The Rx buffer to recycle
    414  *
    415  * This function recycles a finished Rx buffer, and places it on the
    416  * recycle queue (next_to_alloc).
    417  **/
    418 static void i40e_reuse_rx_buffer_zc(struct i40e_ring *rx_ring,
    419 				    struct i40e_rx_buffer *old_bi)
    420 {
    421 	struct i40e_rx_buffer *new_bi = &rx_ring->rx_bi[rx_ring->next_to_alloc];
    422 	unsigned long mask = (unsigned long)rx_ring->xsk_umem->chunk_mask;
    423 	u64 hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM;
    424 	u16 nta = rx_ring->next_to_alloc;
    425 
    426 	/* update, and store next to alloc */
    427 	nta++;
    428 	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
    429 
    430 	/* transfer page from old buffer to new buffer */
    431 	new_bi->dma = old_bi->dma & mask;
    432 	new_bi->dma += hr;
    433 
    434 	new_bi->addr = (void *)((unsigned long)old_bi->addr & mask);
    435 	new_bi->addr += hr;
    436 
    437 	new_bi->handle = old_bi->handle & mask;
    438 	new_bi->handle += rx_ring->xsk_umem->headroom;
    439 
    440 	old_bi->addr = NULL;
    441 }
    442 
    443 /**
    444  * i40e_zca_free - Free callback for MEM_TYPE_ZERO_COPY allocations
    445  * @alloc: Zero-copy allocator
    446  * @handle: Buffer handle
    447  **/
    448 void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle)
    449 {
    450 	struct i40e_rx_buffer *bi;
    451 	struct i40e_ring *rx_ring;
    452 	u64 hr, mask;
    453 	u16 nta;
    454 
    455 	rx_ring = container_of(alloc, struct i40e_ring, zca);
    456 	hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM;
    457 	mask = rx_ring->xsk_umem->chunk_mask;
    458 
    459 	nta = rx_ring->next_to_alloc;
    460 	bi = &rx_ring->rx_bi[nta];
    461 
    462 	nta++;
    463 	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
    464 
    465 	handle &= mask;
    466 
    467 	bi->dma = xdp_umem_get_dma(rx_ring->xsk_umem, handle);
    468 	bi->dma += hr;
    469 
    470 	bi->addr = xdp_umem_get_data(rx_ring->xsk_umem, handle);
    471 	bi->addr += hr;
    472 
    473 	bi->handle = (u64)handle + rx_ring->xsk_umem->headroom;
    474 }
    475 
    476 /**
    477  * i40e_construct_skb_zc - Create skbufff from zero-copy Rx buffer
    478  * @rx_ring: Rx ring
    479  * @bi: Rx buffer
    480  * @xdp: xdp_buff
    481  *
    482  * This functions allocates a new skb from a zero-copy Rx buffer.
    483  *
    484  * Returns the skb, or NULL on failure.
    485  **/
    486 static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring,
    487 					     struct i40e_rx_buffer *bi,
    488 					     struct xdp_buff *xdp)
    489 {
    490 	unsigned int metasize = xdp->data - xdp->data_meta;
    491 	unsigned int datasize = xdp->data_end - xdp->data;
    492 	struct sk_buff *skb;
    493 
    494 	/* allocate a skb to store the frags */
    495 	skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
    496 			       xdp->data_end - xdp->data_hard_start,
    497 			       GFP_ATOMIC | __GFP_NOWARN);
    498 	if (unlikely(!skb))
    499 		return NULL;
    500 
    501 	skb_reserve(skb, xdp->data - xdp->data_hard_start);
    502 	memcpy(__skb_put(skb, datasize), xdp->data, datasize);
    503 	if (metasize)
    504 		skb_metadata_set(skb, metasize);
    505 
    506 	i40e_reuse_rx_buffer_zc(rx_ring, bi);
    507 	return skb;
    508 }
    509 
    510 /**
    511  * i40e_inc_ntc: Advance the next_to_clean index
    512  * @rx_ring: Rx ring
    513  **/
    514 static void i40e_inc_ntc(struct i40e_ring *rx_ring)
    515 {
    516 	u32 ntc = rx_ring->next_to_clean + 1;
    517 
    518 	ntc = (ntc < rx_ring->count) ? ntc : 0;
    519 	rx_ring->next_to_clean = ntc;
    520 	prefetch(I40E_RX_DESC(rx_ring, ntc));
    521 }
    522 
    523 /**
    524  * i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring
    525  * @rx_ring: Rx ring
    526  * @budget: NAPI budget
    527  *
    528  * Returns amount of work completed
    529  **/
    530 int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
    531 {
    532 	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
    533 	u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
    534 	unsigned int xdp_res, xdp_xmit = 0;
    535 	bool failure = false;
    536 	struct sk_buff *skb;
    537 	struct xdp_buff xdp;
    538 
    539 	xdp.rxq = &rx_ring->xdp_rxq;
    540 
    541 	while (likely(total_rx_packets < (unsigned int)budget)) {
    542 		struct i40e_rx_buffer *bi;
    543 		union i40e_rx_desc *rx_desc;
    544 		unsigned int size;
    545 		u64 qword;
    546 
    547 		if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
    548 			failure = failure ||
    549 				  !i40e_alloc_rx_buffers_fast_zc(rx_ring,
    550 								 cleaned_count);
    551 			cleaned_count = 0;
    552 		}
    553 
    554 		rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean);
    555 		qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
    556 
    557 		/* This memory barrier is needed to keep us from reading
    558 		 * any other fields out of the rx_desc until we have
    559 		 * verified the descriptor has been written back.
    560 		 */
    561 		dma_rmb();
    562 
    563 		bi = i40e_clean_programming_status(rx_ring, rx_desc,
    564 						   qword);
    565 		if (unlikely(bi)) {
    566 			i40e_reuse_rx_buffer_zc(rx_ring, bi);
    567 			cleaned_count++;
    568 			continue;
    569 		}
    570 
    571 		size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
    572 		       I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
    573 		if (!size)
    574 			break;
    575 
    576 		bi = i40e_get_rx_buffer_zc(rx_ring, size);
    577 		xdp.data = bi->addr;
    578 		xdp.data_meta = xdp.data;
    579 		xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;
    580 		xdp.data_end = xdp.data + size;
    581 		xdp.handle = bi->handle;
    582 
    583 		xdp_res = i40e_run_xdp_zc(rx_ring, &xdp);
    584 		if (xdp_res) {
    585 			if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR)) {
    586 				xdp_xmit |= xdp_res;
    587 				bi->addr = NULL;
    588 			} else {
    589 				i40e_reuse_rx_buffer_zc(rx_ring, bi);
    590 			}
    591 
    592 			total_rx_bytes += size;
    593 			total_rx_packets++;
    594 
    595 			cleaned_count++;
    596 			i40e_inc_ntc(rx_ring);
    597 			continue;
    598 		}
    599 
    600 		/* XDP_PASS path */
    601 
    602 		/* NB! We are not checking for errors using
    603 		 * i40e_test_staterr with
    604 		 * BIT(I40E_RXD_QW1_ERROR_SHIFT). This is due to that
    605 		 * SBP is *not* set in PRT_SBPVSI (default not set).
    606 		 */
    607 		skb = i40e_construct_skb_zc(rx_ring, bi, &xdp);
    608 		if (!skb) {
    609 			rx_ring->rx_stats.alloc_buff_failed++;
    610 			break;
    611 		}
    612 
    613 		cleaned_count++;
    614 		i40e_inc_ntc(rx_ring);
    615 
    616 		if (eth_skb_pad(skb))
    617 			continue;
    618 
    619 		total_rx_bytes += skb->len;
    620 		total_rx_packets++;
    621 
    622 		i40e_process_skb_fields(rx_ring, rx_desc, skb);
    623 		napi_gro_receive(&rx_ring->q_vector->napi, skb);
    624 	}
    625 
    626 	i40e_finalize_xdp_rx(rx_ring, xdp_xmit);
    627 	i40e_update_rx_stats(rx_ring, total_rx_bytes, total_rx_packets);
    628 	return failure ? budget : (int)total_rx_packets;
    629 }
    630 
    631 /**
    632  * i40e_xmit_zc - Performs zero-copy Tx AF_XDP
    633  * @xdp_ring: XDP Tx ring
    634  * @budget: NAPI budget
    635  *
    636  * Returns true if the work is finished.
    637  **/
    638 static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)
    639 {
    640 	struct i40e_tx_desc *tx_desc = NULL;
    641 	struct i40e_tx_buffer *tx_bi;
    642 	bool work_done = true;
    643 	dma_addr_t dma;
    644 	u32 len;
    645 
    646 	while (budget-- > 0) {
    647 		if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) {
    648 			xdp_ring->tx_stats.tx_busy++;
    649 			work_done = false;
    650 			break;
    651 		}
    652 
    653 		if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &dma, &len))
    654 			break;
    655 
    656 		dma_sync_single_for_device(xdp_ring->dev, dma, len,
    657 					   DMA_BIDIRECTIONAL);
    658 
    659 		tx_bi = &xdp_ring->tx_bi[xdp_ring->next_to_use];
    660 		tx_bi->bytecount = len;
    661 
    662 		tx_desc = I40E_TX_DESC(xdp_ring, xdp_ring->next_to_use);
    663 		tx_desc->buffer_addr = cpu_to_le64(dma);
    664 		tx_desc->cmd_type_offset_bsz =
    665 			build_ctob(I40E_TX_DESC_CMD_ICRC
    666 				   | I40E_TX_DESC_CMD_EOP,
    667 				   0, len, 0);
    668 
    669 		xdp_ring->next_to_use++;
    670 		if (xdp_ring->next_to_use == xdp_ring->count)
    671 			xdp_ring->next_to_use = 0;
    672 	}
    673 
    674 	if (tx_desc) {
    675 		/* Request an interrupt for the last frame and bump tail ptr. */
    676 		tx_desc->cmd_type_offset_bsz |= (I40E_TX_DESC_CMD_RS <<
    677 						 I40E_TXD_QW1_CMD_SHIFT);
    678 		i40e_xdp_ring_update_tail(xdp_ring);
    679 
    680 		xsk_umem_consume_tx_done(xdp_ring->xsk_umem);
    681 	}
    682 
    683 	return !!budget && work_done;
    684 }
    685 
    686 /**
    687  * i40e_clean_xdp_tx_buffer - Frees and unmaps an XDP Tx entry
    688  * @tx_ring: XDP Tx ring
    689  * @tx_bi: Tx buffer info to clean
    690  **/
    691 static void i40e_clean_xdp_tx_buffer(struct i40e_ring *tx_ring,
    692 				     struct i40e_tx_buffer *tx_bi)
    693 {
    694 	xdp_return_frame(tx_bi->xdpf);
    695 	dma_unmap_single(tx_ring->dev,
    696 			 dma_unmap_addr(tx_bi, dma),
    697 			 dma_unmap_len(tx_bi, len), DMA_TO_DEVICE);
    698 	dma_unmap_len_set(tx_bi, len, 0);
    699 }
    700 
    701 /**
    702  * i40e_clean_xdp_tx_irq - Completes AF_XDP entries, and cleans XDP entries
    703  * @tx_ring: XDP Tx ring
    704  * @tx_bi: Tx buffer info to clean
    705  *
    706  * Returns true if cleanup/tranmission is done.
    707  **/
    708 bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi,
    709 			   struct i40e_ring *tx_ring, int napi_budget)
    710 {
    711 	unsigned int ntc, total_bytes = 0, budget = vsi->work_limit;
    712 	u32 i, completed_frames, frames_ready, xsk_frames = 0;
    713 	struct xdp_umem *umem = tx_ring->xsk_umem;
    714 	u32 head_idx = i40e_get_head(tx_ring);
    715 	bool work_done = true, xmit_done;
    716 	struct i40e_tx_buffer *tx_bi;
    717 
    718 	if (head_idx < tx_ring->next_to_clean)
    719 		head_idx += tx_ring->count;
    720 	frames_ready = head_idx - tx_ring->next_to_clean;
    721 
    722 	if (frames_ready == 0) {
    723 		goto out_xmit;
    724 	} else if (frames_ready > budget) {
    725 		completed_frames = budget;
    726 		work_done = false;
    727 	} else {
    728 		completed_frames = frames_ready;
    729 	}
    730 
    731 	ntc = tx_ring->next_to_clean;
    732 
    733 	for (i = 0; i < completed_frames; i++) {
    734 		tx_bi = &tx_ring->tx_bi[ntc];
    735 
    736 		if (tx_bi->xdpf)
    737 			i40e_clean_xdp_tx_buffer(tx_ring, tx_bi);
    738 		else
    739 			xsk_frames++;
    740 
    741 		tx_bi->xdpf = NULL;
    742 		total_bytes += tx_bi->bytecount;
    743 
    744 		if (++ntc >= tx_ring->count)
    745 			ntc = 0;
    746 	}
    747 
    748 	tx_ring->next_to_clean += completed_frames;
    749 	if (unlikely(tx_ring->next_to_clean >= tx_ring->count))
    750 		tx_ring->next_to_clean -= tx_ring->count;
    751 
    752 	if (xsk_frames)
    753 		xsk_umem_complete_tx(umem, xsk_frames);
    754 
    755 	i40e_arm_wb(tx_ring, vsi, budget);
    756 	i40e_update_tx_stats(tx_ring, completed_frames, total_bytes);
    757 
    758 out_xmit:
    759 	xmit_done = i40e_xmit_zc(tx_ring, budget);
    760 
    761 	return work_done && xmit_done;
    762 }
    763 
    764 /**
    765  * i40e_xsk_async_xmit - Implements the ndo_xsk_async_xmit
    766  * @dev: the netdevice
    767  * @queue_id: queue id to wake up
    768  *
    769  * Returns <0 for errors, 0 otherwise.
    770  **/
    771 int i40e_xsk_async_xmit(struct net_device *dev, u32 queue_id)
    772 {
    773 	struct i40e_netdev_priv *np = netdev_priv(dev);
    774 	struct i40e_vsi *vsi = np->vsi;
    775 	struct i40e_ring *ring;
    776 
    777 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
    778 		return -ENETDOWN;
    779 
    780 	if (!i40e_enabled_xdp_vsi(vsi))
    781 		return -ENXIO;
    782 
    783 	if (queue_id >= vsi->num_queue_pairs)
    784 		return -ENXIO;
    785 
    786 	if (!vsi->xdp_rings[queue_id]->xsk_umem)
    787 		return -ENXIO;
    788 
    789 	ring = vsi->xdp_rings[queue_id];
    790 
    791 	/* The idea here is that if NAPI is running, mark a miss, so
    792 	 * it will run again. If not, trigger an interrupt and
    793 	 * schedule the NAPI from interrupt context. If NAPI would be
    794 	 * scheduled here, the interrupt affinity would not be
    795 	 * honored.
    796 	 */
    797 	if (!napi_if_scheduled_mark_missed(&ring->q_vector->napi))
    798 		i40e_force_wb(vsi, ring->q_vector);
    799 
    800 	return 0;
    801 }
    802 
    803 void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring)
    804 {
    805 	u16 i;
    806 
    807 	for (i = 0; i < rx_ring->count; i++) {
    808 		struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i];
    809 
    810 		if (!rx_bi->addr)
    811 			continue;
    812 
    813 		xsk_umem_fq_reuse(rx_ring->xsk_umem, rx_bi->handle);
    814 		rx_bi->addr = NULL;
    815 	}
    816 }
    817 
    818 /**
    819  * i40e_xsk_clean_xdp_ring - Clean the XDP Tx ring on shutdown
    820  * @xdp_ring: XDP Tx ring
    821  **/
    822 void i40e_xsk_clean_tx_ring(struct i40e_ring *tx_ring)
    823 {
    824 	u16 ntc = tx_ring->next_to_clean, ntu = tx_ring->next_to_use;
    825 	struct xdp_umem *umem = tx_ring->xsk_umem;
    826 	struct i40e_tx_buffer *tx_bi;
    827 	u32 xsk_frames = 0;
    828 
    829 	while (ntc != ntu) {
    830 		tx_bi = &tx_ring->tx_bi[ntc];
    831 
    832 		if (tx_bi->xdpf)
    833 			i40e_clean_xdp_tx_buffer(tx_ring, tx_bi);
    834 		else
    835 			xsk_frames++;
    836 
    837 		tx_bi->xdpf = NULL;
    838 
    839 		ntc++;
    840 		if (ntc >= tx_ring->count)
    841 			ntc = 0;
    842 	}
    843 
    844 	if (xsk_frames)
    845 		xsk_umem_complete_tx(umem, xsk_frames);
    846 }
    847 
    848 /**
    849  * i40e_xsk_any_rx_ring_enabled - Checks if Rx rings have AF_XDP UMEM attached
    850  * @vsi: vsi
    851  *
    852  * Returns true if any of the Rx rings has an AF_XDP UMEM attached
    853  **/
    854 bool i40e_xsk_any_rx_ring_enabled(struct i40e_vsi *vsi)
    855 {
    856 	struct net_device *netdev = vsi->netdev;
    857 	int i;
    858 
    859 	for (i = 0; i < vsi->num_queue_pairs; i++) {
    860 		if (xdp_get_umem_from_qid(netdev, i))
    861 			return true;
    862 	}
    863 
    864 	return false;
    865 }